diff --git a/.gitmodules b/.gitmodules index 9d649bcdd3..31c211bf89 100644 --- a/.gitmodules +++ b/.gitmodules @@ -21,4 +21,4 @@ url = https://github.com/taosdata/taosadapter.git [submodule "tools/taosws-rs"] path = tools/taosws-rs - url = https://github.com/taosdata/taosws-rs.git + url = https://github.com/taosdata/taosws-rs diff --git a/docs/zh/10-cluster/01-deploy.md b/docs/zh/10-cluster/01-deploy.md index b44d2942f2..9dbfb50f18 100644 --- a/docs/zh/10-cluster/01-deploy.md +++ b/docs/zh/10-cluster/01-deploy.md @@ -10,7 +10,7 @@ title: 集群部署 ### 第一步 -如果搭建集群的物理节点中，存有之前的测试数据、装过 1.X 的版本，或者装过其他版本的 TDengine，请先将其删除，并清空所有数据（如果需要保留原有数据，请联系涛思交付团队进行旧版本升级、数据迁移），具体步骤请参考博客[《TDengine 多种安装包的安装和卸载》](https://www.taosdata.com/blog/2019/08/09/566.html)。 +如果搭建集群的物理节点中，存有之前的测试数据，或者装过其他版本的 TDengine，请先将其删除，并清空所有数据（如果需要保留原有数据，请联系涛思交付团队进行旧版本升级、数据迁移），具体步骤请参考博客[《TDengine 多种安装包的安装和卸载》](https://www.taosdata.com/blog/2019/08/09/566.html)。 :::note 因为 FQDN 的信息会写进文件，如果之前没有配置或者更改 FQDN，且启动了 TDengine。请一定在确保数据无用或者备份的前提下，清理一下之前的数据（rm -rf /var/lib/taos/\*）； @@ -54,30 +54,16 @@ fqdn h1.taosdata.com // 配置本数据节点的端口号，缺省是 6030 serverPort 6030 -// 副本数为偶数的时候，需要配置，请参考《Arbitrator 的使用》的部分 -arbitrator ha.taosdata.com:6042 -``` - 一定要修改的参数是 firstEp 和 fqdn。在每个数据节点，firstEp 需全部配置成一样，但 fqdn 一定要配置成其所在数据节点的值。其他参数可不做任何修改，除非你很清楚为什么要修改。 -加入到集群中的数据节点 dnode，涉及集群相关的下表 9 项参数必须完全相同，否则不能成功加入到集群中。 +加入到集群中的数据节点 dnode，下表中涉及集群相关的参数必须完全相同，否则不能成功加入到集群中。 | **#** | **配置参数名称** | **含义** | | ----- | ------------------ | ------------------------------------------- | -| 1 | numOfMnodes | 系统中管理节点个数 | -| 2 | mnodeEqualVnodeNum | 一个 mnode 等同于 vnode 消耗的个数 | -| 3 | offlineThreshold | dnode 离线阈值，超过该时间将导致 Dnode 离线 | -| 4 | statusInterval | dnode 向 mnode 报告状态时长 | -| 5 | arbitrator | 系统中裁决器的 End Point | -| 6 | timezone | 时区 | -| 7 | balance | 是否启动负载均衡 | -| 8 | maxTablesPerVnode | 每个 vnode 中能够创建的最大表个数 | -| 9 | maxVgroupsPerDb | 每个 DB 中能够使用的最大 vgroup 个数 | - -:::note -在 2.0.19.0 及更早的版本中，除以上 9 项参数外，dnode 加入集群时，还会要求 locale 和 charset 参数的取值也一致。 - -::: +| 1 | statusInterval | dnode 向 mnode 报告状态时长 | +| 2 | timezone | 时区 | +| 3 | locale | 系统区位信息及编码格式 | +| 4 | charset | 字符集编码 | ## 启动集群 diff --git a/docs/zh/10-cluster/02-cluster-mgmt.md b/docs/zh/10-cluster/02-cluster-mgmt.md index 6ab8ec091b..f02a105826 100644 --- a/docs/zh/10-cluster/02-cluster-mgmt.md +++ b/docs/zh/10-cluster/02-cluster-mgmt.md @@ -24,15 +24,15 @@ SHOW DNODES; ``` taos> show dnodes; - id | end_point | vnodes | cores | status | role | create_time | offline reason | -====================================================================================================================================== - 1 | localhost:6030 | 9 | 8 | ready | any | 2022-04-15 08:27:09.359 | | -Query OK, 1 row(s) in set (0.008298s) + id | endpoint | vnodes | support_vnodes | status | create_time | note | +============================================================================================================================================ + 1 | trd01:6030 | 100 | 1024 | ready | 2022-07-15 16:47:47.726 | | +Query OK, 1 rows affected (0.006684s) ``` ## 查看虚拟节点组 -为充分利用多核技术，并提供 scalability，数据需要分片处理。因此 TDengine 会将一个 DB 的数据切分成多份，存放在多个 vnode 里。这些 vnode 可能分布在多个数据节点 dnode 里，这样就实现了水平扩展。一个 vnode 仅仅属于一个 DB，但一个 DB 可以有多个 vnode。vnode 所在的数据节点是 mnode 根据当前系统资源的情况，自动进行分配的，无需任何人工干预。 +为充分利用多核技术，并提供横向扩展能力，数据需要分片处理。因此 TDengine 会将一个 DB 的数据切分成多份，存放在多个 vnode 里。这些 vnode 可能分布在多个数据节点 dnode 里，这样就实现了水平扩展。一个 vnode 仅仅属于一个 DB，但一个 DB 可以有多个 vnode。vnode 所在的数据节点是 mnode 根据当前系统资源的情况，自动进行分配的，无需任何人工干预。 启动 CLI 程序 taos，然后执行： @@ -44,26 +44,15 @@ SHOW VGROUPS; 输出如下（具体内容仅供参考，取决于实际的集群配置） ``` -taos> show dnodes; - id | end_point | vnodes | cores | status | role | create_time | offline reason | -====================================================================================================================================== - 1 | localhost:6030 | 9 | 8 | ready | any | 2022-04-15 08:27:09.359 | | -Query OK, 1 row(s) in set (0.008298s) - taos> use db; Database changed. taos> show vgroups; - vgId | tables | status | onlines | v1_dnode | v1_status | compacting | -========================================================================================== - 14 | 38000 | ready | 1 | 1 | master | 0 | - 15 | 38000 | ready | 1 | 1 | master | 0 | - 16 | 38000 | ready | 1 | 1 | master | 0 | - 17 | 38000 | ready | 1 | 1 | master | 0 | - 18 | 37001 | ready | 1 | 1 | master | 0 | - 19 | 37000 | ready | 1 | 1 | master | 0 | - 20 | 37000 | ready | 1 | 1 | master | 0 | - 21 | 37000 | ready | 1 | 1 | master | 0 | + vgroup_id | db_name | tables | v1_dnode | v1_status | v2_dnode | v2_status | v3_dnode | v3_status | status | nfiles | file_size | tsma | +================================================================================================================================================================================================ + 2 | db | 0 | 1 | leader | NULL | NULL | NULL | NULL | NULL | NULL | NULL | 0 | + 3 | db | 0 | 1 | leader | NULL | NULL | NULL | NULL | NULL | NULL | NULL | 0 | + 4 | db | 0 | 1 | leader | NULL | NULL | NULL | NULL | NULL | NULL | NULL | 0 | Query OK, 8 row(s) in set (0.001154s) ``` @@ -77,35 +66,21 @@ CREATE DNODE "fqdn:port"; 将新数据节点的 End Point 添加进集群的 EP 列表。“fqdn:port“需要用双引号引起来，否则出错。一个数据节点对外服务的 fqdn 和 port 可以通过配置文件 taos.cfg 进行配置，缺省是自动获取。【强烈不建议用自动获取方式来配置 FQDN，可能导致生成的数据节点的 End Point 不是所期望的】 -示例如下： -``` -taos> create dnode "localhost:7030"; -Query OK, 0 of 0 row(s) in database (0.008203s) - -taos> show dnodes; - id | end_point | vnodes | cores | status | role | create_time | offline reason | -====================================================================================================================================== - 1 | localhost:6030 | 9 | 8 | ready | any | 2022-04-15 08:27:09.359 | | - 2 | localhost:7030 | 0 | 0 | offline | any | 2022-04-19 08:11:42.158 | status not received | -Query OK, 2 row(s) in set (0.001017s) -``` - -在上面的示例中可以看到新创建的 dnode 的状态为 offline，待该 dnode 被启动并连接上配置文件中指定的 firstEp后再次查看，得到如下结果（示例） +然后启动新加入的数据节点的 taosd 进程，再通过 taos 查看数据节点状态： ``` taos> show dnodes; - id | end_point | vnodes | cores | status | role | create_time | offline reason | -====================================================================================================================================== - 1 | localhost:6030 | 3 | 8 | ready | any | 2022-04-15 08:27:09.359 | | - 2 | localhost:7030 | 6 | 8 | ready | any | 2022-04-19 08:14:59.165 | | -Query OK, 2 row(s) in set (0.001316s) + id | endpoint | vnodes | support_vnodes | status | create_time | note | +============================================================================================================================================ + 1 | trd01:6030 | 100 | 1024 | ready | 2022-07-15 16:47:47.726 | | + 2 | trd04:6030 | 0 | 1024 | ready | 2022-07-15 16:56:13.670 | | +Query OK, 2 rows affected (0.007031s) ``` 从中可以看到两个 dnode 状态都为 ready - ## 删除数据节点 -启动 CLI 程序 taos，然后执行： +先停止要删除的数据节点的 taosd 进程，然后启动 CLI 程序 taos，执行： ```sql DROP DNODE "fqdn:port"; @@ -117,26 +92,6 @@ DROP DNODE dnodeId; 通过 “fqdn:port” 或 dnodeID 来指定一个具体的节点都是可以的。其中 fqdn 是被删除的节点的 FQDN，port 是其对外服务器的端口号；dnodeID 可以通过 SHOW DNODES 获得。 -示例如下： -``` -taos> show dnodes; - id | end_point | vnodes | cores | status | role | create_time | offline reason | -====================================================================================================================================== - 1 | localhost:6030 | 9 | 8 | ready | any | 2022-04-15 08:27:09.359 | | - 2 | localhost:7030 | 0 | 0 | offline | any | 2022-04-19 08:11:42.158 | status not received | -Query OK, 2 row(s) in set (0.001017s) - -taos> drop dnode 2; -Query OK, 0 of 0 row(s) in database (0.000518s) - -taos> show dnodes; - id | end_point | vnodes | cores | status | role | create_time | offline reason | -====================================================================================================================================== - 1 | localhost:6030 | 9 | 8 | ready | any | 2022-04-15 08:27:09.359 | | -Query OK, 1 row(s) in set (0.001137s) -``` - -上面的示例中，初次执行 `show dnodes` 列出了两个 dnode, 执行 `drop dnode 2` 删除其中 ID 为 2 的 dnode 之后再次执行 `show dnodes`，可以看到只剩下 ID 为 1 的 dnode 。 :::warning @@ -147,70 +102,4 @@ dnodeID 是集群自动分配的，不得人工指定。它在生成时是递增 ::: -## 手动迁移数据节点 - -手动将某个 vnode 迁移到指定的 dnode。 - -启动 CLI 程序 taos，然后执行： - -```sql -ALTER DNODE BALANCE "VNODE:-DNODE:"; -``` - -其中：source-dnodeId 是源 dnodeId，也就是待迁移的 vnode 所在的 dnodeID；vgId 可以通过 SHOW VGROUPS 获得，列表的第一列；dest-dnodeId 是目标 dnodeId。 - -首先执行 `show vgroups` 查看 vgroup 的分布情况 -``` -taos> show vgroups; - vgId | tables | status | onlines | v1_dnode | v1_status | compacting | -========================================================================================== - 14 | 38000 | ready | 1 | 3 | master | 0 | - 15 | 38000 | ready | 1 | 3 | master | 0 | - 16 | 38000 | ready | 1 | 3 | master | 0 | - 17 | 38000 | ready | 1 | 3 | master | 0 | - 18 | 37001 | ready | 1 | 3 | master | 0 | - 19 | 37000 | ready | 1 | 1 | master | 0 | - 20 | 37000 | ready | 1 | 1 | master | 0 | - 21 | 37000 | ready | 1 | 1 | master | 0 | -Query OK, 8 row(s) in set (0.001314s) -``` - -从中可以看到在 dnode 3 中有5个 vgroup，而 dnode 1 有 3 个 vgroup，假定我们想将其中 vgId 为18 的 vgroup 从 dnode 3 迁移到 dnode 1 - -``` -taos> alter dnode 3 balance "vnode:18-dnode:1"; - -DB error: Balance already enabled (0.00755 -``` - -上面的结果表明目前所在数据库已经启动了 balance 选项，所以无法进行手动迁移。 - -停止整个集群，将两个 dnode 的配置文件中的 balance 都设置为 0 （默认为1）之后，重新启动集群，再次执行 ` alter dnode` 和 `show vgroups` 命令如下 -``` -taos> alter dnode 3 balance "vnode:18-dnode:1"; -Query OK, 0 row(s) in set (0.000575s) - -taos> show vgroups; - vgId | tables | status | onlines | v1_dnode | v1_status | v2_dnode | v2_status | compacting | -================================================================================================================= - 14 | 38000 | ready | 1 | 3 | master | 0 | NULL | 0 | - 15 | 38000 | ready | 1 | 3 | master | 0 | NULL | 0 | - 16 | 38000 | ready | 1 | 3 | master | 0 | NULL | 0 | - 17 | 38000 | ready | 1 | 3 | master | 0 | NULL | 0 | - 18 | 37001 | ready | 2 | 1 | slave | 3 | master | 0 | - 19 | 37000 | ready | 1 | 1 | master | 0 | NULL | 0 | - 20 | 37000 | ready | 1 | 1 | master | 0 | NULL | 0 | - 21 | 37000 | ready | 1 | 1 | master | 0 | NULL | 0 | -Query OK, 8 row(s) in set (0.001242s) -``` - -从上面的输出可以看到 vgId 为 18 的 vnode 被从 dnode 3 迁移到了 dnode 1。 - -:::warning - -只有在集群的自动负载均衡选项关闭时（balance 设置为 0），才允许手动迁移。 -只有处于正常工作状态的 vnode 才能被迁移：master/slave；当处于 offline/unsynced/syncing 状态时，是不能迁移的。 -迁移前，务必核实目标 dnode 的资源足够：CPU、内存、硬盘。 - -::: diff --git a/docs/zh/10-cluster/03-ha-and-lb.md b/docs/zh/10-cluster/03-ha-and-lb.md deleted file mode 100644 index 3d15feb11c..0000000000 --- a/docs/zh/10-cluster/03-ha-and-lb.md +++ /dev/null @@ -1,87 +0,0 @@ ---- -title: 高可用与负载均衡 ---- - -## Vnode 的高可用性 - -TDengine 通过多副本的机制来提供系统的高可用性，包括 vnode 和 mnode 的高可用性。 - -vnode 的副本数是与 DB 关联的，一个集群里可以有多个 DB，根据运营的需求，每个 DB 可以配置不同的副本数。创建数据库时，通过参数 replica 指定副本数（缺省为 1）。如果副本数为 1，系统的可靠性无法保证，只要数据所在的节点宕机，就将无法提供服务。集群的节点数必须大于等于副本数，否则创建表时将返回错误“more dnodes are needed”。比如下面的命令将创建副本数为 3 的数据库 demo： - -```sql -CREATE DATABASE demo replica 3; -``` - -一个 DB 里的数据会被切片分到多个 vnode group，vnode group 里的 vnode 数目就是 DB 的副本数，同一个 vnode group 里各 vnode 的数据是完全一致的。为保证高可用性，vnode group 里的 vnode 一定要分布在不同的数据节点 dnode 里（实际部署时，需要在不同的物理机上），只要一个 vnode group 里超过半数的 vnode 处于工作状态，这个 vnode group 就能正常的对外服务。 - -一个数据节点 dnode 里可能有多个 DB 的数据，因此一个 dnode 离线时，可能会影响到多个 DB。如果一个 vnode group 里的一半或一半以上的 vnode 不工作，那么该 vnode group 就无法对外服务，无法插入或读取数据，这样会影响到它所属的 DB 的一部分表的读写操作。 - -因为 vnode 的引入，无法简单地给出结论：“集群中过半数据节点 dnode 工作，集群就应该工作”。但是对于简单的情形，很好下结论。比如副本数为 3，只有三个 dnode，那如果仅有一个节点不工作，整个集群还是可以正常工作的，但如果有两个数据节点不工作，那整个集群就无法正常工作了。 - -## Mnode 的高可用性 - -TDengine 集群是由 mnode（taosd 的一个模块，管理节点）负责管理的，为保证 mnode 的高可用，可以配置多个 mnode 副本，副本数由系统配置参数 numOfMnodes 决定，有效范围为 1-3。为保证元数据的强一致性，mnode 副本之间是通过同步的方式进行数据复制的。 - -一个集群有多个数据节点 dnode，但一个 dnode 至多运行一个 mnode 实例。多个 dnode 情况下，哪个 dnode 可以作为 mnode 呢？这是完全由系统根据整个系统资源情况，自动指定的。用户可通过 CLI 程序 taos，在 TDengine 的 console 里，执行如下命令： - -```sql -SHOW MNODES; -``` - -来查看 mnode 列表，该列表将列出 mnode 所处的 dnode 的 End Point 和角色（master，slave，unsynced 或 offline）。当集群中第一个数据节点启动时，该数据节点一定会运行一个 mnode 实例，否则该数据节点 dnode 无法正常工作，因为一个系统是必须有至少一个 mnode 的。如果 numOfMnodes 配置为 2，启动第二个 dnode 时，该 dnode 也将运行一个 mnode 实例。 - -为保证 mnode 服务的高可用性，numOfMnodes 必须设置为 2 或更大。因为 mnode 保存的元数据必须是强一致的，如果 numOfMnodes 大于 2，复制参数 quorum 自动设为 2，也就是说，至少要保证有两个副本写入数据成功，才通知客户端应用写入成功。 - -:::note -一个 TDengine 高可用系统，无论是 vnode 还是 mnode，都必须配置多个副本。 - -::: - -## 负载均衡 - -有三种情况，将触发负载均衡，而且都无需人工干预。 - -当一个新数据节点添加进集群时，系统将自动触发负载均衡，一些节点上的数据将被自动转移到新数据节点上，无需任何人工干预。 -当一个数据节点从集群中移除时，系统将自动把该数据节点上的数据转移到其他数据节点，无需任何人工干预。 -如果一个数据节点过热（数据量过大），系统将自动进行负载均衡，将该数据节点的一些 vnode 自动挪到其他节点。 -当上述三种情况发生时，系统将启动各个数据节点的负载计算，从而决定如何挪动。 - -:::tip -负载均衡由参数 balance 控制，它决定是否启动自动负载均衡，0 表示禁用，1 表示启用自动负载均衡。 - -::: - -## 数据节点离线处理 - -如果一个数据节点离线，TDengine 集群将自动检测到。有如下两种情况： - -该数据节点离线超过一定时间（taos.cfg 里配置参数 offlineThreshold 控制时长），系统将自动把该数据节点删除，产生系统报警信息，触发负载均衡流程。如果该被删除的数据节点重新上线时，它将无法加入集群，需要系统管理员重新将其添加进集群才会开始工作。 - -离线后，在 offlineThreshold 的时长内重新上线，系统将自动启动数据恢复流程，等数据完全恢复后，该节点将开始正常工作。 - -:::note -如果一个虚拟节点组（包括 mnode 组）里所归属的每个数据节点都处于离线或 unsynced 状态，必须等该虚拟节点组里的所有数据节点都上线、都能交换状态信息后，才能选出 Master，该虚拟节点组才能对外提供服务。比如整个集群有 3 个数据节点，副本数为 3，如果 3 个数据节点都宕机，然后 2 个数据节点重启，是无法工作的，只有等 3 个数据节点都重启成功，才能对外服务。 - -::: - -## Arbitrator 的使用 - -如果副本数为偶数，当一个 vnode group 里一半或超过一半的 vnode 不工作时，是无法从中选出 master 的。同理，一半或超过一半的 mnode 不工作时，是无法选出 mnode 的 master 的，因为存在“split brain”问题。 - -为解决这个问题，TDengine 引入了 Arbitrator 的概念。Arbitrator 模拟一个 vnode 或 mnode 在工作，但只简单的负责网络连接，不处理任何数据插入或访问。只要包含 Arbitrator 在内，超过半数的 vnode 或 mnode 工作，那么该 vnode group 或 mnode 组就可以正常的提供数据插入或查询服务。比如对于副本数为 2 的情形，如果一个节点 A 离线，但另外一个节点 B 正常，而且能连接到 Arbitrator，那么节点 B 就能正常工作。 - -总之，在目前版本下，TDengine 建议在双副本环境要配置 Arbitrator，以提升系统的可用性。 - -Arbitrator 的执行程序名为 tarbitrator。该程序对系统资源几乎没有要求，只需要保证有网络连接，找任何一台 Linux 服务器运行它即可。以下简要描述安装配置的步骤： - -请点击 安装包下载，在 TDengine Arbitrator Linux 一节中，选择合适的版本下载并安装。 -该应用的命令行参数 -p 可以指定其对外服务的端口号，缺省是 6042。 - -修改每个 taosd 实例的配置文件，在 taos.cfg 里将参数 arbitrator 设置为 tarbitrator 程序所对应的 End Point。（如果该参数配置了，当副本数为偶数时，系统将自动连接配置的 Arbitrator。如果副本数为奇数，即使配置了 Arbitrator，系统也不会去建立连接。） - -在配置文件中配置了的 Arbitrator，会出现在 SHOW DNODES 指令的返回结果中，对应的 role 列的值会是“arb”。 -查看集群 Arbitrator 的状态【2.0.14.0 以后支持】 - -```sql -SHOW DNODES; -``` diff --git a/docs/zh/10-cluster/03-high-availability.md b/docs/zh/10-cluster/03-high-availability.md new file mode 100644 index 0000000000..36f9300366 --- /dev/null +++ b/docs/zh/10-cluster/03-high-availability.md @@ -0,0 +1,33 @@ +--- +title: 高可用 +--- + +## Vnode 的高可用性 + +TDengine 通过多副本的机制来提供系统的高可用性，包括 vnode 和 mnode 的高可用性。 + +vnode 的副本数是与 DB 关联的，一个集群里可以有多个 DB，根据运营的需求，每个 DB 可以配置不同的副本数。创建数据库时，通过参数 replica 指定副本数（缺省为 1）。如果副本数为 1，系统的可靠性无法保证，只要数据所在的节点宕机，就将无法提供服务。集群的节点数必须大于等于副本数，否则创建表时将返回错误“more dnodes are needed”。比如下面的命令将创建副本数为 3 的数据库 demo： + +```sql +CREATE DATABASE demo replica 3; +``` + +一个 DB 里的数据会被切片分到多个 vnode group，vnode group 里的 vnode 数目就是 DB 的副本数，同一个 vnode group 里各 vnode 的数据是完全一致的。为保证高可用性，vnode group 里的 vnode 一定要分布在不同的数据节点 dnode 里（实际部署时，需要在不同的物理机上），只要一个 vnode group 里超过半数的 vnode 处于工作状态，这个 vnode group 就能正常的对外服务。 + +一个数据节点 dnode 里可能有多个 DB 的数据，因此一个 dnode 离线时，可能会影响到多个 DB。如果一个 vnode group 里的一半或一半以上的 vnode 不工作，那么该 vnode group 就无法对外服务，无法插入或读取数据，这样会影响到它所属的 DB 的一部分表的读写操作。 + +因为 vnode 的引入，无法简单地给出结论：“集群中过半数据节点 dnode 工作，集群就应该工作”。但是对于简单的情形，很好下结论。比如副本数为 3，只有三个 dnode，那如果仅有一个节点不工作，整个集群还是可以正常工作的，但如果有两个数据节点不工作，那整个集群就无法正常工作了。 + +## Mnode 的高可用性 + +TDengine 集群是由 mnode（taosd 的一个模块，管理节点）负责管理的，为保证 mnode 的高可用，可以配置多个 mnode 副本，在集群启动时只有一个 mnode，用户可以通过 `create mnode` 来增加新的 mnode。用户可以通过该命令自主决定哪几个 dnode 会承担 mnode 的角色。为保证元数据的强一致性，在有多个 mnode 时，mnode 副本之间是通过同步的方式进行数据复制的。 + +一个集群有多个数据节点 dnode，但一个 dnode 至多运行一个 mnode 实例。用户可通过 CLI 程序 taos，在 TDengine 的 console 里，执行如下命令： + +```sql +SHOW MNODES; +``` + +来查看 mnode 列表，该列表将列出 mnode 所处的 dnode 的 End Point 和角色（leader, follower, candidate）。当集群中第一个数据节点启动时，该数据节点一定会运行一个 mnode 实例，否则该数据节点 dnode 无法正常工作，因为一个系统是必须有至少一个 mnode 的。 + +在 TDengine 3.0 及以后的版本中，数据同步采用 RAFT 协议，所以 mnode 的数量应该被设置为 1 个或者 3 个。 diff --git a/docs/zh/10-cluster/04-load-balance.md b/docs/zh/10-cluster/04-load-balance.md new file mode 100644 index 0000000000..c1a72cf006 --- /dev/null +++ b/docs/zh/10-cluster/04-load-balance.md @@ -0,0 +1,19 @@ +--- +title: 负载均衡 +--- + +TDengine 中的负载均衡主要指对时序数据的处理的负载均衡。TDengine 采用 Hash 一致性算法将一个数据库中的所有表和子表的数据均衡分散在属于该数据库的所有 vgroups 中，每张表或子表只能由一个 vgroups 处理，一个 vgroups 可能负责处理多个表或子表。 + +创建数据库时可以指定其中的 vgroups 的数量： + +```sql +create database db0 vgroups 100; +``` + +如何指定合适的 vgroups 的数量，这取决于系统资源。假定系统中只计划建立一个数据库，则 vgroups 由集群中所有 dnode 所能使用的资源决定。原则上可用的 CPU 和 Memory 越多，可建立的 vgroups 也越多。但也要考虑到磁盘性能，过多的 vgroups 在磁盘性能达到上限后反而会拖累整个系统的性能。假如系统中会建立多个数据库，则多个数据库的 vgoups 之和取决于系统中可用资源的数量。要综合考虑多个数据库之间表的数量、写入频率、数据量等多个因素在多个数据库之间分配 vgroups。实际中建议首先根据系统资源配置选择一个初始的 vgroups 数量，比如 CPU 总核数的 2 倍，以此为起点通过测试找到最佳的 vgroups 数量配置，此为系统中的 vgroups 总数。如果有多个数据库的话，再根据各个数据库的表数和数据量对 vgroups 进行分配。 + +此外，对于任意数据库的 vgroups，TDengine 都是尽可能将其均衡分散在多个 dnode 上。在多副本情况下（replica 3），这种均衡分布尤其复杂，TDengine 的分布策略会尽量避免任意一个 dnode 成为写入的瓶颈。 + +通过以上措施可以最大限度地在整个 TDengine 集群中实现负载均衡，负载均衡也能反过来提升系统总的数据处理能力。 + +在初始的负载均衡建立起来之后，如果由于删库、删表等动作，特别是删库动作会导致属于它的 vnode 都被删除，这有可能会造成一定程度的负载失衡，在后续版本中会提供重新平衡的方法。但如果有新的数据库建立，TDengine 也能够一定程度自我再平衡而无须人工干预。 diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 5a7b32d20b..444a17c7f8 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -57,7 +57,7 @@ enum { // STREAM_INPUT__TABLE_SCAN, STREAM_INPUT__TQ_SCAN, STREAM_INPUT__DATA_RETRIEVE, - STREAM_INPUT__TRIGGER, + STREAM_INPUT__GET_RES, STREAM_INPUT__CHECKPOINT, STREAM_INPUT__DROP, }; @@ -155,10 +155,10 @@ typedef struct SQueryTableDataCond { int32_t numOfCols; SColumnInfo* colList; int32_t type; // data block load type: -// int32_t numOfTWindows; - STimeWindow twindows; - int64_t startVersion; - int64_t endVersion; + // int32_t numOfTWindows; + STimeWindow twindows; + int64_t startVersion; + int64_t endVersion; } SQueryTableDataCond; int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock); diff --git a/include/common/tmsg.h b/include/common/tmsg.h index e2d062911e..5a0ef705fc 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1969,7 +1969,7 @@ typedef struct SVCreateTbReq { int8_t type; union { struct { - char* name; // super table name + char* name; // super table name tb_uid_t suid; SArray* tagName; uint8_t* pTag; @@ -2439,9 +2439,6 @@ typedef struct { int8_t igNotExists; } SMDropStreamReq; -int32_t tSerializeSMDropStreamReq(void* buf, int32_t bufLen, const SMDropStreamReq* pReq); -int32_t tDeserializeSMDropStreamReq(void* buf, int32_t bufLen, SMDropStreamReq* pReq); - typedef struct { int8_t reserved; } SMDropStreamRsp; @@ -2456,6 +2453,27 @@ typedef struct { int8_t reserved; } SVDropStreamTaskRsp; +int32_t tSerializeSMDropStreamReq(void* buf, int32_t bufLen, const SMDropStreamReq* pReq); +int32_t tDeserializeSMDropStreamReq(void* buf, int32_t bufLen, SMDropStreamReq* pReq); + +typedef struct { + char name[TSDB_STREAM_FNAME_LEN]; + int8_t igNotExists; +} SMRecoverStreamReq; + +typedef struct { + int8_t reserved; +} SMRecoverStreamRsp; + +typedef struct { + int64_t recoverObjUid; + int32_t taskId; + int32_t hasCheckPoint; +} SMVStreamGatherInfoReq; + +int32_t tSerializeSMRecoverStreamReq(void* buf, int32_t bufLen, const SMRecoverStreamReq* pReq); +int32_t tDeserializeSMRecoverStreamReq(void* buf, int32_t bufLen, SMRecoverStreamReq* pReq); + typedef struct { int64_t leftForVer; int32_t vgId; @@ -2878,7 +2896,8 @@ static FORCE_INLINE int32_t tEncodeSMqMetaRsp(void** buf, const SMqMetaRsp* pRsp } static FORCE_INLINE void* tDecodeSMqMetaRsp(const void* buf, SMqMetaRsp* pRsp) { - buf = taosDecodeFixedI64(buf, &pRsp->reqOffset);buf = taosDecodeFixedI64(buf, &pRsp->rspOffset); + buf = taosDecodeFixedI64(buf, &pRsp->reqOffset); + buf = taosDecodeFixedI64(buf, &pRsp->rspOffset); buf = taosDecodeFixedI16(buf, &pRsp->resMsgType); buf = taosDecodeFixedI32(buf, &pRsp->metaRspLen); buf = taosDecodeBinary(buf, &pRsp->metaRsp, pRsp->metaRspLen); diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 488bc6346e..a24eda33bb 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -131,6 +131,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_CREATE_STREAM, "create-stream", SCMCreateStreamReq, SCMCreateStreamRsp) TD_DEF_MSG_TYPE(TDMT_MND_ALTER_STREAM, "alter-stream", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_STREAM, "drop-stream", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_RECOVER_STREAM, "recover-stream", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_CREATE_INDEX, "create-index", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_INDEX, "drop-index", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_GET_INDEX, "get-index", NULL, NULL) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index dd64c5bf71..8b0a836ad2 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -192,6 +192,8 @@ int32_t qExtractStreamScanner(qTaskInfo_t tinfo, void** scanner); int32_t qStreamInput(qTaskInfo_t tinfo, void* pItem); +int32_t qStreamPrepareRecover(qTaskInfo_t tinfo, int64_t startVer, int64_t endVer); + #ifdef __cplusplus } #endif diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 299d808017..d55adcb63c 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -190,14 +190,13 @@ bool fmIsUserDefinedFunc(int32_t funcId); bool fmIsDistExecFunc(int32_t funcId); bool fmIsForbidFillFunc(int32_t funcId); bool fmIsForbidStreamFunc(int32_t funcId); -bool fmIsForbidWindowFunc(int32_t funcId); -bool fmIsForbidGroupByFunc(int32_t funcId); bool fmIsIntervalInterpoFunc(int32_t funcId); bool fmIsInterpFunc(int32_t funcId); bool fmIsLastRowFunc(int32_t funcId); bool fmIsSystemInfoFunc(int32_t funcId); bool fmIsImplicitTsFunc(int32_t funcId); bool fmIsClientPseudoColumnFunc(int32_t funcId); +bool fmIsMultiRowsFunc(int32_t funcId); int32_t fmGetDistMethod(const SFunctionNode* pFunc, SFunctionNode** pPartialFunc, SFunctionNode** pMergeFunc); diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 280e9edf33..3bd02b1b48 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -78,6 +78,7 @@ typedef struct SScanLogicNode { SNodeList* pGroupTags; bool groupSort; int8_t cacheLastMode; + bool hasNormalCols; // neither tag column nor primary key tag column } SScanLogicNode; typedef struct SJoinLogicNode { diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index d58d5d4b8b..f8c7024591 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -258,6 +258,7 @@ typedef struct SSelectStmt { bool hasAggFuncs; bool hasRepeatScanFuncs; bool hasIndefiniteRowsFunc; + bool hasMultiRowsFunc; bool hasSelectFunc; bool hasSelectValFunc; bool hasOtherVectorFunc; diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 071c539ff3..5c4d8ce250 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -31,9 +31,18 @@ extern "C" { typedef struct SStreamTask SStreamTask; +enum { + STREAM_STATUS__NORMAL = 0, + STREAM_STATUS__RECOVER, +}; + enum { TASK_STATUS__NORMAL = 0, TASK_STATUS__DROPPING, + TASK_STATUS__FAIL, + TASK_STATUS__STOP, + TASK_STATUS__PREPARE_RECOVER, + TASK_STATUS__RECOVERING, }; enum { @@ -72,6 +81,7 @@ typedef struct { int8_t type; int32_t srcVgId; + int32_t childId; int64_t sourceVer; SArray* blocks; // SArray @@ -222,6 +232,8 @@ typedef struct { int32_t nodeId; int32_t childId; int32_t taskId; + int64_t checkpointVer; + int64_t processedVer; SEpSet epSet; } SStreamChildEpInfo; @@ -232,6 +244,7 @@ typedef struct SStreamTask { int8_t execType; int8_t sinkType; int8_t dispatchType; + int8_t isStreamDistributed; int16_t dispatchMsgType; int8_t taskStatus; @@ -242,6 +255,13 @@ typedef struct SStreamTask { int32_t nodeId; SEpSet epSet; + // used for semi or single task, + // while final task should have processedVer for each child + int64_t recoverSnapVer; + int64_t startVer; + int64_t checkpointVer; + int64_t processedVer; + // children info SArray* childEpInfo; // SArray @@ -316,12 +336,12 @@ static FORCE_INLINE int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem } else if (pItem->type == STREAM_INPUT__CHECKPOINT) { taosWriteQitem(pTask->inputQueue->queue, pItem); // qStreamInput(pTask->exec.executor, pItem); - } else if (pItem->type == STREAM_INPUT__TRIGGER) { + } else if (pItem->type == STREAM_INPUT__GET_RES) { taosWriteQitem(pTask->inputQueue->queue, pItem); // qStreamInput(pTask->exec.executor, pItem); } - if (pItem->type != STREAM_INPUT__TRIGGER && pItem->type != STREAM_INPUT__CHECKPOINT && pTask->triggerParam != 0) { + if (pItem->type != STREAM_INPUT__GET_RES && pItem->type != STREAM_INPUT__CHECKPOINT && pTask->triggerParam != 0) { atomic_val_compare_exchange_8(&pTask->triggerStatus, TASK_TRIGGER_STATUS__IN_ACTIVE, TASK_TRIGGER_STATUS__ACTIVE); } @@ -420,6 +440,36 @@ typedef struct { int8_t inputStatus; } SStreamTaskRecoverRsp; +int32_t tEncodeStreamTaskRecoverReq(SEncoder* pEncoder, const SStreamTaskRecoverReq* pReq); +int32_t tDecodeStreamTaskRecoverReq(SDecoder* pDecoder, SStreamTaskRecoverReq* pReq); + +int32_t tEncodeStreamTaskRecoverRsp(SEncoder* pEncoder, const SStreamTaskRecoverRsp* pRsp); +int32_t tDecodeStreamTaskRecoverRsp(SDecoder* pDecoder, SStreamTaskRecoverRsp* pRsp); + +typedef struct { + int64_t streamId; + int32_t taskId; +} SMStreamTaskRecoverReq; + +typedef struct { + int64_t streamId; + int32_t taskId; +} SMStreamTaskRecoverRsp; + +int32_t tEncodeSMStreamTaskRecoverReq(SEncoder* pEncoder, const SMStreamTaskRecoverReq* pReq); +int32_t tDecodeSMStreamTaskRecoverReq(SDecoder* pDecoder, SMStreamTaskRecoverReq* pReq); + +int32_t tEncodeSMStreamTaskRecoverRsp(SEncoder* pEncoder, const SMStreamTaskRecoverRsp* pRsp); +int32_t tDecodeSMStreamTaskRecoverRsp(SDecoder* pDecoder, SMStreamTaskRecoverRsp* pRsp); + +typedef struct { + int64_t streamId; +} SPStreamTaskRecoverReq; + +typedef struct { + int8_t reserved; +} SPStreamTaskRecoverRsp; + int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq); int32_t tDecodeStreamRetrieveReq(SDecoder* pDecoder, SStreamRetrieveReq* pReq); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 729f3eb3e2..7e61456d45 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -123,7 +123,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TSC_DUP_COL_NAMES TAOS_DEF_ERROR_CODE(0, 0x021D) #define TSDB_CODE_TSC_INVALID_TAG_LENGTH TAOS_DEF_ERROR_CODE(0, 0x021E) #define TSDB_CODE_TSC_INVALID_COLUMN_LENGTH TAOS_DEF_ERROR_CODE(0, 0x021F) -#define TSDB_CODE_TSC_DUP_TAG_NAMES TAOS_DEF_ERROR_CODE(0, 0x0220) +#define TSDB_CODE_TSC_DUP_NAMES TAOS_DEF_ERROR_CODE(0, 0x0220) #define TSDB_CODE_TSC_INVALID_JSON TAOS_DEF_ERROR_CODE(0, 0x0221) #define TSDB_CODE_TSC_INVALID_JSON_TYPE TAOS_DEF_ERROR_CODE(0, 0x0222) #define TSDB_CODE_TSC_VALUE_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x0223) @@ -616,6 +616,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_SML_INVALID_PRECISION_TYPE TAOS_DEF_ERROR_CODE(0, 0x3001) #define TSDB_CODE_SML_INVALID_DATA TAOS_DEF_ERROR_CODE(0, 0x3002) #define TSDB_CODE_SML_INVALID_DB_CONF TAOS_DEF_ERROR_CODE(0, 0x3003) +#define TSDB_CODE_SML_NOT_SAME_TYPE TAOS_DEF_ERROR_CODE(0, 0x3004) //tsma #define TSDB_CODE_TSMA_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x3100) diff --git a/source/client/jni/com_taosdata_jdbc_tmq_TMQConnector.h b/source/client/jni/com_taosdata_jdbc_tmq_TMQConnector.h index b3b098e460..197cd78006 100644 --- a/source/client/jni/com_taosdata_jdbc_tmq_TMQConnector.h +++ b/source/client/jni/com_taosdata_jdbc_tmq_TMQConnector.h @@ -161,7 +161,7 @@ JNIEXPORT jstring JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqGetTableNam * Signature: (JJLcom/taosdata/jdbc/TSDBResultSetBlockData;ILjava/util/List;)I */ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_fetchRawBlockImp(JNIEnv *, jobject, jlong, jlong, - jobject, jint, jobject); + jobject, jobject); #ifdef __cplusplus } diff --git a/source/client/src/TMQConnector.c b/source/client/src/TMQConnector.c index 1d84dcf7a2..3755a591e3 100644 --- a/source/client/src/TMQConnector.c +++ b/source/client/src/TMQConnector.c @@ -278,7 +278,7 @@ JNIEXPORT jstring JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqGetTableNam } JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_fetchRawBlockImp(JNIEnv *env, jobject jobj, jlong con, - jlong res, jobject rowobj, jint flag, + jlong res, jobject rowobj, jobject arrayListObj) { TAOS *tscon = (TAOS *)con; int32_t code = check_for_params(jobj, con, res); @@ -309,16 +309,14 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_fetchRawBlockImp( TAOS_FIELD *fields = taos_fetch_fields(tres); jniDebug("jobj:%p, conn:%p, resultset:%p, fields size is %d", jobj, tscon, tres, numOfFields); - if (flag) { - for (int i = 0; i < numOfFields; ++i) { - jobject metadataObj = (*env)->NewObject(env, g_metadataClass, g_metadataConstructFp); - (*env)->SetIntField(env, metadataObj, g_metadataColtypeField, fields[i].type); - (*env)->SetIntField(env, metadataObj, g_metadataColsizeField, fields[i].bytes); - (*env)->SetIntField(env, metadataObj, g_metadataColindexField, i); - jstring metadataObjColname = (*env)->NewStringUTF(env, fields[i].name); - (*env)->SetObjectField(env, metadataObj, g_metadataColnameField, metadataObjColname); - (*env)->CallBooleanMethod(env, arrayListObj, g_arrayListAddFp, metadataObj); - } + for (int i = 0; i < numOfFields; ++i) { + jobject metadataObj = (*env)->NewObject(env, g_metadataClass, g_metadataConstructFp); + (*env)->SetIntField(env, metadataObj, g_metadataColtypeField, fields[i].type); + (*env)->SetIntField(env, metadataObj, g_metadataColsizeField, fields[i].bytes); + (*env)->SetIntField(env, metadataObj, g_metadataColindexField, i); + jstring metadataObjColname = (*env)->NewStringUTF(env, fields[i].name); + (*env)->SetObjectField(env, metadataObj, g_metadataColnameField, metadataObjColname); + (*env)->CallBooleanMethod(env, arrayListObj, g_arrayListAddFp, metadataObj); } (*env)->CallVoidMethod(env, rowobj, g_blockdataSetNumOfRowsFp, (jint)numOfRows); diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 9ac34cf2df..5f26db67fc 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -274,11 +274,16 @@ static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSm return 0; } -static int32_t smlFindNearestPowerOf2(int32_t length) { +static int32_t smlFindNearestPowerOf2(int32_t length, uint8_t type) { int32_t result = 1; while (result <= length) { result *= 2; } + if (type == TSDB_DATA_TYPE_BINARY && result > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE){ + result = TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE; + } else if (type == TSDB_DATA_TYPE_NCHAR && result > (TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){ + result = (TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + } return result; } @@ -287,7 +292,7 @@ static int32_t smlBuildColumnDescription(SSmlKv *field, char *buf, int32_t bufSi char tname[TSDB_TABLE_NAME_LEN] = {0}; memcpy(tname, field->key, field->keyLen); if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - int32_t bytes = smlFindNearestPowerOf2(field->length); + int32_t bytes = smlFindNearestPowerOf2(field->length, type); int out = snprintf(buf, bufSize, "`%s` %s(%d)", tname, tDataTypes[field->type].name, bytes); *outBytes = out; } else { @@ -834,7 +839,7 @@ static int32_t smlParseTS(SSmlHandle *info, const char *data, int32_t len, SArra ASSERT(0); } - if (ts == -1) return TSDB_CODE_TSC_INVALID_TIME_STAMP; + if (ts == -1) return TSDB_CODE_INVALID_TIMESTAMP; // add ts to SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); @@ -851,35 +856,41 @@ static int32_t smlParseTS(SSmlHandle *info, const char *data, int32_t len, SArra return TSDB_CODE_SUCCESS; } -static bool smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { +static int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { // binary if (smlIsBinary(pVal->value, pVal->length)) { pVal->type = TSDB_DATA_TYPE_BINARY; pVal->length -= BINARY_ADD_LEN; + if (pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE){ + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + } pVal->value += (BINARY_ADD_LEN - 1); - return true; + return TSDB_CODE_SUCCESS; } // nchar if (smlIsNchar(pVal->value, pVal->length)) { pVal->type = TSDB_DATA_TYPE_NCHAR; pVal->length -= NCHAR_ADD_LEN; + if(pVal->length > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){ + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + } pVal->value += (NCHAR_ADD_LEN - 1); - return true; + return TSDB_CODE_SUCCESS; } // bool if (smlParseBool(pVal)) { pVal->type = TSDB_DATA_TYPE_BOOL; pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - return true; + return TSDB_CODE_SUCCESS; } // number if (smlParseNumber(pVal, msg)) { pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - return true; + return TSDB_CODE_SUCCESS; } - return false; + return TSDB_CODE_TSC_INVALID_VALUE; } static int32_t smlParseInfluxString(const char *sql, SSmlLineInfo *elements, SSmlMsgBuf *msg) { @@ -906,7 +917,7 @@ static int32_t smlParseInfluxString(const char *sql, SSmlLineInfo *elements, SSm elements->measureLen = sql - elements->measure; if (IS_INVALID_TABLE_LEN(elements->measureLen)) { smlBuildInvalidDataMsg(msg, "measure is empty or too large than 192", NULL); - return TSDB_CODE_SML_INVALID_DATA; + return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; } // parse tag @@ -1001,11 +1012,11 @@ static int32_t smlParseTelnetTags(const char *data, SArray *cols, char *childTab if (IS_INVALID_COL_LEN(keyLen)) { smlBuildInvalidDataMsg(msg, "invalid key or key is too long than 64", key); - return TSDB_CODE_SML_INVALID_DATA; + return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; } if (smlCheckDuplicateKey(key, keyLen, dumplicateKey)) { smlBuildInvalidDataMsg(msg, "dumplicate key", key); - return TSDB_CODE_TSC_DUP_TAG_NAMES; + return TSDB_CODE_TSC_DUP_NAMES; } // parse value @@ -1026,7 +1037,7 @@ static int32_t smlParseTelnetTags(const char *data, SArray *cols, char *childTab if (valueLen == 0) { smlBuildInvalidDataMsg(msg, "invalid value", value); - return TSDB_CODE_SML_INVALID_DATA; + return TSDB_CODE_TSC_INVALID_VALUE; } // handle child table name @@ -1059,7 +1070,7 @@ static int32_t smlParseTelnetString(SSmlHandle *info, const char *sql, SSmlTable smlParseTelnetElement(&sql, &tinfo->sTableName, &tinfo->sTableNameLen); if (!(tinfo->sTableName) || IS_INVALID_TABLE_LEN(tinfo->sTableNameLen)) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); - return TSDB_CODE_SML_INVALID_DATA; + return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; } // parse timestamp @@ -1074,7 +1085,7 @@ static int32_t smlParseTelnetString(SSmlHandle *info, const char *sql, SSmlTable int32_t ret = smlParseTS(info, timestamp, tLen, cols); if (ret != TSDB_CODE_SUCCESS) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql); - return TSDB_CODE_SML_INVALID_DATA; + return ret; } // parse value @@ -1083,7 +1094,7 @@ static int32_t smlParseTelnetString(SSmlHandle *info, const char *sql, SSmlTable smlParseTelnetElement(&sql, &value, &valueLen); if (!value || valueLen == 0) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", sql); - return TSDB_CODE_SML_INVALID_DATA; + return TSDB_CODE_TSC_INVALID_VALUE; } SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); @@ -1093,15 +1104,15 @@ static int32_t smlParseTelnetString(SSmlHandle *info, const char *sql, SSmlTable kv->keyLen = VALUE_LEN; kv->value = value; kv->length = valueLen; - if (!smlParseValue(kv, &info->msgBuf)) { - return TSDB_CODE_SML_INVALID_DATA; + if ((ret = smlParseValue(kv, &info->msgBuf)) != TSDB_CODE_SUCCESS) { + return ret; } // parse tags ret = smlParseTelnetTags(sql, tinfo->tags, tinfo->childTableName, info->dumplicateKey, &info->msgBuf); if (ret != TSDB_CODE_SUCCESS) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); - return TSDB_CODE_SML_INVALID_DATA; + return ret; } return TSDB_CODE_SUCCESS; @@ -1135,11 +1146,11 @@ static int32_t smlParseCols(const char *data, int32_t len, SArray *cols, char *c if (IS_INVALID_COL_LEN(keyLen)) { smlBuildInvalidDataMsg(msg, "invalid key or key is too long than 64", key); - return TSDB_CODE_SML_INVALID_DATA; + return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; } if (smlCheckDuplicateKey(key, keyLen, dumplicateKey)) { smlBuildInvalidDataMsg(msg, "dumplicate key", key); - return TSDB_CODE_TSC_DUP_TAG_NAMES; + return TSDB_CODE_TSC_DUP_NAMES; } // parse value @@ -1195,8 +1206,9 @@ static int32_t smlParseCols(const char *data, int32_t len, SArray *cols, char *c if (isTag) { kv->type = TSDB_DATA_TYPE_NCHAR; } else { - if (!smlParseValue(kv, msg)) { - return TSDB_CODE_SML_INVALID_DATA; + int32_t ret = smlParseValue(kv, msg); + if (ret != TSDB_CODE_SUCCESS) { + return ret; } } } @@ -1204,8 +1216,8 @@ static int32_t smlParseCols(const char *data, int32_t len, SArray *cols, char *c return TSDB_CODE_SUCCESS; } -static bool smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, SSmlMsgBuf *msg) { - for (int i = 0; i < taosArrayGetSize(cols); ++i) { // jump timestamp +static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, SSmlMsgBuf *msg) { + for (int i = 0; i < taosArrayGetSize(cols); ++i) { SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); int16_t *index = (int16_t *)taosHashGet(metaHash, kv->key, kv->keyLen); @@ -1213,7 +1225,7 @@ static bool smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, S SSmlKv **value = (SSmlKv **)taosArrayGet(metaArray, *index); if (kv->type != (*value)->type) { smlBuildInvalidDataMsg(msg, "the type is not the same like before", kv->key); - return false; + return TSDB_CODE_SML_NOT_SAME_TYPE; } else { if (IS_VAR_DATA_TYPE(kv->type)) { // update string len, if bigger if (kv->length > (*value)->length) { @@ -1230,7 +1242,7 @@ static bool smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, S } } - return true; + return TSDB_CODE_SUCCESS; } static void smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols) { @@ -1564,10 +1576,16 @@ static int32_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int64_t *tsV double timeDouble = value->valuedouble; if (smlDoubleToInt64OverFlow(timeDouble)) { smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_TSC_INVALID_TIME_STAMP; + return TSDB_CODE_INVALID_TIMESTAMP; } - if (timeDouble <= 0) { - return TSDB_CODE_TSC_INVALID_TIME_STAMP; + + if (timeDouble == 0) { + *tsVal = taosGetTimestampNs(); + return TSDB_CODE_SUCCESS; + } + + if (timeDouble < 0) { + return TSDB_CODE_INVALID_TIMESTAMP; } *tsVal = timeDouble; @@ -1578,7 +1596,7 @@ static int32_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int64_t *tsV timeDouble = timeDouble * NANOSECOND_PER_SEC; if (smlDoubleToInt64OverFlow(timeDouble)) { smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_TSC_INVALID_TIME_STAMP; + return TSDB_CODE_INVALID_TIMESTAMP; } } else if (typeLen == 2 && (type->valuestring[1] == 's' || type->valuestring[1] == 'S')) { switch (type->valuestring[0]) { @@ -1589,7 +1607,7 @@ static int32_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int64_t *tsV timeDouble = timeDouble * NANOSECOND_PER_MSEC; if (smlDoubleToInt64OverFlow(timeDouble)) { smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_TSC_INVALID_TIME_STAMP; + return TSDB_CODE_INVALID_TIMESTAMP; } break; case 'u': @@ -1599,7 +1617,7 @@ static int32_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int64_t *tsV timeDouble = timeDouble * NANOSECOND_PER_USEC; if (smlDoubleToInt64OverFlow(timeDouble)) { smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_TSC_INVALID_TIME_STAMP; + return TSDB_CODE_INVALID_TIMESTAMP; } break; case 'n': @@ -1634,11 +1652,11 @@ static int32_t smlParseTSFromJSON(SSmlHandle *info, cJSON *root, SArray *cols) { double timeDouble = timestamp->valuedouble; if (smlDoubleToInt64OverFlow(timeDouble)) { smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_TSC_INVALID_TIME_STAMP; + return TSDB_CODE_INVALID_TIMESTAMP; } if (timeDouble < 0) { - return TSDB_CODE_TSC_INVALID_TIME_STAMP; + return TSDB_CODE_INVALID_TIMESTAMP; } uint8_t tsLen = smlGetTimestampLen((int64_t)timeDouble); @@ -1648,19 +1666,19 @@ static int32_t smlParseTSFromJSON(SSmlHandle *info, cJSON *root, SArray *cols) { timeDouble = timeDouble * NANOSECOND_PER_SEC; if (smlDoubleToInt64OverFlow(timeDouble)) { smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_TSC_INVALID_TIME_STAMP; + return TSDB_CODE_INVALID_TIMESTAMP; } } else if (tsLen == TSDB_TIME_PRECISION_MILLI_DIGITS) { tsVal = tsVal * NANOSECOND_PER_MSEC; timeDouble = timeDouble * NANOSECOND_PER_MSEC; if (smlDoubleToInt64OverFlow(timeDouble)) { smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_TSC_INVALID_TIME_STAMP; + return TSDB_CODE_INVALID_TIMESTAMP; } } else if (timeDouble == 0) { tsVal = taosGetTimestampNs(); } else { - return TSDB_CODE_TSC_INVALID_TIME_STAMP; + return TSDB_CODE_INVALID_TIMESTAMP; } } else if (cJSON_IsObject(timestamp)) { int32_t ret = smlParseTSFromJSONObj(info, timestamp, &tsVal); @@ -1779,6 +1797,14 @@ static int32_t smlConvertJSONString(SSmlKv *pVal, char *typeStr, cJSON *value) { return TSDB_CODE_TSC_INVALID_JSON_TYPE; } pVal->length = (int16_t)strlen(value->valuestring); + + if (pVal->type == TSDB_DATA_TYPE_BINARY && pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE){ + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + } + if (pVal->type == TSDB_DATA_TYPE_NCHAR && pVal->length > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){ + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + } + return smlJsonCreateSring(&pVal->value, value->valuestring, pVal->length); } @@ -1913,7 +1939,7 @@ static int32_t smlParseTagsFromJSON(cJSON *root, SArray *pKVs, char *childTableN } // check duplicate keys if (smlCheckDuplicateKey(tag->string, keyLen, dumplicateKey)) { - return TSDB_CODE_TSC_DUP_TAG_NAMES; + return TSDB_CODE_TSC_DUP_NAMES; } // handle child table name @@ -2033,7 +2059,7 @@ static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql) { } if (taosArrayGetSize(cols) > TSDB_MAX_COLUMNS) { smlBuildInvalidDataMsg(&info->msgBuf, "too many columns than 4096", NULL); - return TSDB_CODE_SML_INVALID_DATA; + return TSDB_CODE_PAR_TOO_MANY_COLUMNS; } bool hasTable = true; @@ -2065,7 +2091,7 @@ static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql) { if (taosArrayGetSize((*oneTable)->tags) > TSDB_MAX_TAGS) { smlBuildInvalidDataMsg(&info->msgBuf, "too many tags than 128", NULL); - return TSDB_CODE_SML_INVALID_DATA; + return TSDB_CODE_PAR_INVALID_TAGS_NUM; } (*oneTable)->sTableName = elements.measure; @@ -2084,12 +2110,12 @@ static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql) { SSmlSTableMeta **tableMeta = (SSmlSTableMeta **)taosHashGet(info->superTables, elements.measure, elements.measureLen); if (tableMeta) { // update meta ret = smlUpdateMeta((*tableMeta)->colHash, (*tableMeta)->cols, cols, &info->msgBuf); - if (!hasTable && ret) { + if (!hasTable && ret == TSDB_CODE_SUCCESS) { ret = smlUpdateMeta((*tableMeta)->tagHash, (*tableMeta)->tags, (*oneTable)->tags, &info->msgBuf); } - if (!ret) { + if (ret != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id); - return TSDB_CODE_SML_INVALID_DATA; + return ret; } } else { SSmlSTableMeta *meta = smlBuildSTableMeta(); @@ -2138,7 +2164,7 @@ static int32_t smlParseTelnetLine(SSmlHandle *info, void *data) { smlDestroyTableInfo(info, tinfo); smlDestroyCols(cols); taosArrayDestroy(cols); - return TSDB_CODE_SML_INVALID_DATA; + return TSDB_CODE_PAR_INVALID_TAGS_NUM; } taosHashClear(info->dumplicateKey); @@ -2169,9 +2195,9 @@ static int32_t smlParseTelnetLine(SSmlHandle *info, void *data) { if (!hasTable && ret) { ret = smlUpdateMeta((*tableMeta)->tagHash, (*tableMeta)->tags, (*oneTable)->tags, &info->msgBuf); } - if (!ret) { + if (ret != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id); - return TSDB_CODE_SML_INVALID_DATA; + return ret; } } else { SSmlSTableMeta *meta = smlBuildSTableMeta(); diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index a6062efb98..17025db730 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -623,8 +623,12 @@ TEST(testCase, smlParseLine_error_Test) { taos_free_result(pRes); const char *sql[] = { - "measure,t1=3 c1=8", - "measure,t2=3 c1=8u8" + "st123456,t1=3i64,t2=4f64,t3=\"t3\" c1=3i64,c3=L\"passit\",c2=false,c4=4f64 1626006833639000000", + "st123456,t1=4i64,t3=\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin\",c2=true,c4=5f64,c5=5f64 1626006833640000000", + "test_stb,t2=5f64,t3=L\"ste\" c1=true,c2=4i64,c3=\"iam\" 1626056811823316532" +// "mqjqbesqqq,t0=t c0=f,c1=\"grigbrpnjctnyhamnwfqsjrywdforgpwpabdisdnymlboguwxuoscfyyajiyusxrocjndexxcvcqrzgxceqolvtdrpeabrcokpmcnhduylzxxljospclwuebutrdbpklpdbtkrdppamenzlmkkzttacrfxaozvwodpxzralhmdhvgaurtacnyhlsaojjglfnrylswactjumeldmuuafnmwsuuyiwhpdzqludgpluvllfowkwhfbtgsjsnxdbfbcrqnrxllmokbzrkiuxkhumfcjogeugbbjowmckoeyrilsoenowqwjpuufprqnxxzjlwfxnoljtodghyfdtyyptxafertndevhewboikewxwtwvbusnjpxwpnhhcrqqyicuuxmadxqjhbodsbexgpuaicbxduewqnogdhyjhcyjyftfbvbctgbjrwrkqtmqhzwxnilkmorotbiuwsimuvloeykzxqdepdkvvdcjtzmsvdseygtprbvhuikvomoafwnfojzaojxbkbpwbjqasazgokjjpktofqhjqhxxplkdvttwflbekawvozxiuhoahajwpimnjsbzjfhqgbbcgjgrjszmuqmwupxqlosfdsqnpkertnamcipfanxxewygtkeiaqopvykygkfbihdqqvhwapyctmxbjvzdndobrooemwtotrjzuknaupwxrjbrjzmnmupbwcdwkoghsilyfrjeefwrmgordzlafyjweavrapqqsicqnmkjulambrjxcmmsnvcjbbbwrloifqnmcmqlndubcynhumpikddliddyduafrpfgcltiymwlpbhtukmyawxdaaqiscvpfvsacjdljlfaeqhearjyczdpjsyjygfwaegqtylpibtdqinncmttbtiifbsesqbhpieectocontqhoyggjgbgjiyegoypfxorfqgbewhfqhkqftjdwtcnaiconxwjwryxqyexmlauoiysodziwfyyzyfewnfjyvfnvvxkkxeajmwbypoodsfrfygadcwpcjhzvemaplczgqxpsxkgxuqbxqhchpybojemqgxlhcyxmddjvwnbkvykkhwebfdpoovtvzgpkuwneodbochwxwauggxulmkynhoohchnkkcybhtwelotxpzoqzuczhwbjxsuqckuzsapdfkeiwxkcutimncyfpuaovhzwaebebkxgbognzpcxldjptnnldzqwtzzsiyjambnbrbyuyptxdkyxhlvigovllmkylbyzecxhdxczlkvsconlfvnafqvrwhcughqbtlmwgumeponoonhqqbklqaxvslkxowcuztjikgbutrnkmizschzrjxbmzvkkdlcuchpnrbxhgvxjqxawftlbirksyqbnltbaxtpyivrrqgxcjjgbhhvlltfqogehhddmgzivwlznkfuqgfbxrtixuonnywsxsdunhsmziyitecmrmxjkzmqhivinqkqywggffljpoxnofmzxkrmnxbdkokaleaqwbqizzhvywrweklybntszygkvuvypmukyxawhgbtnltemjchaytpqjplavcbypkwjwdmfogdxiddrtwrvvoqlmhsqlrdmmibawotbouzosrmbzocqvqdhoamuzvrfeyxkrnzpzyuacffbuvlcqupmbmqughegvjrobyzcyhynnpvvjntdcyplahaajwcbbleblkhjyauehbuoudyzrsgrtqnufijaawllbiexvhveipkaxffuiyczbkpcpzdnajwkkbbfrfchpedgabsraaalfbddgypeayprqwjzfvifjmgwaexrezitgaqgjmgizaohcfizhocckuzysshxzwqolddumeomghgsoaaerfxsapupejhywucurrhgctmmlgbyfjigveayriyvdmapvafzeydlxiwxcgnnajwjaqzkecczrdlbxgtdvehelzibmogdijpdiatcafnediqldwszonasgodqnnvajqxuwuftuvcqwtvayeiyysihhckitlimuimjllslrcnbobmumpbtoakqxallkqhszloxzpogfxxclnkmbfnqqomtzfpzdgxnfyvppeybjnekchjsafvhkrpvxpiumfvraeqcwqneatdrxdykoyscehvputknetluvfexdvtnnnaitrbdwyrzwnymuimydmadqbncdfixsgrcxkdnjzzyimcfbomioddimdxzxbmqwgnrezaquhiqytxmgsqpmywzxqksahlgwpxprtuzxtghkdbgxwpmdqxastvqggkhaqkxhjdsfcwyljwlleymorfgkezzroxaalrguityckdxsgqkcpaxxcsqvttvmmmmxezdztmkgcpnxbobuggzuqaetqfnmttjbshhfqqfsmfylavatksdchwvfvhyipfsepkwzqtprzogoxxohvibkwwiuwpsybbqqgbvziecnulnudzpudxvtcosvedrdxkhnkprghljzltucqljhdwpsfrsfryxpzybmybockeswpyihgossicvoxroiuzvgkbtduxzgsmgrohrxjbdvpnqwhgtvfkjrgpmirfdqddoyaztlooxlzllljsniwxfbihodjfywxallozikruusmzigztbzlyofrxtghhjwgptdbntmqkoxmrgzaznesgsgjnbmgarjqsqvswzygkbgquhbxsulabxzfpfnopzfnbiqeivuzayjbaikqrxrhyysttcafyxfdzgbbadcxiqltlwyhbcibkcxildnhmgwskeztcnmzdncqlyzpbzifjrhflsahlecmxwxlzmvpkqdexbfflmjhdqymxmjrktxaratynebetkfaltnrjgvsbdbvdcdyqujuypensmjnjskovbeweuwnqfjueeylefnqvmdmkwjqfvbjcuaibosymddysdymzhscroykljydnfvwosgplfphpznaqsddtbcjmyxhmcnxwdesycovtwrlmixqmpnzsbyfwpgnujxxqillwpbasdnbfxzokimfkujvlabycfwlplzpcgangjagrrhhjbrtddgitpoemixmobwyabyhsnkjtbeasdejawrueegmijbupygyciwbrhiwisguhlthnkpjqyzhiwvfrpgglqphhjtirtjxsjqxvqjpmokcgqbtjpsvravymmrrpyedruuncsjbjyrysjowqsnwtmvbakadxkxbyfunxnrkqhqvoeuzffmbpzfiiwrfdaekcrevffoxpsauhzziuyyjodsisaadbnyuugaxadvyxhfhbwhbmsgaklslihzwgvprpcawdtrniispfdnjoxkatlwebopgdqnaemwsflgfcuhkdnofblftofqzsphykpirzuckdnuxarzakvwurtrtbprdryikxmzytqhdmoyuizplphpvoliisgzhiganghwvqhzdmijccnfqqvboifovqxqvziktibyzpbffaguffgaqpbujvrvxecmaqygoyyptgzmwlnrwbeyuiiapgazdrgyrobtkcmsoheumgzjjpztatlpjkckxjqgfwvlhojdwztjgjdfdvalsglxjggmlfrbvtfeyhdbkggzukvdjnjtoytyrvxgrlvbqkkgrixhmjvwmojeiugbcyetihdtsizatgeukaczqllddwfqtwzsdquxmjmsypnftypppdsrqmkrfwxpwasrbtbeaaflqiatngmxylmhzwfoczsvcvwkgmxvhzyaoxrbblpqhbcozesrkjqncpyjukhppbubshyhwclceaefzhlbncxwdgglbtmzlksugrgnwjghgscqxfydztoraxrnthpqfojlgnablbxsovkcvpboujoczpihxrdblfirvlpxzgjhgiueyinhzasfelqnwmyhwwiaahrwoivetpfyyeeponmaqofwcbpvagruzshxaugnfzpaognklwcmfjmojrmjgmhroomwinouwdosuwtbrpkrzqtjfyspdnzgtbybsjyuohmchoukdyjfgovroyigpxqavcpmwnccdouskjxpqmpkjzkmcouwmauimkpatyfgkerqazsjuhctrbmqvqfdjfogajgrjnskzmrwnfjjfszebtbsioumdvhvqzgdkkhmsciutobqaefncvepnwqhvrfajmmrqnjryniwrckbaampnegmzoiqibwszbrqcpfgvtnlzemcmzaooywydmonegybzpdtukduxedpyquadxslnvirvewqihhnarvkpsbhmoggmoypwbimrnkuuiztvdqltnrytvvzlvhovoaekomlkqacgvlhdjaxhusehccgzjljjxjdpzpfnsrfnrxbzhoopziyrcmtpuvaqpvrevjjvmucezpecyckcmyvgnzgvitbkkdoptciamgkovowlhfcjmraynfyvlepowelkcjmjnibcsnabchcesnrwiplkavzgvdjyhulhthtbjgeckloshfcgobqovmxpryfbaaxfemkkpmtllovhqncrsbgbhjaozoycdnbcilhhlyfxbzvcmumpspgjszohxqhdocwnoxatmtnkqkpvupobukdudumdpsspzjxrcxstvajlarmicnsnjgdyyxcliqftvftmjmztbktbfmddbqtrfrygqzzzplqgemtvgijkydpshxiajzgcpmxsuamhtucpnejafrjqiwdxxflmaeyhntqfftvmsovtzunqszbvmvjhxcemtorseiariixtbnmxhkcwrghzposhvfnorlcwipsolpmkmvfpwdjswietamqfggxhpwfnsbkooocopbjzzxuhqxbtkklsxmmsgqvxldrfutlgntrewlyksrxdfexgkburyxbuqzhjmvqsqdzwppzyoqibdbhavyhexuybqhstktgtvtrckzqezauehcoxlnntilnkqekvdachlmvuxcowizzbqrldzaggpbvvlfwhsqfdyqvwqwrbkqvrqpzdihtnnafxbxqulzfswevlvxsjugrsaombysnngstmnlyayizrynmofiwbggehbfugsufhmyogsctxkfzlwwwshxnvoaqvstgpjtvyczlgoueutienayowbzwuhzearmhhbukmebpyewdrlmflwbvrzfhrkixvgewburjiqfovxrkiwvvbdrswvbcsznriinohlfeukcxmgmoyrlpqzjtgpjpvsnzbriifdyljkbqqiketrpvmvimmxhmpxlfzqluenskwrtshagizqrxigmmynfppfxfzxcvwbogamdxfipiqdasphwixefwvgrihkqjcflqvqqfvzxdtqyvvnnfzeucqhwlmxjconjuqkachpnysbnhrcfadculwgxcruihnuixxuvdmztugpvesdddargavwiudrtybxwmvywqleepplrchioqkyomusvamfawlxcwdkdjnydcmgfrlmkxpvqhcuioilsahnzrvlxnrfyxmjxvtlliyilcjtcwwcuucgurbbcshnlzrzilgkhdojcivhgltssezykltyzcubevrbapzmnhfhtntgnmjytjubvasdfiagwlzzwohzaibzqwqdlsikaodfljcgnhyckowudmfbqimtuszqgyxxzvipniipgsotrpkzamiwpkngnvwmjjivjtxhpzlmrwcjznavijhjjmvhxkjdahleprpynjnqltqhyamkfdfspbridpbuphtqxkncpognjgxwwyzxnkizrzvobpdxepncwvuhdspajmooiybeksqkhpncluiwwgsapihnkvgmwektybpzlnizkhtxtgeqqgaphditecptyoquueofaleodgsvfjxhokmzgjwflceebrbbjkxvqvkymjatpvdcvatnvkecfpxrpvwgnusmuetshyeyphgzjwktlwycqjqmsfjtiqkkbhndslyfxdegaejuzfylnbqlvacephpbuytqmxvwosukulbwdoofqomqlgdptocqlnjkikcvwcvyrpubzoeegonjhdtuibklelcgtacvovyntmucnzknumratvvwcphkfcxzjfmzwbqluzpexancupokekqnykxmwnyxvclvvxstnbbylaqknrgfegxfgkrnipkrstthxkkyborfgciqgksruwjzxwfuztgizrjrilmshcmnfzxwucrsscgotmniegribamhyzwjwyeuminjukrurpspcjmfllgceyuivmqfgegjjjpbswhjijrlajtbtevijdyanduyhbtedmihjaadtwbnjgrhlxgbvxxmtqzinsclkctlvhocntuppgfeaubksbwxouqsmdeaijulvlpawxuuvadmroswmaceodnqnxaxnwxwsoogqctfkadzabezoeufgskhtxgeefigmjcwrsoymyardzujtpejrsjslnorwixaawuqkhtgtqgrbjrzoxdpgetayqwsvptbwoljgypbkaxjcfujykdtikngwvnmwlpefdecpkywsbkoqjuyiaaizknmygqiqdjhfxfzpsdnlzqosmcdgacngjdrmhhnmltesihrwsfrfjvhctfjinwolonpeuibvxhhunarulabdrrwpipkczhxaxrqxvydmuerawuoshzupvvhfhlvbdahibhygftjmfqostlufujpwrfduppuhidftnjegdoqjyfekysuglomymoybrcypfkabcgiddimrpahbmtjwropodagfdfrpffqqgffriqcmvqsbnrjqwkqrpappefsabbjkotyspncbzjdlqjobgzkxzebhuliwikfvhfroqotbwsyywapztlwnnumngdwuinqefmgmndpvfsmmzrozkzplzmgjojgkzwkfwgljxrvvfuvozeihsiwqvksibqdkbsqslxwydowhsekwuslrppizukfcvvfxuffrnnceoriukxnqoujatnhqvgjaertcqcdfccsttyirwzxytgflyoedmkhzufythspclmyrwzxlvvhhqohxdppsvzoqgcvclykgadmtkwxfnzpcoziukoajwjjaiufyzormcokrwbdpnhcotdmvyihscatzmotgqoqthdcdegnxxsxdqgtbdirmvujyvssdvpztvhzaklkqvvhkpqmqyrwbfwcygnvbjjvrfmccrmjmspvqmxadbpipprbcurcjcjyjjbnzbjdnpgobvckrdcbjiphtgmavthjedrkulplgedfiavvdupwfugxvrowmuipujzqdkzebvfgzqxxznnbdfjmfrrgjwpqkudgscpotdhtguvgyymhhwkrctnvuphhjnrwcqzwargqxxpsdvsvfynlhxrzekjfgtdmcaspmtmzdaojduyhqieipeetptyfuhrynsszfnxcgtvnfahfgkjfbxmgnuhxtifzhgtlmjlgayybpshyzixkvocjlorxlpvsjqgssxlwmxwpmwouocgylxbmyfrezwpubyewxsnqalzgetnpdfwrgxsawaargjclnfxoucwljnuqaiokxgixwogrmfhegurpyzitefhejtqawnmglkhlhxoxblmgdhzkavxnqhoeagcrbbqlssotgphffqtcgkupzvlkmljmjomnqxgcmiyysmkvziridmuijdrzozgzxsuiudhjzuxxjoatipfcpjsqqckmvcgsjdaoecooposrptdwtrdwvfltbtczbnyqhvdrkphccwyyponubffazdikxuifbxnqmoubdtqbpxrpsfyoevuwgmwlnvgblxlvshhdavmdhbmurkmlhsiepzyiqoaiugfdzwkpmtjozzpqfrxpafkiebadrglatgpoiargnyofrhsdrpfgdipxnlsxopmbhxupantpxyasrvqziefcarckihgxkbfszzgtjpoazjuuuxxccegqhjtsjqdhgshczrznrbyjrraxeyzdgciyvaeapkwgvkejkrckdsbyekoukliqozslwgghnjrfbzpqkrfwjawoutztlnasoecujozksrefzdduhnvnskvziighbejokbqyrdespapyqbidgkzwlfvapyjcxcoybgwxweivmzblrdyumcxcnddqgvlthtfjwmefwzkzvnycnfduawgvsqmullejnpapzeujmmwkbmtalkrpunhjlargfhxpjphesgxdvldteileyzxpftdikjyyqgldfwrzglixzuegwslfyhrqjceeeggllgbvfeaefztngfpjncbjeyfmyvcmdashzponstxigskortcevevfpqcbwzmqrbvbniwjwajbdhdfqlyujnwiuveihahtbakokmzkpznqqrqdbbivaettleiciafubnklnowubzzhvzhyhkfhzvvcsajxkqnruuyoaxmrahzmqnuedlmjyiioucsaxvhspmrmglcmpoxvqzwssgxgptdcclstkjxwwaqekdwkixnowusxbnftnzjectfsckbeeevhytludfcdzwdiujywcsgwrvmbecqwibvusgqhhvmztiavlsmvlwztgburxaaotbcslvxnffaohthhwhaatkyvaptdxwoztfcqovimzbpsbxwuwbjwkbdvrkuytovzsvcmkporgabibniqiiobhljsbgeqsdbofcdpuxgdiqlmpwpadfuymdmauguvvewtnrkbkgfogitcidofpaduxeetslyqppgsquivqvvmfmdpyfvmqfliuhkasezljpmlagfgqcqahtfojamfwjmptsuvgbslskjmvqhmdlhouymghfngfysjiqkjfcwbjjtorzpjblzuabghntwyxrcqrrtviijbcknzjolpatwpssnzmobrpxwyaubjgakgdzydkkvsisnfscwklbmkdhrzbopcdmimqleofwvfugtbtogbdmazqjmlslpfeukuqcpmpwggseebnoqpadfpudcnriiwlhojpzpbbqdgqoweijlyjplkxxpxawanihmdkxmmdsdlknwcxrbsmrpsxawxxoepzckcilssqxntruzwmtqqjrxsupdaedboovfkecckmdxtymhagyoweznpgtwxkpbnoqfkrnzvsxpdlgynleqcpyrodfqngjgmkweiotmvpmbujluktefwwhhprfqtusjzebtnhyztjhbhlnmfzdrcsxktxbzqsoczgwoydpcssgksstfeslmesjkdbwhlorqtswfcfsxkysbedidqzsxorpgnhgieonzdzlpyqxjkkncypuhjtgwzxvrqmpleelcampexgswcdtezuqdghfzzxkzzyulqpfojwsdgcdniblomxrxflbnylwqxtifxxfkembyxhkvhfjnmpdinrpodvticucowipekvthfobnkdvgfhoobhhtwdtppcogtwqyynixndujqclzrvwfirjqsmvfjxbhisdaugeaswspcljkdigdqcekcftqcemsjlxhplmrxootbcsjylvkvwtvvnusaxtkxcjrxazsjeheguoxrebicpecuuorpwzsgpfgztgtfpilvauzikosbtzbhrwafktgltkteknizcioxefizyvwfgyfwhbgkssmvobxrzvqfkdhcvezdmyvqqedjvspyvsgwqwovdxrecdanapoydetgehibxaslvllrqkxdzhsebmrdflqxylvgfaaghcstzrlutizgxkgfjzatylehdqcctkhqahctbyazuibdkvvgyyoqlmiocgkripiofrbmjvkavkebaelrhrizmzbskptanrhwzcpzrtofjxzkrushctxejlaziteklpjakzskzklmdgukiabxxduslretgbomoexppmgimlfhfehoswtixefjffecudfmacfvlguvvbzcbtgywrxbwifkrxlhoqvtslpwhbcanoaynjonlyiobcwstxshesdowbviqdejatogcfbllmnctasbeininbnwmtpdhmuvurvtpnkqpscvwtlzhtlpvoztdqbncxxmqymjojjnllivocansiodawzlcygkejjgisvzvvdlmacnffffhxyodgtmmlevrjhnplezrfidsuygsariqdqbyvntpqnurmtrxtentgnopsipnayoxipkvysbunxqjisyjevmjvgxoqruhxvsqedcsimagxmsbjslwohsckiivuhbjnegobkpxjdoqfnicgunugidyfngasefvcbwltaljvxamhnuefkvhgbwyozaggdszyqghnnfmcyjfvhfcamcxjrggysglomdptedlthpfxmmbqbfzlzgodcsahagnuepupqbrfxjgqldwbuenabygoeduhwgtxnfmzlsojbvxmmavdbmxivmdozdratbytpyjysrzpejdggqguhyeshcobbfodtuqnwwundapkfkblfzdlnsbylsufiuycoejkljrcovadehyazpwqordifrsomfskmjzogqciiluldojkxfgtwrlbqjekbqotuhffowjptmjkitgolgsofzkvjasgzktoophkpnidqujvcdxofcfuwwwihpgitnsfsrgxxqzvzfjlabwqptlvsusszjajgxshshzncuhafxndwqcxujigvkymfztczglcuwbzhgomvqxkdmxilzewacpnffzlkxezzpxbfvlfosxkvmdopnuwoqkbjrogfecxpzcqvyzeuadikskcwpgyknryrgcumvspxtgzzdsoebizpsehtpqfmtgnwjhrcoqthrjxjugvzyhvoglnerbyffgastsyoizzzrmmeawztdizcebilasdsthmujjvmjsssvhwlyglddnljtigltporpjaiokkoeuqreawmpbvbnjiuvhdslieeanfazyxubwacizffpahfndinebzcqdrnqnbrwdddcorvatawhqeacjtfikkastvtluavsyixwldxuifyxpmgtxqpdcpyggdiztwihzsvhtotqgtscvmwtpsakuuyuastebtsivnoemlzhdllyvyifirqcvxfapegnfyaxepsvvqhdrztwzgbbtbslrtifugxhrsiidptyafyaxbtbrpxlsvwmxvcmrpgatnpnqoghnjqqxwtfpsicpwrtwtqrxgxrdzlkamgspznezzlezvaftrvbvjatefhsrtrqusxnrxrahdckrsdgyzbtflaaelpkwfddzgapzlktcrizyawqeazasrtkcsryowkcjvmsbkvhkmdxrudjjpczpzfxtjbmgpvwhchvtlctrhdqqjijrnkunalsucruwhhfrrdsjztcrkivvrlszopymvuxnnlqklatzgcjjuxmhrmydtcyxhisvxepljzwjuhinuxvsmkdtmrrojutimnivlxxcjvgbpclzuxcppfopckrvndccoelzzmzcdyqrkuxdwompgshazcuzxwytnjeejmpwpabiuaorkhctezqydizuuontnukrkvithhctnmwwivqbabuvqwvjyxpwsgpsoyszfsnjeeofmqoyxyakfcmwrwkisglzadmtcolpwhrnpasmxbkozdfgtuchqhvdnfahlxbzqqxgfisdjrwwqsjihtcgflpnskznnfdzeotcrzylojcuvsabyngjoetcptkdbihowprmxokppjfjvxsztypzkzgwuurqmlwdzapowwsaozebryypltamqzmduirxskstryqrdaagwerlbnwgteibjiktrladowyuhsuasqppzkqtsvpxzdcxyulrqgjzspppjqujffcwrtovaxaflttvrwdlojqdmmcvgeoiieifzkpzfusoozkunhnxaafpnrnhsfraglsbylzbigxjxqbjgxfbtevzeuqrewyjywvmedtwajobluxrsdlvaghovxhfcieuudwrhffehgfuwkqvgpofqijpklraclahqmewxggvrboqxveabgovkfylybrbrxqnvljafuooyscossddrmuosmcxbthyynfuhytjyhkkvwiaqpicrfjxvwftatdwhwuxxqessofyecjfzbzhpqblvqooasrwnlaqrlzindcvxzunpizmgpgsmnangmfargzqcvclgphenedrlpkfnhcccwzkhsgswrtqnqidkaleqitfqyikkjkcjokeelqfldfbzmtmvcyfzbpcgbsjfedviylpheoilpddgtohbywwuuqdcvihhcqtkrmntgfkeytjeytnfjzxongdmvahbyubjbsdkrejpiyezopkkvrfeiwycizgexqcnrpbqwrksztcjqlrhbyhenwqlxxjkicoajfphdxjpndnkfsjrfoqsmntuenabiufbjkyhemewernlberrrlivflnehtterrvdgnrlaosrljjggogsyxpguzinyohitcbcaqebmogtkamdzhgtiufxeshimfdrcmfqrqtcbapddsmerewofiqrwprfcgdmwzuddpavlnohnybgibsnxtsnzilexfehiphpbpqghnocawhmkzakmotjkeuuilvkagummlcclpuwxbyeoubwqtqsokrnxgotuajewgabdyzzyglufirtdfebmvafinbboecugtxacqdxwmbxyhcksiygftwubfrnxlivjiofvjctzygkjqsnjlhhmoshoxpbrkhbqkztkhjcmeqxgpzxymlfwozldnpllxboixvivoquplfrvtwxljpbmyjlbvbgujcczqhjwdvqtgatnvcuwmncazwmsykjsgjpvhkgusfzyctzmigqowhmdicguijmatupdjzxqbunxbeqardupokgfbtgnkwmajdacajwzsuvpiwjmtzyimluenlcrybwpvtuztfxfsvgrgndhljizthoceovimkxsxyneohxnbzbkmnxlidsczqkknlhrbqdlhxsfypqviucqiywljmiqlzlaofolpmtkvhzgvscoowmzlkehvfidefmcfeqssjquavrehjhugjoeeuqrrskpnsituzqjxoydxxssszyzgmczdtdahjisrjjgdwlnjglrqrzrnyudairljibcutnfcojuyjzuhazszfepncfyvoxgnbiyyixzspcnlhclxddafebdukcvdkblnqmzqkonheehbszjkhhyvecpizqjdnwraosmbfntitxhocadbbniuqzxfuyjqfrpvocwnrziazjitmtxxvkewhfdcewxqfovliuxzilicokdjmuncxipcixlipdcmuwukshsotjjcabvewtorjckmmqtknmdrsvvgqzilbwnxuzlogloepyrsaiqyoxwjwmxbnwckvbiesvybwdqvjnywbwthuadbgeieblmdboggqwxugtiporxgkbroidfuykuypwyavecwgfskshqogbvajbthoemgryusercuxztwgtbzofcsiduoavtafszohwwchuqjpjbbrbxsudotmxprrtavzddwxijonauwgscsvtrjwomoqchhwxtohwatxghcvsbaqxsntzsluhxsmrajjefralceyhhympznjuzmwqummdxuwwqzwdffrgkjggfnjnyebxegzzbujfyeivmlwwgwrglrooznuhlfvguwezrqwnekgnahtocwbjamdtrtowwyyohusnsmznehzpieuayritybnlrldihnbdsbsbdqtpdpqyjkdrcecfwlljsljpdgifxetsajmymzcdlefllhtcotecgnbtputyabsmeigdjxwxywoyvyimdleebaadxpsfeadudsvebxbjrnotvqldkxutesdeimkhwpdbbyvhxsgjalsoosgpikstmbapoffdkljthvhlagsjtnglpuomrvejsdvfcxlgwhitnekotzcmagrjnvqdumqohzpshypkcijkgwozgyxvdozkaasbuohhkzaabuhllmnvtxtwqooxzkkcfaveprjtvklmaoxtftwzkdbpvvuezwbgohnzcomtjsudbbdpowrrtvqixxfellzkloxbrxdroctzwywujgzzptupqmfpstlpiowfnmdgvgkciyzlvskiinwoxsxvbgyprttxjgasztpuvjvwztcnutyxplebjnsgipbarhlcnwjkaspbohchtiurjfykknkslygfkomhqnaiocohyccfguufzebncmchjsapecxdbkouugsmtnipfmdxdamfhfoxcdoqjnjnzfpqsgdirbcaszqchlqhxupypvepxgxecyrwpkaziqndjjkjrpqjowpspvbizerthixqznivlbaflzhtujtkmqgcjdkpnjdrxktphtwfbwpcwcavxaxdrojjteqsajvizogsvgcctyinjqzsjplfkjajuxaprouznlyepxtvfswdsglgbaclhnpoiwkfqrggbmlmpdavzubxdcifoxaokwfwonulygizsuvxqxnomczdjcrcgxfduosvazmwzbzlhcuvxywlzguxjjkkvyutqwwlvtgxljaiercxbzmlwgudfdhseusqifvoxksxpbxublditsfyiflzcvzfdfpdeibmoekyjpddexbfnudsusdxbthmtfxhrgwtiirccxhbizvqffcwghjuusqfcbynbfewdsskwexmpvtrilyqsgraromzsuhbqnjldrpchnclecocjykihgzlwynfcrnhigbfxkrwblbphkdjttqjihergujyickvhoaomtnkmsjpzkyvzljexphylviqyhnbuxrgqdirpcfbevfjtmmodmarupbgicdefifsprpfqszlgjpnhzowtorkanvprqqtjiausxyhtjmtiiwfidmasztdcpynqacphntdtmfpdvpjtaekaggbevqmyxymtiokdspbzgnxubwikzaapehiabcktjhwkgjzhzldgrxjsfyuwgmghenfrtzsdauuaodxqvvyerjuebapknrmhwjhbrojwzcodwhpdbgaeninbtyrhzqxsfpdwzrvfnbruccjfqfupcdsiqjlnrfjasrkhznssbintubxchjhhiqahjtkfawlfyonocudtjpgkgjlyrrkohgrzzvqcwfwgprofintyzpwiregtwyxjywrvrusvnsqyvciubsqaotawxlmromuszooghkkfxwjpsdmvxdkjukxjwjdksmrxpkkpxtpbwbfisfqneuohxhbinrbxmaklfdjzhhpzrfnzrkpegzqnjmdlngvthppmovnrbclgpmiqnzzcwxgstfbrtmealyfigyxfnogdxpoxonzlrzjrvoplzkaimklngxqvhkiijuecthgeqrtxfsajsimbwyknlohabegrkrrwfytemczxogrdrrgibzankeqfddiufslellsggwthsvwkentzyrfppyacqczprryimfnhzowoxtrlpvmtfkstlbcgicqsnkgpysifmykfdzreydxneuydzjhqpmbwrbxgefmsojgbhuxkdfhpfxzvdbpfgdhekmnmaokhssvbsdbgqisfcpwsfzsvojfjsrqhuuduwifaywnthkiuhsrgnkrvuknmilvrowfwsqohmrusibdhuhcjvzjlvrufbtypotgjqagipmmhlcmliieerwhuizsvjnxtubqwabqaifcvsrzlklwxbgwfmxrmimdgxjnlaxtctdrerfpxvifkwbxuskrybktiyeambeebcptsvvmsmhgdxollkhlomdzlyjvyvvnrbaddfrujpvzngaisvfluqjscncriugpimqlcinkebcrtczhiyyirdanhddlusnoezbziuwphjeejhfivvznkemfbtcoiyahtljlynrwzearpvekmzhlguwvmgmmbwzadorelfxidnoiwiehpzgzefmppajnmttvdyemgzwfodtlpirdsmnzkitryomcyfukylxoinaornrtmdisoiuddnzwqitqzwhjecrmyhoretzgxciqngpsxcfgfzyneoxresrogmeebiqrcnpyehfriprzueajqfnrczmullahnexfebqaqfnzzkysvbagwemvxttmwvrvflcfjenjoizhuubutzmsxogboepyyezibsqbmgkwkwrcjyqhikbfpiqsmrjmqriwppdbijldaqzxpuiawhxkaujicxchftemfyfmscxhbxweswtjgtlmtkhpyvpybrkmtgtqvtocnqvaxpkjwkedgvvgsjiftgdqdbukackiefopjqpnhzezgrgrzpyvttugsedhmjcmrvnkeofqqignddniiazspgwgfbxolzwwklvairwvqchjxybwfjugmyflkkuuulqzgqkgsuymvrlemwrblieexszuzkygujowopflsaadzidkrqgsnmntbipofuwrahnypixrpzp\" 1626006833639000000", +// "measure,t1=3 c1=8", +// "measure,t2=3 c1=8u8" }; pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql)/sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); ASSERT_NE(taos_errno(pRes), 0); @@ -692,7 +696,7 @@ TEST(testCase, smlProcess_json1_Test) { "[\n" " {\n" " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 1346846400,\n" + " \"timestamp\": 0,\n" " \"value\": 18,\n" " \"tags\": {\n" " \"host\": \"web01\",\n" @@ -910,10 +914,11 @@ TEST(testCase, smlParseTelnetLine_error_Test) { pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, TSDB_SQL_INSERT); + SRequestObj *request = (SRequestObj *)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); - SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + STscObj* pTscObj = acquireTscObj(*(int64_t*)taos); + SSmlHandle *info = smlBuildSmlInfo(pTscObj, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); ASSERT_NE(info, nullptr); int32_t ret = 0; @@ -959,9 +964,35 @@ TEST(testCase, smlParseTelnetLine_diff_type_Test) { pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - const char *sql[2] = { + const char *sql[] = { "sys.procs.running 1479496104000 42 host=web01", - "sys.procs.running 1479496104000 42u8 host=web01" + "sys.procs.running 1479496104000 42u8 host=web01", + "appywjnuct 1626006833641 True id=\"appywjnuct_40601_49808_1\" t0=t t1=127i8 id=\"appywjnuct_40601_49808_2\" t2=32767i16 t3=2147483647i32 t4=9223372036854775807i64 t5=11.12345f32 t6=22.123456789f64 t7=\"binaryTagValue\" t8=L\"ncharTagValue\"" +// "meters 1601481600000 -863431872.000000f32 t0=-418706150i64 t1=844637295i64 t2=482576837i64 t3=736261541i64 t4=L\"5S6jypOYDYkALfeXCf2gbUEio7iTM9vFOrMcGqYae0yNeDAEIrKHacOo0U7JTrev\"", +// "meters 1601481600010 742480256.000000f32 t0=-418706150i64 t1=844637295i64 t2=482576837i64 t3=736261541i64 t4=L\"5S6jypOYDYkALfeXCf2gbUEio7iTM9vFOrMcGqYae0yNeDAEIrKHacOo0U7JTrev\"", +// "meters 1601481600020 -163715920.000000f32 t0=-418706150i64 t1=844637295i64 t2=482576837i64 t3=736261541i64 t4=L\"5S6jypOYDYkALfeXCf2gbUEio7iTM9vFOrMcGqYae0yNeDAEIrKHacOo0U7JTrev\"", +// "meters 1601481600030 63386372.000000f32 t0=-418706150i64 t1=844637295i64 t2=482576837i64 t3=736261541i64 t4=L\"5S6jypOYDYkALfeXCf2gbUEio7iTM9vFOrMcGqYae0yNeDAEIrKHacOo0U7JTrev\"", +// "meters 1601481600040 -82687824.000000f32 t0=-418706150i64 t1=844637295i64 t2=482576837i64 t3=736261541i64 t4=L\"5S6jypOYDYkALfeXCf2gbUEio7iTM9vFOrMcGqYae0yNeDAEIrKHacOo0U7JTrev\"", +// "meters 1601481600000 -683842112.000000f32 t0=354941102i64 t1=-228279853i64 t2=-78283134i64 t3=91718788i64 t4=L\"wQyjbkfama3csU7N9TPIVAzx3v5ZUoMg3bn3jq3tqSuHAqky8X8QnwbeQ64AjGEa\"", +// "meters 1601481600010 362312416.000000f32 t0=354941102i64 t1=-228279853i64 t2=-78283134i64 t3=91718788i64 t4=L\"wQyjbkfama3csU7N9TPIVAzx3v5ZUoMg3bn3jq3tqSuHAqky8X8QnwbeQ64AjGEa\"", +// "meters 1601481600020 178229296.000000f32 t0=354941102i64 t1=-228279853i64 t2=-78283134i64 t3=91718788i64 t4=L\"wQyjbkfama3csU7N9TPIVAzx3v5ZUoMg3bn3jq3tqSuHAqky8X8QnwbeQ64AjGEa\"", +// "meters 1601481600030 977283136.000000f32 t0=354941102i64 t1=-228279853i64 t2=-78283134i64 t3=91718788i64 t4=L\"wQyjbkfama3csU7N9TPIVAzx3v5ZUoMg3bn3jq3tqSuHAqky8X8QnwbeQ64AjGEa\"", +// "meters 1601481600040 -774479360.000000f32 t0=354941102i64 t1=-228279853i64 t2=-78283134i64 t3=91718788i64 t4=L\"wQyjbkfama3csU7N9TPIVAzx3v5ZUoMg3bn3jq3tqSuHAqky8X8QnwbeQ64AjGEa\"", +// "meters 1601481600000 -863431872.000000f32 t0=-503950941i64 t1=-1008101453i64 t2=800907871i64 t3=688116272i64 t4=L\"5kb9hzKk1aOxqn5qnGCmryWaOYtkDPlx1ku8I5hy3UVi6OwikZvBlfzX4R7wwfUm\"", +// "meters 1601481600010 742480256.000000f32 t0=-503950941i64 t1=-1008101453i64 t2=800907871i64 t3=688116272i64 t4=L\"5kb9hzKk1aOxqn5qnGCmryWaOYtkDPlx1ku8I5hy3UVi6OwikZvBlfzX4R7wwfUm\"", +// "meters 1601481600020 -163715920.000000f32 t0=-503950941i64 t1=-1008101453i64 t2=800907871i64 t3=688116272i64 t4=L\"5kb9hzKk1aOxqn5qnGCmryWaOYtkDPlx1ku8I5hy3UVi6OwikZvBlfzX4R7wwfUm\"", +// "meters 1601481600030 63386372.000000f32 t0=-503950941i64 t1=-1008101453i64 t2=800907871i64 t3=688116272i64 t4=L\"5kb9hzKk1aOxqn5qnGCmryWaOYtkDPlx1ku8I5hy3UVi6OwikZvBlfzX4R7wwfUm\"", +// "meters 1601481600040 -82687824.000000f32 t0=-503950941i64 t1=-1008101453i64 t2=800907871i64 t3=688116272i64 t4=L\"5kb9hzKk1aOxqn5qnGCmryWaOYtkDPlx1ku8I5hy3UVi6OwikZvBlfzX4R7wwfUm\"", +// "meters 1601481600000 -863431872.000000f32 t0=28805371i64 t1=-231884121i64 t2=940124207i64 t3=176395723i64 t4=L\"7pkY8763Ir0QeugozDbqk6NHbvRpx2drfndch74No3sqmyCJZCZaxAFwVmLgcMvh\"", +// "meters 1601481600010 742480256.000000f32 t0=28805371i64 t1=-231884121i64 t2=940124207i64 t3=176395723i64 t4=L\"7pkY8763Ir0QeugozDbqk6NHbvRpx2drfndch74No3sqmyCJZCZaxAFwVmLgcMvh\"", +// "meters 1601481600020 -163715920.000000f32 t0=28805371i64 t1=-231884121i64 t2=940124207i64 t3=176395723i64 t4=L\"7pkY8763Ir0QeugozDbqk6NHbvRpx2drfndch74No3sqmyCJZCZaxAFwVmLgcMvh\"", +// "meters 1601481600030 63386372.000000f32 t0=28805371i64 t1=-231884121i64 t2=940124207i64 t3=176395723i64 t4=L\"7pkY8763Ir0QeugozDbqk6NHbvRpx2drfndch74No3sqmyCJZCZaxAFwVmLgcMvh\"", +// "meters 1601481600040 -82687824.000000f32 t0=28805371i64 t1=-231884121i64 t2=940124207i64 t3=176395723i64 t4=L\"7pkY8763Ir0QeugozDbqk6NHbvRpx2drfndch74No3sqmyCJZCZaxAFwVmLgcMvh\"", +// "meters 1601481600000 -863431872.000000f32 t0=-208520225i64 t1=-254703350i64 t2=-1059776552i64 t3=1056267931i64 t4=L\"1zWxWvHNZYailPvb4XxafeA6QvrUrKUf8ECU1axNWvV9ae851s34wqZcMeU2ME7J\"", +// "meters 1601481600010 742480256.000000f32 t0=-208520225i64 t1=-254703350i64 t2=-1059776552i64 t3=1056267931i64 t4=L\"1zWxWvHNZYailPvb4XxafeA6QvrUrKUf8ECU1axNWvV9ae851s34wqZcMeU2ME7J\"", +// "meters 1601481600020 -163715920.000000f32 t0=-208520225i64 t1=-254703350i64 t2=-1059776552i64 t3=1056267931i64 t4=L\"1zWxWvHNZYailPvb4XxafeA6QvrUrKUf8ECU1axNWvV9ae851s34wqZcMeU2ME7J\"", +// "meters 1601481600030 63386372.000000f32 t0=-208520225i64 t1=-254703350i64 t2=-1059776552i64 t3=1056267931i64 t4=L\"1zWxWvHNZYailPvb4XxafeA6QvrUrKUf8ECU1axNWvV9ae851s34wqZcMeU2ME7J\"", +// "meters 1601481600040 -82687824.000000f32 t0=-208520225i64 t1=-254703350i64 t2=-1059776552i64 t3=1056267931i64 t4=L\"1zWxWvHNZYailPvb4XxafeA6QvrUrKUf8ECU1axNWvV9ae851s34wqZcMeU2ME7J\"" }; pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql)/sizeof(sql[0]), TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); ASSERT_NE(taos_errno(pRes), 0); @@ -979,10 +1010,11 @@ TEST(testCase, smlParseTelnetLine_json_error_Test) { pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, TSDB_SQL_INSERT); + SRequestObj *request = (SRequestObj *)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); - SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + STscObj* pTscObj = acquireTscObj(*(int64_t*)taos); + SSmlHandle *info = smlBuildSmlInfo(pTscObj, request, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); ASSERT_NE(info, nullptr); int32_t ret = 0; @@ -1069,7 +1101,7 @@ TEST(testCase, smlParseTelnetLine_diff_json_type1_Test) { " },\n" "]", }; - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql)/sizeof(sql[0]), TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql)/sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); ASSERT_NE(taos_errno(pRes), 0); taos_free_result(pRes); } @@ -1106,7 +1138,7 @@ TEST(testCase, smlParseTelnetLine_diff_json_type2_Test) { " },\n" "]", }; - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql)/sizeof(sql[0]), TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + pRes = taos_schemaless_insert(taos, (char **)sql, 0, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); ASSERT_NE(taos_errno(pRes), 0); taos_free_result(pRes); } @@ -1257,7 +1289,7 @@ TEST(testCase, sml_16368_Test) { "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833839006, \"type\": \"us\"}, \"value\": {\"value\": 8, \"type\": \"double\"}, \"tags\": {\"t1\": {\"value\": 4, \"type\": \"double\"}, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}, \"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, \"type\": \"double\"}}},\n" "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833939007, \"type\": \"us\"}, \"value\": {\"value\": 9, \"type\": \"double\"}, \"tags\": {\"t1\": 4, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}, \"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, \"type\": \"double\"}}}]" }; - pRes = taos_schemaless_insert(taos, (char**)sql, sizeof(sql)/sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_MICRO_SECONDS); + pRes = taos_schemaless_insert(taos, (char**)sql, 0, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_MICRO_SECONDS); ASSERT_EQ(taos_errno(pRes), 0); taos_free_result(pRes); } @@ -1490,4 +1522,4 @@ pRes = taos_schemaless_insert(taos, (char**)sql, sizeof(sql)/sizeof(sql[0]), TSD ASSERT_EQ(taos_errno(pRes), 0); taos_free_result(pRes); } -*/ \ No newline at end of file +*/ diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 47cfa5d410..cd21de52ed 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1752,7 +1752,7 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock); int32_t rows = pDataBlock->info.rows; int32_t len = 0; - len += snprintf(dumpBuf + len, size - len, "\n%s |block type %d |child id %d|group id:%" PRIu64 "| uid:%ld\n", flag, + len += snprintf(dumpBuf + len, size - len, "%s |block type %d |child id %d|group id:%" PRIu64 "| uid:%ld|======\n", "dumpBlockData", (int32_t)pDataBlock->info.type, pDataBlock->info.childId, pDataBlock->info.groupId, pDataBlock->info.uid); if (len >= size - 1) return dumpBuf; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 35d3513efa..aeb83d3425 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -4827,6 +4827,35 @@ int32_t tDeserializeSMDropStreamReq(void *buf, int32_t bufLen, SMDropStreamReq * return 0; } +int32_t tSerializeSMRecoverStreamReq(void *buf, int32_t bufLen, const SMRecoverStreamReq *pReq) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; + if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1; + + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSMRecoverStreamReq(void *buf, int32_t bufLen, SMRecoverStreamReq *pReq) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1; + + tEndDecode(&decoder); + + tDecoderClear(&decoder); + return 0; +} + void tFreeSCMCreateStreamReq(SCMCreateStreamReq *pReq) { taosMemoryFreeClear(pReq->sql); taosMemoryFreeClear(pReq->ast); @@ -4949,8 +4978,8 @@ int tEncodeSVCreateTbReq(SEncoder *pCoder, const SVCreateTbReq *pReq) { if (tEncodeTag(pCoder, (const STag *)pReq->ctb.pTag) < 0) return -1; int32_t len = taosArrayGetSize(pReq->ctb.tagName); if (tEncodeI32(pCoder, len) < 0) return -1; - for (int32_t i = 0; i < len; i++){ - char* name = taosArrayGet(pReq->ctb.tagName, i); + for (int32_t i = 0; i < len; i++) { + char *name = taosArrayGet(pReq->ctb.tagName, i); if (tEncodeCStr(pCoder, name) < 0) return -1; } } else if (pReq->type == TSDB_NORMAL_TABLE) { @@ -4986,9 +5015,9 @@ int tDecodeSVCreateTbReq(SDecoder *pCoder, SVCreateTbReq *pReq) { int32_t len = 0; if (tDecodeI32(pCoder, &len) < 0) return -1; pReq->ctb.tagName = taosArrayInit(len, TSDB_COL_NAME_LEN); - if(pReq->ctb.tagName == NULL) return -1; - for (int32_t i = 0; i < len; i++){ - char name[TSDB_COL_NAME_LEN] = {0}; + if (pReq->ctb.tagName == NULL) return -1; + for (int32_t i = 0; i < len; i++) { + char name[TSDB_COL_NAME_LEN] = {0}; char *tmp = NULL; if (tDecodeCStr(pCoder, &tmp) < 0) return -1; strcpy(name, tmp); diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c index 84491a82b0..49207225a5 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c @@ -148,9 +148,9 @@ static int32_t mmStart(SMnodeMgmt *pMgmt) { static void mmStop(SMnodeMgmt *pMgmt) { dDebug("mnode-mgmt start to stop"); + mndPreClose(pMgmt->pMnode); taosThreadRwlockWrlock(&pMgmt->lock); pMgmt->stopped = 1; - mndPreClose(pMgmt->pMnode); taosThreadRwlockUnlock(&pMgmt->lock); mndStop(pMgmt->pMnode); diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 9052fb20ca..7efa46c514 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -221,11 +221,11 @@ int32_t dmInitMsgHandle(SDnode *pDnode) { static inline int32_t dmSendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) { SDnode *pDnode = dmInstance(); - if (pDnode->status != DND_STAT_RUNNING) { + if (pDnode->status != DND_STAT_RUNNING && pMsg->msgType < TDMT_SYNC_MSG) { rpcFreeCont(pMsg->pCont); pMsg->pCont = NULL; terrno = TSDB_CODE_NODE_OFFLINE; - dError("failed to send rpc msg since %s, handle:%p", terrstr(), pMsg->info.handle); + dError("failed to send rpc msg:%s since %s, handle:%p", TMSG_INFO(pMsg->msgType), terrstr(), pMsg->info.handle); return -1; } else { rpcSendRequest(pDnode->trans.clientRpc, pEpSet, pMsg, NULL); diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 7ac991451e..5cf6e26ee8 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -559,6 +559,7 @@ typedef struct { // info int64_t uid; int8_t status; + int8_t isDistributed; // config int8_t igExpired; int8_t trigger; @@ -586,6 +587,23 @@ typedef struct { int32_t tEncodeSStreamObj(SEncoder* pEncoder, const SStreamObj* pObj); int32_t tDecodeSStreamObj(SDecoder* pDecoder, SStreamObj* pObj); +typedef struct { + char streamName[TSDB_STREAM_FNAME_LEN]; + int64_t uid; + int64_t streamUid; + SArray* childInfo; // SArray +} SStreamCheckpointObj; + +#if 0 +typedef struct { + int64_t uid; + int64_t streamId; + int8_t isDistributed; + int8_t status; + int8_t stage; +} SStreamRecoverObj; +#endif + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 9f168e2c83..8ad1ac56e4 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -50,8 +50,8 @@ extern "C" { // clang-format on #define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE) -#define SYSTABLE_SCH_DB_NAME_LEN ((TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE) -#define SYSTABLE_SCH_COL_NAME_LEN ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE) +#define SYSTABLE_SCH_DB_NAME_LEN ((TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE) +#define SYSTABLE_SCH_COL_NAME_LEN ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE) typedef int32_t (*MndMsgFp)(SRpcMsg *pMsg); typedef int32_t (*MndInitFp)(SMnode *pMnode); @@ -61,7 +61,7 @@ typedef void (*ShowFreeIterFp)(SMnode *pMnode, void *pIter); typedef struct SQWorker SQHandle; typedef struct { - const char * name; + const char *name; MndInitFp initFp; MndCleanupFp cleanupFp; } SMnodeStep; @@ -70,7 +70,7 @@ typedef struct { int64_t showId; ShowRetrieveFp retrieveFps[TSDB_MGMT_TABLE_MAX]; ShowFreeIterFp freeIterFps[TSDB_MGMT_TABLE_MAX]; - SCacheObj * cache; + SCacheObj *cache; } SShowMgmt; typedef struct { @@ -84,12 +84,13 @@ typedef struct { } STelemMgmt; typedef struct { - tsem_t syncSem; + tsem_t syncSem; int64_t sync; bool standby; SReplica replica; int32_t errCode; int32_t transId; + int8_t leaderTransferFinish; } SSyncMgmt; typedef struct { @@ -107,14 +108,14 @@ typedef struct SMnode { bool stopped; bool restored; bool deploy; - char * path; + char *path; int64_t checkTime; - SSdb * pSdb; - SArray * pSteps; - SQHandle * pQuery; - SHashObj * infosMeta; - SHashObj * perfsMeta; - SWal * pWal; + SSdb *pSdb; + SArray *pSteps; + SQHandle *pQuery; + SHashObj *infosMeta; + SHashObj *perfsMeta; + SWal *pWal; SShowMgmt showMgmt; SProfileMgmt profileMgmt; STelemMgmt telemMgmt; diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index 1497bba11c..6e367fbe24 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -27,6 +27,7 @@ typedef enum { TRANS_STOP_FUNC_TEST = 2, TRANS_START_FUNC_MQ_REB = 3, TRANS_STOP_FUNC_MQ_REB = 4, + TRANS_FUNC_RECOVER_STREAM_STEP_NEXT = 5, } ETrnFunc; typedef enum { diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index 727b20ef8a..c26424e049 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -27,6 +27,7 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) { if (tEncodeI64(pEncoder, pObj->uid) < 0) return -1; if (tEncodeI8(pEncoder, pObj->status) < 0) return -1; + if (tEncodeI8(pEncoder, pObj->isDistributed) < 0) return -1; if (tEncodeI8(pEncoder, pObj->igExpired) < 0) return -1; if (tEncodeI8(pEncoder, pObj->trigger) < 0) return -1; @@ -72,6 +73,7 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj) { if (tDecodeI64(pDecoder, &pObj->uid) < 0) return -1; if (tDecodeI8(pDecoder, &pObj->status) < 0) return -1; + if (tDecodeI8(pDecoder, &pObj->isDistributed) < 0) return -1; if (tDecodeI8(pDecoder, &pObj->igExpired) < 0) return -1; if (tDecodeI8(pDecoder, &pObj->trigger) < 0) return -1; diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 29e68ce4e8..041fc2a2d1 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -368,7 +368,18 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) { void mndPreClose(SMnode *pMnode) { if (pMnode != NULL) { + atomic_store_8(&(pMnode->syncMgmt.leaderTransferFinish), 0); syncLeaderTransfer(pMnode->syncMgmt.sync); + + /* + mDebug("vgId:1, mnode start leader transfer"); + // wait for leader transfer finish + while (!atomic_load_8(&(pMnode->syncMgmt.leaderTransferFinish))) { + taosMsleep(10); + mDebug("vgId:1, mnode waiting for leader transfer"); + } + mDebug("vgId:1, mnode finish leader transfer"); + */ } } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index d7eaa72202..13655ac21f 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -218,7 +218,6 @@ bool mndIsMnode(SMnode *pMnode, int32_t dnodeId) { } void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) { -#if 0 SSdb *pSdb = pMnode->pSdb; int32_t totalMnodes = sdbGetSize(pSdb, SDB_MNODE); void *pIter = NULL; @@ -238,9 +237,10 @@ void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) { addEpIntoEpSet(pEpSet, pObj->pDnode->fqdn, pObj->pDnode->port); sdbRelease(pSdb, pObj); } -#else - syncGetRetryEpSet(pMnode->syncMgmt.sync, pEpSet); -#endif + + if (pEpSet->numOfEps == 0) { + syncGetRetryEpSet(pMnode->syncMgmt.sync, pEpSet); + } } static int32_t mndSetCreateMnodeRedoLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) { diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index aa6e7192fc..ec0ea90f46 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -319,6 +319,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { int32_t totLevel = LIST_LENGTH(pPlan->pSubplans); ASSERT(totLevel <= 2); pStream->tasks = taosArrayInit(totLevel, sizeof(void*)); + pStream->isDistributed = totLevel == 2; bool hasExtraSink = false; bool externalTargetDB = strcmp(pStream->sourceDb, pStream->targetDb) != 0; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 5777df4fa6..5e8867fdfb 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -36,7 +36,7 @@ static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream); static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pStream, SStreamObj *pNewStream); static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq); static int32_t mndProcessDropStreamReq(SRpcMsg *pReq); -/*static int32_t mndProcessDropStreamInRsp(SRpcMsg *pRsp);*/ +static int32_t mndProcessRecoverStreamReq(SRpcMsg *pReq); static int32_t mndProcessStreamMetaReq(SRpcMsg *pReq); static int32_t mndGetStreamMeta(SRpcMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); @@ -55,6 +55,7 @@ int32_t mndInitStream(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_CREATE_STREAM, mndProcessCreateStreamReq); mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM, mndProcessDropStreamReq); + mndSetMsgHandle(pMnode, TDMT_MND_RECOVER_STREAM, mndProcessRecoverStreamReq); mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_DEPLOY_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_DROP_RSP, mndTransProcessRsp); @@ -176,7 +177,7 @@ static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pOldStream, SStream taosWLockLatch(&pOldStream->lock); - // TODO handle update + pOldStream->status = pNewStream->status; taosWUnLockLatch(&pOldStream->lock); return 0; @@ -394,6 +395,20 @@ int32_t mndPersistDropStreamLog(SMnode *pMnode, STrans *pTrans, SStreamObj *pStr return 0; } +static int32_t mndSetStreamRecover(SMnode *pMnode, STrans *pTrans, const SStreamObj *pStream) { + SStreamObj streamObj = {0}; + memcpy(streamObj.name, pStream->name, TSDB_STREAM_FNAME_LEN); + streamObj.status = STREAM_STATUS__RECOVER; + SSdbRaw *pCommitRaw = mndStreamActionEncode(&streamObj); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + mError("stream trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); + return 0; +} + static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStreamObj *pStream, const char *user) { SStbObj *pStb = NULL; SDbObj *pDb = NULL; @@ -491,6 +506,76 @@ static int32_t mndPersistTaskDropReq(STrans *pTrans, SStreamTask *pTask) { return 0; } +static int32_t mndPersistTaskRecoverReq(STrans *pTrans, SStreamTask *pTask) { + SMStreamTaskRecoverReq *pReq = taosMemoryCalloc(1, sizeof(SMStreamTaskRecoverReq)); + if (pReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pReq->streamId = pTask->streamId; + pReq->taskId = pTask->taskId; + int32_t len; + int32_t code; + tEncodeSize(tEncodeSMStreamTaskRecoverReq, pReq, len, code); + if (code != 0) { + return -1; + } + void *buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len); + if (buf == NULL) { + return -1; + } + void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + SEncoder encoder; + tEncoderInit(&encoder, abuf, len); + tEncodeSMStreamTaskRecoverReq(&encoder, pReq); + ((SMsgHead *)buf)->vgId = pTask->nodeId; + + STransAction action = {0}; + memcpy(&action.epSet, &pTask->epSet, sizeof(SEpSet)); + action.pCont = buf; + action.contLen = sizeof(SMsgHead) + len; + action.msgType = TDMT_STREAM_TASK_RECOVER; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(buf); + return -1; + } + return 0; +} + +int32_t mndRecoverStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { + if (pStream->isDistributed) { + int32_t lv = taosArrayGetSize(pStream->tasks); + for (int32_t i = 0; i < lv; i++) { + SArray *pTasks = taosArrayGetP(pStream->tasks, i); + int32_t sz = taosArrayGetSize(pTasks); + SStreamTask *pTask = taosArrayGetP(pTasks, 0); + if (!pTask->isDataScan && pTask->execType != TASK_EXEC__NONE) { + ASSERT(sz == 1); + if (mndPersistTaskRecoverReq(pTrans, pTask) < 0) { + return -1; + } + } else { + continue; + } + } + } else { + int32_t lv = taosArrayGetSize(pStream->tasks); + for (int32_t i = 0; i < lv; i++) { + SArray *pTasks = taosArrayGetP(pStream->tasks, i); + int32_t sz = taosArrayGetSize(pTasks); + for (int32_t j = 0; j < sz; j++) { + SStreamTask *pTask = taosArrayGetP(pTasks, j); + if (!pTask->isDataScan) break; + ASSERT(pTask->execType != TASK_EXEC__NONE); + if (mndPersistTaskRecoverReq(pTrans, pTask) < 0) { + return -1; + } + } + } + } + return 0; +} + int32_t mndDropStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { int32_t lv = taosArrayGetSize(pStream->tasks); for (int32_t i = 0; i < lv; i++) { @@ -672,6 +757,69 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { return TSDB_CODE_ACTION_IN_PROGRESS; } +static int32_t mndProcessRecoverStreamReq(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; + SStreamObj *pStream = NULL; + /*SDbObj *pDb = NULL;*/ + /*SUserObj *pUser = NULL;*/ + + SMRecoverStreamReq recoverReq = {0}; + if (tDeserializeSMRecoverStreamReq(pReq->pCont, pReq->contLen, &recoverReq) < 0) { + ASSERT(0); + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + pStream = mndAcquireStream(pMnode, recoverReq.name); + + if (pStream == NULL) { + if (recoverReq.igNotExists) { + mDebug("stream:%s, not exist, ignore not exist is set", recoverReq.name); + sdbRelease(pMnode->pSdb, pStream); + return 0; + } else { + terrno = TSDB_CODE_MND_STREAM_NOT_EXIST; + return -1; + } + } + + if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pStream->targetDb) != 0) { + return -1; + } + + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq); + if (pTrans == NULL) { + mError("stream:%s, failed to recover since %s", recoverReq.name, terrstr()); + sdbRelease(pMnode->pSdb, pStream); + return -1; + } + mDebug("trans:%d, used to drop stream:%s", pTrans->id, recoverReq.name); + + // broadcast to recover all tasks + if (mndRecoverStreamTasks(pMnode, pTrans, pStream) < 0) { + mError("stream:%s, failed to recover task since %s", recoverReq.name, terrstr()); + sdbRelease(pMnode->pSdb, pStream); + return -1; + } + + // update stream status + if (mndSetStreamRecover(pMnode, pTrans, pStream) < 0) { + sdbRelease(pMnode->pSdb, pStream); + return -1; + } + + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("trans:%d, failed to prepare recover stream trans since %s", pTrans->id, terrstr()); + sdbRelease(pMnode->pSdb, pStream); + mndTransDrop(pTrans); + return -1; + } + + sdbRelease(pMnode->pSdb, pStream); + + return TSDB_CODE_ACTION_IN_PROGRESS; +} + int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 3c3864b620..2811aeb43a 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -56,23 +56,24 @@ void mndSyncCommitMsg(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbM sdbSetApplyInfo(pMnode->pSdb, cbMeta.index, cbMeta.term, cbMeta.lastConfigIndex); } - if (pMgmt->transId == transId && transId != 0) { + if (transId <= 0) { + mError("trans:%d, invalid commit msg", transId); + } else if (transId == pMgmt->transId) { if (pMgmt->errCode != 0) { mError("trans:%d, failed to propose since %s", transId, tstrerror(pMgmt->errCode)); } pMgmt->transId = 0; tsem_post(&pMgmt->syncSem); } else { -#if 1 - mError("trans:%d, invalid commit msg since trandId not match with %d", transId, pMgmt->transId); -#else STrans *pTrans = mndAcquireTrans(pMnode, transId); if (pTrans != NULL) { + mDebug("trans:%d, execute in mnode which not leader", transId); mndTransExecute(pMnode, pTrans); mndReleaseTrans(pMnode, pTrans); + // sdbWriteFile(pMnode->pSdb, SDB_WRITE_DELTA); + } else { + mError("trans:%d, not found while execute in mnode since %s", transId, terrstr()); } - // sdbWriteFile(pMnode->pSdb, SDB_WRITE_DELTA); -#endif } } @@ -153,6 +154,12 @@ int32_t mndSnapshotDoWrite(struct SSyncFSM *pFsm, void *pWriter, void *pBuf, int return sdbDoWrite(pMnode->pSdb, pWriter, pBuf, len); } +void mndLeaderTransfer(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { + SMnode *pMnode = pFsm->data; + atomic_store_8(&(pMnode->syncMgmt.leaderTransferFinish), 1); + mDebug("vgId:1, mnd leader transfer finish"); +} + SSyncFSM *mndSyncMakeFsm(SMnode *pMnode) { SSyncFSM *pFsm = taosMemoryCalloc(1, sizeof(SSyncFSM)); pFsm->data = pMnode; @@ -160,6 +167,7 @@ SSyncFSM *mndSyncMakeFsm(SMnode *pMnode) { pFsm->FpPreCommitCb = NULL; pFsm->FpRollBackCb = NULL; pFsm->FpRestoreFinishCb = mndRestoreFinish; + pFsm->FpLeaderTransferCb = mndLeaderTransfer; pFsm->FpReConfigCb = mndReConfig; pFsm->FpGetSnapshot = mndSyncGetSnapshot; pFsm->FpGetSnapshotInfo = mndSyncGetSnapshotInfo; diff --git a/source/dnode/mnode/sdb/inc/sdb.h b/source/dnode/mnode/sdb/inc/sdb.h index 1294f0cff3..991dd2c4f9 100644 --- a/source/dnode/mnode/sdb/inc/sdb.h +++ b/source/dnode/mnode/sdb/inc/sdb.h @@ -137,17 +137,18 @@ typedef enum { SDB_USER = 7, SDB_AUTH = 8, SDB_ACCT = 9, - SDB_STREAM = 10, - SDB_OFFSET = 11, - SDB_SUBSCRIBE = 12, - SDB_CONSUMER = 13, - SDB_TOPIC = 14, - SDB_VGROUP = 15, - SDB_SMA = 16, - SDB_STB = 17, - SDB_DB = 18, - SDB_FUNC = 19, - SDB_MAX = 20 + SDB_STREAM_CK = 10, + SDB_STREAM = 11, + SDB_OFFSET = 12, + SDB_SUBSCRIBE = 13, + SDB_CONSUMER = 14, + SDB_TOPIC = 15, + SDB_VGROUP = 16, + SDB_SMA = 17, + SDB_STB = 18, + SDB_DB = 19, + SDB_FUNC = 20, + SDB_MAX = 21 } ESdbType; typedef struct SSdbRaw { @@ -308,7 +309,7 @@ void sdbRelease(SSdb *pSdb, void *pObj); * @return void* The next iterator of the table. */ void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj); -void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStatus *status) ; +void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStatus *status); /** * @brief Cancel a traversal diff --git a/source/dnode/snode/inc/sndInt.h b/source/dnode/snode/inc/sndInt.h index 8916e2a31c..1f0019ef46 100644 --- a/source/dnode/snode/inc/sndInt.h +++ b/source/dnode/snode/inc/sndInt.h @@ -30,15 +30,6 @@ extern "C" { #endif -enum { - STREAM_STATUS__RUNNING = 1, - STREAM_STATUS__STOPPED, - STREAM_STATUS__CREATING, - STREAM_STATUS__STOPING, - STREAM_STATUS__RESTORING, - STREAM_STATUS__DELETING, -}; - typedef struct { SHashObj* pHash; // taskId -> SStreamTask } SStreamMeta; diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 87148a8450..81177fd5d8 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -536,6 +536,10 @@ static int32_t vnodeSnapshotDoWrite(struct SSyncFSM *pFsm, void *pWriter, void * #endif } +static void vnodeLeaderTransfer(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { + SVnode *pVnode = pFsm->data; +} + static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) { SSyncFSM *pFsm = taosMemoryCalloc(1, sizeof(SSyncFSM)); pFsm->data = pVnode; @@ -544,6 +548,7 @@ static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) { pFsm->FpRollBackCb = vnodeSyncRollBackMsg; pFsm->FpGetSnapshotInfo = vnodeSyncGetSnapshot; pFsm->FpRestoreFinishCb = NULL; + pFsm->FpLeaderTransferCb = vnodeLeaderTransfer; pFsm->FpReConfigCb = vnodeSyncReconfig; pFsm->FpSnapshotStartRead = vnodeSnapshotStartRead; pFsm->FpSnapshotStopRead = vnodeSnapshotStopRead; @@ -579,8 +584,8 @@ int32_t vnodeSyncOpen(SVnode *pVnode, char *path) { } setPingTimerMS(pVnode->sync, 5000); - setElectTimerMS(pVnode->sync, 500); - setHeartbeatTimerMS(pVnode->sync, 100); + setElectTimerMS(pVnode->sync, 1300); + setHeartbeatTimerMS(pVnode->sync, 900); return 0; } diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 0d57a6370c..cd50a13edd 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -139,6 +139,12 @@ typedef struct STaskIdInfo { char* str; } STaskIdInfo; +enum { + STREAM_RECOVER_STEP__NONE = 0, + STREAM_RECOVER_STEP__PREPARE, + STREAM_RECOVER_STEP__SCAN, +}; + typedef struct { //TODO remove prepareStatus STqOffsetVal prepareStatus; // for tmq @@ -147,6 +153,10 @@ typedef struct { SSDataBlock* pullOverBlk; // for streaming SWalFilterCond cond; int64_t lastScanUid; + int8_t recoverStep; + SQueryTableDataCond tableCond; + int64_t recoverStartVer; + int64_t recoverEndVer; } SStreamTaskInfo; typedef struct SExecTaskInfo { @@ -361,6 +371,13 @@ typedef struct SessionWindowSupporter { uint8_t parentType; } SessionWindowSupporter; +typedef struct STimeWindowSupp { + int8_t calTrigger; + int64_t waterMark; + TSKEY maxTs; + SColumnInfoData timeWindowData; // query time window info for scalar function execution. +} STimeWindowAggSupp; + typedef struct SStreamScanInfo { uint64_t tableUid; // queried super table uid SExprInfo* pPseudoExpr; @@ -397,6 +414,7 @@ typedef struct SStreamScanInfo { SSDataBlock* pDeleteDataRes; // delete data SSDataBlock int32_t deleteDataIndex; STimeWindow updateWin; + STimeWindowAggSupp twAggSup; // status for tmq // SSchemaWrapper schema; @@ -442,13 +460,6 @@ typedef struct SAggSupporter { int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row } SAggSupporter; -typedef struct STimeWindowSupp { - int8_t calTrigger; - int64_t waterMark; - TSKEY maxTs; - SColumnInfoData timeWindowData; // query time window info for scalar function execution. -} STimeWindowAggSupp; - typedef struct SIntervalAggOperatorInfo { // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode SOptrBasicInfo binfo; // basic info @@ -942,6 +953,7 @@ bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap); int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pStartTs, TSKEY* pEndTs, int32_t rows, int32_t start, int64_t gap, SHashObj* pStDeleted); bool functionNeedToExecute(SqlFunctionCtx* pCtx); +bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup); int32_t finalizeResultRowIntoResultDataBlock(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition, SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, int32_t numOfExprs, const int32_t* rowCellOffset, diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 78a7e58ee1..395b42aa86 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -139,18 +139,29 @@ SSDataBlock* doScanLastrow(SOperatorInfo* pOperator) { colDataAppend(pDst, 0, p, isNull); } + pInfo->pRes->info.uid = *(tb_uid_t*)taosArrayGet(pInfo->pUidList, pInfo->indexOfBufferedRes); + pInfo->pRes->info.rows = 1; + if (pInfo->pseudoExprSup.numOfExprs > 0) { SExprSupp* pSup = &pInfo->pseudoExprSup; - addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pInfo->pRes, + int32_t code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pInfo->pRes, GET_TASKID(pTaskInfo)); + if (code != TSDB_CODE_SUCCESS) { + pTaskInfo->code = code; + return NULL; + } } - pInfo->pRes->info.uid = *(tb_uid_t*)taosArrayGet(pInfo->pUidList, pInfo->indexOfBufferedRes); - int64_t* groupId = taosHashGet(pTableList->map, &pInfo->pRes->info.uid, sizeof(int64_t)); - pInfo->pRes->info.groupId = *groupId; + if (pTableList->map != NULL) { + int64_t* groupId = taosHashGet(pTableList->map, &pInfo->pRes->info.uid, sizeof(int64_t)); + pInfo->pRes->info.groupId = *groupId; + } else { + ASSERT(taosArrayGetSize(pTableList->pTableList) == 1); + STableKeyInfo* pKeyInfo = taosArrayGet(pTableList->pTableList, 0); + pInfo->pRes->info.groupId = pKeyInfo->groupId; + } pInfo->indexOfBufferedRes += 1; - pInfo->pRes->info.rows = 1; return pInfo->pRes; } else { doSetOperatorCompleted(pOperator); @@ -182,8 +193,12 @@ SSDataBlock* doScanLastrow(SOperatorInfo* pOperator) { STableKeyInfo* pKeyInfo = taosArrayGet(pGroupTableList, 0); pInfo->pRes->info.groupId = pKeyInfo->groupId; - addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pInfo->pRes, + code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pInfo->pRes, GET_TASKID(pTaskInfo)); + if (code != TSDB_CODE_SUCCESS) { + pTaskInfo->code = code; + return NULL; + } } tsdbLastrowReaderClose(pInfo->pLastrowReader); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 76d853ef3e..a8ff3188c8 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -65,7 +65,7 @@ size_t getResultRowSize(SqlFunctionCtx* pCtx, int32_t numOfOutput) { } rowSize += - (numOfOutput * sizeof(bool)); // expand rowSize to mark if col is null for top/bottom result(saveTupleData) + (numOfOutput * sizeof(bool)); // expand rowSize to mark if col is null for top/bottom result(doSaveTupleData) return rowSize; } diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index d910b8be34..37d2521e5d 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -261,6 +261,15 @@ int32_t qStreamInput(qTaskInfo_t tinfo, void* pItem) { } #endif +int32_t qStreamPrepareRecover(qTaskInfo_t tinfo, int64_t startVer, int64_t endVer) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM); + pTaskInfo->streamInfo.recoverStartVer = startVer; + pTaskInfo->streamInfo.recoverEndVer = endVer; + pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__PREPARE; + return 0; +} + void* qExtractReaderFromStreamScanner(void* scanner) { SStreamScanInfo* pInfo = scanner; return (void*)pInfo->tqReader; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 6aacb0ee01..a81a796fa7 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -516,10 +516,14 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { } SArray* tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, pInfo->currentGroupId); + tsdbReaderClose(pInfo->dataReader); int32_t code = tsdbReaderOpen(pInfo->readHandle.vnode, &pInfo->cond, tableList, (STsdbReader**)&pInfo->dataReader, GET_TASKID(pTaskInfo)); + if (code != 0) { + // TODO + } } SSDataBlock* result = doTableScanGroup(pOperator); @@ -798,7 +802,12 @@ static bool isStateWindow(SStreamScanInfo* pInfo) { static bool isIntervalWindow(SStreamScanInfo* pInfo) { return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL || - pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL; + pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL || + pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL; +} + +static bool isSignleIntervalWindow(SStreamScanInfo* pInfo) { + return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL; } static uint64_t getGroupId(SOperatorInfo* pOperator, uint64_t uid) { @@ -871,6 +880,7 @@ static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_ } resetTableScanInfo(pInfo->pTableScanOp->info, &win); + pInfo->pTableScanOp->status = OP_OPENED; return true; } @@ -1125,9 +1135,14 @@ static void setUpdateData(SStreamScanInfo* pInfo, SSDataBlock* pBlock, SSDataBlo static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock* pBlock, bool out) { SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex); ASSERT(pColDataInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP); - TSKEY* ts = (TSKEY*)pColDataInfo->pData; + TSKEY* tsCol = (TSKEY*)pColDataInfo->pData; for (int32_t rowId = 0; rowId < pBlock->info.rows; rowId++) { - if (updateInfoIsUpdated(pInfo->pUpdateInfo, pBlock->info.uid, ts[rowId]) && out) { + SResultRowInfo dumyInfo; + dumyInfo.cur.pageId = -1; + STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCol[rowId], &pInfo->interval, TSDB_ORDER_ASC); + // must check update info first. + bool update = updateInfoIsUpdated(pInfo->pUpdateInfo, pBlock->info.uid, tsCol[rowId]); + if ( (update || (isSignleIntervalWindow(pInfo) && isCloseWindow(&win, &pInfo->twAggSup)) ) && out) { taosArrayPush(pInfo->tsArray, &rowId); } } @@ -1193,8 +1208,6 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock } } - ASSERT(pInfo->pRes->pDataBlock != NULL); - // currently only the tbname pseudo column if (pInfo->numOfPseudoExpr > 0) { int32_t code = addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes, @@ -1259,6 +1272,24 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { return NULL; } + if (pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__PREPARE) { + STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; + memcpy(&pTSInfo->cond, &pTaskInfo->streamInfo.tableCond, sizeof(SQueryTableDataCond)); + pTSInfo->scanTimes = 0; + pTSInfo->currentGroupId = -1; + pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__SCAN; + } + + if (pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__SCAN) { + SSDataBlock* pBlock = doTableScan(pInfo->pTableScanOp); + if (pBlock != NULL) { + return pBlock; + } + // TODO fill in bloom filter + pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__NONE; + return NULL; + } + size_t total = taosArrayGetSize(pInfo->pBlockLists); // TODO: refactor if (pInfo->blockType == STREAM_INPUT__DATA_BLOCK) { @@ -1392,6 +1423,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { pInfo->tsArrayIndex = 0; checkUpdateData(pInfo, true, pInfo->pRes, true); setUpdateData(pInfo, pInfo->pRes, pInfo->pUpdateRes); + pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlockInfo->window.ekey); if (pInfo->pUpdateRes->info.rows > 0) { if (pInfo->pUpdateRes->info.type == STREAM_CLEAR) { pInfo->updateResIndex = 0; @@ -1551,6 +1583,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys goto _error; } taosArrayDestroy(tableIdList); + memcpy(&pTaskInfo->streamInfo.tableCond, &pTSInfo->cond, sizeof(SQueryTableDataCond)); } // create the pseduo columns info @@ -1562,13 +1595,14 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pInfo->pUpdateRes = createResDataBlock(pDescNode); pInfo->pCondition = pScanPhyNode->node.pConditions; pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; - pInfo->sessionSup = (SessionWindowSupporter){.pStreamAggSup = NULL, .gap = -1}; + pInfo->sessionSup = (SessionWindowSupporter){.pStreamAggSup = NULL, .gap = -1, .parentType = QUERY_NODE_PHYSICAL_PLAN}; pInfo->groupId = 0; pInfo->pPullDataRes = createPullDataBlock(); pInfo->pStreamScanOp = pOperator; pInfo->deleteDataIndex = 0; pInfo->pDeleteDataRes = createPullDataBlock(); pInfo->updateWin = (STimeWindow){.skey = INT64_MAX, .ekey = INT64_MAX}; + pInfo->twAggSup = *pTwSup; pOperator->name = "StreamScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index acc56af5cd..71eb221c64 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -652,8 +652,8 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num } void printDataBlock(SSDataBlock* pBlock, const char* flag) { - if (pBlock == NULL) { - qDebug("======printDataBlock Block is Null"); + if (!pBlock || pBlock->info.rows == 0) { + qDebug("======printDataBlock: Block is Null or Empty"); return; } char* pBuf = NULL; @@ -1355,7 +1355,7 @@ static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, int32_t size = taosArrayGetSize(chAy); qDebug("window %" PRId64 " wait child size:%d", win.skey, size); for (int32_t i = 0; i < size; i++) { - qDebug("window %" PRId64 " wait chid id:%d", win.skey, *(int32_t*)taosArrayGet(chAy, i)); + qDebug("window %" PRId64 " wait child id:%d", win.skey, *(int32_t*)taosArrayGet(chAy, i)); } continue; } else if (pPullDataMap) { @@ -1626,6 +1626,12 @@ SSDataBlock* createDeleteBlock() { return pBlock; } +void initIntervalDownStream(SOperatorInfo* downstream, uint8_t type) { + ASSERT(downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN); + SStreamScanInfo* pScanInfo = downstream->info; + pScanInfo->sessionSup.parentType = type; +} + SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, STimeWindowAggSupp* pTwAggSupp, SIntervalPhysiNode* pPhyNode, @@ -1701,6 +1707,10 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, doStreamIntervalAgg, NULL, destroyIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); + if (nodeType(pPhyNode) == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL) { + initIntervalDownStream(downstream, QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL); + } + code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -2476,12 +2486,14 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc } else { int32_t index = -1; SArray* chArray = NULL; + int32_t chId = 0; if (chIds) { chArray = *(void**)chIds; - int32_t chId = getChildIndex(pSDataBlock); + chId = getChildIndex(pSDataBlock); index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ); } if (index != -1 && pSDataBlock->info.type == STREAM_PULL_DATA) { + qDebug("======delete child id %d", chId); taosArrayRemove(chArray, index); if (taosArrayGetSize(chArray) == 0) { // pull data is over @@ -3010,6 +3022,7 @@ void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t num pDummy[i].functionId = pCtx[i].functionId; } } + void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, int64_t gap, int64_t waterMark, uint8_t type) { ASSERT(downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN); diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 8779fe5415..9b15cc7adb 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -621,7 +621,7 @@ static int32_t createInitialSources(SSortHandle* pHandle) { pHandle->sortElapsed += el; // All sorted data can fit in memory, external memory sort is not needed. Return to directly - if (size <= sortBufSize) { + if (size <= sortBufSize && pHandle->pBuf == NULL) { pHandle->cmpParam.numOfSources = 1; pHandle->inMemSort = true; diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 30fdbb245d..35669b3e42 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -106,7 +106,7 @@ bool irateFuncSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo); int32_t irateFunction(SqlFunctionCtx *pCtx); int32_t irateFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); -int32_t cacheLastRowFunction(SqlFunctionCtx* pCtx); +int32_t cachedLastRowFunction(SqlFunctionCtx* pCtx); bool getFirstLastFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); int32_t firstFunction(SqlFunctionCtx *pCtx); diff --git a/source/libs/function/inc/functionMgtInt.h b/source/libs/function/inc/functionMgtInt.h index da5dd0433d..0b4fdefec4 100644 --- a/source/libs/function/inc/functionMgtInt.h +++ b/source/libs/function/inc/functionMgtInt.h @@ -44,10 +44,9 @@ extern "C" { #define FUNC_MGT_FORBID_FILL_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(15) #define FUNC_MGT_INTERVAL_INTERPO_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(16) #define FUNC_MGT_FORBID_STREAM_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(17) -#define FUNC_MGT_FORBID_WINDOW_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(18) -#define FUNC_MGT_FORBID_GROUP_BY_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(19) -#define FUNC_MGT_SYSTEM_INFO_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(20) -#define FUNC_MGT_CLIENT_PC_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(21) +#define FUNC_MGT_SYSTEM_INFO_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(18) +#define FUNC_MGT_CLIENT_PC_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(19) +#define FUNC_MGT_MULTI_ROWS_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(20) #define FUNC_MGT_TEST_MASK(val, mask) (((val) & (mask)) != 0) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 208022ad1a..c458e2caec 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2081,7 +2081,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "top", .type = FUNCTION_TYPE_TOP, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateTopBot, .getEnvFunc = getTopBotFuncEnv, .initFunc = topBotFunctionSetup, @@ -2095,7 +2095,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "bottom", .type = FUNCTION_TYPE_BOTTOM, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateTopBot, .getEnvFunc = getTopBotFuncEnv, .initFunc = topBotFunctionSetup, @@ -2236,7 +2236,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .translateFunc = translateFirstLast, .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, - .processFunc = cacheLastRowFunction, + .processFunc = cachedLastRowFunction, .finalizeFunc = firstLastFinalize }, { @@ -2323,7 +2323,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "histogram", .type = FUNCTION_TYPE_HISTOGRAM, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_FORBID_FILL_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_FORBID_FILL_FUNC, .translateFunc = translateHistogram, .getEnvFunc = getHistogramFuncEnv, .initFunc = histogramFunctionSetup, @@ -2399,7 +2399,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "diff", .type = FUNCTION_TYPE_DIFF, - .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_WINDOW_FUNC, + .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateDiff, .getEnvFunc = getDiffFuncEnv, .initFunc = diffFunctionSetup, @@ -2409,7 +2409,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "statecount", .type = FUNCTION_TYPE_STATE_COUNT, - .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_WINDOW_FUNC, + .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateStateCount, .getEnvFunc = getStateFuncEnv, .initFunc = functionSetup, @@ -2419,7 +2419,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "stateduration", .type = FUNCTION_TYPE_STATE_DURATION, - .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_WINDOW_FUNC, + .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateStateDuration, .getEnvFunc = getStateFuncEnv, .initFunc = functionSetup, @@ -2429,7 +2429,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "csum", .type = FUNCTION_TYPE_CSUM, - .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_WINDOW_FUNC, + .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateCsum, .getEnvFunc = getCsumFuncEnv, .initFunc = functionSetup, @@ -2439,7 +2439,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "mavg", .type = FUNCTION_TYPE_MAVG, - .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_WINDOW_FUNC, + .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateMavg, .getEnvFunc = getMavgFuncEnv, .initFunc = mavgFunctionSetup, @@ -2449,7 +2449,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "sample", .type = FUNCTION_TYPE_SAMPLE, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateSample, .getEnvFunc = getSampleFuncEnv, .initFunc = sampleFunctionSetup, @@ -2460,7 +2460,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .name = "tail", .type = FUNCTION_TYPE_TAIL, .classification = FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | - FUNC_MGT_FORBID_WINDOW_FUNC | FUNC_MGT_FORBID_GROUP_BY_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, + FUNC_MGT_IMPLICIT_TS_FUNC, .translateFunc = translateTail, .getEnvFunc = getTailFuncEnv, .initFunc = tailFunctionSetup, @@ -2471,7 +2471,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .name = "unique", .type = FUNCTION_TYPE_UNIQUE, .classification = FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | - FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_WINDOW_FUNC | FUNC_MGT_FORBID_GROUP_BY_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, + FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, .translateFunc = translateUnique, .getEnvFunc = getUniqueFuncEnv, .initFunc = uniqueFunctionSetup, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index e2288d9f70..12b796c5ca 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -80,13 +80,14 @@ typedef struct STopBotRes { } STopBotRes; typedef struct SFirstLastRes { - bool hasResult; + bool hasResult; // used for last_row function only, isNullRes in SResultRowEntry can not be passed to downstream.So, // this attribute is required - bool isNull; - int32_t bytes; - int64_t ts; - char buf[]; + bool isNull; + int32_t bytes; + int64_t ts; + STuplePos pos; + char buf[]; } SFirstLastRes; typedef struct SStddevRes { @@ -1141,8 +1142,8 @@ bool getMinmaxFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { return true; } -static void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos); -static void copyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos); +static void doSaveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos); +static void doCopyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos); static int32_t findRowIndex(int32_t start, int32_t num, SColumnInfoData* pCol, const char* tval) { // the data is loaded, not only the block SMA value @@ -1195,7 +1196,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { pBuf->v = *(int64_t*)tval; if (pCtx->subsidiaries.num > 0) { index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval); - saveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); } } else { if (IS_SIGNED_NUMERIC_TYPE(type)) { @@ -1207,7 +1208,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { pBuf->v = val; if (pCtx->subsidiaries.num > 0) { index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval); - saveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); } } @@ -1220,7 +1221,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { pBuf->v = val; if (pCtx->subsidiaries.num > 0) { index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval); - saveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); } } } else if (type == TSDB_DATA_TYPE_DOUBLE) { @@ -1232,7 +1233,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { pBuf->v = val; if (pCtx->subsidiaries.num > 0) { index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval); - saveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); } } } else if (type == TSDB_DATA_TYPE_FLOAT) { @@ -1246,7 +1247,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (pCtx->subsidiaries.num > 0) { index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval); - saveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); } } } @@ -1271,7 +1272,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (!pBuf->assign) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } pBuf->assign = true; } else { @@ -1283,7 +1284,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if ((*val < pData[i]) ^ isMinFunc) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - copyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doCopyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } } } @@ -1302,7 +1303,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (!pBuf->assign) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } pBuf->assign = true; } else { @@ -1314,7 +1315,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if ((*val < pData[i]) ^ isMinFunc) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - copyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doCopyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } } } @@ -1333,7 +1334,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (!pBuf->assign) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } pBuf->assign = true; } else { @@ -1345,7 +1346,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if ((*val < pData[i]) ^ isMinFunc) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - copyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doCopyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } } } @@ -1364,7 +1365,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (!pBuf->assign) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } pBuf->assign = true; } else { @@ -1376,7 +1377,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if ((*val < pData[i]) ^ isMinFunc) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - copyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doCopyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } } } @@ -1397,7 +1398,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (!pBuf->assign) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } pBuf->assign = true; } else { @@ -1409,7 +1410,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if ((*val < pData[i]) ^ isMinFunc) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - copyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doCopyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } } } @@ -1428,7 +1429,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (!pBuf->assign) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } pBuf->assign = true; } else { @@ -1440,7 +1441,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if ((*val < pData[i]) ^ isMinFunc) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - copyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doCopyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } } } @@ -1459,7 +1460,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (!pBuf->assign) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } pBuf->assign = true; } else { @@ -1471,7 +1472,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if ((*val < pData[i]) ^ isMinFunc) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - copyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doCopyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } } } @@ -1490,7 +1491,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (!pBuf->assign) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } pBuf->assign = true; } else { @@ -1502,7 +1503,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if ((*val < pData[i]) ^ isMinFunc) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - copyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doCopyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } } } @@ -1522,7 +1523,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (!pBuf->assign) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } pBuf->assign = true; } else { @@ -1534,7 +1535,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if ((*val < pData[i]) ^ isMinFunc) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - copyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doCopyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } } } @@ -1553,7 +1554,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (!pBuf->assign) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } pBuf->assign = true; } else { @@ -1565,7 +1566,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if ((*val < pData[i]) ^ isMinFunc) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - copyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doCopyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } } } @@ -2640,7 +2641,7 @@ int32_t apercentileCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) } int32_t getFirstLastInfoSize(int32_t resBytes) { - return sizeof(SFirstLastRes) + resBytes + sizeof(int64_t) + sizeof(STuplePos); + return sizeof(SFirstLastRes) + resBytes; } bool getFirstLastFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { @@ -2669,6 +2670,33 @@ static FORCE_INLINE TSKEY getRowPTs(SColumnInfoData* pTsColInfo, int32_t rowInde return *(TSKEY*)colDataGetData(pTsColInfo, rowIndex); } +static void saveTupleData(const SSDataBlock* pSrcBlock, int32_t rowIndex, SqlFunctionCtx* pCtx, SFirstLastRes* pInfo) { + if (pCtx->subsidiaries.num <= 0) { + return; + } + + if (!pInfo->hasResult) { + doSaveTupleData(pCtx, rowIndex, pSrcBlock, &pInfo->pos); + } else { + doCopyTupleData(pCtx, rowIndex, pSrcBlock, &pInfo->pos); + } +} + +static void doSaveCurrentVal(SqlFunctionCtx* pCtx, int32_t rowIndex, int64_t currentTs, int32_t type, char* pData) { + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); + SFirstLastRes* pInfo = GET_ROWCELL_INTERBUF(pResInfo); + + if (IS_VAR_DATA_TYPE(type)) { + pInfo->bytes = varDataTLen(pData); + } + + memcpy(pInfo->buf, pData, pInfo->bytes); + pInfo->ts = currentTs; + saveTupleData(pCtx->pSrcBlock, rowIndex, pCtx, pInfo); + + pInfo->hasResult = true; +} + // This ordinary first function does not care if current scan is ascending order or descending order scan // the OPTIMIZED version of first function will only handle the ascending order scan int32_t firstFunction(SqlFunctionCtx* pCtx) { @@ -2680,9 +2708,7 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pInputCol = pInput->pData[0]; - int32_t type = pInputCol->info.type; - int32_t bytes = pInputCol->info.bytes; - pInfo->bytes = bytes; + pInfo->bytes = pInputCol->info.bytes; // All null data column, return directly. if (pInput->colDataAggIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows)) { @@ -2700,8 +2726,7 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { if (blockDataOrder == TSDB_ORDER_ASC) { // filter according to current result firstly if (pResInfo->numOfRes > 0) { - TSKEY ts = *(TSKEY*)(pInfo->buf + bytes); - if (ts < startKey) { + if (pInfo->ts < startKey) { return TSDB_CODE_SUCCESS; } } @@ -2715,26 +2740,8 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { char* data = colDataGetData(pInputCol, i); TSKEY cts = getRowPTs(pInput->pPTS, i); - - if (pResInfo->numOfRes == 0 || *(TSKEY*)(pInfo->buf + bytes) > cts) { - if (IS_VAR_DATA_TYPE(type)) { - bytes = varDataTLen(data); - pInfo->bytes = bytes; - } - memcpy(pInfo->buf, data, bytes); - *(TSKEY*)(pInfo->buf + bytes) = cts; - // handle selectivity - if (pCtx->subsidiaries.num > 0) { - STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY)); - if (!pInfo->hasResult) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } else { - copyTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } - } - pInfo->hasResult = true; - // DO_UPDATE_TAG_COLUMNS(pCtx, ts); - pResInfo->numOfRes = 1; + if (pResInfo->numOfRes == 0 || pInfo->ts > cts) { + doSaveCurrentVal(pCtx, i, cts, pInputCol->info.type, data); break; } } @@ -2742,8 +2749,7 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { // in case of descending order time stamp serial, which usually happens as the results of the nest query, // all data needs to be check. if (pResInfo->numOfRes > 0) { - TSKEY ts = *(TSKEY*)(pInfo->buf + bytes); - if (ts < endKey) { + if (pInfo->ts < endKey) { return TSDB_CODE_SUCCESS; } } @@ -2758,24 +2764,8 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { char* data = colDataGetData(pInputCol, i); TSKEY cts = getRowPTs(pInput->pPTS, i); - if (pResInfo->numOfRes == 0 || *(TSKEY*)(pInfo->buf + bytes) > cts) { - if (IS_VAR_DATA_TYPE(type)) { - bytes = varDataTLen(data); - pInfo->bytes = bytes; - } - memcpy(pInfo->buf, data, bytes); - *(TSKEY*)(pInfo->buf + bytes) = cts; - // handle selectivity - if (pCtx->subsidiaries.num > 0) { - STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY)); - if (!pInfo->hasResult) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } else { - copyTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } - } - pInfo->hasResult = true; - pResInfo->numOfRes = 1; + if (pResInfo->numOfRes == 0 || pInfo->ts > cts) { + doSaveCurrentVal(pCtx, i, cts, pInputCol->info.type, data); break; } } @@ -2821,26 +2811,10 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { char* data = colDataGetData(pInputCol, i); TSKEY cts = getRowPTs(pInput->pPTS, i); - if (pResInfo->numOfRes == 0 || *(TSKEY*)(pInfo->buf + bytes) < cts) { - if (IS_VAR_DATA_TYPE(type)) { - bytes = varDataTLen(data); - pInfo->bytes = bytes; - } - memcpy(pInfo->buf, data, bytes); - *(TSKEY*)(pInfo->buf + bytes) = cts; - // handle selectivity - if (pCtx->subsidiaries.num > 0) { - STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY)); - if (!pInfo->hasResult) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } else { - copyTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } - } - pInfo->hasResult = true; - // DO_UPDATE_TAG_COLUMNS(pCtx, ts); - pResInfo->numOfRes = 1; + if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { + doSaveCurrentVal(pCtx, i, cts, type, data); } + break; } } else { // descending order @@ -2853,24 +2827,8 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { char* data = colDataGetData(pInputCol, i); TSKEY cts = getRowPTs(pInput->pPTS, i); - if (pResInfo->numOfRes == 0 || *(TSKEY*)(pInfo->buf + bytes) < cts) { - if (IS_VAR_DATA_TYPE(type)) { - bytes = varDataTLen(data); - pInfo->bytes = bytes; - } - memcpy(pInfo->buf, data, bytes); - *(TSKEY*)(pInfo->buf + bytes) = cts; - // handle selectivity - if (pCtx->subsidiaries.num > 0) { - STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY)); - if (!pInfo->hasResult) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } else { - copyTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } - } - pInfo->hasResult = true; - pResInfo->numOfRes = 1; + if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { + doSaveCurrentVal(pCtx, i, cts, type, data); } break; } @@ -2885,8 +2843,9 @@ static void firstLastTransferInfo(SqlFunctionCtx* pCtx, SFirstLastRes* pInput, S int32_t start = pColInfo->startRowIndex; pOutput->bytes = pInput->bytes; - TSKEY* tsIn = (TSKEY*)(pInput->buf + pInput->bytes); - TSKEY* tsOut = (TSKEY*)(pOutput->buf + pInput->bytes); + TSKEY* tsIn = &pInput->ts; + TSKEY* tsOut = &pOutput->ts; + if (pOutput->hasResult) { if (isFirst) { if (*tsIn > *tsOut) { @@ -2898,20 +2857,12 @@ static void firstLastTransferInfo(SqlFunctionCtx* pCtx, SFirstLastRes* pInput, S } } } + *tsOut = *tsIn; memcpy(pOutput->buf, pInput->buf, pOutput->bytes); - // handle selectivity - STuplePos* pTuplePos = (STuplePos*)(pOutput->buf + pOutput->bytes + sizeof(TSKEY)); - if (pCtx->subsidiaries.num > 0) { - if (!pOutput->hasResult) { - saveTupleData(pCtx, start, pCtx->pSrcBlock, pTuplePos); - } else { - copyTupleData(pCtx, start, pCtx->pSrcBlock, pTuplePos); - } - } - pOutput->hasResult = true; + saveTupleData(pCtx->pSrcBlock, start, pCtx, pOutput); - return; + pOutput->hasResult = true; } static int32_t firstLastFunctionMergeImpl(SqlFunctionCtx* pCtx, bool isFirstQuery) { @@ -2953,34 +2904,34 @@ int32_t firstLastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { colDataAppend(pCol, pBlock->info.rows, pRes->buf, pRes->isNull || pResInfo->isNullRes); // handle selectivity - STuplePos* pTuplePos = (STuplePos*)(pRes->buf + pRes->bytes + sizeof(TSKEY)); - setSelectivityValue(pCtx, pBlock, pTuplePos, pBlock->info.rows); + setSelectivityValue(pCtx, pBlock, &pRes->pos, pBlock->info.rows); return pResInfo->numOfRes; } int32_t firstLastPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(pCtx); - SFirstLastRes* pRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + SFirstLastRes* pRes = GET_ROWCELL_INTERBUF(pEntryInfo); int32_t resultBytes = getFirstLastInfoSize(pRes->bytes); - char* res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char)); + // todo check for failure + char* res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char)); memcpy(varDataVal(res), pRes, resultBytes); + varDataSetLen(res, resultBytes); int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); colDataAppend(pCol, pBlock->info.rows, res, false); - // handle selectivity - STuplePos* pTuplePos = (STuplePos*)(pRes->buf + pRes->bytes + sizeof(TSKEY)); - setSelectivityValue(pCtx, pBlock, pTuplePos, pBlock->info.rows); + setSelectivityValue(pCtx, pBlock, &pRes->pos, pBlock->info.rows); taosMemoryFree(res); return 1; } +//todo rewrite: int32_t lastCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx); char* pDBuf = GET_ROWCELL_INTERBUF(pDResInfo); @@ -2998,6 +2949,28 @@ int32_t lastCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { return TSDB_CODE_SUCCESS; } +static void doSaveLastrow(SqlFunctionCtx *pCtx, char* pData, int32_t rowIndex, int64_t cts, SFirstLastRes* pInfo) { + SInputColumnInfoData* pInput = &pCtx->input; + SColumnInfoData* pInputCol = pInput->pData[0]; + + if (colDataIsNull_s(pInputCol, rowIndex)) { + pInfo->isNull = true; + } else { + pInfo->isNull = false; + + if (IS_VAR_DATA_TYPE(pInputCol->info.type)) { + pInfo->bytes = varDataTLen(pData); + } + + memcpy(pInfo->buf, pData, pInfo->bytes); + } + + pInfo->ts = cts; + saveTupleData(pCtx->pSrcBlock, rowIndex, pCtx, pInfo); + + pInfo->hasResult = true; +} + int32_t lastRowFunction(SqlFunctionCtx* pCtx) { int32_t numOfElems = 0; @@ -3007,12 +2980,9 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pInputCol = pInput->pData[0]; - int32_t type = pInputCol->info.type; int32_t bytes = pInputCol->info.bytes; pInfo->bytes = bytes; - SColumnDataAgg* pColAgg = (pInput->colDataAggIsSet) ? pInput->pColumnDataAgg[0] : NULL; - TSKEY startKey = getRowPTs(pInput->pPTS, 0); TSKEY endKey = getRowPTs(pInput->pPTS, pInput->totalRows - 1); @@ -3022,31 +2992,10 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) { for (int32_t i = pInput->numOfRows + pInput->startRowIndex - 1; i >= pInput->startRowIndex; --i) { char* data = colDataGetData(pInputCol, i); TSKEY cts = getRowPTs(pInput->pPTS, i); - if (pResInfo->numOfRes == 0 || *(TSKEY*)(pInfo->buf) < cts) { - if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) { - pInfo->isNull = true; - } else { - pInfo->isNull = false; - if (IS_VAR_DATA_TYPE(type)) { - bytes = varDataTLen(data); - pInfo->bytes = bytes; - } - memcpy(pInfo->buf + sizeof(TSKEY), data, bytes); - } - *(TSKEY*)(pInfo->buf) = cts; - numOfElems++; - // handle selectivity - if (pCtx->subsidiaries.num > 0) { - STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY)); - if (!pInfo->hasResult) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } else { - copyTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } - } - pInfo->hasResult = true; - // DO_UPDATE_TAG_COLUMNS(pCtx, ts); - pResInfo->numOfRes = 1; + numOfElems++; + + if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { + doSaveLastrow(pCtx, data, i, cts, pInfo); } break; } @@ -3054,31 +3003,10 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) { for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { char* data = colDataGetData(pInputCol, i); TSKEY cts = getRowPTs(pInput->pPTS, i); - if (pResInfo->numOfRes == 0 || *(TSKEY*)(pInfo->buf) < cts) { - if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) { - pInfo->isNull = true; - } else { - pInfo->isNull = false; - if (IS_VAR_DATA_TYPE(type)) { - bytes = varDataTLen(data); - pInfo->bytes = bytes; - } - memcpy(pInfo->buf + sizeof(TSKEY), data, bytes); - } - *(TSKEY*)(pInfo->buf) = cts; - numOfElems++; - // handle selectivity - if (pCtx->subsidiaries.num > 0) { - STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY)); - if (!pInfo->hasResult) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } else { - copyTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } - } - pInfo->hasResult = true; - pResInfo->numOfRes = 1; - // DO_UPDATE_TAG_COLUMNS(pCtx, ts); + numOfElems++; + + if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { + doSaveLastrow(pCtx, data, i, cts, pInfo); } break; } @@ -3088,21 +3016,6 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) { return TSDB_CODE_SUCCESS; } -int32_t lastRowFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { - int32_t slotId = pCtx->pExpr->base.resSchema.slotId; - SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); - - SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); - - SFirstLastRes* pRes = GET_ROWCELL_INTERBUF(pResInfo); - colDataAppend(pCol, pBlock->info.rows, pRes->buf + sizeof(TSKEY), pRes->isNull); - // handle selectivity - STuplePos* pTuplePos = (STuplePos*)(pRes->buf + pRes->bytes + sizeof(TSKEY)); - setSelectivityValue(pCtx, pBlock, pTuplePos, pBlock->info.rows); - - return pResInfo->numOfRes; -} - bool getDiffFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { pEnv->calcMemSize = sizeof(SDiffInfo); return true; @@ -3425,7 +3338,7 @@ void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSData // save the data of this tuple if (pCtx->subsidiaries.num > 0) { - saveTupleData(pCtx, rowIndex, pSrcBlock, &pItem->tuplePos); + doSaveTupleData(pCtx, rowIndex, pSrcBlock, &pItem->tuplePos); } #ifdef BUF_PAGE_DEBUG qDebug("page_saveTuple i:%d, item:%p,pageId:%d, offset:%d\n", pEntryInfo->numOfRes, pItem, pItem->tuplePos.pageId, @@ -3449,7 +3362,7 @@ void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSData // save the data of this tuple by over writing the old data if (pCtx->subsidiaries.num > 0) { - copyTupleData(pCtx, rowIndex, pSrcBlock, &pItem->tuplePos); + doCopyTupleData(pCtx, rowIndex, pSrcBlock, &pItem->tuplePos); } #ifdef BUF_PAGE_DEBUG qDebug("page_copyTuple pageId:%d, offset:%d", pItem->tuplePos.pageId, pItem->tuplePos.offset); @@ -3466,7 +3379,7 @@ void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSData * |(n columns, one bit for each column)| src column #1| src column #2| * +------------------------------------+--------------+--------------+ */ -void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos) { +void doSaveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos) { SFilePage* pPage = NULL; // todo refactor: move away @@ -3527,7 +3440,7 @@ void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pS #endif } -void copyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos) { +void doCopyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos) { SFilePage* pPage = getBufPage(pCtx->pBuf, pPos->pageId); int32_t numOfCols = pCtx->subsidiaries.num; @@ -4843,7 +4756,7 @@ static void doReservoirSample(SqlFunctionCtx* pCtx, SSampleInfo* pInfo, char* da if (pInfo->numSampled < pInfo->samples) { sampleAssignResult(pInfo, data, pInfo->numSampled); if (pCtx->subsidiaries.num > 0) { - saveTupleData(pCtx, index, pCtx->pSrcBlock, &pInfo->tuplePos[pInfo->numSampled]); + doSaveTupleData(pCtx, index, pCtx->pSrcBlock, &pInfo->tuplePos[pInfo->numSampled]); } pInfo->numSampled++; } else { @@ -4851,7 +4764,7 @@ static void doReservoirSample(SqlFunctionCtx* pCtx, SSampleInfo* pInfo, char* da if (j < pInfo->samples) { sampleAssignResult(pInfo, data, j); if (pCtx->subsidiaries.num > 0) { - copyTupleData(pCtx, index, pCtx->pSrcBlock, &pInfo->tuplePos[j]); + doCopyTupleData(pCtx, index, pCtx->pSrcBlock, &pInfo->tuplePos[j]); } } } @@ -5993,7 +5906,7 @@ int32_t interpFunction(SqlFunctionCtx* pCtx) { return TSDB_CODE_SUCCESS; } -int32_t cacheLastRowFunction(SqlFunctionCtx* pCtx) { +int32_t cachedLastRowFunction(SqlFunctionCtx* pCtx) { int32_t numOfElems = 0; SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); @@ -6002,9 +5915,7 @@ int32_t cacheLastRowFunction(SqlFunctionCtx* pCtx) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pInputCol = pInput->pData[0]; - int32_t type = pInputCol->info.type; int32_t bytes = pInputCol->info.bytes; - pInfo->bytes = bytes; // last_row function does not ignore the null value @@ -6014,28 +5925,7 @@ int32_t cacheLastRowFunction(SqlFunctionCtx* pCtx) { char* data = colDataGetData(pInputCol, i); TSKEY cts = getRowPTs(pInput->pPTS, i); if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { - if (colDataIsNull_s(pInputCol, i)) { - pInfo->isNull = true; - } else { - if (IS_VAR_DATA_TYPE(type)) { - bytes = varDataTLen(data); - pInfo->bytes = bytes; - } - - memcpy(pInfo->buf, data, bytes); - } - - pInfo->ts = cts; - if (pCtx->subsidiaries.num > 0) { - STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY)); - if (!pInfo->hasResult) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } else { - copyTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } - } - - pInfo->hasResult = true; + doSaveLastrow(pCtx, data, i, cts, pInfo); } } diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index f73a61fa60..ff74c880e3 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -175,16 +175,14 @@ bool fmIsIntervalInterpoFunc(int32_t funcId) { return isSpecificClassifyFunc(fun bool fmIsForbidStreamFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_FORBID_STREAM_FUNC); } -bool fmIsForbidWindowFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_FORBID_WINDOW_FUNC); } - -bool fmIsForbidGroupByFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_FORBID_GROUP_BY_FUNC); } - bool fmIsSystemInfoFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SYSTEM_INFO_FUNC); } bool fmIsImplicitTsFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_IMPLICIT_TS_FUNC); } bool fmIsClientPseudoColumnFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_CLIENT_PC_FUNC); } +bool fmIsMultiRowsFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_MULTI_ROWS_FUNC); } + bool fmIsInterpFunc(int32_t funcId) { if (funcId < 0 || funcId >= funcMgtBuiltinsNum) { return false; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index b0a012e20b..8960380925 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1058,7 +1058,9 @@ static int32_t translateAggFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { if (hasInvalidFuncNesting(pFunc->pParameterList)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_AGG_FUNC_NESTING); } - if (isSelectStmt(pCxt->pCurrStmt) && ((SSelectStmt*)pCxt->pCurrStmt)->hasIndefiniteRowsFunc) { + // The auto-generated COUNT function in the DELETE statement is legal + if (isSelectStmt(pCxt->pCurrStmt) && + (((SSelectStmt*)pCxt->pCurrStmt)->hasIndefiniteRowsFunc || ((SSelectStmt*)pCxt->pCurrStmt)->hasMultiRowsFunc)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC); } @@ -1093,7 +1095,27 @@ static int32_t translateIndefiniteRowsFunc(STranslateContext* pCxt, SFunctionNod return TSDB_CODE_SUCCESS; } if (!isSelectStmt(pCxt->pCurrStmt) || SQL_CLAUSE_SELECT != pCxt->currClause || - ((SSelectStmt*)pCxt->pCurrStmt)->hasIndefiniteRowsFunc || ((SSelectStmt*)pCxt->pCurrStmt)->hasAggFuncs) { + ((SSelectStmt*)pCxt->pCurrStmt)->hasIndefiniteRowsFunc || ((SSelectStmt*)pCxt->pCurrStmt)->hasAggFuncs || + ((SSelectStmt*)pCxt->pCurrStmt)->hasMultiRowsFunc) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC); + } + if (NULL != ((SSelectStmt*)pCxt->pCurrStmt)->pWindow || NULL != ((SSelectStmt*)pCxt->pCurrStmt)->pGroupByList) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC, + "%s function is not supported in window query or group query", pFunc->functionName); + } + if (hasInvalidFuncNesting(pFunc->pParameterList)) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_AGG_FUNC_NESTING); + } + return TSDB_CODE_SUCCESS; +} + +static int32_t translateMultiRowsFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { + if (!fmIsMultiRowsFunc(pFunc->funcId)) { + return TSDB_CODE_SUCCESS; + } + if (!isSelectStmt(pCxt->pCurrStmt) || SQL_CLAUSE_SELECT != pCxt->currClause || + ((SSelectStmt*)pCxt->pCurrStmt)->hasIndefiniteRowsFunc || ((SSelectStmt*)pCxt->pCurrStmt)->hasAggFuncs || + ((SSelectStmt*)pCxt->pCurrStmt)->hasMultiRowsFunc) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC); } if (hasInvalidFuncNesting(pFunc->pParameterList)) { @@ -1131,16 +1153,6 @@ static int32_t translateWindowPseudoColumnFunc(STranslateContext* pCxt, SFunctio return TSDB_CODE_SUCCESS; } -static int32_t translateForbidWindowFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { - if (!fmIsForbidWindowFunc(pFunc->funcId)) { - return TSDB_CODE_SUCCESS; - } - if (isSelectStmt(pCxt->pCurrStmt) && NULL != ((SSelectStmt*)pCxt->pCurrStmt)->pWindow) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_WINDOW_NOT_ALLOWED_FUNC, pFunc->functionName); - } - return TSDB_CODE_SUCCESS; -} - static int32_t translateForbidStreamFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { if (!fmIsForbidStreamFunc(pFunc->funcId)) { return TSDB_CODE_SUCCESS; @@ -1151,16 +1163,6 @@ static int32_t translateForbidStreamFunc(STranslateContext* pCxt, SFunctionNode* return TSDB_CODE_SUCCESS; } -static int32_t translateForbidGroupByFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { - if (!fmIsForbidGroupByFunc(pFunc->funcId)) { - return TSDB_CODE_SUCCESS; - } - if (isSelectStmt(pCxt->pCurrStmt) && NULL != ((SSelectStmt*)pCxt->pCurrStmt)->pGroupByList) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GROUP_BY_NOT_ALLOWED_FUNC, pFunc->functionName); - } - return TSDB_CODE_SUCCESS; -} - static int32_t translateRepeatScanFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { if (!fmIsRepeatScanFunc(pFunc->funcId)) { return TSDB_CODE_SUCCESS; @@ -1195,7 +1197,7 @@ static int32_t translateMultiResFunc(STranslateContext* pCxt, SFunctionNode* pFu if (!fmIsMultiResFunc(pFunc->funcId)) { return TSDB_CODE_SUCCESS; } - if (SQL_CLAUSE_SELECT != pCxt->currClause ) { + if (SQL_CLAUSE_SELECT != pCxt->currClause) { SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); if (isStar(pPara) || isTableStar(pPara)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC, @@ -1210,6 +1212,7 @@ static void setFuncClassification(SNode* pCurrStmt, SFunctionNode* pFunc) { pSelect->hasAggFuncs = pSelect->hasAggFuncs ? true : fmIsAggFunc(pFunc->funcId); pSelect->hasRepeatScanFuncs = pSelect->hasRepeatScanFuncs ? true : fmIsRepeatScanFunc(pFunc->funcId); pSelect->hasIndefiniteRowsFunc = pSelect->hasIndefiniteRowsFunc ? true : fmIsIndefiniteRowsFunc(pFunc->funcId); + pSelect->hasMultiRowsFunc = pSelect->hasMultiRowsFunc ? true : fmIsMultiRowsFunc(pFunc->funcId); if (fmIsSelectFunc(pFunc->funcId)) { pSelect->hasSelectFunc = true; ++(pSelect->selectFuncNum); @@ -1326,21 +1329,18 @@ static int32_t translateNoramlFunction(STranslateContext* pCxt, SFunctionNode* p if (TSDB_CODE_SUCCESS == code) { code = translateWindowPseudoColumnFunc(pCxt, pFunc); } - if (TSDB_CODE_SUCCESS == code) { - code = translateForbidWindowFunc(pCxt, pFunc); - } if (TSDB_CODE_SUCCESS == code) { code = translateForbidStreamFunc(pCxt, pFunc); } - if (TSDB_CODE_SUCCESS == code) { - code = translateForbidGroupByFunc(pCxt, pFunc); - } if (TSDB_CODE_SUCCESS == code) { code = translateRepeatScanFunc(pCxt, pFunc); } if (TSDB_CODE_SUCCESS == code) { code = translateMultiResFunc(pCxt, pFunc); } + if (TSDB_CODE_SUCCESS == code) { + code = translateMultiRowsFunc(pCxt, pFunc); + } if (TSDB_CODE_SUCCESS == code) { setFuncClassification(pCxt->pCurrStmt, pFunc); } diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index e682059793..b9a760d342 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -152,9 +152,9 @@ TEST_F(ParserSelectTest, IndefiniteRowsFuncSemanticCheck) { run("SELECT DIFF(c1), CSUM(c1) FROM t1", TSDB_CODE_PAR_NOT_ALLOWED_FUNC); - run("SELECT CSUM(c3) FROM t1 STATE_WINDOW(c1)", TSDB_CODE_PAR_WINDOW_NOT_ALLOWED_FUNC); + run("SELECT CSUM(c3) FROM t1 STATE_WINDOW(c1)", TSDB_CODE_PAR_NOT_ALLOWED_FUNC); - run("SELECT DIFF(c1) FROM t1 INTERVAL(10s)", TSDB_CODE_PAR_WINDOW_NOT_ALLOWED_FUNC); + run("SELECT DIFF(c1) FROM t1 INTERVAL(10s)", TSDB_CODE_PAR_NOT_ALLOWED_FUNC); } TEST_F(ParserSelectTest, useDefinedFunc) { @@ -178,9 +178,9 @@ TEST_F(ParserSelectTest, uniqueFunc) { TEST_F(ParserSelectTest, uniqueFuncSemanticCheck) { useDb("root", "test"); - run("SELECT UNIQUE(c1) FROM t1 INTERVAL(10S)", TSDB_CODE_PAR_WINDOW_NOT_ALLOWED_FUNC); + run("SELECT UNIQUE(c1) FROM t1 INTERVAL(10S)", TSDB_CODE_PAR_NOT_ALLOWED_FUNC); - run("SELECT UNIQUE(c1) FROM t1 GROUP BY c2", TSDB_CODE_PAR_GROUP_BY_NOT_ALLOWED_FUNC); + run("SELECT UNIQUE(c1) FROM t1 GROUP BY c2", TSDB_CODE_PAR_NOT_ALLOWED_FUNC); } TEST_F(ParserSelectTest, tailFunc) { @@ -194,9 +194,9 @@ TEST_F(ParserSelectTest, tailFunc) { TEST_F(ParserSelectTest, tailFuncSemanticCheck) { useDb("root", "test"); - run("SELECT TAIL(c1, 10) FROM t1 INTERVAL(10S)", TSDB_CODE_PAR_WINDOW_NOT_ALLOWED_FUNC); + run("SELECT TAIL(c1, 10) FROM t1 INTERVAL(10S)", TSDB_CODE_PAR_NOT_ALLOWED_FUNC); - run("SELECT TAIL(c1, 10) FROM t1 GROUP BY c2", TSDB_CODE_PAR_GROUP_BY_NOT_ALLOWED_FUNC); + run("SELECT TAIL(c1, 10) FROM t1 GROUP BY c2", TSDB_CODE_PAR_NOT_ALLOWED_FUNC); } TEST_F(ParserSelectTest, partitionBy) { diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 4a062a86db..e0b020ce21 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -278,6 +278,10 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect pScan->scanType = getScanType(pCxt, pScan->pScanPseudoCols, pScan->pScanCols, pScan->tableType); + if (NULL != pScan->pScanCols) { + pScan->hasNormalCols = true; + } + if (TSDB_CODE_SUCCESS == code) { code = addPrimaryKeyCol(pScan->tableId, &pScan->pScanCols); } diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 97858f83fb..cdbe74bf04 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1993,7 +1993,8 @@ static bool lastRowScanOptMayBeOptimized(SLogicNode* pNode) { SNode* pFunc = NULL; FOREACH(pFunc, ((SAggLogicNode*)pNode)->pAggFuncs) { if (FUNCTION_TYPE_LAST_ROW != ((SFunctionNode*)pFunc)->funcType && - FUNCTION_TYPE_SELECT_VALUE != ((SFunctionNode*)pFunc)->funcType) { + FUNCTION_TYPE_SELECT_VALUE != ((SFunctionNode*)pFunc)->funcType && + FUNCTION_TYPE_GROUP_KEY != ((SFunctionNode*)pFunc)->funcType) { return false; } } @@ -2112,7 +2113,7 @@ static bool tagScanMayBeOptimized(SLogicNode* pNode) { return false; } SScanLogicNode* pScan = (SScanLogicNode*)pNode; - if (NULL != pScan->pScanCols) { + if (pScan->hasNormalCols) { return false; } if (NULL == pNode->pParent || QUERY_NODE_LOGIC_PLAN_AGG != nodeType(pNode->pParent) || diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 871bcf015b..ae0ccb1c51 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -198,8 +198,7 @@ static bool stbSplHasGatherExecFunc(const SNodeList* pFuncs) { } static bool stbSplIsMultiTbScan(bool streamQuery, SScanLogicNode* pScan) { - return (NULL != pScan->pVgroupList && pScan->pVgroupList->numOfVgroups > 1) || - (streamQuery && TSDB_SUPER_TABLE == pScan->tableType); + return (NULL != pScan->pVgroupList && pScan->pVgroupList->numOfVgroups > 1); } static bool stbSplHasMultiTbScan(bool streamQuery, SLogicNode* pNode) { diff --git a/source/libs/stream/inc/streamInc.h b/source/libs/stream/inc/streamInc.h index f9f4e62774..d10ea76c83 100644 --- a/source/libs/stream/inc/streamInc.h +++ b/source/libs/stream/inc/streamInc.h @@ -33,6 +33,8 @@ typedef struct { static SStreamGlobalEnv streamEnv; int32_t streamExec(SStreamTask* pTask, SMsgCb* pMsgCb); +int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum); + int32_t streamDispatch(SStreamTask* pTask, SMsgCb* pMsgCb); int32_t streamDispatchReqToData(const SStreamDispatchReq* pReq, SStreamDataBlock* pData); int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock* pData); diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 566d9209a8..2a96db3bfc 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -57,7 +57,7 @@ void streamTriggerByTimer(void* param, void* tmrId) { if (atomic_load_8(&pTask->triggerStatus) == TASK_TRIGGER_STATUS__ACTIVE) { SStreamTrigger* trigger = taosAllocateQitem(sizeof(SStreamTrigger), DEF_QITEM); if (trigger == NULL) return; - trigger->type = STREAM_INPUT__TRIGGER; + trigger->type = STREAM_INPUT__GET_RES; trigger->pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); if (trigger->pBlock == NULL) { taosFreeQitem(trigger); @@ -183,8 +183,11 @@ int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, S // 2.1. idle: exec // 2.2. executing: return // 2.3. closing: keep trying +#if 0 if (pTask->execType != TASK_EXEC__NONE) { - streamExec(pTask, pTask->pMsgCb); +#endif + streamExec(pTask, pTask->pMsgCb); +#if 0 } else { ASSERT(pTask->sinkType != TASK_SINK__NONE); while (1) { @@ -195,11 +198,13 @@ int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, S } } } +#endif // 3. handle output // 3.1 check and set status // 3.2 dispatch / sink if (pTask->dispatchType != TASK_DISPATCH__NONE) { + ASSERT(pTask->sinkType == TASK_SINK__NONE); streamDispatch(pTask, pTask->pMsgCb); } diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index 8e16b23e56..dbf6350c93 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -112,7 +112,7 @@ int32_t streamAppendQueueItem(SStreamQueueItem* dst, SStreamQueueItem* elem) { void streamFreeQitem(SStreamQueueItem* data) { int8_t type = data->type; - if (type == STREAM_INPUT__TRIGGER) { + if (type == STREAM_INPUT__GET_RES) { blockDataDestroy(((SStreamTrigger*)data)->pBlock); taosFreeQitem(data); } else if (type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__DATA_RETRIEVE) { diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 1e46a9622b..9d22da8662 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -20,7 +20,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) // set input SStreamQueueItem* pItem = (SStreamQueueItem*)data; - if (pItem->type == STREAM_INPUT__TRIGGER) { + if (pItem->type == STREAM_INPUT__GET_RES) { SStreamTrigger* pTrigger = (SStreamTrigger*)data; qSetMultiStreamInput(exec, pTrigger->pBlock, 1, STREAM_INPUT__DATA_BLOCK, false); } else if (pItem->type == STREAM_INPUT__DATA_SUBMIT) { @@ -73,6 +73,72 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) return 0; } +static FORCE_INLINE int32_t streamUpdateVer(SStreamTask* pTask, SStreamDataBlock* pBlock) { + ASSERT(pBlock->type == STREAM_INPUT__DATA_BLOCK); + int32_t childId = pBlock->childId; + int64_t ver = pBlock->sourceVer; + SStreamChildEpInfo* pChildInfo = taosArrayGetP(pTask->childEpInfo, childId); + pChildInfo->processedVer = ver; + return 0; +} + +int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum) { + ASSERT(pTask->execType != TASK_EXEC__NONE); + + void* exec = pTask->exec.executor; + + while (1) { + SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); + if (pRes == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + int32_t batchCnt = 0; + while (1) { + SSDataBlock* output = NULL; + uint64_t ts = 0; + if (qExecTask(exec, &output, &ts) < 0) { + ASSERT(0); + } + if (output == NULL) break; + + SSDataBlock block = {0}; + assignOneDataBlock(&block, output); + block.info.childId = pTask->selfChildId; + taosArrayPush(pRes, &block); + + if (++batchCnt >= batchNum) break; + } + if (taosArrayGetSize(pRes) == 0) { + taosArrayDestroy(pRes); + break; + } + SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM); + if (qRes == NULL) { + taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); + return -1; + } + + qRes->type = STREAM_INPUT__DATA_BLOCK; + qRes->blocks = pRes; + qRes->childId = pTask->selfChildId; + + if (streamTaskOutput(pTask, qRes) < 0) { + taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); + taosFreeQitem(qRes); + return -1; + } + + if (pTask->dispatchType != TASK_DISPATCH__NONE) { + ASSERT(pTask->sinkType == TASK_SINK__NONE); + streamDispatch(pTask, pTask->pMsgCb); + } + } + + return 0; +} + static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) { int32_t cnt = 0; void* data = NULL; @@ -84,14 +150,17 @@ static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) { } if (data == NULL) { data = qItem; + if (qItem->type == STREAM_INPUT__DATA_BLOCK) { + /*streamUpdateVer(pTask, (SStreamDataBlock*)qItem);*/ + } streamQueueProcessSuccess(pTask->inputQueue); - continue; } else { if (streamAppendQueueItem(data, qItem) < 0) { streamQueueProcessFail(pTask->inputQueue); break; } else { cnt++; + /*streamUpdateVer(pTask, (SStreamDataBlock*)qItem);*/ streamQueueProcessSuccess(pTask->inputQueue); taosArrayDestroy(((SStreamDataBlock*)qItem)->blocks); taosFreeQitem(qItem); @@ -106,6 +175,12 @@ static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) { if (data == NULL) return pRes; + if (pTask->execType == TASK_EXEC__NONE) { + ASSERT(((SStreamQueueItem*)data)->type == STREAM_INPUT__DATA_BLOCK); + streamTaskOutput(pTask, data); + return pRes; + } + qDebug("stream task %d exec begin, msg batch: %d", pTask->taskId, cnt); streamTaskExecImpl(pTask, data, pRes); qDebug("stream task %d exec end", pTask->taskId); @@ -125,6 +200,11 @@ static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) { taosFreeQitem(qRes); return NULL; } + if (((SStreamQueueItem*)data)->type == STREAM_INPUT__DATA_SUBMIT) { + SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)data; + qRes->childId = pTask->selfChildId; + qRes->sourceVer = pSubmit->ver; + } /*streamQueueProcessSuccess(pTask->inputQueue);*/ pRes = taosArrayInit(0, sizeof(SSDataBlock)); } diff --git a/source/libs/stream/src/streamRecover.c b/source/libs/stream/src/streamRecover.c new file mode 100644 index 0000000000..87b27daf60 --- /dev/null +++ b/source/libs/stream/src/streamRecover.c @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "streamInc.h" + +int32_t tEncodeStreamTaskRecoverReq(SEncoder* pEncoder, const SStreamTaskRecoverReq* pReq) { + if (tStartEncode(pEncoder) < 0) return -1; + if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->taskId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->sourceTaskId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->sourceVg) < 0) return -1; + tEndEncode(pEncoder); + return pEncoder->pos; +} + +int32_t tDecodeStreamTaskRecoverReq(SDecoder* pDecoder, SStreamTaskRecoverReq* pReq) { + if (tStartDecode(pDecoder) < 0) return -1; + if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->taskId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->sourceTaskId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->sourceVg) < 0) return -1; + tEndDecode(pDecoder); + return 0; +} + +int32_t tEncodeStreamTaskRecoverRsp(SEncoder* pEncoder, const SStreamTaskRecoverRsp* pRsp) { + if (tStartEncode(pEncoder) < 0) return -1; + if (tEncodeI64(pEncoder, pRsp->streamId) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->taskId) < 0) return -1; + if (tEncodeI8(pEncoder, pRsp->inputStatus) < 0) return -1; + tEndEncode(pEncoder); + return pEncoder->pos; +} + +int32_t tDecodeStreamTaskRecoverRsp(SDecoder* pDecoder, SStreamTaskRecoverRsp* pReq) { + if (tStartDecode(pDecoder) < 0) return -1; + if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->taskId) < 0) return -1; + if (tDecodeI8(pDecoder, &pReq->inputStatus) < 0) return -1; + tEndDecode(pDecoder); + return 0; +} + +int32_t tEncodeSMStreamTaskRecoverReq(SEncoder* pEncoder, const SMStreamTaskRecoverReq* pReq) { + if (tStartEncode(pEncoder) < 0) return -1; + if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->taskId) < 0) return -1; + tEndEncode(pEncoder); + return pEncoder->pos; +} + +int32_t tDecodeSMStreamTaskRecoverReq(SDecoder* pDecoder, SMStreamTaskRecoverReq* pReq) { + if (tStartDecode(pDecoder) < 0) return -1; + if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->taskId) < 0) return -1; + tEndDecode(pDecoder); + return 0; +} + +int32_t tEncodeSMStreamTaskRecoverRsp(SEncoder* pEncoder, const SMStreamTaskRecoverRsp* pRsp) { + if (tStartEncode(pEncoder) < 0) return -1; + if (tEncodeI64(pEncoder, pRsp->streamId) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->taskId) < 0) return -1; + tEndEncode(pEncoder); + return pEncoder->pos; +} + +int32_t tDecodeSMStreamTaskRecoverRsp(SDecoder* pDecoder, SMStreamTaskRecoverRsp* pReq) { + if (tStartDecode(pDecoder) < 0) return -1; + if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->taskId) < 0) return -1; + tEndDecode(pDecoder); + return 0; +} + +int32_t streamProcessFailRecoverReq(SStreamTask* pTask, SMStreamTaskRecoverReq* pReq, SRpcMsg* pRsp) { + if (pTask->taskStatus != TASK_STATUS__FAIL) { + return 0; + } + + if (pTask->isStreamDistributed) { + if (pTask->isDataScan) { + pTask->taskStatus = TASK_STATUS__PREPARE_RECOVER; + } else if (pTask->execType != TASK_EXEC__NONE) { + pTask->taskStatus = TASK_STATUS__PREPARE_RECOVER; + bool hasCheckpoint = false; + int32_t childSz = taosArrayGetSize(pTask->childEpInfo); + for (int32_t i = 0; i < childSz; i++) { + SStreamChildEpInfo* pEpInfo = taosArrayGetP(pTask->childEpInfo, i); + if (pEpInfo->checkpointVer == -1) { + hasCheckpoint = true; + break; + } + } + if (hasCheckpoint) { + // load from checkpoint + } else { + // recover child + } + } + } else { + if (pTask->isDataScan) { + if (pTask->checkpointVer != -1) { + // load from checkpoint + } else { + // reset stream query task info + // TODO get snapshot ver + pTask->recoverSnapVer = -1; + qStreamPrepareRecover(pTask->exec.executor, pTask->startVer, pTask->recoverSnapVer); + pTask->taskStatus = TASK_STATUS__RECOVERING; + } + } + } + + if (pTask->taskStatus == TASK_STATUS__RECOVERING) { + if (streamPipelineExec(pTask, 10) < 0) { + // set fail + return -1; + } + } + + return 0; +} diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 7488c009bd..2d15c31bf1 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -34,6 +34,7 @@ int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo) if (tEncodeI32(pEncoder, pInfo->taskId) < 0) return -1; if (tEncodeI32(pEncoder, pInfo->nodeId) < 0) return -1; if (tEncodeI32(pEncoder, pInfo->childId) < 0) return -1; + if (tEncodeI64(pEncoder, pInfo->processedVer) < 0) return -1; if (tEncodeSEpSet(pEncoder, &pInfo->epSet) < 0) return -1; return 0; } @@ -42,6 +43,7 @@ int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo) { if (tDecodeI32(pDecoder, &pInfo->taskId) < 0) return -1; if (tDecodeI32(pDecoder, &pInfo->nodeId) < 0) return -1; if (tDecodeI32(pDecoder, &pInfo->childId) < 0) return -1; + if (tDecodeI64(pDecoder, &pInfo->processedVer) < 0) return -1; if (tDecodeSEpSet(pDecoder, &pInfo->epSet) < 0) return -1; return 0; } diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index e361c8021c..1f26033f63 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -253,6 +253,7 @@ bool syncNodeCheckNewConfig(SSyncNode* pSyncNode, const SSyncCfg* pNewCfg); int32_t syncNodeLeaderTransfer(SSyncNode* pSyncNode); int32_t syncNodeLeaderTransferTo(SSyncNode* pSyncNode, SNodeInfo newLeader); +int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* pEntry); // for debug -------------- void syncNodePrint(SSyncNode* pObj); diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 8409e1c711..dca80f6826 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -477,6 +477,13 @@ static int32_t syncNodeDoMakeLogSame(SSyncNode* ths, SyncIndex FromIndex) { static int32_t syncNodePreCommit(SSyncNode* ths, SSyncRaftEntry* pEntry) { SRpcMsg rpcMsg; syncEntry2OriginalRpc(pEntry, &rpcMsg); + + // leader transfer + if (pEntry->originalRpcType == TDMT_SYNC_LEADER_TRANSFER) { + int32_t code = syncDoLeaderTransfer(ths, &rpcMsg, pEntry); + ASSERT(code == 0); + } + if (ths->pFsm != NULL) { if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pEntry->originalRpcType)) { SFsmCbMeta cbMeta = {0}; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 80b2032ab7..b6c01f2923 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -2598,9 +2598,13 @@ const char* syncStr(ESyncState state) { } } -static int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* pEntry) { +int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* pEntry) { SyncLeaderTransfer* pSyncLeaderTransfer = syncLeaderTransferFromRpcMsg2(pRpcMsg); + if (ths->state != TAOS_SYNC_STATE_FOLLOWER) { + syncNodeEventLog(ths, "I am not follower, can not do leader transfer"); + return 0; + } syncNodeEventLog(ths, "do leader transfer"); bool sameId = syncUtilSameId(&(pSyncLeaderTransfer->newLeaderId), &(ths->myRaftId)); @@ -2811,11 +2815,14 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, ASSERT(code == 0); } +#if 0 + // execute in pre-commit // leader transfer if (pEntry->originalRpcType == TDMT_SYNC_LEADER_TRANSFER) { code = syncDoLeaderTransfer(ths, &rpcMsg, pEntry); ASSERT(code == 0); } +#endif // restore finish // if only snapshot, a noop entry will be append, so syncLogLastIndex is always ok diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 0de35a44cf..cc5c9eb651 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -127,7 +127,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_META_CACHED, "No table meta cached" TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DUP_COL_NAMES, "duplicated column names") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_TAG_LENGTH, "Invalid tag length") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_COLUMN_LENGTH, "Invalid column length") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DUP_TAG_NAMES, "duplicated tag names") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DUP_NAMES, "duplicated names") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_JSON, "Invalid JSON format") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_JSON_TYPE, "Invalid JSON data type") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_VALUE_OUT_OF_RANGE, "Value out of range") @@ -582,8 +582,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_OUTPUT_TYPE, "udf invalid output //schemaless TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PROTOCOL_TYPE, "Invalid line protocol type") TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PRECISION_TYPE, "Invalid timestamp precision type") -TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_DATA, "Invalid data type") +TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_DATA, "Invalid data format") TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_DB_CONF, "Invalid schemaless db config") +TAOS_DEFINE_ERROR(TSDB_CODE_SML_NOT_SAME_TYPE, "Not the same type like before") //tsma TAOS_DEFINE_ERROR(TSDB_CODE_TSMA_ALREADY_EXIST, "Tsma already exists") diff --git a/tests/script/tsim/stream/sliding.sim b/tests/script/tsim/stream/sliding.sim index 1ffca2a67b..f34a50de9d 100644 --- a/tests/script/tsim/stream/sliding.sim +++ b/tests/script/tsim/stream/sliding.sim @@ -366,4 +366,143 @@ if $data32 != 8 then goto loop1 endi +sql drop database IF EXISTS test2; +sql drop stream IF EXISTS streams21; +sql drop stream IF EXISTS streams22; + +sql create database test2 vgroups 2; +sql use test2; +sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +sql create stream streams21 trigger at_once into streamt as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s); +sql create stream streams22 trigger at_once into streamt2 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s); + +sql insert into t1 values(1648791213000,1,1,1,1.0); +sql insert into t1 values(1648791223001,2,2,2,1.1); +sql insert into t1 values(1648791233002,3,3,3,2.1); +sql insert into t1 values(1648791243003,4,4,4,3.1); +sql insert into t1 values(1648791213004,4,5,5,4.1); + +sql insert into t2 values(1648791213000,1,6,6,1.0); +sql insert into t2 values(1648791223001,2,7,7,1.1); +sql insert into t2 values(1648791233002,3,8,8,2.1); +sql insert into t2 values(1648791243003,4,9,9,3.1); +sql insert into t2 values(1648791213004,4,10,10,4.1); + +$loop_count = 0 + +loop2: +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from streamt; + +# row 0 +if $data01 != 2 then + print =====data01=$data01 + goto loop2 +endi + +if $data02 != 5 then + print =====data02=$data02 + goto loop2 +endi + +# row 1 +if $data11 != 1 then + print =====data11=$data11 + goto loop2 +endi + +if $data12 != 2 then + print =====data12=$data12 + goto loop2 +endi + +# row 2 +if $data21 != 1 then + print =====data21=$data21 + goto loop2 +endi + +if $data22 != 3 then + print =====data22=$data22 + goto loop2 +endi + +# row 3 +if $data31 != 1 then + print =====data31=$data31 + goto loop2 +endi + +if $data32 != 4 then + print =====data32=$data32 + goto loop2 +endi + +print step 6 + +$loop_count = 0 + +loop3: +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from streamt2; + +# row 0 +if $data01 != 4 then + print =====data01=$data01 + # goto loop3 +endi + +if $data02 != 10 then + print =====data02=$data02 + goto loop3 +endi + +# row 1 +if $data11 != 2 then + print =====data11=$data11 + goto loop3 +endi + +if $data12 != 4 then + print =====data12=$data12 + goto loop3 +endi + +# row 2 +if $data21 != 2 then + print =====data21=$data21 + goto loop3 +endi + +if $data22 != 6 then + print =====data22=$data22 + goto loop3 +endi + +# row 3 +if $data31 != 2 then + print =====data31=$data31 + goto loop3 +endi + +if $data32 != 8 then + print =====data32=$data32 + goto loop3 +endi + system sh/stop_dnodes.sh \ No newline at end of file diff --git a/tests/script/tsim/sync/mnodeLeaderTransfer.sim b/tests/script/tsim/sync/mnodeLeaderTransfer.sim new file mode 100644 index 0000000000..00329ced76 --- /dev/null +++ b/tests/script/tsim/sync/mnodeLeaderTransfer.sim @@ -0,0 +1,41 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +sql connect + +print =============== show dnodes +sql show mnodes; +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data02 != leader then + return -1 +endi + +print =============== create dnodes +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 +sleep 3000 + + +print =============== create mnode 2, 3 +sql create mnode on dnode 2 +sql create mnode on dnode 3 +sleep 3000 + +print =============== create user +sql create user user1 PASS 'user1' +sql show users +if $rows != 2 then + return -1 +endi + diff --git a/tests/script/tsim/sync/vnode-insert.sim b/tests/script/tsim/sync/vnode-insert.sim new file mode 100644 index 0000000000..221e3dd2b2 --- /dev/null +++ b/tests/script/tsim/sync/vnode-insert.sim @@ -0,0 +1,181 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 + +system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 + +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start + +sql connect +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 +sql create dnode $hostname port 7400 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +if $rows != 4 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi +if $data(3)[4] != ready then + goto step1 +endi +if $data(4)[4] != ready then + goto step1 +endi + +$replica = 3 +$vgroups = 1 + +print ============= create database +sql create database db replica $replica vgroups $vgroups + +$loop_cnt = 0 +check_db_ready: +$loop_cnt = $loop_cnt + 1 +sleep 200 +if $loop_cnt == 100 then + print ====> db not ready! + return -1 +endi +sql show databases +print ===> rows: $rows +print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] $data[2][7] $data[2][8] $data[2][9] $data[2][6] $data[2][11] $data[2][12] $data[2][13] $data[2][14] $data[2][15] $data[2][16] $data[2][17] $data[2][18] $data[2][19] +if $rows != 3 then + return -1 +endi +if $data[2][19] != ready then + goto check_db_ready +endi + +sql use db + +$loop_cnt = 0 +check_vg_ready: +$loop_cnt = $loop_cnt + 1 +sleep 200 +if $loop_cnt == 300 then + print ====> vgroups not ready! + return -1 +endi + +sql show vgroups +print ===> rows: $rows +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] $data[0][7] $data[0][8] $data[0][9] $data[0][10] $data[0][11] + +if $rows != $vgroups then + return -1 +endi + +if $data[0][4] == leader then + if $data[0][6] == follower then + if $data[0][8] == follower then + print ---- vgroup $data[0][0] leader locate on dnode $data[0][3] + endi + endi +elif $data[0][6] == leader then + if $data[0][4] == follower then + if $data[0][8] == follower then + print ---- vgroup $data[0][0] leader locate on dnode $data[0][5] + endi + endi +elif $data[0][8] == leader then + if $data[0][4] == follower then + if $data[0][6] == follower then + print ---- vgroup $data[0][0] leader locate on dnode $data[0][7] + endi + endi +else + goto check_vg_ready +endi + + +vg_ready: +print ====> create stable/child table +sql create table stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int) + +sql show stables +if $rows != 1 then + return -1 +endi + +sql create table ct1 using stb tags(1000) + + +print ====> step1 insert 1000 records +$N = 1000 +$count = 0 +while $count < $N + $ms = 1591200000000 + $count + sql insert into ct1 values( $ms , $count , 2.1, 3.1) + $count = $count + 1 +endw + +print ====> step2 sleep 20s, checking data +sleep 20000 + + +print ====> step3 sleep 30s, kill leader +sleep 30000 + +print ====> step4 insert 1000 records +$N = 1000 +$count = 0 +while $count < $N + $ms = 1591201000000 + $count + sql insert into ct1 values( $ms , $count , 2.1, 3.1) + $count = $count + 1 +endw + +print ====> step5 sleep 20s, checking data +sleep 20000 + +print ====> step6 stop all +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT + +print ====> step7 start all +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start + +print ====> step8 sleep 20s, checking data +sleep 20000 + +print ====> step9 insert 1000 records +$N = 1000 +$count = 0 +while $count < $N + $ms = 1591202000000 + $count + sql insert into ct1 values( $ms , $count , 2.1, 3.1) + $count = $count + 1 +endw + +print ====> step10 sleep 20s, checking data +sleep 20000 + + diff --git a/tests/script/tsim/sync/vnodeLeaderTransfer.sim b/tests/script/tsim/sync/vnodeLeaderTransfer.sim new file mode 100644 index 0000000000..cbeb7063ff --- /dev/null +++ b/tests/script/tsim/sync/vnodeLeaderTransfer.sim @@ -0,0 +1,141 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 + +system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 + +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start + +sql connect +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 +sql create dnode $hostname port 7400 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +if $rows != 4 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi +if $data(3)[4] != ready then + goto step1 +endi +if $data(4)[4] != ready then + goto step1 +endi + +$replica = 3 +$vgroups = 6 + +print ============= create database +sql create database db replica $replica vgroups $vgroups + +$loop_cnt = 0 +check_db_ready: +$loop_cnt = $loop_cnt + 1 +sleep 200 +if $loop_cnt == 100 then + print ====> db not ready! + return -1 +endi +sql show databases +print ===> rows: $rows +print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] $data[2][7] $data[2][8] $data[2][9] $data[2][6] $data[2][11] $data[2][12] $data[2][13] $data[2][14] $data[2][15] $data[2][16] $data[2][17] $data[2][18] $data[2][19] +if $rows != 3 then + return -1 +endi +if $data[2][19] != ready then + goto check_db_ready +endi + +sql use db + +$loop_cnt = 0 +check_vg_ready: +$loop_cnt = $loop_cnt + 1 +sleep 200 +if $loop_cnt == 300 then + print ====> vgroups not ready! + return -1 +endi + +sql show vgroups +print ===> rows: $rows +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] $data[0][7] $data[0][8] $data[0][9] $data[0][10] $data[0][11] + +if $rows != $vgroups then + return -1 +endi + +if $data[0][4] == leader then + if $data[0][6] == follower then + if $data[0][8] == follower then + print ---- vgroup $data[0][0] leader locate on dnode $data[0][3] + endi + endi +elif $data[0][6] == leader then + if $data[0][4] == follower then + if $data[0][8] == follower then + print ---- vgroup $data[0][0] leader locate on dnode $data[0][5] + endi + endi +elif $data[0][8] == leader then + if $data[0][4] == follower then + if $data[0][6] == follower then + print ---- vgroup $data[0][0] leader locate on dnode $data[0][7] + endi + endi +else + goto check_vg_ready +endi + + +vg_ready: +print ====> create stable/child table +sql create table stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int) + +sql show stables +if $rows != 1 then + return -1 +endi + +sql create table ct1 using stb tags(1000) + + +print ====> step1 insert 1000 records +$N = 1000 +$count = 0 +while $count < $N + $ms = 1591200000000 + $count + sql insert into ct1 values( $ms , $count , 2.1, 3.1) + $count = $count + 1 +endw + +print ====> step2 sleep 20s, checking data +sleep 20000 + + +print ====> step3 sleep 30s, kill leader +sleep 30000 + diff --git a/tests/script/tsim/valgrind/basic1.sim b/tests/script/tsim/valgrind/basic1.sim index 784b83d96b..86926509bb 100644 --- a/tests/script/tsim/valgrind/basic1.sim +++ b/tests/script/tsim/valgrind/basic1.sim @@ -62,26 +62,26 @@ system sh/exec.sh -n dnode1 -s start -v print =============== stepa: query data sql select * from c1 -#sql select * from stb -#sql select * from stb_1 -#sql select ts, c1, c2, c3 from c1 -#sql select ts, c1, c2, c3 from stb -#sql select ts, c1 from stb_2 -#sql select ts, c1, t1 from c1 -#sql select ts, c1, t1 from stb -#sql select ts, c1, t1 from stb_2 +sql select * from stb +sql select * from stb_1 +sql select ts, c1, c2, c3 from c1 +sql select ts, c1, c2, c3 from stb +sql select ts, c1 from stb_2 +sql select ts, c1, t1 from c1 +sql select ts, c1, t1 from stb +sql select ts, c1, t1 from stb_2 print =============== stepb: count -#sql select count(*) from c1; -#sql select count(*) from stb; -#sql select count(ts), count(c1), count(c2), count(c3) from c1 -#sql select count(ts), count(c1), count(c2), count(c3) from stb +sql select count(*) from c1; +sql select count(*) from stb; +sql select count(ts), count(c1), count(c2), count(c3) from c1 +sql select count(ts), count(c1), count(c2), count(c3) from stb print =============== stepc: func -#sql select first(ts), first(c1), first(c2), first(c3) from c1 -#sql select min(c2), min(c3), min(c4) from c1 -#sql select max(c2), max(c3), max(c4) from c1 -#sql select sum(c2), sum(c3), sum(c4) from c1 +sql select first(ts), first(c1), first(c2), first(c3) from c1 +sql select min(c2), min(c3), min(c4) from c1 +sql select max(c2), max(c3), max(c4) from c1 +sql select sum(c2), sum(c3), sum(c4) from c1 _OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/valgrind/checkError3.sim b/tests/script/tsim/valgrind/checkError3.sim index 10a8f01fb3..86926509bb 100644 --- a/tests/script/tsim/valgrind/checkError3.sim +++ b/tests/script/tsim/valgrind/checkError3.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c debugflag -v 131 -system sh/exec.sh -n dnode1 -s start -v +system sh/exec.sh -n dnode1 -s start sql connect print =============== step1: create drop show dnodes @@ -56,6 +56,10 @@ print =============== step6: alter insert sql insert into c3 using stb tags(true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) values(now-1s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) sql insert into c3 using stb tags(true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) values(now+0s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) +print =============== restart +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start -v + print =============== stepa: query data sql select * from c1 sql select * from stb diff --git a/tests/system-test/2-query/json_tag_large_tables.py b/tests/system-test/2-query/json_tag_large_tables.py new file mode 100644 index 0000000000..fc41858580 --- /dev/null +++ b/tests/system-test/2-query/json_tag_large_tables.py @@ -0,0 +1,672 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, db_test.stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import imp +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +import json +import os + + +class TDTestCase: + def caseDescription(self): + ''' + Json tag test case, include create table with json tag, select json tag and query with json tag in where condition, besides, include json tag in group by/order by/join/subquery. + case1: [TD-12452] fix error if json tag is NULL + case2: [TD-12389] describe child table, tag length error if the tag is json tag + ''' + return + + def init(self, conn, logSql): + self.testcasePath = os.path.split(__file__)[0] + self.testcaseFilename = os.path.split(__file__)[-1] + # os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename)) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + + def run(self): + # tdSql.prepare() + tdSql.execute('drop database if exists db') + tdSql.execute('create database db vgroups 1') + tdSql.execute('use db') + print("============== STEP 1 ===== prepare data & validate json string") + i = 0 + tdSql.execute("create table if not exists jsons1(ts timestamp, dataInt int, dataBool bool, dataStr nchar(50), dataStrBin binary(150)) tags(jtag json)") + while i <= 100000: + f = "insert into jsons1_{} using jsons1 tags('{\"tag1\":\"fff\",\"tag2\":{}, \"tag3\":true}') values(1591060618000, 1, false, 'json1', '你是') (1591060608000, 23, true, '等等', 'json')".format + sql = f(i, i) + tdSql.execute(sql) + i = i + 1 + + # test duplicate key using the first one. elimate empty key + #tdSql.execute("CREATE TABLE if not exists jsons1_8 using jsons1 tags('{\"tag1\":null, \"tag1\":true, \"tag1\":45, \"1tag$\":2, \" \":90, \"\":32}')") tdSql.query("select jtag from jsons1_8") tdSql.checkRows(0); + #tdSql.query("select ts,jtag from jsons1 order by ts limit 2,3") + #tdSql.checkData(0, 0, '2020-06-02 09:17:08.000') + #tdSql.checkData(0, 1, '{"tag1":5,"tag2":"beijing"}') + #tdSql.checkData(1, 0, '2020-06-02 09:17:48.000') + #tdSql.checkData(1, 1, '{"tag1":false,"tag2":"beijing"}') + #tdSql.checkData(2, 0, '2020-06-02 09:18:48.000') + #tdSql.checkData(2, 1, '{"tag1":null,"tag2":"shanghai","tag3":"hello"}') + + #tdSql.query("select ts,jtag->'tag1' from jsons1 order by ts limit 2,3") + #tdSql.checkData(0, 0, '2020-06-02 09:17:08.000') + #tdSql.checkData(0, 1, '5.000000000') + #tdSql.checkData(1, 0, '2020-06-02 09:17:48.000') + #tdSql.checkData(1, 1, 'false') + #tdSql.checkData(2, 0, '2020-06-02 09:18:48.000') + #tdSql.checkData(2, 1, 'null') + + ## test empty json string, save as jtag is NULL + #tdSql.execute("insert into jsons1_9 using jsons1 tags('\t') values (1591062328000, 24, NULL, '你就会', '2sdw')") + #tdSql.execute("CREATE TABLE if not exists jsons1_10 using jsons1 tags('')") + #tdSql.execute("CREATE TABLE if not exists jsons1_11 using jsons1 tags(' ')") + #tdSql.execute("CREATE TABLE if not exists jsons1_12 using jsons1 tags('{}')") + #tdSql.execute("CREATE TABLE if not exists jsons1_13 using jsons1 tags('null')") + + ## test invalidate json + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('\"efwewf\"')") + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('3333')") + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags(76)") + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags(hell)") + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('33.33')") + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('false')") + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('[1,true]')") + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{222}')") + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"fe\"}')") + + ## test invalidate json key, key must can be printed assic char + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"tag1\":[1,true]}')") + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"tag1\":{}}')") + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"。loc\":\"fff\"}')") + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"\t\":\"fff\"}')") + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"试试\":\"fff\"}')") + + ## test invalidate json value, value number can not be inf,nan TD-12166 + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"k\":1.8e308}')") + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"k\":-1.8e308}')") + + ##test length limit + #char1= ''.join(['abcd']*64) + #char3= ''.join(['abcd']*1021) + #print(len(char3)) # 4084 + #tdSql.error("CREATE TABLE if not exists jsons1_15 using jsons1 tags('{\"%s1\":5}')" % char1) # len(key)=257 + #tdSql.execute("CREATE TABLE if not exists jsons1_15 using jsons1 tags('{\"%s\":5}')" % char1) # len(key)=256 + #tdSql.error("CREATE TABLE if not exists jsons1_16 using jsons1 tags('{\"TSSSS\":\"%s\"}')" % char3) # len(object)=4096 + #tdSql.execute("CREATE TABLE if not exists jsons1_16 using jsons1 tags('{\"TSSS\":\"%s\"}')" % char3) # len(object)=4095 + #tdSql.execute("drop table if exists jsons1_15") + #tdSql.execute("drop table if exists jsons1_16") + + #print("============== STEP 2 ===== alter table json tag") + #tdSql.error("ALTER STABLE jsons1 add tag tag2 nchar(20)") + #tdSql.error("ALTER STABLE jsons1 drop tag jtag") + #tdSql.error("ALTER TABLE jsons1 MODIFY TAG jtag nchar(128)") + + #tdSql.execute("ALTER TABLE jsons1_1 SET TAG jtag='{\"tag1\":\"femail\",\"tag2\":35,\"tag3\":true}'") + #tdSql.query("select jtag from jsons1_1") + #tdSql.checkData(0, 0, '{"tag1":"femail","tag2":35,"tag3":true}') + #tdSql.execute("ALTER TABLE jsons1 rename TAG jtag jtag_new") + #tdSql.execute("ALTER TABLE jsons1 rename TAG jtag_new jtag") + + #tdSql.execute("create table st(ts timestamp, i int) tags(t int)") + #tdSql.error("ALTER STABLE st add tag jtag json") + #tdSql.error("ALTER STABLE st add column jtag json") + + #print("============== STEP 3 ===== query table") + ## test error syntax + #tdSql.error("select * from jsons1 where jtag->tag1='beijing'") + #tdSql.error("select -> from jsons1") + #tdSql.error("select * from jsons1 where contains") + #tdSql.error("select * from jsons1 where jtag->") + #tdSql.error("select jtag->location from jsons1") + #tdSql.error("select jtag contains location from jsons1") + #tdSql.error("select * from jsons1 where jtag contains location") + #tdSql.query("select * from jsons1 where jtag contains''") + #tdSql.error("select * from jsons1 where jtag contains 'location'='beijing'") + + ## test function error + #tdSql.error("select avg(jtag->'tag1') from jsons1") + #tdSql.error("select avg(jtag) from jsons1") + #tdSql.error("select min(jtag->'tag1') from jsons1") + #tdSql.error("select min(jtag) from jsons1") + #tdSql.error("select ceil(jtag->'tag1') from jsons1") + #tdSql.error("select ceil(jtag) from jsons1") + + + ##test scalar operation + #tdSql.query("select jtag contains 'tag1',jtag->'tag1' from jsons1 order by jtag->'tag1'") + #tdSql.checkRows(9) + #tdSql.query("select jtag->'tag1' like 'fe%',jtag->'tag1' from jsons1 order by jtag->'tag1'") + #tdSql.checkRows(9) + #tdSql.query("select jtag->'tag1' not like 'fe%',jtag->'tag1' from jsons1 order by jtag->'tag1'") + #tdSql.checkRows(9) + #tdSql.query("select jtag->'tag1' match 'fe',jtag->'tag1' from jsons1 order by jtag->'tag1'") + #tdSql.checkRows(9) + #tdSql.query("select jtag->'tag1' nmatch 'fe',jtag->'tag1' from jsons1 order by jtag->'tag1'") + #tdSql.checkRows(9) + #tdSql.query("select jtag->'tag1',jtag->'tag1'>='a' from jsons1 order by jtag->'tag1'") + #tdSql.checkRows(9) + + ## test select normal column + #tdSql.query("select dataint from jsons1 order by dataint") + #tdSql.checkRows(9) + #tdSql.checkData(1, 0, 1) + + ## test select json tag + #tdSql.query("select * from jsons1") + #tdSql.checkRows(9) + #tdSql.query("select jtag from jsons1") + #tdSql.checkRows(9) + #tdSql.query("select * from jsons1 where jtag is null") + #tdSql.checkRows(1) + #tdSql.query("select * from jsons1 where jtag is not null") + #tdSql.checkRows(8) + + ## test jtag is NULL + #tdSql.query("select jtag from jsons1_9") + #tdSql.checkData(0, 0, None) + + ## test select json tag->'key', value is string + #tdSql.query("select jtag->'tag1' from jsons1_1") + #tdSql.checkData(0, 0, '"femail"') + #tdSql.query("select jtag->'tag2' from jsons1_6") + #tdSql.checkData(0, 0, '""') + ## test select json tag->'key', value is int + #tdSql.query("select jtag->'tag2' from jsons1_1") + #tdSql.checkData(0, 0, "35.000000000") + ## test select json tag->'key', value is bool + #tdSql.query("select jtag->'tag3' from jsons1_1") + #tdSql.checkData(0, 0, "true") + ## test select json tag->'key', value is null + #tdSql.query("select jtag->'tag1' from jsons1_4") + #tdSql.checkData(0, 0, None) + ## test select json tag->'key', value is double + #tdSql.query("select jtag->'tag1' from jsons1_5") + #tdSql.checkData(0, 0, "1.232000000") + ## test select json tag->'key', key is not exist + #tdSql.query("select jtag->'tag10' from jsons1_4") + #tdSql.checkData(0, 0, None) + + #tdSql.query("select jtag->'tag1' from jsons1") + #tdSql.checkRows(9) + ## test header name + #res = tdSql.getColNameList("select jtag->'tag1' from jsons1") + #cname_list = [] + #cname_list.append("jtag->'tag1'") + #tdSql.checkColNameList(res, cname_list) + + + ## test where with json tag + #tdSql.query("select * from jsons1_1 where jtag is not null") + #tdSql.query("select * from jsons1 where jtag='{\"tag1\":11,\"tag2\":\"\"}'") + #tdSql.error("select * from jsons1 where jtag->'tag1'={}") + + ## test json error + #tdSql.error("select jtag + 1 from jsons1") + #tdSql.error("select jtag > 1 from jsons1") + #tdSql.error("select jtag like \"1\" from jsons1") + #tdSql.error("select jtag in (\"1\") from jsons1") + ##tdSql.error("select jtag from jsons1 where jtag > 1") + ##tdSql.error("select jtag from jsons1 where jtag like 'fsss'") + ##tdSql.error("select jtag from jsons1 where jtag in (1)") + + + ## where json value is string + #tdSql.query("select * from jsons1 where jtag->'tag2'='beijing'") + #tdSql.checkRows(2) + #tdSql.query("select dataint,tbname,jtag->'tag1',jtag from jsons1 where jtag->'tag2'='beijing' order by dataint") + #tdSql.checkRows(2) + #tdSql.checkData(0, 0, 2) + #tdSql.checkData(0, 1, 'jsons1_2') + #tdSql.checkData(0, 2, "5.000000000") + #tdSql.checkData(0, 3, '{"tag1":5,"tag2":"beijing"}') + #tdSql.checkData(1, 0, 3) + #tdSql.checkData(1, 1, 'jsons1_3') + #tdSql.checkData(1, 2, 'false') + + + #tdSql.query("select * from jsons1 where jtag->'tag1'='beijing'") + #tdSql.checkRows(0) + #tdSql.query("select * from jsons1 where jtag->'tag1'='收到货'") + #tdSql.checkRows(1) + #tdSql.query("select * from jsons1 where jtag->'tag2'>'beijing'") + #tdSql.checkRows(1) + #tdSql.query("select * from jsons1 where jtag->'tag2'>='beijing'") + #tdSql.checkRows(3) + #tdSql.query("select * from jsons1 where jtag->'tag2'<'beijing'") + #tdSql.checkRows(2) + #tdSql.query("select * from jsons1 where jtag->'tag2'<='beijing'") + #tdSql.checkRows(4) + #tdSql.query("select * from jsons1 where jtag->'tag2'!='beijing'") + #tdSql.checkRows(3) + #tdSql.query("select * from jsons1 where jtag->'tag2'=''") + #tdSql.checkRows(2) + + ## where json value is int + #tdSql.query("select * from jsons1 where jtag->'tag1'=5") + #tdSql.checkRows(1) + #tdSql.checkData(0, 1, 2) + #tdSql.query("select * from jsons1 where jtag->'tag1'=10") + #tdSql.checkRows(0) + #tdSql.query("select * from jsons1 where jtag->'tag1'<54") + #tdSql.checkRows(3) + #tdSql.query("select * from jsons1 where jtag->'tag1'<=11") + #tdSql.checkRows(3) + #tdSql.query("select * from jsons1 where jtag->'tag1'>4") + #tdSql.checkRows(2) + #tdSql.query("select * from jsons1 where jtag->'tag1'>=5") + #tdSql.checkRows(2) + #tdSql.query("select * from jsons1 where jtag->'tag1'!=5") + #tdSql.checkRows(2) + #tdSql.query("select * from jsons1 where jtag->'tag1'!=55") + #tdSql.checkRows(3) + + ## where json value is double + #tdSql.query("select * from jsons1 where jtag->'tag1'=1.232") + #tdSql.checkRows(1) + #tdSql.query("select * from jsons1 where jtag->'tag1'<1.232") + #tdSql.checkRows(0) + #tdSql.query("select * from jsons1 where jtag->'tag1'<=1.232") + #tdSql.checkRows(1) + #tdSql.query("select * from jsons1 where jtag->'tag1'>1.23") + #tdSql.checkRows(3) + #tdSql.query("select * from jsons1 where jtag->'tag1'>=1.232") + #tdSql.checkRows(3) + #tdSql.query("select * from jsons1 where jtag->'tag1'!=1.232") + #tdSql.checkRows(2) + #tdSql.query("select * from jsons1 where jtag->'tag1'!=3.232") + #tdSql.checkRows(3) + #tdSql.query("select * from jsons1 where jtag->'tag1'/0=3") + #tdSql.checkRows(0) + #tdSql.query("select * from jsons1 where jtag->'tag1'/5=1") + #tdSql.checkRows(1) + + ## where json value is bool + #tdSql.query("select * from jsons1 where jtag->'tag1'=true") + #tdSql.checkRows(0) + #tdSql.query("select * from jsons1 where jtag->'tag1'=false") + #tdSql.checkRows(1) + #tdSql.query("select * from jsons1 where jtag->'tag1'!=false") + #tdSql.checkRows(0) + #tdSql.query("select * from jsons1 where jtag->'tag1'>false") + #tdSql.checkRows(0) + + ## where json value is null + #tdSql.query("select * from jsons1 where jtag->'tag1'=null") + #tdSql.checkRows(0) + + ## where json key is null + #tdSql.query("select * from jsons1 where jtag->'tag_no_exist'=3") + #tdSql.checkRows(0) + + ## where json value is not exist + #tdSql.query("select * from jsons1 where jtag->'tag1' is null") + #tdSql.checkData(0, 0, 'jsons1_9') + #tdSql.checkRows(2) + #tdSql.query("select * from jsons1 where jtag->'tag4' is null") + #tdSql.checkRows(9) + #tdSql.query("select * from jsons1 where jtag->'tag3' is not null") + #tdSql.checkRows(3) + + ## test contains + #tdSql.query("select * from jsons1 where jtag contains 'tag1'") + #tdSql.checkRows(8) + #tdSql.query("select * from jsons1 where jtag contains 'tag3'") + #tdSql.checkRows(4) + #tdSql.query("select * from jsons1 where jtag contains 'tag_no_exist'") + #tdSql.checkRows(0) + + ## test json tag in where condition with and/or + #tdSql.query("select * from jsons1 where jtag->'tag1'=false and jtag->'tag2'='beijing'") + #tdSql.checkRows(1) + #tdSql.query("select * from jsons1 where jtag->'tag1'=false or jtag->'tag2'='beijing'") + #tdSql.checkRows(2) + #tdSql.query("select * from jsons1 where jtag->'tag1'=false and jtag->'tag2'='shanghai'") + #tdSql.checkRows(0) + #tdSql.query("select * from jsons1 where jtag->'tag1'=false and jtag->'tag2'='shanghai'") + #tdSql.checkRows(0) + #tdSql.query("select * from jsons1 where jtag->'tag1'=13 or jtag->'tag2'>35") + #tdSql.checkRows(0) + #tdSql.query("select * from jsons1 where jtag->'tag1'=13 or jtag->'tag2'>35") + #tdSql.checkRows(0) + #tdSql.query("select * from jsons1 where jtag->'tag1' is not null and jtag contains 'tag3'") + #tdSql.checkRows(3) + #tdSql.query("select * from jsons1 where jtag->'tag1'='femail' and jtag contains 'tag3'") + #tdSql.checkRows(2) + + + ## test with between and + #tdSql.query("select * from jsons1 where jtag->'tag1' between 1 and 30") + #tdSql.checkRows(3) + #tdSql.query("select * from jsons1 where jtag->'tag1' between 'femail' and 'beijing'") + #tdSql.checkRows(2) + + ## test with tbname/normal column + #tdSql.query("select * from jsons1 where tbname = 'jsons1_1'") + #tdSql.checkRows(2) + #tdSql.query("select * from jsons1 where tbname = 'jsons1_1' and jtag contains 'tag3'") + #tdSql.checkRows(2) + #tdSql.query("select * from jsons1 where tbname = 'jsons1_1' and jtag contains 'tag3' and dataint=3") + #tdSql.checkRows(0) + #tdSql.query("select * from jsons1 where tbname = 'jsons1_1' and jtag contains 'tag3' and dataint=23") + #tdSql.checkRows(1) + + + ## test where condition like + #tdSql.query("select * from jsons1 where jtag->'tag2' like 'bei%'") + #tdSql.checkRows(2) + #tdSql.query("select * from jsons1 where jtag->'tag1' like 'fe%' and jtag->'tag2' is not null") + #tdSql.checkRows(2) + + ## test where condition in no support in + #tdSql.error("select * from jsons1 where jtag->'tag1' in ('beijing')") + + ## test where condition match/nmath + #tdSql.query("select * from jsons1 where jtag->'tag1' match 'ma'") + #tdSql.checkRows(2) + #tdSql.query("select * from jsons1 where jtag->'tag1' match 'ma$'") + #tdSql.checkRows(0) + #tdSql.query("select * from jsons1 where jtag->'tag2' match 'jing$'") + #tdSql.checkRows(2) + #tdSql.query("select * from jsons1 where jtag->'tag1' match '收到'") + #tdSql.checkRows(1) + #tdSql.query("select * from jsons1 where jtag->'tag1' nmatch 'ma'") + #tdSql.checkRows(1) + + ## test distinct + #tdSql.execute("insert into jsons1_14 using jsons1 tags('{\"tag1\":\"收到货\",\"tag2\":\"\",\"tag3\":null}') values(1591062628000, 2, NULL, '你就会', 'dws')") + #tdSql.query("select distinct jtag->'tag1' from jsons1") + #tdSql.checkRows(7) + ## tdSql.query("select distinct jtag from jsons1") + ## tdSql.checkRows(9) + + ##test dumplicate key with normal colomn + #tdSql.execute("INSERT INTO jsons1_15 using jsons1 tags('{\"tbname\":\"tt\",\"databool\":true,\"datastr\":\"是是是\"}') values(1591060828000, 4, false, 'jjsf', \"你就会\")") + #tdSql.query("select * from jsons1 where jtag->'datastr' match '是' and datastr match 'js'") + #tdSql.checkRows(1) + ## tdSql.query("select tbname,jtag->'tbname' from jsons1 where jtag->'tbname'='tt' and tbname='jsons1_14'") + ## tdSql.checkRows(1) + + ## test join + #tdSql.execute("create table if not exists jsons2(ts timestamp, dataInt int, dataBool bool, dataStr nchar(50), dataStrBin binary(150)) tags(jtag json)") + #tdSql.execute("insert into jsons2_1 using jsons2 tags('{\"tag1\":\"fff\",\"tag2\":5, \"tag3\":true}') values(1591060618000, 2, false, 'json2', '你是2')") + #tdSql.execute("insert into jsons2_2 using jsons2 tags('{\"tag1\":5,\"tag2\":null}') values (1591060628000, 2, true, 'json2', 'sss')") + + #tdSql.execute("create table if not exists jsons3(ts timestamp, dataInt int, dataBool bool, dataStr nchar(50), dataStrBin binary(150)) tags(jtag json)") + #tdSql.execute("insert into jsons3_1 using jsons3 tags('{\"tag1\":\"fff\",\"tag2\":5, \"tag3\":true}') values(1591060618000, 3, false, 'json3', '你是3')") + #tdSql.execute("insert into jsons3_2 using jsons3 tags('{\"tag1\":5,\"tag2\":\"beijing\"}') values (1591060638000, 2, true, 'json3', 'sss')") + #tdSql.query("select 'sss',33,a.jtag->'tag3' from jsons2 a,jsons3 b where a.ts=b.ts and a.jtag->'tag1'=b.jtag->'tag1'") + #tdSql.checkData(0, 0, "sss") + #tdSql.checkData(0, 2, "true") + + #res = tdSql.getColNameList("select 'sss',33,a.jtag->'tag3' from jsons2 a,jsons3 b where a.ts=b.ts and a.jtag->'tag1'=b.jtag->'tag1'") + #cname_list = [] + #cname_list.append("'sss'") + #cname_list.append("33") + #cname_list.append("a.jtag->'tag3'") + #tdSql.checkColNameList(res, cname_list) + ## + ## test group by & order by json tag + #tdSql.query("select ts,jtag->'tag1' from jsons1 partition by jtag->'tag1' order by jtag->'tag1' desc") + #tdSql.checkRows(11) + #tdSql.checkData(0, 1, '"femail"') + #tdSql.checkData(2, 1, '"收到货"') + #tdSql.checkData(7, 1, "false") + + + #tdSql.error("select count(*) from jsons1 group by jtag") + #tdSql.error("select count(*) from jsons1 partition by jtag") + #tdSql.error("select count(*) from jsons1 group by jtag order by jtag") + #tdSql.error("select count(*) from jsons1 group by jtag->'tag1' order by jtag->'tag2'") + #tdSql.error("select count(*) from jsons1 group by jtag->'tag1' order by jtag") + #tdSql.query("select count(*),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1' desc") + #tdSql.checkRows(8) + #tdSql.checkData(0, 0, 2) + #tdSql.checkData(0, 1, '"femail"') + #tdSql.checkData(1, 0, 2) + #tdSql.checkData(1, 1, '"收到货"') + #tdSql.checkData(2, 0, 1) + #tdSql.checkData(2, 1, "11.000000000") + #tdSql.checkData(5, 0, 1) + #tdSql.checkData(5, 1, "false") + + #tdSql.query("select count(*),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1' asc") + #tdSql.checkRows(8) + #tdSql.checkData(0, 1, None) + #tdSql.checkData(2, 0, 1) + #tdSql.checkData(2, 1, "false") + #tdSql.checkData(5, 0, 1) + #tdSql.checkData(5, 1, "11.000000000") + #tdSql.checkData(7, 0, 2) + #tdSql.checkData(7, 1, '"femail"') + + ## test stddev with group by json tag + #tdSql.query("select stddev(dataint),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1'") + #tdSql.checkRows(8) + #tdSql.checkData(0, 1, None) + #tdSql.checkData(4, 0, 0) + #tdSql.checkData(4, 1, "5.000000000") + #tdSql.checkData(7, 0, 11) + #tdSql.checkData(7, 1, '"femail"') + + #res = tdSql.getColNameList("select stddev(dataint),jsons1.jtag->'tag1' from jsons1 group by jsons1.jtag->'tag1' order by jtag->'tag1'") + #cname_list = [] + #cname_list.append("stddev(dataint)") + #cname_list.append("jsons1.jtag->'tag1'") + #tdSql.checkColNameList(res, cname_list) + + ## test top/bottom with group by json tag + #tdSql.query("select top(dataint,2),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1'") + #tdSql.checkRows(11) + #tdSql.checkData(0, 1, None) + ##tdSql.checkData(2, 0, 24) + ##tdSql.checkData(3, 0, 3) + ##tdSql.checkData(3, 1, "false") + ##tdSql.checkData(8, 0, 2) + ##tdSql.checkData(10, 1, '"femail"') + + ## test having + ## tdSql.query("select count(*),jtag->'tag1' from jsons1 group by jtag->'tag1' having count(*) > 1") + ## tdSql.checkRows(3) + + ## subquery with json tag + #tdSql.query("select * from (select jtag, dataint from jsons1) order by dataint") + #tdSql.checkRows(11) + #tdSql.checkData(1, 1, 1) + #tdSql.checkData(5, 0, '{"tag1":false,"tag2":"beijing"}') + + #tdSql.error("select jtag->'tag1' from (select jtag->'tag1', dataint from jsons1)") + #tdSql.error("select t->'tag1' from (select jtag->'tag1' as t, dataint from jsons1)") + ## tdSql.query("select ts,jtag->'tag1' from (select jtag->'tag1',tbname,ts from jsons1 order by ts)") + ## tdSql.checkRows(11) + ## tdSql.checkData(1, 1, "jsons1_1") + ## tdSql.checkData(1, 2, '"femail"') + + ## union all + #tdSql.query("select jtag->'tag1' from jsons1 union all select jtag->'tag2' from jsons2") + #tdSql.checkRows(13) + #tdSql.query("select jtag->'tag1' from jsons1_1 union all select jtag->'tag2' from jsons2_1") + #tdSql.checkRows(3) + + #tdSql.query("select jtag->'tag1' from jsons1_1 union all select jtag->'tag1' from jsons2_1") + #tdSql.checkRows(3) + #tdSql.query("select dataint,jtag->'tag1',tbname from jsons1 union all select dataint,jtag->'tag1',tbname from jsons2") + #tdSql.checkRows(13) + #tdSql.query("select dataint,jtag,tbname from jsons1 union all select dataint,jtag,tbname from jsons2") + #tdSql.checkRows(13) + + ##show create table + #tdSql.query("show create table jsons1") + #tdSql.checkData(0, 1, 'CREATE STABLE `jsons1` (`ts` TIMESTAMP, `dataint` INT, `databool` BOOL, `datastr` NCHAR(50), `datastrbin` VARCHAR(150)) TAGS (`jtag` JSON) WATERMARK 5000a, 5000a') + + ##test aggregate function:count/avg/twa/irate/sum/stddev/leastsquares + #tdSql.query("select count(*) from jsons1 where jtag is not null") + #tdSql.checkData(0, 0, 10) + #tdSql.query("select avg(dataint) from jsons1 where jtag is not null") + #tdSql.checkData(0, 0, 5.3) + ## tdSql.query("select twa(dataint) from jsons1 where jtag is not null") + ## tdSql.checkData(0, 0, 28.386363636363637) + ## tdSql.query("select irate(dataint) from jsons1 where jtag is not null") + + #tdSql.query("select sum(dataint) from jsons1 where jtag->'tag1' is not null") + #tdSql.checkData(0, 0, 45) + #tdSql.query("select stddev(dataint) from jsons1 where jtag->'tag1'>1") + #tdSql.checkData(0, 0, 4.496912521) + #tdSql.query("SELECT LEASTSQUARES(dataint, 1, 1) from jsons1 where jtag is not null") + + ##test selection function:min/max/first/last/top/bottom/percentile/apercentile/last_row/interp + #tdSql.query("select min(dataint) from jsons1 where jtag->'tag1'>1") + #tdSql.checkData(0, 0, 1) + #tdSql.query("select max(dataint) from jsons1 where jtag->'tag1'>1") + #tdSql.checkData(0, 0, 11) + #tdSql.query("select first(dataint) from jsons1 where jtag->'tag1'>1") + #tdSql.checkData(0, 0, 2) + #tdSql.query("select last(dataint) from jsons1 where jtag->'tag1'>1") + #tdSql.checkData(0, 0, 11) + #tdSql.query("select top(dataint,100) from jsons1 where jtag->'tag1'>1") + #tdSql.checkRows(3) + #tdSql.query("select bottom(dataint,100) from jsons1 where jtag->'tag1'>1") + #tdSql.checkRows(3) + ##tdSql.query("select percentile(dataint,20) from jsons1 where jtag->'tag1'>1") + #tdSql.query("select apercentile(dataint, 50) from jsons1 where jtag->'tag1'>1") + #tdSql.checkData(0, 0, 1.5) + ## tdSql.query("select last_row(dataint) from jsons1 where jtag->'tag1'>1") + ## tdSql.query("select interp(dataint) from jsons1 where ts = '2020-06-02 09:17:08.000' and jtag->'tag1'>1") + + ##test calculation function:diff/derivative/spread/ceil/floor/round/ + #tdSql.query("select diff(dataint) from jsons1 where jtag->'tag1'>1") + #tdSql.checkRows(2) + ## tdSql.checkData(0, 0, -1) + ## tdSql.checkData(1, 0, 10) + #tdSql.query("select derivative(dataint, 10m, 0) from jsons1 where jtag->'tag1'>1") + ## tdSql.checkData(0, 0, -2) + #tdSql.query("select spread(dataint) from jsons1 where jtag->'tag1'>1") + #tdSql.checkData(0, 0, 10) + #tdSql.query("select ceil(dataint) from jsons1 where jtag->'tag1'>1") + #tdSql.checkRows(3) + #tdSql.query("select floor(dataint) from jsons1 where jtag->'tag1'>1") + #tdSql.checkRows(3) + #tdSql.query("select round(dataint) from jsons1 where jtag->'tag1'>1") + #tdSql.checkRows(3) + + ##math function + #tdSql.query("select sin(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select cos(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select tan(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select asin(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select acos(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select atan(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select ceil(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select floor(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select round(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select abs(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select pow(dataint,5) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select log(dataint,10) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select sqrt(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select HISTOGRAM(dataint,'user_input','[1, 33, 555, 7777]',1) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select csum(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select mavg(dataint,1) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select statecount(dataint,'GE',10) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select stateduration(dataint,'GE',0) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select sample(dataint,3) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select HYPERLOGLOG(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(1) + #tdSql.query("select twa(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(1) + + ## function not ready + #tdSql.query("select tail(dataint,1) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(1) + #tdSql.query("select unique(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select mode(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(1) + #tdSql.query("select irate(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(1) + + ##str function + #tdSql.query("select upper(dataStr) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select ltrim(dataStr) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select lower(dataStr) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select rtrim(dataStr) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select LENGTH(dataStr) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select CHAR_LENGTH(dataStr) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select SUBSTR(dataStr,5) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select CONCAT(dataStr,dataStrBin) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select CONCAT_ws('adad!@!@%$^$%$^$%^a',dataStr,dataStrBin) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select CAST(dataStr as bigint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + + ##time function + #tdSql.query("select now() from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select today() from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select TIMEZONE() from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select TO_ISO8601(ts) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select TO_UNIXTIMESTAMP(datastr) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select TIMETRUNCATE(ts,1s) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select TIMEDIFF(ts,_c0) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select TIMEDIFF(ts,1u) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select ELAPSED(ts,1h) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(1) + + ##test TD-12077 + #tdSql.execute("insert into jsons1_16 using jsons1 tags('{\"tag1\":\"收到货\",\"tag2\":\"\",\"tag3\":-2.111}') values(1591062628000, 2, NULL, '你就会', 'dws')") + #tdSql.query("select jtag->'tag3' from jsons1_16") + #tdSql.checkData(0, 0, '-2.111000000') + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) + diff --git a/tests/system-test/2-query/statecount.py b/tests/system-test/2-query/statecount.py index 90e8bebab4..91e2aa9e47 100644 --- a/tests/system-test/2-query/statecount.py +++ b/tests/system-test/2-query/statecount.py @@ -335,7 +335,7 @@ class TDTestCase: # tdSql.checkRows(21) # group by - tdSql.query("select statecount(c1,'GT',1) from ct1 group by c1") + tdSql.error("select statecount(c1,'GT',1) from ct1 group by c1") tdSql.error("select statecount(c1,'GT',1) from ct1 group by tbname") # super table diff --git a/tools/taos-tools b/tools/taos-tools index bd496f76b6..d807c3ffa6 160000 --- a/tools/taos-tools +++ b/tools/taos-tools @@ -1 +1 @@ -Subproject commit bd496f76b64931c66da2f8b0f24143a98a881cde +Subproject commit d807c3ffa6f750f7765e102917d1328cadf21c13 diff --git a/tools/taosws-rs b/tools/taosws-rs index 7a94ffab45..c5fded266d 160000 --- a/tools/taosws-rs +++ b/tools/taosws-rs @@ -1 +1 @@ -Subproject commit 7a94ffab45f08e16f09b3f430fe75d717054adb6 +Subproject commit c5fded266d3b10508e38bf3285bb7ecf798bc343