diff --git a/docs/en/07-develop/03-insert-data/03-opentsdb-telnet.mdx b/docs/en/07-develop/03-insert-data/03-opentsdb-telnet.mdx index d88a6335cb..a71efde735 100644 --- a/docs/en/07-develop/03-insert-data/03-opentsdb-telnet.mdx +++ b/docs/en/07-develop/03-insert-data/03-opentsdb-telnet.mdx @@ -64,10 +64,10 @@ taos> use test; Database changed. taos> show stables; - name | created_time | columns | tags | tables | -============================================================================================ - meters.current | 2022-03-30 17:04:10.877 | 2 | 2 | 2 | - meters.voltage | 2022-03-30 17:04:10.882 | 2 | 2 | 2 | + name | +================================= + meters.current | + meters.voltage | Query OK, 2 row(s) in set (0.002544s) taos> select tbname, * from `meters.current`; diff --git a/docs/en/07-develop/03-insert-data/04-opentsdb-json.mdx b/docs/en/07-develop/03-insert-data/04-opentsdb-json.mdx index e2e6d6fc9f..c0a287e331 100644 --- a/docs/en/07-develop/03-insert-data/04-opentsdb-json.mdx +++ b/docs/en/07-develop/03-insert-data/04-opentsdb-json.mdx @@ -81,10 +81,10 @@ taos> use test; Database changed. taos> show stables; - name | created_time | columns | tags | tables | -============================================================================================ - meters.current | 2022-03-29 16:05:25.193 | 2 | 2 | 1 | - meters.voltage | 2022-03-29 16:05:25.200 | 2 | 2 | 1 | + name | +================================= + meters.current | + meters.voltage | Query OK, 2 row(s) in set (0.001954s) taos> select * from `meters.current`; diff --git a/docs/en/14-reference/03-connector/_verify_linux.mdx b/docs/en/14-reference/03-connector/_verify_linux.mdx index 3438b1578e..220fccfef3 100644 --- a/docs/en/14-reference/03-connector/_verify_linux.mdx +++ b/docs/en/14-reference/03-connector/_verify_linux.mdx @@ -4,11 +4,11 @@ Execute TDengine CLI program `taos` directly from the Linux shell to connect to $ taos taos> show databases; - name | create_time | vgroups | ntables | replica | strict | duration | keep | buffer | pagesize | pages | minrows | maxrows | comp | precision | status | retention | single_stable | cachemodel | cachesize | wal_level | wal_fsync_period | wal_retention_period | wal_retention_size | wal_roll_period | wal_seg_size | -========================================================================================================================================================================================================================================================================================================================================================================================================================================================================= - information_schema | NULL | NULL | 14 | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | ready | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | - performance_schema | NULL | NULL | 3 | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | ready | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | - db | 2022-08-04 14:14:49.385 | 2 | 4 | 1 | off | 14400m | 5254560m,5254560m,5254560m | 96 | 4 | 256 | 100 | 4096 | 2 | ms | ready | NULL | false | none | 1 | 1 | 3000 | 0 | 0 | 0 | 0 | + name | +================================= + information_schema | + performance_schema | + db | Query OK, 3 rows in database (0.019154s) taos> diff --git a/docs/en/14-reference/03-connector/_verify_windows.mdx b/docs/en/14-reference/03-connector/_verify_windows.mdx index 402b170511..65d10f0875 100644 --- a/docs/en/14-reference/03-connector/_verify_windows.mdx +++ b/docs/en/14-reference/03-connector/_verify_windows.mdx @@ -2,12 +2,11 @@ Go to the `C:\TDengine` directory from `cmd` and execute TDengine CLI program `t ```text taos> show databases; - name | create_time | vgroups | ntables | replica | strict | duration | keep | buffer | pagesize | pages | minrows | maxrows | comp | precision | status | retention | single_stable | cachemodel | cachesize | wal_level | wal_fsync_period | wal_retention_period | wal_retention_size | wal_roll_period | wal_seg_size | -========================================================================================================================================================================================================================================================================================================================================================================================================================================================================= - information_schema | NULL | NULL | 14 | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | ready | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | - performance_schema | NULL | NULL | 3 | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | ready | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | - test | 2022-08-04 16:46:40.506 | 2 | 0 | 1 | off | 14400m | 5256000m,5256000m,5256000m | 96 | 4 | 256 | -100 | 4096 | 2 | ms | ready | NULL | false | none | 1 | 1 | 3000 | 0 | 0 | 0 | 0 | + name | +================================= + information_schema | + performance_schema | + test | Query OK, 3 rows in database (0.123000s) taos> diff --git a/docs/en/14-reference/11-docker/index.md b/docs/en/14-reference/11-docker/index.md index 7cd1e810dc..89987c456f 100644 --- a/docs/en/14-reference/11-docker/index.md +++ b/docs/en/14-reference/11-docker/index.md @@ -25,10 +25,11 @@ The TDengine client taos can be executed in this container to access TDengine us $ docker exec -it tdengine taos taos> show databases; - name | created_time | ntables | vgroups | replica | quorum | days | keep | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | cachelast | precision | update | status | -==================================================================================================================================================================================================================================================================================== - log | 2022-01-17 13:57:22.270 | 10 | 1 | 1 | 1 | 10 | 30 | 1 | 3 | 100 | 4096 | 1 | 3000 | 2 | 0 | us | 0 | ready | -Query OK, 1 row(s) in set (0.002843s) + name | +================================= + information_schema | + performance_schema | +Query OK, 2 row(s) in set (0.002843s) ``` The TDengine server running in the container uses the container's hostname to establish a connection. Using TDengine CLI or various connectors (such as JDBC-JNI) to access the TDengine inside the container from outside the container is more complicated. So the above is the simplest way to access the TDengine service in the container and is suitable for some simple scenarios. Please refer to the next section if you want to access the TDengine service in the container from outside the container using TDengine CLI or various connectors for complex scenarios. diff --git a/docs/en/14-reference/_statsd.mdx b/docs/en/14-reference/_statsd.mdx index d0721a16fb..d068304a1d 100644 --- a/docs/en/14-reference/_statsd.mdx +++ b/docs/en/14-reference/_statsd.mdx @@ -51,5 +51,6 @@ port: 8125 Start StatsD after adding the following (assuming the config file is modified to config.js) ``` +npm install node stats.js config.js & ``` diff --git a/docs/en/20-third-party/02-prometheus.md b/docs/en/20-third-party/02-prometheus.md index c46b0381b9..ef9b9cb637 100644 --- a/docs/en/20-third-party/02-prometheus.md +++ b/docs/en/20-third-party/02-prometheus.md @@ -30,21 +30,20 @@ After restarting Prometheus, you can refer to the following example to verify th ``` taos> show databases; - name | created_time | ntables | vgroups | replica | quorum | days | keep | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | cachelast | precision | update | status | -==================================================================================================================================================================================================================================================================================== - test | 2022-04-12 08:07:58.756 | 1 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ms | 0 | ready | - log | 2022-04-20 07:19:50.260 | 2 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ms | 0 | ready | - prometheus_data | 2022-04-20 07:21:09.202 | 158 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ns | 2 | ready | - db | 2022-04-15 06:37:08.512 | 1 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ms | 0 | ready | -Query OK, 4 row(s) in set (0.000585s) + name | +================================= + information_schema | + performance_schema | + prometheus_data | +Query OK, 3 row(s) in set (0.000585s) taos> use prometheus_data; Database changed. taos> show stables; - name | created_time | columns | tags | tables | -============================================================================================ - metrics | 2022-04-20 07:21:09.209 | 2 | 1 | 1389 | + name | +================================= + metrics | Query OK, 1 row(s) in set (0.000487s) taos> select * from metrics limit 10; diff --git a/docs/en/20-third-party/03-telegraf.md b/docs/en/20-third-party/03-telegraf.md index ab06ab0379..8f3cab0e57 100644 --- a/docs/en/20-third-party/03-telegraf.md +++ b/docs/en/20-third-party/03-telegraf.md @@ -43,16 +43,16 @@ taos> use telegraf; Database changed. taos> show stables; - name | created_time | columns | tags | tables | -============================================================================================ - swap | 2022-04-20 08:47:53.532 | 7 | 1 | 1 | - cpu | 2022-04-20 08:48:03.488 | 11 | 2 | 5 | - system | 2022-04-20 08:47:53.512 | 8 | 1 | 1 | - diskio | 2022-04-20 08:47:53.550 | 12 | 2 | 15 | - kernel | 2022-04-20 08:47:53.503 | 6 | 1 | 1 | - mem | 2022-04-20 08:47:53.521 | 35 | 1 | 1 | - processes | 2022-04-20 08:47:53.555 | 12 | 1 | 1 | - disk | 2022-04-20 08:47:53.541 | 8 | 5 | 2 | + name | +================================= + swap | + cpu | + system | + diskio | + kernel | + mem | + processes | + disk | Query OK, 8 row(s) in set (0.000521s) taos> select * from telegraf.system limit 10; diff --git a/docs/en/20-third-party/05-collectd.md b/docs/en/20-third-party/05-collectd.md index 295c08c307..5b52e3b7bc 100644 --- a/docs/en/20-third-party/05-collectd.md +++ b/docs/en/20-third-party/05-collectd.md @@ -32,28 +32,29 @@ Use the TDengine CLI to verify that collectd's data is written to TDengine and c ``` taos> show databases; - name | created_time | ntables | vgroups | replica | quorum | days | keep | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | cachelast | precision | update | status | -==================================================================================================================================================================================================================================================================================== - collectd | 2022-04-20 09:27:45.460 | 95 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ns | 2 | ready | - log | 2022-04-20 07:19:50.260 | 11 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ms | 0 | ready | -Query OK, 2 row(s) in set (0.003266s) + name | +================================= + information_schema | + performance_schema | + collectd | +Query OK, 3 row(s) in set (0.003266s) taos> use collectd; Database changed. taos> show stables; - name | created_time | columns | tags | tables | -============================================================================================ - load_1 | 2022-04-20 09:27:45.492 | 2 | 2 | 1 | - memory_value | 2022-04-20 09:27:45.463 | 2 | 3 | 6 | - df_value | 2022-04-20 09:27:45.463 | 2 | 4 | 25 | - load_2 | 2022-04-20 09:27:45.501 | 2 | 2 | 1 | - load_0 | 2022-04-20 09:27:45.485 | 2 | 2 | 1 | - interface_1 | 2022-04-20 09:27:45.488 | 2 | 3 | 12 | - irq_value | 2022-04-20 09:27:45.476 | 2 | 3 | 31 | - interface_0 | 2022-04-20 09:27:45.480 | 2 | 3 | 12 | - entropy_value | 2022-04-20 09:27:45.473 | 2 | 2 | 1 | - swap_value | 2022-04-20 09:27:45.477 | 2 | 3 | 5 | + name | +================================= + load_1 | + memory_value | + df_value | + load_2 | + load_0 | + interface_1 | + irq_value | + interface_0 | + entropy_value | + swap_value | Query OK, 10 row(s) in set (0.002236s) taos> select * from collectd.memory_value limit 10; diff --git a/docs/en/20-third-party/06-statsd.md b/docs/en/20-third-party/06-statsd.md index 32b1bbb97a..b861a48ecd 100644 --- a/docs/en/20-third-party/06-statsd.md +++ b/docs/en/20-third-party/06-statsd.md @@ -26,7 +26,7 @@ Start StatsD: ``` $ node stats.js config.js & [1] 8546 -$ 20 Apr 09:54:41 - [8546] reading config file: exampleConfig.js +$ 20 Apr 09:54:41 - [8546] reading config file: config.js 20 Apr 09:54:41 - server is up INFO ``` @@ -40,19 +40,20 @@ Use the TDengine CLI to verify that StatsD data is written to TDengine and can r ``` taos> show databases; - name | created_time | ntables | vgroups | replica | quorum | days | keep | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | cachelast | precision | update | status | -==================================================================================================================================================================================================================================================================================== - log | 2022-04-20 07:19:50.260 | 11 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ms | 0 | ready | - statsd | 2022-04-20 09:54:51.220 | 1 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ns | 2 | ready | -Query OK, 2 row(s) in set (0.003142s) + name | +================================= + information_schema | + performance_schema | + statsd | +Query OK, 3 row(s) in set (0.003142s) taos> use statsd; Database changed. taos> show stables; - name | created_time | columns | tags | tables | -============================================================================================ - foo | 2022-04-20 09:54:51.234 | 2 | 1 | 1 | + name | +================================= + foo | Query OK, 1 row(s) in set (0.002161s) taos> select * from foo; @@ -63,3 +64,8 @@ Query OK, 1 row(s) in set (0.004179s) taos> ``` + +:::note + +- TDengine will automatically create unique IDs for sub-table names by the rule. +::: diff --git a/docs/en/20-third-party/07-icinga2.md b/docs/en/20-third-party/07-icinga2.md index e98f209803..167b6a4303 100644 --- a/docs/en/20-third-party/07-icinga2.md +++ b/docs/en/20-third-party/07-icinga2.md @@ -47,30 +47,30 @@ taos> use icinga2; Database changed. taos> show stables; - name | created_time | columns | tags | tables | -============================================================================================ - icinga.service.users.state_... | 2022-04-20 12:11:39.726 | 2 | 1 | 1 | - icinga.service.users.acknow... | 2022-04-20 12:11:39.756 | 2 | 1 | 1 | - icinga.service.procs.downti... | 2022-04-20 12:11:44.541 | 2 | 1 | 1 | - icinga.service.users.users | 2022-04-20 12:11:39.770 | 2 | 1 | 1 | - icinga.service.procs.procs_min | 2022-04-20 12:11:44.599 | 2 | 1 | 1 | - icinga.service.users.users_min | 2022-04-20 12:11:39.809 | 2 | 1 | 1 | - icinga.check.max_check_atte... | 2022-04-20 12:11:39.847 | 2 | 3 | 2 | - icinga.service.procs.state_... | 2022-04-20 12:11:44.522 | 2 | 1 | 1 | - icinga.service.procs.procs_... | 2022-04-20 12:11:44.576 | 2 | 1 | 1 | - icinga.service.users.users_... | 2022-04-20 12:11:39.796 | 2 | 1 | 1 | - icinga.check.latency | 2022-04-20 12:11:39.869 | 2 | 3 | 2 | - icinga.service.procs.procs_... | 2022-04-20 12:11:44.588 | 2 | 1 | 1 | - icinga.service.users.downti... | 2022-04-20 12:11:39.746 | 2 | 1 | 1 | - icinga.service.users.users_... | 2022-04-20 12:11:39.783 | 2 | 1 | 1 | - icinga.service.users.reachable | 2022-04-20 12:11:39.736 | 2 | 1 | 1 | - icinga.service.procs.procs | 2022-04-20 12:11:44.565 | 2 | 1 | 1 | - icinga.service.procs.acknow... | 2022-04-20 12:11:44.554 | 2 | 1 | 1 | - icinga.service.procs.state | 2022-04-20 12:11:44.509 | 2 | 1 | 1 | - icinga.service.procs.reachable | 2022-04-20 12:11:44.532 | 2 | 1 | 1 | - icinga.check.current_attempt | 2022-04-20 12:11:39.825 | 2 | 3 | 2 | - icinga.check.execution_time | 2022-04-20 12:11:39.898 | 2 | 3 | 2 | - icinga.service.users.state | 2022-04-20 12:11:39.704 | 2 | 1 | 1 | + name | +================================= + icinga.service.users.state_... | + icinga.service.users.acknow... | + icinga.service.procs.downti... | + icinga.service.users.users | + icinga.service.procs.procs_min | + icinga.service.users.users_min | + icinga.check.max_check_atte... | + icinga.service.procs.state_... | + icinga.service.procs.procs_... | + icinga.service.users.users_... | + icinga.check.latency | + icinga.service.procs.procs_... | + icinga.service.users.downti... | + icinga.service.users.users_... | + icinga.service.users.reachable | + icinga.service.procs.procs | + icinga.service.procs.acknow... | + icinga.service.procs.state | + icinga.service.procs.reachable | + icinga.check.current_attempt | + icinga.check.execution_time | + icinga.service.users.state | Query OK, 22 row(s) in set (0.002317s) ``` diff --git a/docs/en/20-third-party/08-tcollector.md b/docs/en/20-third-party/08-tcollector.md index 16b73c2394..b604a2d712 100644 --- a/docs/en/20-third-party/08-tcollector.md +++ b/docs/en/20-third-party/08-tcollector.md @@ -33,35 +33,41 @@ Wait for a few seconds and then use the TDengine CLI to query whether the corres ``` taos> show databases; - name | created_time | ntables | vgroups | replica | quorum | days | keep | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | cachelast | precision | update | status | -==================================================================================================================================================================================================================================================================================== - tcollector | 2022-04-20 12:44:49.604 | 88 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ns | 2 | ready | - log | 2022-04-20 07:19:50.260 | 11 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ms | 0 | ready | -Query OK, 2 row(s) in set (0.002679s) + name | +================================= + information_schema | + performance_schema | + tcollector | +Query OK, 3 rows in database (0.001647s) taos> use tcollector; Database changed. taos> show stables; - name | created_time | columns | tags | tables | -============================================================================================ - proc.meminfo.hugepages_rsvd | 2022-04-20 12:44:53.945 | 2 | 1 | 1 | - proc.meminfo.directmap1g | 2022-04-20 12:44:54.110 | 2 | 1 | 1 | - proc.meminfo.vmallocchunk | 2022-04-20 12:44:53.724 | 2 | 1 | 1 | - proc.meminfo.hugepagesize | 2022-04-20 12:44:54.004 | 2 | 1 | 1 | - tcollector.reader.lines_dro... | 2022-04-20 12:44:49.675 | 2 | 1 | 1 | - proc.meminfo.sunreclaim | 2022-04-20 12:44:53.437 | 2 | 1 | 1 | - proc.stat.ctxt | 2022-04-20 12:44:55.363 | 2 | 1 | 1 | - proc.meminfo.swaptotal | 2022-04-20 12:44:53.158 | 2 | 1 | 1 | - proc.uptime.total | 2022-04-20 12:44:52.813 | 2 | 1 | 1 | - tcollector.collector.lines_... | 2022-04-20 12:44:49.895 | 2 | 2 | 51 | - proc.meminfo.vmallocused | 2022-04-20 12:44:53.704 | 2 | 1 | 1 | - proc.meminfo.memavailable | 2022-04-20 12:44:52.939 | 2 | 1 | 1 | - sys.numa.foreign_allocs | 2022-04-20 12:44:57.929 | 2 | 2 | 1 | - proc.meminfo.committed_as | 2022-04-20 12:44:53.639 | 2 | 1 | 1 | - proc.vmstat.pswpin | 2022-04-20 12:44:54.177 | 2 | 1 | 1 | - proc.meminfo.cmafree | 2022-04-20 12:44:53.865 | 2 | 1 | 1 | - proc.meminfo.mapped | 2022-04-20 12:44:53.349 | 2 | 1 | 1 | - proc.vmstat.pgmajfault | 2022-04-20 12:44:54.251 | 2 | 1 | 1 | + name | +================================= + proc.meminfo.hugepages_rsvd | + proc.meminfo.directmap1g | + proc.meminfo.vmallocchunk | + proc.meminfo.hugepagesize | + tcollector.reader.lines_dro... | + proc.meminfo.sunreclaim | + proc.stat.ctxt | + proc.meminfo.swaptotal | + proc.uptime.total | + tcollector.collector.lines_... | + proc.meminfo.vmallocused | + proc.meminfo.memavailable | + sys.numa.foreign_allocs | + proc.meminfo.committed_as | + proc.vmstat.pswpin | + proc.meminfo.cmafree | + proc.meminfo.mapped | + proc.vmstat.pgmajfault | ... ``` + +:::note + +- TDengine will automatically create unique IDs for sub-table names by the rule. +::: diff --git a/docs/zh/07-develop/03-insert-data/03-opentsdb-telnet.mdx b/docs/zh/07-develop/03-insert-data/03-opentsdb-telnet.mdx index 58bca7f843..8d097e3f65 100644 --- a/docs/zh/07-develop/03-insert-data/03-opentsdb-telnet.mdx +++ b/docs/zh/07-develop/03-insert-data/03-opentsdb-telnet.mdx @@ -66,10 +66,10 @@ taos> use test; Database changed. taos> show stables; - name | created_time | columns | tags | tables | -============================================================================================ - meters.current | 2022-03-30 17:04:10.877 | 2 | 2 | 2 | - meters.voltage | 2022-03-30 17:04:10.882 | 2 | 2 | 2 | + name | +================================= + meters.current | + meters.voltage | Query OK, 2 row(s) in set (0.002544s) taos> select tbname, * from `meters.current`; diff --git a/docs/zh/07-develop/03-insert-data/04-opentsdb-json.mdx b/docs/zh/07-develop/03-insert-data/04-opentsdb-json.mdx index aa3e5980cf..e2e7d7c8fa 100644 --- a/docs/zh/07-develop/03-insert-data/04-opentsdb-json.mdx +++ b/docs/zh/07-develop/03-insert-data/04-opentsdb-json.mdx @@ -82,10 +82,10 @@ taos> use test; Database changed. taos> show stables; - name | created_time | columns | tags | tables | -============================================================================================ - meters.current | 2022-03-29 16:05:25.193 | 2 | 2 | 1 | - meters.voltage | 2022-03-29 16:05:25.200 | 2 | 2 | 1 | + name | +================================= + meters.current | + meters.voltage | Query OK, 2 row(s) in set (0.001954s) taos> select * from `meters.current`; diff --git a/docs/zh/08-connector/_verify_linux.mdx b/docs/zh/08-connector/_verify_linux.mdx index 28957e77aa..4137b2880b 100644 --- a/docs/zh/08-connector/_verify_linux.mdx +++ b/docs/zh/08-connector/_verify_linux.mdx @@ -4,11 +4,11 @@ $ taos taos> show databases; - name | create_time | vgroups | ntables | replica | strict | duration | keep | buffer | pagesize | pages | minrows | maxrows | comp | precision | status | retention | single_stable | cachemodel | cachesize | wal_level | wal_fsync_period | wal_retention_period | wal_retention_size | wal_roll_period | wal_seg_size | -========================================================================================================================================================================================================================================================================================================================================================================================================================================================================= - information_schema | NULL | NULL | 14 | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | ready | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | - performance_schema | NULL | NULL | 3 | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | ready | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | - db | 2022-08-04 14:14:49.385 | 2 | 4 | 1 | off | 14400m | 5254560m,5254560m,5254560m | 96 | 4 | 256 | 100 | 4096 | 2 | ms | ready | NULL | false | none | 1 | 1 | 3000 | 0 | 0 | 0 | 0 | + name | +================================= + information_schema | + performance_schema | + db | Query OK, 3 rows in database (0.019154s) taos> diff --git a/docs/zh/14-reference/_statsd.mdx b/docs/zh/14-reference/_statsd.mdx index b225c44267..dc4f19b37c 100644 --- a/docs/zh/14-reference/_statsd.mdx +++ b/docs/zh/14-reference/_statsd.mdx @@ -51,5 +51,6 @@ port: 8125 增加如下内容后启动 StatsD（假设配置文件修改为 config.js）。 ``` +npm install node stats.js config.js & ``` diff --git a/docs/zh/20-third-party/02-prometheus.md b/docs/zh/20-third-party/02-prometheus.md index 59bab1823a..0f95053a78 100644 --- a/docs/zh/20-third-party/02-prometheus.md +++ b/docs/zh/20-third-party/02-prometheus.md @@ -29,21 +29,20 @@ Prometheus 提供了 `remote_write` 和 `remote_read` 接口来利用其它数 ### 使用 TDengine CLI 查询写入数据 ``` taos> show databases; - name | created_time | ntables | vgroups | replica | quorum | days | keep | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | cachelast | precision | update | status | -==================================================================================================================================================================================================================================================================================== - test | 2022-04-12 08:07:58.756 | 1 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ms | 0 | ready | - log | 2022-04-20 07:19:50.260 | 2 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ms | 0 | ready | - prometheus_data | 2022-04-20 07:21:09.202 | 158 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ns | 2 | ready | - db | 2022-04-15 06:37:08.512 | 1 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ms | 0 | ready | -Query OK, 4 row(s) in set (0.000585s) + name | +================================= + information_schema | + performance_schema | + prometheus_data | +Query OK, 3 row(s) in set (0.000585s) taos> use prometheus_data; Database changed. taos> show stables; - name | created_time | columns | tags | tables | -============================================================================================ - metrics | 2022-04-20 07:21:09.209 | 2 | 1 | 1389 | + name | +================================= + metrics | Query OK, 1 row(s) in set (0.000487s) taos> select * from metrics limit 10; diff --git a/docs/zh/20-third-party/03-telegraf.md b/docs/zh/20-third-party/03-telegraf.md index 71bb1b3885..bb688a6ab6 100644 --- a/docs/zh/20-third-party/03-telegraf.md +++ b/docs/zh/20-third-party/03-telegraf.md @@ -44,16 +44,16 @@ taos> use telegraf; Database changed. taos> show stables; - name | created_time | columns | tags | tables | -============================================================================================ - swap | 2022-04-20 08:47:53.532 | 7 | 1 | 1 | - cpu | 2022-04-20 08:48:03.488 | 11 | 2 | 5 | - system | 2022-04-20 08:47:53.512 | 8 | 1 | 1 | - diskio | 2022-04-20 08:47:53.550 | 12 | 2 | 15 | - kernel | 2022-04-20 08:47:53.503 | 6 | 1 | 1 | - mem | 2022-04-20 08:47:53.521 | 35 | 1 | 1 | - processes | 2022-04-20 08:47:53.555 | 12 | 1 | 1 | - disk | 2022-04-20 08:47:53.541 | 8 | 5 | 2 | + name | +================================= + swap | + cpu | + system | + diskio | + kernel | + mem | + processes | + disk | Query OK, 8 row(s) in set (0.000521s) taos> select * from telegraf.system limit 10; diff --git a/docs/zh/20-third-party/05-collectd.md b/docs/zh/20-third-party/05-collectd.md index d55b981fb9..bda66b3cce 100644 --- a/docs/zh/20-third-party/05-collectd.md +++ b/docs/zh/20-third-party/05-collectd.md @@ -32,28 +32,29 @@ sudo systemctl restart collectd ``` taos> show databases; - name | created_time | ntables | vgroups | replica | quorum | days | keep | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | cachelast | precision | update | status | -==================================================================================================================================================================================================================================================================================== - collectd | 2022-04-20 09:27:45.460 | 95 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ns | 2 | ready | - log | 2022-04-20 07:19:50.260 | 11 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ms | 0 | ready | -Query OK, 2 row(s) in set (0.003266s) + name | +================================= + information_schema | + performance_schema | + collectd | +Query OK, 3 row(s) in set (0.003266s) taos> use collectd; Database changed. taos> show stables; - name | created_time | columns | tags | tables | -============================================================================================ - load_1 | 2022-04-20 09:27:45.492 | 2 | 2 | 1 | - memory_value | 2022-04-20 09:27:45.463 | 2 | 3 | 6 | - df_value | 2022-04-20 09:27:45.463 | 2 | 4 | 25 | - load_2 | 2022-04-20 09:27:45.501 | 2 | 2 | 1 | - load_0 | 2022-04-20 09:27:45.485 | 2 | 2 | 1 | - interface_1 | 2022-04-20 09:27:45.488 | 2 | 3 | 12 | - irq_value | 2022-04-20 09:27:45.476 | 2 | 3 | 31 | - interface_0 | 2022-04-20 09:27:45.480 | 2 | 3 | 12 | - entropy_value | 2022-04-20 09:27:45.473 | 2 | 2 | 1 | - swap_value | 2022-04-20 09:27:45.477 | 2 | 3 | 5 | + name | +================================= + load_1 | + memory_value | + df_value | + load_2 | + load_0 | + interface_1 | + irq_value | + interface_0 | + entropy_value | + swap_value | Query OK, 10 row(s) in set (0.002236s) taos> select * from collectd.memory_value limit 10; diff --git a/docs/zh/20-third-party/06-statsd.md b/docs/zh/20-third-party/06-statsd.md index 122c9fd94c..20b4d800be 100644 --- a/docs/zh/20-third-party/06-statsd.md +++ b/docs/zh/20-third-party/06-statsd.md @@ -27,7 +27,7 @@ StatsD 是汇总和总结应用指标的一个简单的守护进程，近些年 ``` $ node stats.js config.js & [1] 8546 -$ 20 Apr 09:54:41 - [8546] reading config file: exampleConfig.js +$ 20 Apr 09:54:41 - [8546] reading config file: config.js 20 Apr 09:54:41 - server is up INFO ``` @@ -41,19 +41,20 @@ $ echo "foo:1|c" | nc -u -w0 127.0.0.1 8125 ``` taos> show databases; - name | created_time | ntables | vgroups | replica | quorum | days | keep | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | cachelast | precision | update | status | -==================================================================================================================================================================================================================================================================================== - log | 2022-04-20 07:19:50.260 | 11 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ms | 0 | ready | - statsd | 2022-04-20 09:54:51.220 | 1 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ns | 2 | ready | -Query OK, 2 row(s) in set (0.003142s) + name | +================================= + information_schema | + performance_schema | + statsd | +Query OK, 3 row(s) in set (0.003142s) taos> use statsd; Database changed. taos> show stables; - name | created_time | columns | tags | tables | -============================================================================================ - foo | 2022-04-20 09:54:51.234 | 2 | 1 | 1 | + name | +================================= + foo | Query OK, 1 row(s) in set (0.002161s) taos> select * from foo; @@ -64,3 +65,8 @@ Query OK, 1 row(s) in set (0.004179s) taos> ``` + +:::note + +- TDengine will automatically create unique IDs for sub-table names by the rule. +::: diff --git a/docs/zh/20-third-party/07-icinga2.md b/docs/zh/20-third-party/07-icinga2.md index 03d733a326..d112a7a5e6 100644 --- a/docs/zh/20-third-party/07-icinga2.md +++ b/docs/zh/20-third-party/07-icinga2.md @@ -48,30 +48,30 @@ taos> use icinga2; Database changed. taos> show stables; - name | created_time | columns | tags | tables | -============================================================================================ - icinga.service.users.state_... | 2022-04-20 12:11:39.726 | 2 | 1 | 1 | - icinga.service.users.acknow... | 2022-04-20 12:11:39.756 | 2 | 1 | 1 | - icinga.service.procs.downti... | 2022-04-20 12:11:44.541 | 2 | 1 | 1 | - icinga.service.users.users | 2022-04-20 12:11:39.770 | 2 | 1 | 1 | - icinga.service.procs.procs_min | 2022-04-20 12:11:44.599 | 2 | 1 | 1 | - icinga.service.users.users_min | 2022-04-20 12:11:39.809 | 2 | 1 | 1 | - icinga.check.max_check_atte... | 2022-04-20 12:11:39.847 | 2 | 3 | 2 | - icinga.service.procs.state_... | 2022-04-20 12:11:44.522 | 2 | 1 | 1 | - icinga.service.procs.procs_... | 2022-04-20 12:11:44.576 | 2 | 1 | 1 | - icinga.service.users.users_... | 2022-04-20 12:11:39.796 | 2 | 1 | 1 | - icinga.check.latency | 2022-04-20 12:11:39.869 | 2 | 3 | 2 | - icinga.service.procs.procs_... | 2022-04-20 12:11:44.588 | 2 | 1 | 1 | - icinga.service.users.downti... | 2022-04-20 12:11:39.746 | 2 | 1 | 1 | - icinga.service.users.users_... | 2022-04-20 12:11:39.783 | 2 | 1 | 1 | - icinga.service.users.reachable | 2022-04-20 12:11:39.736 | 2 | 1 | 1 | - icinga.service.procs.procs | 2022-04-20 12:11:44.565 | 2 | 1 | 1 | - icinga.service.procs.acknow... | 2022-04-20 12:11:44.554 | 2 | 1 | 1 | - icinga.service.procs.state | 2022-04-20 12:11:44.509 | 2 | 1 | 1 | - icinga.service.procs.reachable | 2022-04-20 12:11:44.532 | 2 | 1 | 1 | - icinga.check.current_attempt | 2022-04-20 12:11:39.825 | 2 | 3 | 2 | - icinga.check.execution_time | 2022-04-20 12:11:39.898 | 2 | 3 | 2 | - icinga.service.users.state | 2022-04-20 12:11:39.704 | 2 | 1 | 1 | + name | +================================= + icinga.service.users.state_... | + icinga.service.users.acknow... | + icinga.service.procs.downti... | + icinga.service.users.users | + icinga.service.procs.procs_min | + icinga.service.users.users_min | + icinga.check.max_check_atte... | + icinga.service.procs.state_... | + icinga.service.procs.procs_... | + icinga.service.users.users_... | + icinga.check.latency | + icinga.service.procs.procs_... | + icinga.service.users.downti... | + icinga.service.users.users_... | + icinga.service.users.reachable | + icinga.service.procs.procs | + icinga.service.procs.acknow... | + icinga.service.procs.state | + icinga.service.procs.reachable | + icinga.check.current_attempt | + icinga.check.execution_time | + icinga.service.users.state | Query OK, 22 row(s) in set (0.002317s) ``` diff --git a/docs/zh/20-third-party/08-tcollector.md b/docs/zh/20-third-party/08-tcollector.md index 78d0b4a5df..a6ed0f89e2 100644 --- a/docs/zh/20-third-party/08-tcollector.md +++ b/docs/zh/20-third-party/08-tcollector.md @@ -34,35 +34,42 @@ sudo systemctl restart taosadapter ``` taos> show databases; - name | created_time | ntables | vgroups | replica | quorum | days | keep | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | cachelast | precision | update | status | -==================================================================================================================================================================================================================================================================================== - tcollector | 2022-04-20 12:44:49.604 | 88 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ns | 2 | ready | - log | 2022-04-20 07:19:50.260 | 11 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ms | 0 | ready | -Query OK, 2 row(s) in set (0.002679s) + name | +================================= + information_schema | + performance_schema | + tcollector | +Query OK, 3 rows in database (0.001647s) + taos> use tcollector; Database changed. taos> show stables; - name | created_time | columns | tags | tables | -============================================================================================ - proc.meminfo.hugepages_rsvd | 2022-04-20 12:44:53.945 | 2 | 1 | 1 | - proc.meminfo.directmap1g | 2022-04-20 12:44:54.110 | 2 | 1 | 1 | - proc.meminfo.vmallocchunk | 2022-04-20 12:44:53.724 | 2 | 1 | 1 | - proc.meminfo.hugepagesize | 2022-04-20 12:44:54.004 | 2 | 1 | 1 | - tcollector.reader.lines_dro... | 2022-04-20 12:44:49.675 | 2 | 1 | 1 | - proc.meminfo.sunreclaim | 2022-04-20 12:44:53.437 | 2 | 1 | 1 | - proc.stat.ctxt | 2022-04-20 12:44:55.363 | 2 | 1 | 1 | - proc.meminfo.swaptotal | 2022-04-20 12:44:53.158 | 2 | 1 | 1 | - proc.uptime.total | 2022-04-20 12:44:52.813 | 2 | 1 | 1 | - tcollector.collector.lines_... | 2022-04-20 12:44:49.895 | 2 | 2 | 51 | - proc.meminfo.vmallocused | 2022-04-20 12:44:53.704 | 2 | 1 | 1 | - proc.meminfo.memavailable | 2022-04-20 12:44:52.939 | 2 | 1 | 1 | - sys.numa.foreign_allocs | 2022-04-20 12:44:57.929 | 2 | 2 | 1 | - proc.meminfo.committed_as | 2022-04-20 12:44:53.639 | 2 | 1 | 1 | - proc.vmstat.pswpin | 2022-04-20 12:44:54.177 | 2 | 1 | 1 | - proc.meminfo.cmafree | 2022-04-20 12:44:53.865 | 2 | 1 | 1 | - proc.meminfo.mapped | 2022-04-20 12:44:53.349 | 2 | 1 | 1 | - proc.vmstat.pgmajfault | 2022-04-20 12:44:54.251 | 2 | 1 | 1 | + name | +================================= + proc.meminfo.hugepages_rsvd | + proc.meminfo.directmap1g | + proc.meminfo.vmallocchunk | + proc.meminfo.hugepagesize | + tcollector.reader.lines_dro... | + proc.meminfo.sunreclaim | + proc.stat.ctxt | + proc.meminfo.swaptotal | + proc.uptime.total | + tcollector.collector.lines_... | + proc.meminfo.vmallocused | + proc.meminfo.memavailable | + sys.numa.foreign_allocs | + proc.meminfo.committed_as | + proc.vmstat.pswpin | + proc.meminfo.cmafree | + proc.meminfo.mapped | + proc.vmstat.pgmajfault | ... ``` + +:::note + +- TDengine 默认生成的子表名是根据规则生成的唯一 ID 值。 +::: diff --git a/include/libs/index/index.h b/include/libs/index/index.h index c1fdc4df52..0d31ca2f68 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -206,12 +206,6 @@ void indexJsonRebuild(SIndexJson* idx, void* iter); **/ bool indexJsonIsRebuild(SIndexJson* idx); -/* - * init index env - * - */ -void indexInit(); - /* index filter */ typedef struct SIndexMetaArg { void* metaEx; @@ -225,6 +219,12 @@ typedef enum { SFLT_NOT_INDEX, SFLT_COARSE_INDEX, SFLT_ACCURATE_INDEX } SIdxFltS SIdxFltStatus idxGetFltStatus(SNode* pFilterNode); int32_t doFilterTag(SNode* pFilterNode, SIndexMetaArg* metaArg, SArray* result, SIdxFltStatus* status); + +/* + * init index env + * + */ +void indexInit(int32_t threads); /* * destory index env * diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h index 0adcf976f0..c2b3a7977c 100644 --- a/include/libs/stream/streamState.h +++ b/include/libs/stream/streamState.h @@ -33,6 +33,7 @@ typedef struct { TTB* pFuncStateDb; TTB* pFillStateDb; // todo refactor TXN txn; + int32_t number; } SStreamState; SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath); @@ -42,7 +43,8 @@ int32_t streamStateCommit(SStreamState* pState); int32_t streamStateAbort(SStreamState* pState); typedef struct { - TBC* pCur; + TBC* pCur; + int64_t number; } SStreamStateCur; int32_t streamStateFuncPut(SStreamState* pState, const STupleKey* key, const void* value, int32_t vLen); @@ -52,6 +54,8 @@ int32_t streamStateFuncDel(SStreamState* pState, const STupleKey* key); int32_t streamStatePut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); int32_t streamStateGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); int32_t streamStateDel(SStreamState* pState, const SWinKey* key); +int32_t streamStateClear(SStreamState* pState); +void streamStateSetNumber(SStreamState* pState, int32_t number); int32_t streamStateFillPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); int32_t streamStateFillGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); @@ -63,6 +67,7 @@ void streamFreeVal(void* val); SStreamStateCur* streamStateGetCur(SStreamState* pState, const SWinKey* key); SStreamStateCur* streamStateGetAndCheckCur(SStreamState* pState, SWinKey* key); +SStreamStateCur* streamStateSeekKeyNext(SStreamState* pState, const SWinKey* key); SStreamStateCur* streamStateFillSeekKeyNext(SStreamState* pState, const SWinKey* key); SStreamStateCur* streamStateFillSeekKeyPrev(SStreamState* pState, const SWinKey* key); void streamStateFreeCur(SStreamStateCur* pCur); @@ -70,6 +75,7 @@ void streamStateFreeCur(SStreamStateCur* pCur); int32_t streamStateGetGroupKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); int32_t streamStateGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); +int32_t streamStateGetFirst(SStreamState* pState, SWinKey* key); int32_t streamStateSeekFirst(SStreamState* pState, SStreamStateCur* pCur); int32_t streamStateSeekLast(SStreamState* pState, SStreamStateCur* pCur); diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index 582b16ce99..2e149be3b6 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -193,6 +193,8 @@ int32_t dmInitDnode(SDnode *pDnode, EDndNodeType rtype) { goto _OVER; } + indexInit(tsNumOfCommitThreads); + dmReportStartup("dnode-transport", "initialized"); dDebug("dnode is created, ptr:%p", pDnode); code = 0; diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 96748b3620..3502ecf604 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -237,6 +237,7 @@ static int32_t mndCreateDefaultCluster(SMnode *pMnode) { STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, NULL, "create-cluster"); if (pTrans == NULL) { + sdbFreeRaw(pRaw); mError("cluster:%" PRId64 ", failed to create since %s", clusterObj.id, terrstr()); return -1; } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index a05d8dd739..1d70f43834 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1594,7 +1594,7 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, break; } char precVstr[10] = {0}; - STR_WITH_SIZE_TO_VARSTR(precVstr, precStr, 2); + STR_WITH_MAXSIZE_TO_VARSTR(precVstr, precStr, 10); char *statusStr = "ready"; if (objStatus == SDB_STATUS_CREATING) { @@ -1607,7 +1607,7 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, } } char statusVstr[24] = {0}; - STR_WITH_SIZE_TO_VARSTR(statusVstr, statusStr, strlen(statusStr)); + STR_WITH_MAXSIZE_TO_VARSTR(statusVstr, statusStr, 24); if (sysDb || !sysinfo) { for (int32_t i = 0; i < pShow->numOfColumns; ++i) { @@ -1644,7 +1644,7 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, const char *strictStr = pDb->cfg.strict ? "on" : "off"; char strictVstr[24] = {0}; - STR_WITH_SIZE_TO_VARSTR(strictVstr, strictStr, strlen(strictStr)); + STR_WITH_MAXSIZE_TO_VARSTR(strictVstr, strictStr, 24); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, rows, (const char *)strictVstr, false); @@ -1704,7 +1704,7 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, const char *cacheModelStr = getCacheModelStr(pDb->cfg.cacheLast); char cacheModelVstr[24] = {0}; - STR_WITH_SIZE_TO_VARSTR(cacheModelVstr, cacheModelStr, strlen(cacheModelStr)); + STR_WITH_MAXSIZE_TO_VARSTR(cacheModelVstr, cacheModelStr, 24); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, rows, (const char *)cacheModelVstr, false); diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 4a8436513f..c8995eb6bc 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -102,7 +102,7 @@ static int32_t mndCreateDefaultDnode(SMnode *pMnode) { dnodeObj.updateTime = dnodeObj.createdTime; dnodeObj.port = tsServerPort; memcpy(&dnodeObj.fqdn, tsLocalFqdn, TSDB_FQDN_LEN); - snprintf(dnodeObj.ep, TSDB_EP_LEN, "%s:%u", dnodeObj.fqdn, dnodeObj.port); + snprintf(dnodeObj.ep, TSDB_EP_LEN - 1, "%s:%u", dnodeObj.fqdn, dnodeObj.port); pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, NULL, "create-dnode"); if (pTrans == NULL) goto _OVER; @@ -190,7 +190,7 @@ _OVER: static int32_t mndDnodeActionInsert(SSdb *pSdb, SDnodeObj *pDnode) { mTrace("dnode:%d, perform insert action, row:%p", pDnode->id, pDnode); pDnode->offlineReason = DND_REASON_STATUS_NOT_RECEIVED; - snprintf(pDnode->ep, TSDB_EP_LEN, "%s:%u", pDnode->fqdn, pDnode->port); + snprintf(pDnode->ep, TSDB_EP_LEN - 1, "%s:%u", pDnode->fqdn, pDnode->port); return 0; } @@ -253,7 +253,7 @@ int32_t mndGetDnodeSize(SMnode *pMnode) { bool mndIsDnodeOnline(SDnodeObj *pDnode, int64_t curMs) { int64_t interval = TABS(pDnode->lastAccessTime - curMs); - if (interval > 5000 * tsStatusInterval) { + if (interval > 5000 * (int64_t)tsStatusInterval) { if (pDnode->rebootTime > 0) { pDnode->offlineReason = DND_REASON_STATUS_MSG_TIMEOUT; } @@ -486,7 +486,7 @@ static int32_t mndCreateDnode(SMnode *pMnode, SRpcMsg *pReq, SCreateDnodeReq *pC dnodeObj.updateTime = dnodeObj.createdTime; dnodeObj.port = pCreate->port; memcpy(dnodeObj.fqdn, pCreate->fqdn, TSDB_FQDN_LEN); - snprintf(dnodeObj.ep, TSDB_EP_LEN, "%s:%u", dnodeObj.fqdn, dnodeObj.port); + snprintf(dnodeObj.ep, TSDB_EP_LEN - 1, "%s:%u", dnodeObj.fqdn, dnodeObj.port); pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_GLOBAL, pReq, "create-dnode"); if (pTrans == NULL) goto _OVER; @@ -673,12 +673,14 @@ static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SM mInfo("trans:%d, used to drop dnode:%d", pTrans->id, pDnode->id); pRaw = mndDnodeActionEncode(pDnode); - if (pRaw == NULL || mndTransAppendRedolog(pTrans, pRaw) != 0) goto _OVER; + if (pRaw == NULL) goto _OVER; + if (mndTransAppendRedolog(pTrans, pRaw) != 0) goto _OVER; sdbSetRawStatus(pRaw, SDB_STATUS_DROPPING); pRaw = NULL; pRaw = mndDnodeActionEncode(pDnode); - if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER; + if (pRaw == NULL) goto _OVER; + if (mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER; sdbSetRawStatus(pRaw, SDB_STATUS_DROPPED); pRaw = NULL; diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index 727f7b0cc9..ccab4774e2 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -257,15 +257,18 @@ static int32_t mndDropFunc(SMnode *pMnode, SRpcMsg *pReq, SFuncObj *pFunc) { mInfo("trans:%d, used to drop user:%s", pTrans->id, pFunc->name); SSdbRaw *pRedoRaw = mndFuncActionEncode(pFunc); - if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) goto _OVER; + if (pRedoRaw == NULL) goto _OVER; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) goto _OVER; sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING); SSdbRaw *pUndoRaw = mndFuncActionEncode(pFunc); - if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) goto _OVER; + if (pUndoRaw == NULL) goto _OVER; + if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) goto _OVER; sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY); SSdbRaw *pCommitRaw = mndFuncActionEncode(pFunc); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) goto _OVER; + if (pCommitRaw == NULL) goto _OVER; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) goto _OVER; sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndGrant.c b/source/dnode/mnode/impl/src/mndGrant.c index a608a1e40f..7262f23657 100644 --- a/source/dnode/mnode/impl/src/mndGrant.c +++ b/source/dnode/mnode/impl/src/mndGrant.c @@ -30,85 +30,85 @@ static int32_t mndRetrieveGrant(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl cols = 0; SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols); const char *src = "community"; - STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); + STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); colDataAppend(pColInfo, numOfRows, tmp, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); src = "unlimited"; - STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); + STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); colDataAppend(pColInfo, numOfRows, tmp, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); src = "false"; - STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); + STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); colDataAppend(pColInfo, numOfRows, tmp, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); src = "unlimited"; - STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); + STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); colDataAppend(pColInfo, numOfRows, tmp, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); src = "unlimited"; - STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); + STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); colDataAppend(pColInfo, numOfRows, tmp, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); src = "unlimited"; - STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); + STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); colDataAppend(pColInfo, numOfRows, tmp, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); src = "unlimited"; - STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); + STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); colDataAppend(pColInfo, numOfRows, tmp, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); src = "unlimited"; - STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); + STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); colDataAppend(pColInfo, numOfRows, tmp, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); src = "unlimited"; - STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); + STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); colDataAppend(pColInfo, numOfRows, tmp, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); src = "unlimited"; - STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); + STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); colDataAppend(pColInfo, numOfRows, tmp, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); src = "unlimited"; - STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); + STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); colDataAppend(pColInfo, numOfRows, tmp, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); src = "unlimited"; - STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); + STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); colDataAppend(pColInfo, numOfRows, tmp, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); src = "unlimited"; - STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); + STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); colDataAppend(pColInfo, numOfRows, tmp, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); src = "unlimited"; - STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); + STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); colDataAppend(pColInfo, numOfRows, tmp, false); numOfRows++; diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index a628cefa65..b3b9927a34 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -649,7 +649,7 @@ int32_t mndProcessRpcMsg(SRpcMsg *pMsg) { void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp) { tmsg_t type = TMSG_INDEX(msgType); - if (type >= 0 && type < TDMT_MAX) { + if (type < TDMT_MAX) { pMnode->msgFp[type] = fp; } } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index a41f958c0f..cfb362c51c 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -93,6 +93,7 @@ static int32_t mndCreateDefaultMnode(SMnode *pMnode) { STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, NULL, "create-mnode"); if (pTrans == NULL) { + sdbFreeRaw(pRaw); mError("mnode:%d, failed to create since %s", mnodeObj.id, terrstr()); return -1; } @@ -220,8 +221,12 @@ bool mndIsMnode(SMnode *pMnode, int32_t dnodeId) { void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) { SSdb *pSdb = pMnode->pSdb; int32_t totalMnodes = sdbGetSize(pSdb, SDB_MNODE); - void *pIter = NULL; + if (totalMnodes == 0) { + syncGetRetryEpSet(pMnode->syncMgmt.sync, pEpSet); + return; + } + void *pIter = NULL; while (1) { SMnodeObj *pObj = NULL; pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pObj); @@ -658,7 +663,7 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB colDataAppend(pColInfo, numOfRows, (const char *)&pObj->id, false); char b1[TSDB_EP_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_WITH_MAXSIZE_TO_VARSTR(b1, pObj->pDnode->ep, pShow->pMeta->pSchemas[cols].bytes); + STR_WITH_MAXSIZE_TO_VARSTR(b1, pObj->pDnode->ep, TSDB_EP_LEN + VARSTR_HEADER_SIZE); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, b1, false); @@ -667,7 +672,7 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB if (pObj->id == pMnode->selfDnodeId) { roles = syncStr(TAOS_SYNC_STATE_LEADER); } - if (pObj->pDnode && mndIsDnodeOnline(pObj->pDnode, curMs)) { + if (mndIsDnodeOnline(pObj->pDnode, curMs)) { roles = syncStr(pObj->state); if (pObj->state == TAOS_SYNC_STATE_LEADER && pObj->id != pMnode->selfDnodeId) { roles = syncStr(TAOS_SYNC_STATE_ERROR); diff --git a/source/dnode/mnode/impl/src/mndPerfSchema.c b/source/dnode/mnode/impl/src/mndPerfSchema.c index fe8d1a0a0e..cb636dc037 100644 --- a/source/dnode/mnode/impl/src/mndPerfSchema.c +++ b/source/dnode/mnode/impl/src/mndPerfSchema.c @@ -26,7 +26,7 @@ int32_t mndInitPerfsTableSchema(const SSysDbTableSchema *pSrc, int32_t colNum, S } for (int32_t i = 0; i < colNum; ++i) { - strcpy(schema[i].name, pSrc[i].name); + tstrncpy(schema[i].name, pSrc[i].name, sizeof(schema[i].name)); schema[i].type = pSrc[i].type; schema[i].colId = i + 1; @@ -40,7 +40,7 @@ int32_t mndInitPerfsTableSchema(const SSysDbTableSchema *pSrc, int32_t colNum, S int32_t mndPerfsInitMeta(SHashObj *hash) { STableMetaRsp meta = {0}; - strcpy(meta.dbFName, TSDB_INFORMATION_SCHEMA_DB); + tstrncpy(meta.dbFName, TSDB_INFORMATION_SCHEMA_DB, sizeof(meta.dbFName)); meta.tableType = TSDB_SYSTEM_TABLE; meta.sversion = 1; meta.tversion = 1; @@ -50,7 +50,7 @@ int32_t mndPerfsInitMeta(SHashObj *hash) { getPerfDbMeta(&pSysDbTableMeta, &size); for (int32_t i = 0; i < size; ++i) { - strcpy(meta.tbName, pSysDbTableMeta[i].name); + tstrncpy(meta.tbName, pSysDbTableMeta[i].name, sizeof(meta.tbName)); meta.numOfColumns = pSysDbTableMeta[i].colNum; if (mndInitPerfsTableSchema(pSysDbTableMeta[i].schema, pSysDbTableMeta[i].colNum, &meta.pSchemas)) { diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index e8737e30c9..1909b168bd 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -664,7 +664,7 @@ static int32_t mndProcessKillConnReq(SRpcMsg *pReq) { static int32_t mndProcessSvrVerReq(SRpcMsg *pReq) { int32_t code = -1; SServerVerRsp rsp = {0}; - strcpy(rsp.ver, version); + tstrncpy(rsp.ver, version, sizeof(rsp.ver)); int32_t contLen = tSerializeSServerVerRsp(NULL, 0, &rsp); if (contLen < 0) goto _over; @@ -702,7 +702,7 @@ static int32_t mndRetrieveConns(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl break; } - if ((taosGetTimestampMs() - pConn->lastAccessTimeMs) > (keepTime * 1000)) { + if ((taosGetTimestampMs() - pConn->lastAccessTimeMs) > ((int64_t)keepTime * 1000)) { continue; } diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 21e45407fd..f2fec27bc5 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -123,7 +123,7 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const SMqSubscri req.subType = pSub->subType; req.withMeta = pSub->withMeta; req.suid = pSub->stbUid; - strncpy(req.subKey, pSub->key, TSDB_SUBSCRIBE_KEY_LEN); + tstrncpy(req.subKey, pSub->key, TSDB_SUBSCRIBE_KEY_LEN); int32_t tlen = sizeof(SMsgHead) + tEncodeSMqRebVgReq(NULL, &req); void *buf = taosMemoryMalloc(tlen); diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index da9dab1e24..fec050b5ea 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -62,7 +62,9 @@ void vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot); void vnodeGetInfo(SVnode *pVnode, const char **dbname, int32_t *vgId); int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen); int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list); + int32_t vnodeGetCtbIdList(SVnode *pVnode, int64_t suid, SArray *list); +int32_t vnodeGetCtbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg); int32_t vnodeGetStbIdList(SVnode *pVnode, int64_t suid, SArray *list); void *vnodeGetIdx(SVnode *pVnode); void *vnodeGetIvtIdx(SVnode *pVnode); @@ -94,9 +96,12 @@ void metaReaderClear(SMetaReader *pReader); int32_t metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid); int metaGetTableEntryByName(SMetaReader *pReader, const char *name); int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList, SHashObj *tags); +int32_t metaGetTableTagsOpt(SMeta *pMeta, uint64_t suid, SArray *uidList, SHashObj *tags); int32_t metaReadNext(SMetaReader *pReader); const void *metaGetTableTagVal(void *tag, int16_t type, STagVal *tagVal); int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName); +int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid); +int metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType); bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid); typedef struct SMetaFltParam { @@ -154,7 +159,7 @@ uint64_t getReaderMaxVersion(STsdbReader *pReader); int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, SArray *pTableIdList, int32_t numOfCols, void **pReader); int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, SArray *pTableUids); -int32_t tsdbCacherowsReaderClose(void *pReader); +void* tsdbCacherowsReaderClose(void *pReader); int32_t tsdbGetTableSchema(SVnode *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid); void tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 079efd7097..ec570b4540 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -110,22 +110,24 @@ int metaAlterTable(SMeta* pMeta, int64_t version, SVAlterTbReq* pReq SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, bool isinline); STSchema* metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver); int32_t metaGetTbTSchemaEx(SMeta* pMeta, tb_uid_t suid, tb_uid_t uid, int32_t sver, STSchema** ppTSchema); -tb_uid_t metaGetTableEntryUidByName(SMeta* pMeta, const char* name); -int64_t metaGetTbNum(SMeta* pMeta); -int64_t metaGetTimeSeriesNum(SMeta* pMeta); -SMCtbCursor* metaOpenCtbCursor(SMeta* pMeta, tb_uid_t uid); -void metaCloseCtbCursor(SMCtbCursor* pCtbCur); -tb_uid_t metaCtbCursorNext(SMCtbCursor* pCtbCur); -SMStbCursor* metaOpenStbCursor(SMeta* pMeta, tb_uid_t uid); -void metaCloseStbCursor(SMStbCursor* pStbCur); -tb_uid_t metaStbCursorNext(SMStbCursor* pStbCur); -STSma* metaGetSmaInfoByIndex(SMeta* pMeta, int64_t indexUid); -STSmaWrapper* metaGetSmaInfoByTable(SMeta* pMeta, tb_uid_t uid, bool deepCopy); -SArray* metaGetSmaIdsByTable(SMeta* pMeta, tb_uid_t uid); -SArray* metaGetSmaTbUids(SMeta* pMeta); -void* metaGetIdx(SMeta* pMeta); -void* metaGetIvtIdx(SMeta* pMeta); -int metaTtlSmaller(SMeta* pMeta, uint64_t time, SArray* uidList); +int metaGetTableEntryByName(SMetaReader* pReader, const char* name); + +tb_uid_t metaGetTableEntryUidByName(SMeta* pMeta, const char* name); +int64_t metaGetTbNum(SMeta* pMeta); +int64_t metaGetTimeSeriesNum(SMeta* pMeta); +SMCtbCursor* metaOpenCtbCursor(SMeta* pMeta, tb_uid_t uid); +void metaCloseCtbCursor(SMCtbCursor* pCtbCur); +tb_uid_t metaCtbCursorNext(SMCtbCursor* pCtbCur); +SMStbCursor* metaOpenStbCursor(SMeta* pMeta, tb_uid_t uid); +void metaCloseStbCursor(SMStbCursor* pStbCur); +tb_uid_t metaStbCursorNext(SMStbCursor* pStbCur); +STSma* metaGetSmaInfoByIndex(SMeta* pMeta, int64_t indexUid); +STSmaWrapper* metaGetSmaInfoByTable(SMeta* pMeta, tb_uid_t uid, bool deepCopy); +SArray* metaGetSmaIdsByTable(SMeta* pMeta, tb_uid_t uid); +SArray* metaGetSmaTbUids(SMeta* pMeta); +void* metaGetIdx(SMeta* pMeta); +void* metaGetIvtIdx(SMeta* pMeta); +int metaTtlSmaller(SMeta* pMeta, uint64_t time, SArray* uidList); int32_t metaCreateTSma(SMeta* pMeta, int64_t version, SSmaCfg* pCfg); int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid); @@ -183,8 +185,9 @@ int32_t tqProcessTaskRecoverRsp(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskRetrieveRsp(STQ* pTq, SRpcMsg* pMsg); -SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchema* pSchema, SSchemaWrapper* pTagSchemaWrapper, bool createTb, int64_t suid, - const char* stbFullName, SBatchDeleteReq* pDeleteReq); +SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchema* pSchema, + SSchemaWrapper* pTagSchemaWrapper, bool createTb, int64_t suid, const char* stbFullName, + SBatchDeleteReq* pDeleteReq); // sma int32_t smaInit(); diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index f8ecd17cb7..034d6260da 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -200,36 +200,30 @@ int metaClose(SMeta *pMeta) { int32_t metaRLock(SMeta *pMeta) { int32_t ret = 0; - metaTrace("meta rlock %p B", &pMeta->lock); + metaTrace("meta rlock %p", &pMeta->lock); ret = taosThreadRwlockRdlock(&pMeta->lock); - metaTrace("meta rlock %p E", &pMeta->lock); - return ret; } int32_t metaWLock(SMeta *pMeta) { int32_t ret = 0; - metaTrace("meta wlock %p B", &pMeta->lock); + metaTrace("meta wlock %p", &pMeta->lock); ret = taosThreadRwlockWrlock(&pMeta->lock); - metaTrace("meta wlock %p E", &pMeta->lock); - return ret; } int32_t metaULock(SMeta *pMeta) { int32_t ret = 0; - metaTrace("meta ulock %p B", &pMeta->lock); + metaTrace("meta ulock %p", &pMeta->lock); ret = taosThreadRwlockUnlock(&pMeta->lock); - metaTrace("meta ulock %p E", &pMeta->lock); - return ret; } diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 2ef4e7510e..d499db4958 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -202,6 +202,38 @@ int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName) { return 0; } +int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid) { + int code = 0; + SMetaReader mr = {0}; + metaReaderInit(&mr, (SMeta *)meta, 0); + + SMeta *pMeta = mr.pMeta; + SMetaReader *pReader = &mr; + + // query name.idx + if (tdbTbGet(pMeta->pNameIdx, tbName, strlen(tbName) + 1, &pReader->pBuf, &pReader->szBuf) < 0) { + terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; + metaReaderClear(&mr); + return -1; + } + + *uid = *(tb_uid_t *)pReader->pBuf; + metaReaderClear(&mr); + + return 0; +} + +int metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType) { + int code = 0; + SMetaReader mr = {0}; + metaReaderInit(&mr, (SMeta *)meta, 0); + + if (metaGetTableEntryByName(&mr, tbName) == 0) { + *tbType = mr.me.type; + } + metaReaderClear(&mr); + return 0; +} int metaReadNext(SMetaReader *pReader) { SMeta *pMeta = pReader->pMeta; @@ -1102,6 +1134,21 @@ END: return ret; } +int32_t metaGetTableTagsOpt(SMeta *pMeta, uint64_t suid, SArray *uidList, SHashObj *tags) { + int32_t sz = uidList ? taosArrayGetSize(uidList) : 0; + for (int i = 0; i < sz; i++) { + tb_uid_t *id = taosArrayGet(uidList, i); + SCtbIdxKey ctbIdxKey = {.suid = suid, .uid = *id}; + + void *val = NULL; + int32_t len = 0; + if (taosHashGet(tags, id, sizeof(tb_uid_t)) == NULL && + 0 == tdbTbGet(pMeta->pCtbIdx, &ctbIdxKey, sizeof(SCtbIdxKey), &val, &len)) { + taosHashPut(tags, id, sizeof(tb_uid_t), val, len); + } + } + return 0; +} int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList, SHashObj *tags) { SMCtbCursor *pCur = metaOpenCtbCursor(pMeta, suid); diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index 5f981649f3..7a000253d5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -101,7 +101,7 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, SArray* pTableIdList return TSDB_CODE_SUCCESS; } -int32_t tsdbCacherowsReaderClose(void* pReader) { +void* tsdbCacherowsReaderClose(void* pReader) { SCacheRowsReader* p = pReader; if (p->pSchema != NULL) { @@ -114,7 +114,7 @@ int32_t tsdbCacherowsReaderClose(void* pReader) { } taosMemoryFree(pReader); - return TSDB_CODE_SUCCESS; + return NULL; } static int32_t doExtractCacheRow(SCacheRowsReader* pr, SLRUCache* lruCache, uint64_t uid, STSRow** pRow, diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 825bf8a0a7..6806c034dc 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -409,6 +409,9 @@ int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list) { return TSDB_CODE_SUCCESS; } +int32_t vnodeGetCtbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg) { + return 0; +} int32_t vnodeGetCtbIdList(SVnode *pVnode, int64_t suid, SArray *list) { SMCtbCursor *pCur = metaOpenCtbCursor(pVnode->pMeta, suid); diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index baf7d447cc..9dba29811e 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -15,7 +15,6 @@ #ifndef TDENGINE_QUERYUTIL_H #define TDENGINE_QUERYUTIL_H -#include "vnode.h" #include "function.h" #include "nodes.h" #include "plannodes.h" @@ -23,6 +22,7 @@ #include "tcommon.h" #include "tpagedbuf.h" #include "tsimplehash.h" +#include "vnode.h" #define T_LONG_JMP(_obj, _c) \ do { \ @@ -93,7 +93,7 @@ void resetResultRow(SResultRow* pResultRow, size_t entrySize); struct SResultRowEntryInfo* getResultEntryInfo(const SResultRow* pRow, int32_t index, const int32_t* offset); static FORCE_INLINE SResultRow* getResultRowByPos(SDiskbasedBuf* pBuf, SResultRowPosition* pos, bool forUpdate) { - SFilePage* bufPage = (SFilePage*)getBufPage(pBuf, pos->pageId); + SFilePage* bufPage = (SFilePage*)getBufPage(pBuf, pos->pageId); if (forUpdate) { setBufPageDirty(bufPage, true); } @@ -101,11 +101,6 @@ static FORCE_INLINE SResultRow* getResultRowByPos(SDiskbasedBuf* pBuf, SResultRo return pRow; } -static FORCE_INLINE void setResultBufPageDirty(SDiskbasedBuf* pBuf, SResultRowPosition* pos) { - void* pPage = getBufPage(pBuf, pos->pageId); - setBufPageDirty(pPage, true); -} - void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, int32_t order); void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo); @@ -117,17 +112,18 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo); SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode); EDealRes doTranslateTagExpr(SNode** pNode, void* pContext); -int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond, STableListInfo* pListInfo); -int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId); -int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableListInfo* pTableListInfo); -size_t getTableTagsBufLen(const SNodeList* pGroups); +int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond, + STableListInfo* pListInfo); +int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId); +int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableListInfo* pTableListInfo); +size_t getTableTagsBufLen(const SNodeList* pGroups); -SArray* createSortInfo(SNodeList* pNodeList); -SArray* extractPartitionColInfo(SNodeList* pNodeList); -SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols, - int32_t type); +SArray* createSortInfo(SNodeList* pNodeList); +SArray* extractPartitionColInfo(SNodeList* pNodeList); +SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols, + int32_t type); -void createExprFromTargetNode(SExprInfo* pExp, STargetNode* pTargetNode); +void createExprFromTargetNode(SExprInfo* pExp, STargetNode* pTargetNode); SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* numOfExprs); SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowEntryInfoOffset); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 56470f0668..897015c4d3 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -577,13 +577,7 @@ typedef struct SIntervalAggOperatorInfo { int32_t inputOrder; // input data ts order EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model] STimeWindowAggSupp twAggSup; - bool invertible; SArray* pPrevValues; // SArray used to keep the previous not null value for interpolation. - bool ignoreExpiredData; - SArray* pRecycledPages; - SArray* pDelWins; // SWinRes - int32_t delIndex; - SSDataBlock* pDelRes; SNode* pCondition; } SIntervalAggOperatorInfo; @@ -609,38 +603,21 @@ typedef struct SStreamIntervalOperatorInfo { STimeWindowAggSupp twAggSup; bool invertible; bool ignoreExpiredData; - SArray* pRecycledPages; SArray* pDelWins; // SWinRes int32_t delIndex; SSDataBlock* pDelRes; - bool isFinal; -} SStreamIntervalOperatorInfo; - -typedef struct SStreamFinalIntervalOperatorInfo { - // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode - SOptrBasicInfo binfo; // basic info - SAggSupporter aggSup; // aggregate supporter - SExprSupp scalarSupp; // supporter for perform scalar function - SGroupResInfo groupResInfo; // multiple results build supporter - SInterval interval; // interval info - int32_t primaryTsIndex; // primary time stamp slot id from result of downstream operator. - int32_t order; // current SSDataBlock scan order - STimeWindowAggSupp twAggSup; - SArray* pChildren; SSDataBlock* pUpdateRes; bool returnUpdate; - SPhysiNode* pPhyNode; // create new child - bool isFinal; + SPhysiNode* pPhyNode; // create new child SHashObj* pPullDataMap; - SArray* pPullWins; // SPullWindowInfo + SArray* pPullWins; // SPullWindowInfo int32_t pullIndex; SSDataBlock* pPullDataRes; - bool ignoreExpiredData; - SArray* pRecycledPages; - SArray* pDelWins; // SWinRes - int32_t delIndex; - SSDataBlock* pDelRes; -} SStreamFinalIntervalOperatorInfo; + bool isFinal; + SArray* pChildren; + SStreamState* pState; + SWinKey delKey; +} SStreamIntervalOperatorInfo; typedef struct SAggOperatorInfo { // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode @@ -1086,7 +1063,7 @@ bool functionNeedToExecute(SqlFunctionCtx* pCtx); bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup); bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup); bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup); -bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SOperatorInfo* pOperator, STimeWindowAggSupp* pTwSup); +bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SStreamState* pState, STimeWindowAggSupp* pTwSup); void appendOneRow(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid, uint64_t* pGp); void printDataBlock(SSDataBlock* pBlock, const char* flag); uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId); @@ -1108,13 +1085,12 @@ void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsCol bool groupbyTbname(SNodeList* pGroupList); int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* groupKey); void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput); -int32_t buildDataBlockFromGroupRes(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, +int32_t buildDataBlockFromGroupRes(SExecTaskInfo* pTaskInfo, SStreamState* pState, SSDataBlock* pBlock, SExprSupp* pSup, SGroupResInfo* pGroupResInfo); -int32_t setOutputBuf(STimeWindow* win, SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx, - int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup, - SExecTaskInfo* pTaskInfo); -int32_t releaseOutputBuf(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult); -int32_t saveOutputBuf(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult, int32_t resSize); +int32_t setOutputBuf(SStreamState* pState, STimeWindow* win, SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx, + int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup); +int32_t releaseOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult); +int32_t saveOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult, int32_t resSize); void getNextIntervalWindow(SInterval* pInterval, STimeWindow* tw, int32_t order); #ifdef __cplusplus diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 94d9d0cadb..1b7448712e 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -207,7 +207,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { } } - tsdbCacherowsReaderClose(pInfo->pLastrowReader); + pInfo->pLastrowReader = tsdbCacherowsReaderClose(pInfo->pLastrowReader); return pInfo->pRes; } } @@ -220,6 +220,15 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { void destroyLastrowScanOperator(void* param) { SLastrowScanInfo* pInfo = (SLastrowScanInfo*)param; blockDataDestroy(pInfo->pRes); + blockDataDestroy(pInfo->pBufferredRes); + taosMemoryFree(pInfo->pSlotIds); + taosArrayDestroy(pInfo->pUidList); + taosArrayDestroy(pInfo->pColMatchInfo); + + if (pInfo->pLastrowReader != NULL) { + pInfo->pLastrowReader = tsdbCacherowsReaderClose(pInfo->pLastrowReader); + } + taosMemoryFreeClear(param); } diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index b26603b394..38304bcc76 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -26,6 +26,9 @@ #include "executorimpl.h" #include "tcompression.h" +static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list, SNode* pTagCond); +static int32_t optimizeTbnameInCondImpl(void* metaHandle, int64_t suid, SArray* list, SNode* pTagCond); + void initResultRowInfo(SResultRowInfo* pResultRowInfo) { pResultRowInfo->size = 0; pResultRowInfo->cur.pageId = -1; @@ -407,11 +410,18 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, uint64_t suid, SArray // int64_t stt = taosGetTimestampUs(); tags = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); - code = metaGetTableTags(metaHandle, suid, uidList, tags); - if (code != TSDB_CODE_SUCCESS) { - qError("failed to get table tags from meta, reason:%s, suid:%" PRIu64, tstrerror(code), suid); - terrno = code; - goto end; + + int32_t filter = optimizeTbnameInCond(metaHandle, suid, uidList, pTagCond); + if (filter == -1) { + code = metaGetTableTags(metaHandle, suid, uidList, tags); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to get table tags from meta, reason:%s, suid:%" PRIu64, tstrerror(code), suid); + terrno = code; + goto end; + } + } else { + metaGetTableTagsOpt(metaHandle, suid, uidList, tags); + qInfo("succ to get table from meta idx, suid:%" PRIu64, suid); } int32_t rows = taosArrayGetSize(uidList); @@ -742,6 +752,94 @@ end: return code; } +static int tableUidCompare(const void* a, const void* b) { + uint64_t u1 = *(uint64_t*)a; + uint64_t u2 = *(uint64_t*)b; + if (u1 == u2) { + return 0; + } + return u1 < u2 ? -1 : 1; +} +static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list, SNode* cond) { + if (nodeType(cond) == QUERY_NODE_OPERATOR) { + return optimizeTbnameInCondImpl(metaHandle, suid, list, cond); + } + + if (nodeType(cond) != QUERY_NODE_LOGIC_CONDITION || ((SLogicConditionNode*)cond)->condType != LOGIC_COND_TYPE_AND) { + return -1; + } + + bool hasTbnameCond = false; + SLogicConditionNode* pNode = (SLogicConditionNode*)cond; + SNodeList* pList = (SNodeList*)pNode->pParameterList; + + int32_t len = LIST_LENGTH(pList); + if (len <= 0) return -1; + + SListCell* cell = pList->pHead; + for (int i = 0; i < len; i++) { + if (cell == NULL) break; + if (optimizeTbnameInCondImpl(metaHandle, suid, list, cell->pNode) == 0) { + hasTbnameCond = true; + } + cell = cell->pNext; + } + taosArraySort(list, tableUidCompare); + taosArrayRemoveDuplicate(list, tableUidCompare, NULL); + + return hasTbnameCond == true ? 0 : -1; +} +static int32_t optimizeTbnameInCondImpl(void* metaHandle, int64_t suid, SArray* list, SNode* pTagCond) { + if (nodeType(pTagCond) != QUERY_NODE_OPERATOR) { + return -1; + } + SOperatorNode* pNode = (SOperatorNode*)pTagCond; + if (pNode->opType != OP_TYPE_IN) { + return -1; + } + if ((pNode->pLeft != NULL && nodeType(pNode->pLeft) == QUERY_NODE_COLUMN && + ((SColumnNode*)pNode->pLeft)->colType == COLUMN_TYPE_TBNAME) && + (pNode->pRight != NULL && nodeType(pNode->pRight) == QUERY_NODE_NODE_LIST)) { + SNodeListNode* pList = (SNodeListNode*)pNode->pRight; + + int32_t len = LIST_LENGTH(pList->pNodeList); + if (len <= 0) return -1; + + SListCell* cell = pList->pNodeList->pHead; + + SArray* pTbList = taosArrayInit(len, sizeof(void*)); + for (int i = 0; i < pList->pNodeList->length; i++) { + SValueNode* valueNode = (SValueNode*)cell->pNode; + if (!IS_VAR_DATA_TYPE(valueNode->node.resType.type)) { + taosArrayDestroy(pTbList); + return -1; + } + char* name = varDataVal(valueNode->datum.p); + taosArrayPush(pTbList, &name); + cell = cell->pNext; + } + + for (int i = 0; i < taosArrayGetSize(pTbList); i++) { + char* name = taosArrayGetP(pTbList, i); + uint64_t uid = 0; + if (metaGetTableUidByName(metaHandle, name, &uid) == 0) { + ETableType tbType = TSDB_TABLE_MAX; + if (metaGetTableTypeByName(metaHandle, name, &tbType) == 0 && tbType == TSDB_CHILD_TABLE) { + taosArrayPush(list, &uid); + } else { + taosArrayDestroy(pTbList); + return -1; + } + } else { + qWarn("failed to get tableIds from by table name: %s, reason: %s", name, tstrerror(terrno)); + terrno = 0; + } + } + taosArrayDestroy(pTbList); + return 0; + } + return -1; +} int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond, STableListInfo* pListInfo) { int32_t code = TSDB_CODE_SUCCESS; @@ -767,9 +865,6 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, qError("failed to get tableIds from index, reason:%s, suid:%" PRIu64, tstrerror(code), tableUid); code = TDB_CODE_SUCCESS; } - - // int64_t stt1 = taosGetTimestampUs(); - // qDebug("generate table list, cost:%ld us", stt1-stt); } else if (!pTagCond) { vnodeGetCtbIdList(pVnode, pScanNode->suid, res); } @@ -813,7 +908,7 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, size_t numOfTables = taosArrayGetSize(res); for (int i = 0; i < numOfTables; i++) { STableKeyInfo info = {.uid = *(uint64_t*)taosArrayGet(res, i), .groupId = 0}; - void* p = taosArrayPush(pListInfo->pTableList, &info); + void* p = taosArrayPush(pListInfo->pTableList, &info); if (p == NULL) { taosArrayDestroy(res); return TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index ff28199712..5db55b02f8 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4183,9 +4183,8 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlF return code; } -int32_t setOutputBuf(STimeWindow* win, SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx, - int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup, - SExecTaskInfo* pTaskInfo) { +int32_t setOutputBuf(SStreamState* pState, STimeWindow* win, SResultRow** pResult, int64_t tableGroupId, + SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup) { SWinKey key = { .ts = win->skey, .groupId = tableGroupId, @@ -4194,7 +4193,7 @@ int32_t setOutputBuf(STimeWindow* win, SResultRow** pResult, int64_t tableGroupI int32_t size = pAggSup->resultRowSize; tSimpleHashPut(pAggSup->pResultRowHashTable, &key, sizeof(SWinKey), NULL, 0); - if (streamStateAddIfNotExist(pTaskInfo->streamInfo.pState, &key, (void**)&value, &size) < 0) { + if (streamStateAddIfNotExist(pState, &key, (void**)&value, &size) < 0) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } *pResult = (SResultRow*)value; @@ -4205,18 +4204,17 @@ int32_t setOutputBuf(STimeWindow* win, SResultRow** pResult, int64_t tableGroupI return TSDB_CODE_SUCCESS; } -int32_t releaseOutputBuf(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult) { - streamStateReleaseBuf(pTaskInfo->streamInfo.pState, pKey, pResult); - /*taosMemoryFree((*(void**)pResult));*/ +int32_t releaseOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult) { + streamStateReleaseBuf(pState, pKey, pResult); return TSDB_CODE_SUCCESS; } -int32_t saveOutputBuf(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult, int32_t resSize) { - streamStatePut(pTaskInfo->streamInfo.pState, pKey, pResult, resSize); +int32_t saveOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult, int32_t resSize) { + streamStatePut(pState, pKey, pResult, resSize); return TSDB_CODE_SUCCESS; } -int32_t buildDataBlockFromGroupRes(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, +int32_t buildDataBlockFromGroupRes(SExecTaskInfo* pTaskInfo, SStreamState* pState, SSDataBlock* pBlock, SExprSupp* pSup, SGroupResInfo* pGroupResInfo) { SExprInfo* pExprInfo = pSup->pExprInfo; int32_t numOfExprs = pSup->numOfExprs; @@ -4233,14 +4231,14 @@ int32_t buildDataBlockFromGroupRes(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock .ts = *(TSKEY*)pPos->key, .groupId = pPos->groupId, }; - int32_t code = streamStateGet(pTaskInfo->streamInfo.pState, &key, &pVal, &size); + int32_t code = streamStateGet(pState, &key, &pVal, &size); ASSERT(code == 0); SResultRow* pRow = (SResultRow*)pVal; doUpdateNumOfRows(pCtx, pRow, numOfExprs, rowEntryOffset); // no results, continue to check the next one if (pRow->numOfRows == 0) { pGroupResInfo->index += 1; - releaseOutputBuf(pTaskInfo, &key, pRow); + releaseOutputBuf(pState, &key, pRow); continue; } @@ -4249,14 +4247,14 @@ int32_t buildDataBlockFromGroupRes(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock } else { // current value belongs to different group, it can't be packed into one datablock if (pBlock->info.groupId != pPos->groupId) { - releaseOutputBuf(pTaskInfo, &key, pRow); + releaseOutputBuf(pState, &key, pRow); break; } } if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { ASSERT(pBlock->info.rows > 0); - releaseOutputBuf(pTaskInfo, &key, pRow); + releaseOutputBuf(pState, &key, pRow); break; } @@ -4286,7 +4284,7 @@ int32_t buildDataBlockFromGroupRes(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock } pBlock->info.rows += pRow->numOfRows; - releaseOutputBuf(pTaskInfo, &key, pRow); + releaseOutputBuf(pState, &key, pRow); } blockDataUpdateTsWindow(pBlock, 0); return TSDB_CODE_SUCCESS; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 9694ccadb7..40fc8cb9a5 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1357,7 +1357,8 @@ static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock // must check update info first. bool update = updateInfoIsUpdated(pInfo->pUpdateInfo, pBlock->info.uid, tsCol[rowId]); bool closedWin = isClosed && isSignleIntervalWindow(pInfo) && - isDeletedStreamWindow(&win, pBlock->info.groupId, pInfo->pTableScanOp, &pInfo->twAggSup); + isDeletedStreamWindow(&win, pBlock->info.groupId, + pInfo->pTableScanOp->pTaskInfo->streamInfo.pState, &pInfo->twAggSup); if ((update || closedWin) && out) { qDebug("stream update check not pass, update %d, closedWin %d", update, closedWin); uint64_t gpId = 0; @@ -2135,6 +2136,9 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pInfo->pUpdateInfo = NULL; pInfo->pTableScanOp = pTableScanOp; + if (pInfo->pTableScanOp->pTaskInfo->streamInfo.pState) { + streamStateSetNumber(pInfo->pTableScanOp->pTaskInfo->streamInfo.pState, -1); + } pInfo->readHandle = *pHandle; pInfo->tableUid = pScanPhyNode->uid; diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index ea0d26f4de..f6fd8c8920 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -1620,9 +1620,7 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi goto _error; } - SInterval* pInterval = QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL == downstream->operatorType - ? &((SStreamFinalIntervalOperatorInfo*)downstream->info)->interval - : &((SStreamIntervalOperatorInfo*)downstream->info)->interval; + SInterval* pInterval = &((SStreamIntervalOperatorInfo*)downstream->info)->interval; int32_t numOfFillCols = 0; SExprInfo* pFillExprInfo = createExprInfo(pPhyFillNode->pFillExprs, NULL, &numOfFillCols); pInfo->pFillSup = initStreamFillSup(pPhyFillNode, pInterval, pFillExprInfo, numOfFillCols); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index b4e2b73889..9b4b7f1ee8 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -867,10 +867,6 @@ static int32_t saveWinResult(int64_t ts, int32_t pageId, int32_t offset, uint64_ return TSDB_CODE_SUCCESS; } -static int32_t saveWinResultRow(SResultRow* result, uint64_t groupId, SHashObj* pUpdatedMap) { - return saveWinResult(result->win.skey, result->pageId, result->offset, groupId, pUpdatedMap); -} - static int32_t saveWinResultInfo(TSKEY ts, uint64_t groupId, SHashObj* pUpdatedMap) { return saveWinResult(ts, -1, -1, groupId, pUpdatedMap); } @@ -1390,7 +1386,7 @@ static bool doDeleteWindow(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, SStreamIntervalOperatorInfo* pInfo = pOperator->info; SWinKey key = {.ts = ts, .groupId = groupId}; tSimpleHashRemove(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey)); - streamStateDel(pOperator->pTaskInfo->streamInfo.pState, &key); + streamStateDel(pInfo->pState, &key); return true; } @@ -1406,7 +1402,7 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, int3 SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTsCols[i], pInterval, TSDB_ORDER_ASC); - while (win.ekey <= endTsCols[i]) { + while (win.skey <= endTsCols[i]) { uint64_t winGpId = pGpDatas[i]; bool res = doDeleteWindow(pOperator, win.skey, winGpId, numOfOutput); SWinKey winRes = {.ts = win.skey, .groupId = winGpId}; @@ -1505,64 +1501,13 @@ static int32_t getAllIntervalWindow(SSHashObj* pHashMap, SHashObj* resWins) { return TSDB_CODE_SUCCESS; } -static int32_t closeIntervalWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pSup, SInterval* pInterval, - SHashObj* pPullDataMap, SHashObj* closeWins, SArray* pRecyPages, - SDiskbasedBuf* pDiscBuf) { - qDebug("===stream===close interval window"); - void* pIte = NULL; - size_t keyLen = 0; - int32_t iter = 0; - while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) { - void* key = tSimpleHashGetKey(pIte, &keyLen); - uint64_t groupId = *(uint64_t*)key; - ASSERT(keyLen == GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))); - TSKEY ts = *(int64_t*)((char*)key + sizeof(uint64_t)); - STimeWindow win; - win.skey = ts; - win.ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; - SWinKey winRe = { - .ts = win.skey, - .groupId = groupId, - }; - void* chIds = taosHashGet(pPullDataMap, &winRe, sizeof(SWinKey)); - if (isCloseWindow(&win, pSup)) { - if (chIds && pPullDataMap) { - SArray* chAy = *(SArray**)chIds; - int32_t size = taosArrayGetSize(chAy); - qDebug("===stream===window %" PRId64 " wait child size:%d", win.skey, size); - for (int32_t i = 0; i < size; i++) { - qDebug("===stream===window %" PRId64 " wait child id:%d", win.skey, *(int32_t*)taosArrayGet(chAy, i)); - } - continue; - } else if (pPullDataMap) { - qDebug("===stream===close window %" PRId64, win.skey); - } - SResultRowPosition* pPos = (SResultRowPosition*)pIte; - if (pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { - int32_t code = saveWinResult(ts, pPos->pageId, pPos->offset, groupId, closeWins); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - ASSERT(pRecyPages != NULL); - taosArrayPush(pRecyPages, &pPos->pageId); - } else { - // SFilePage* bufPage = getBufPage(pDiscBuf, pPos->pageId); - // dBufSetBufPageRecycled(pDiscBuf, bufPage); - } - char keyBuf[GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))]; - SET_RES_WINDOW_KEY(keyBuf, &ts, sizeof(TSKEY), groupId); - tSimpleHashIterateRemove(pHashMap, keyBuf, keyLen, &pIte, &iter); - } - } - return TSDB_CODE_SUCCESS; -} - static int32_t closeStreamIntervalWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SInterval* pInterval, SHashObj* pPullDataMap, SHashObj* closeWins, SOperatorInfo* pOperator) { qDebug("===stream===close interval window"); - void* pIte = NULL; - size_t keyLen = 0; - int32_t iter = 0; + void* pIte = NULL; + size_t keyLen = 0; + int32_t iter = 0; + SStreamIntervalOperatorInfo* pInfo = pOperator->info; while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) { void* key = tSimpleHashGetKey(pIte, &keyLen); SWinKey* pWinKey = (SWinKey*)key; @@ -1591,24 +1536,85 @@ static int32_t closeStreamIntervalWindow(SSHashObj* pHashMap, STimeWindowAggSupp } } tSimpleHashIterateRemove(pHashMap, pWinKey, sizeof(SWinKey), &pIte, &iter); - - if (needDeleteWindowBuf(&win, pTwSup)) { - streamStateDel(pOperator->pTaskInfo->streamInfo.pState, pWinKey); - } } } return TSDB_CODE_SUCCESS; } -static void closeChildIntervalWindow(SArray* pChildren, TSKEY maxTs) { +STimeWindow getFinalTimeWindow(int64_t ts, SInterval* pInterval) { + STimeWindow w = {.skey = ts, .ekey = INT64_MAX}; + w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; + return w; +} + +static void deleteIntervalDiscBuf(SStreamState* pState, SHashObj* pPullDataMap, TSKEY mark, SInterval* pInterval, + SWinKey* key) { + STimeWindow tw = getFinalTimeWindow(key->ts, pInterval); + SWinKey next = {0}; + while (tw.ekey < mark) { + SStreamStateCur* pCur = streamStateSeekKeyNext(pState, key); + int32_t code = streamStateGetKVByCur(pCur, &next, NULL, 0); + streamStateFreeCur(pCur); + + void* chIds = taosHashGet(pPullDataMap, key, sizeof(SWinKey)); + if (chIds && pPullDataMap) { + SArray* chAy = *(SArray**)chIds; + int32_t size = taosArrayGetSize(chAy); + qDebug("===stream===window %" PRId64 " wait child size:%d", key->ts, size); + for (int32_t i = 0; i < size; i++) { + qDebug("===stream===window %" PRId64 " wait child id:%d", key->ts, *(int32_t*)taosArrayGet(chAy, i)); + } + break; + } + qDebug("===stream===delete window %" PRId64, key->ts); + int32_t codeDel = streamStateDel(pState, key); + if (codeDel != TSDB_CODE_SUCCESS) { + code = streamStateGetFirst(pState, key); + if (code != TSDB_CODE_SUCCESS) { + qDebug("===stream===stream state first key: empty-empty"); + return; + } + continue; + } + if (code == TSDB_CODE_SUCCESS) { + *key = next; + tw = getFinalTimeWindow(key->ts, pInterval); + } + } + + if (qDebugFlag & DEBUG_DEBUG) { + SStreamStateCur* pCur = streamStateGetCur(pState, key); + int32_t code = streamStateCurPrev(pState, pCur); + if (code == TSDB_CODE_SUCCESS) { + SWinKey tmpKey = {0}; + code = streamStateGetKVByCur(pCur, &tmpKey, NULL, 0); + if (code == TSDB_CODE_SUCCESS) { + STimeWindow tw = getFinalTimeWindow(tmpKey.ts, pInterval); + qDebug("===stream===error stream state first key:%" PRId64 "-%" PRId64 ",%" PRId64 ",mark %" PRId64, tw.skey, + tw.ekey, tmpKey.groupId, mark); + } else { + STimeWindow tw = getFinalTimeWindow(key->ts, pInterval); + qDebug("===stream===stream state first key:%" PRId64 "-%" PRId64 ",%" PRId64 ",mark %" PRId64, tw.skey, tw.ekey, + key->groupId, mark); + } + } else { + STimeWindow tw = getFinalTimeWindow(key->ts, pInterval); + qDebug("===stream===stream state first key:%" PRId64 "-%" PRId64 ",%" PRId64 ",mark %" PRId64, tw.skey, tw.ekey, + key->groupId, mark); + } + streamStateFreeCur(pCur); + } +} + +static void closeChildIntervalWindow(SOperatorInfo* pOperator, SArray* pChildren, TSKEY maxTs) { int32_t size = taosArrayGetSize(pChildren); for (int32_t i = 0; i < size; i++) { - SOperatorInfo* pChildOp = taosArrayGetP(pChildren, i); - SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info; + SOperatorInfo* pChildOp = taosArrayGetP(pChildren, i); + SStreamIntervalOperatorInfo* pChInfo = pChildOp->info; ASSERT(pChInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE); pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, maxTs); - closeIntervalWindow(pChInfo->aggSup.pResultRowHashTable, &pChInfo->twAggSup, &pChInfo->interval, NULL, NULL, NULL, - pChInfo->aggSup.pResultBuf); + closeStreamIntervalWindow(pChInfo->aggSup.pResultRowHashTable, &pChInfo->twAggSup, &pChInfo->interval, NULL, NULL, + pOperator); } } @@ -1660,13 +1666,10 @@ void destroyIntervalOperatorInfo(void* param) { tdListFree(pInfo->binfo.resultRowInfo.openWindow); - pInfo->pRecycledPages = taosArrayDestroy(pInfo->pRecycledPages); pInfo->pInterpCols = taosArrayDestroy(pInfo->pInterpCols); taosArrayDestroyEx(pInfo->pPrevValues, freeItem); pInfo->pPrevValues = NULL; - pInfo->pDelWins = taosArrayDestroy(pInfo->pDelWins); - pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes); cleanupGroupResInfo(&pInfo->groupResInfo); colDataDestroy(&pInfo->twAggSup.timeWindowData); @@ -1674,17 +1677,17 @@ void destroyIntervalOperatorInfo(void* param) { } void destroyStreamFinalIntervalOperatorInfo(void* param) { - SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)param; + SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)param; cleanupBasicInfo(&pInfo->binfo); cleanupAggSup(&pInfo->aggSup); // it should be empty. taosHashCleanup(pInfo->pPullDataMap); taosArrayDestroy(pInfo->pPullWins); blockDataDestroy(pInfo->pPullDataRes); - taosArrayDestroy(pInfo->pRecycledPages); blockDataDestroy(pInfo->pUpdateRes); taosArrayDestroy(pInfo->pDelWins); blockDataDestroy(pInfo->pDelRes); + taosMemoryFreeClear(pInfo->pState); if (pInfo->pChildren) { int32_t size = taosArrayGetSize(pInfo->pChildren); @@ -1794,7 +1797,6 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pInfo->interval = *pInterval; pInfo->execModel = pTaskInfo->execModel; pInfo->twAggSup = *pTwAggSupp; - pInfo->ignoreExpiredData = pPhyNode->window.igExpired; pInfo->pCondition = pPhyNode->window.node.pConditions; pInfo->binfo.mergeResultBlock = pPhyNode->window.mergeDataBlock; @@ -1827,9 +1829,6 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win); - pInfo->invertible = allInvertible(pSup->pCtx, numOfCols); - pInfo->invertible = false; // Todo(liuyao): Dependent TSDB API - pInfo->timeWindowInterpo = timeWindowinterpNeeded(pSup->pCtx, numOfCols, pInfo); if (pInfo->timeWindowInterpo) { pInfo->binfo.resultRowInfo.openWindow = tdListNew(sizeof(SOpenWindowInfo)); @@ -1838,10 +1837,6 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* } } - pInfo->pRecycledPages = taosArrayInit(4, sizeof(int32_t)); - pInfo->pDelWins = taosArrayInit(4, sizeof(SWinKey)); - pInfo->delIndex = 0; - pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT); initResultRowInfo(&pInfo->binfo.resultRowInfo); pOperator->name = "TimeIntervalAggOperator"; @@ -2100,7 +2095,7 @@ static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp for (int32_t j = 0; j < pExprSup->numOfExprs; ++j) { SExprInfo* pExprInfo = &pExprSup->pExprInfo[j]; - int32_t dstSlot = pExprInfo->base.resSchema.slotId; + int32_t dstSlot = pExprInfo->base.resSchema.slotId; SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot); if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) { @@ -2356,13 +2351,13 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) { SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j]; - int32_t dstSlot = pExprInfo->base.resSchema.slotId; + int32_t dstSlot = pExprInfo->base.resSchema.slotId; SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot); if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) { - colDataAppend(pDst, pResBlock->info.rows, (char *)&pSliceInfo->current, false); + colDataAppend(pDst, pResBlock->info.rows, (char*)&pSliceInfo->current, false); } else { - int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; + int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); if (colDataIsNull_s(pSrc, i)) { @@ -2493,13 +2488,13 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) { SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j]; - int32_t dstSlot = pExprInfo->base.resSchema.slotId; + int32_t dstSlot = pExprInfo->base.resSchema.slotId; SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot); if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) { - colDataAppend(pDst, pResBlock->info.rows, (char *)&pSliceInfo->current, false); + colDataAppend(pDst, pResBlock->info.rows, (char*)&pSliceInfo->current, false); } else { - int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; + int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); if (colDataIsNull_s(pSrc, i)) { @@ -2837,24 +2832,15 @@ void compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int3 } } -bool hasIntervalWindow(SAggSupporter* pSup, TSKEY ts, uint64_t groupId) { - int32_t bytes = sizeof(TSKEY); - SET_RES_WINDOW_KEY(pSup->keyBuf, &ts, bytes, groupId); - SResultRowPosition* p1 = - (SResultRowPosition*)tSimpleHashGet(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); - return p1 != NULL; +bool hasIntervalWindow(SStreamState* pState, SWinKey* pKey) { + return TSDB_CODE_SUCCESS == streamStateGet(pState, pKey, NULL, 0); } -STimeWindow getFinalTimeWindow(int64_t ts, SInterval* pInterval) { - STimeWindow w = {.skey = ts, .ekey = INT64_MAX}; - w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; - return w; -} - -static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SExprSupp* pSup, SArray* pWinArray, - int32_t groupId, int32_t numOfOutput, SExecTaskInfo* pTaskInfo, - SHashObj* pUpdatedMap) { - int32_t size = taosArrayGetSize(pWinArray); +static void rebuildIntervalWindow(SOperatorInfo* pOperator, SExprSupp* pSup, SArray* pWinArray, SHashObj* pUpdatedMap) { + SStreamIntervalOperatorInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + int32_t size = taosArrayGetSize(pWinArray); + int32_t numOfOutput = pOperator->exprSupp.numOfExprs; if (!pInfo->pChildren) { return; } @@ -2862,31 +2848,35 @@ static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SExpr SWinKey* pWinRes = taosArrayGet(pWinArray, i); SResultRow* pCurResult = NULL; STimeWindow parentWin = getFinalTimeWindow(pWinRes->ts, &pInfo->interval); - if (isDeletedWindow(&parentWin, pWinRes->groupId, &pInfo->aggSup) && isCloseWindow(&parentWin, &pInfo->twAggSup)) { + if (isDeletedStreamWindow(&parentWin, pWinRes->groupId, pInfo->pState, &pInfo->twAggSup) && + isCloseWindow(&parentWin, &pInfo->twAggSup)) { continue; } - setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &parentWin, true, &pCurResult, pWinRes->groupId, pSup->pCtx, - numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); + int32_t code = setOutputBuf(pInfo->pState, &parentWin, &pCurResult, pWinRes->groupId, pSup->pCtx, numOfOutput, + pSup->rowEntryInfoOffset, &pInfo->aggSup); + if (code != TSDB_CODE_SUCCESS || pCurResult == NULL) { + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren); int32_t num = 0; for (int32_t j = 0; j < numOfChildren; j++) { - SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, j); - SIntervalAggOperatorInfo* pChInfo = pChildOp->info; - SExprSupp* pChildSup = &pChildOp->exprSupp; - if (!hasIntervalWindow(&pChInfo->aggSup, pWinRes->ts, pWinRes->groupId)) { + SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, j); + SStreamIntervalOperatorInfo* pChInfo = pChildOp->info; + SExprSupp* pChildSup = &pChildOp->exprSupp; + if (!hasIntervalWindow(pChInfo->pState, pWinRes)) { continue; } num++; SResultRow* pChResult = NULL; - setTimeWindowOutputBuf(&pChInfo->binfo.resultRowInfo, &parentWin, true, &pChResult, pWinRes->groupId, - pChildSup->pCtx, pChildSup->numOfExprs, pChildSup->rowEntryInfoOffset, &pChInfo->aggSup, - pTaskInfo); + setOutputBuf(pChInfo->pState, &parentWin, &pChResult, pWinRes->groupId, pChildSup->pCtx, pChildSup->numOfExprs, + pChildSup->rowEntryInfoOffset, &pChInfo->aggSup); updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &parentWin, true); compactFunctions(pSup->pCtx, pChildSup->pCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData); } if (num > 0 && pUpdatedMap) { - saveWinResultRow(pCurResult, pWinRes->groupId, pUpdatedMap); - setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pInfo->binfo.resultRowInfo.cur); + saveWinResultInfo(pCurResult->win.skey, pWinRes->groupId, pUpdatedMap); + saveOutputBuf(pInfo->pState, pWinRes, pCurResult, pInfo->aggSup.resultRowSize); + releaseOutputBuf(pInfo->pState, pWinRes, pCurResult); } } } @@ -2898,15 +2888,15 @@ bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup) { return p1 == NULL; } -bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SOperatorInfo* pOperator, STimeWindowAggSupp* pTwSup) { +bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SStreamState* pState, STimeWindowAggSupp* pTwSup) { if (pWin->ekey < pTwSup->maxTs - pTwSup->deleteMark) { SWinKey key = {.ts = pWin->skey, .groupId = groupId}; void* pVal = NULL; int32_t size = 0; - if (streamStateGet(pOperator->pTaskInfo->streamInfo.pState, &key, &pVal, &size) < 0) { + if (streamStateGet(pState, &key, &pVal, &size) == TSDB_CODE_SUCCESS) { return false; } - streamStateReleaseBuf(pOperator->pTaskInfo->streamInfo.pState, &key, pVal); + return true; } return false; } @@ -2929,109 +2919,12 @@ void addPullWindow(SHashObj* pMap, SWinKey* pWinRes, int32_t size) { static int32_t getChildIndex(SSDataBlock* pBlock) { return pBlock->info.childId; } -static void doHashIntervalAgg(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t tableGroupId, - SHashObj* pUpdatedMap) { - SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)pOperatorInfo->info; - SResultRowInfo* pResultRowInfo = &(pInfo->binfo.resultRowInfo); - SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; - SExprSupp* pSup = &pOperatorInfo->exprSupp; - int32_t numOfOutput = pSup->numOfExprs; - int32_t step = 1; - TSKEY* tsCols = NULL; - SResultRow* pResult = NULL; - int32_t forwardRows = 0; - - ASSERT(pSDataBlock->pDataBlock != NULL); - SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); - tsCols = (int64_t*)pColDataInfo->pData; - - int32_t startPos = 0; - TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols); - STimeWindow nextWin = {0}; - if (IS_FINAL_OP(pInfo)) { - nextWin = getFinalTimeWindow(ts, &pInfo->interval); - } else { - nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, pInfo->order); - } - while (1) { - bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup); - if ((pInfo->ignoreExpiredData && isClosed) || !inSlidingWindow(&pInfo->interval, &nextWin, &pSDataBlock->info)) { - startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin); - if (startPos < 0) { - break; - } - continue; - } - if (IS_FINAL_OP(pInfo) && isClosed && pInfo->pChildren) { - bool ignore = true; - SWinKey winRes = { - .ts = nextWin.skey, - .groupId = tableGroupId, - }; - void* chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinKey)); - if (isDeletedWindow(&nextWin, tableGroupId, &pInfo->aggSup) && !chIds) { - SPullWindowInfo pull = {.window = nextWin, .groupId = tableGroupId}; - // add pull data request - savePullWindow(&pull, pInfo->pPullWins); - int32_t size = taosArrayGetSize(pInfo->pChildren); - addPullWindow(pInfo->pPullDataMap, &winRes, size); - qDebug("===stream===prepare retrive %" PRId64 ", size:%d", winRes.ts, size); - } else { - int32_t index = -1; - SArray* chArray = NULL; - int32_t chId = 0; - if (chIds) { - chArray = *(void**)chIds; - chId = getChildIndex(pSDataBlock); - index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ); - } - if (index == -1 || pSDataBlock->info.type == STREAM_PULL_DATA) { - ignore = false; - } - } - - if (ignore) { - startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin); - if (startPos < 0) { - break; - } - continue; - } - } - - int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, true, &pResult, tableGroupId, pSup->pCtx, - numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); - if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - if (IS_FINAL_OP(pInfo)) { - forwardRows = 1; - } else { - forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, - NULL, TSDB_ORDER_ASC); - } - if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) { - saveWinResultRow(pResult, tableGroupId, pUpdatedMap); - setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pResultRowInfo->cur); - } - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); - doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, - pSDataBlock->info.rows, numOfOutput); - int32_t prevEndPos = (forwardRows - 1) * step + startPos; - ASSERT(pSDataBlock->info.window.skey > 0 && pSDataBlock->info.window.ekey > 0); - startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, pInfo->order); - if (startPos < 0) { - break; - } - } -} - -static void clearStreamIntervalOperator(SStreamFinalIntervalOperatorInfo* pInfo) { +static void clearStreamIntervalOperator(SStreamIntervalOperatorInfo* pInfo) { tSimpleHashClear(pInfo->aggSup.pResultRowHashTable); clearDiskbasedBuf(pInfo->aggSup.pResultBuf); initResultRowInfo(&pInfo->binfo.resultRowInfo); pInfo->aggSup.currentPageId = -1; + streamStateClear(pInfo->pState); } static void clearSpecialDataBlock(SSDataBlock* pBlock) { @@ -3115,12 +3008,12 @@ void processPullOver(SSDataBlock* pBlock, SHashObj* pMap) { } } -static void addRetriveWindow(SArray* wins, SStreamFinalIntervalOperatorInfo* pInfo) { +static void addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo) { int32_t size = taosArrayGetSize(wins); for (int32_t i = 0; i < size; i++) { SWinKey* winKey = taosArrayGet(wins, i); STimeWindow nextWin = getFinalTimeWindow(winKey->ts, &pInfo->interval); - if (isCloseWindow(&nextWin, &pInfo->twAggSup) && !pInfo->ignoreExpiredData) { + if (needDeleteWindowBuf(&nextWin, &pInfo->twAggSup) && !pInfo->ignoreExpiredData) { void* chIds = taosHashGet(pInfo->pPullDataMap, winKey, sizeof(SWinKey)); if (!chIds) { SPullWindowInfo pull = {.window = nextWin, .groupId = winKey->groupId}; @@ -3140,8 +3033,124 @@ static void clearFunctionContext(SExprSupp* pSup) { } } +void doBuildResult(SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, SGroupResInfo* pGroupResInfo) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + // set output datablock version + pBlock->info.version = pTaskInfo->version; + + blockDataCleanup(pBlock); + if (!hasRemainResults(pGroupResInfo)) { + return; + } + + // clear the existed group id + pBlock->info.groupId = 0; + buildDataBlockFromGroupRes(pTaskInfo, pState, pBlock, &pOperator->exprSupp, pGroupResInfo); +} + +static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t groupId, + SHashObj* pUpdatedMap) { + SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperatorInfo->info; + + SResultRowInfo* pResultRowInfo = &(pInfo->binfo.resultRowInfo); + SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; + SExprSupp* pSup = &pOperatorInfo->exprSupp; + int32_t numOfOutput = pSup->numOfExprs; + int32_t step = 1; + TSKEY* tsCols = NULL; + SResultRow* pResult = NULL; + int32_t forwardRows = 0; + + ASSERT(pSDataBlock->pDataBlock != NULL); + SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); + tsCols = (int64_t*)pColDataInfo->pData; + + int32_t startPos = 0; + TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols); + STimeWindow nextWin = + getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, TSDB_ORDER_ASC); + while (1) { + bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup); + if ((pInfo->ignoreExpiredData && isClosed) || !inSlidingWindow(&pInfo->interval, &nextWin, &pSDataBlock->info)) { + startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin); + if (startPos < 0) { + break; + } + continue; + } + + if (IS_FINAL_OP(pInfo) && isClosed && pInfo->pChildren) { + bool ignore = true; + SWinKey winRes = { + .ts = nextWin.skey, + .groupId = groupId, + }; + void* chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinKey)); + if (isDeletedStreamWindow(&nextWin, groupId, pInfo->pState, &pInfo->twAggSup) && !chIds) { + SPullWindowInfo pull = {.window = nextWin, .groupId = groupId}; + // add pull data request + savePullWindow(&pull, pInfo->pPullWins); + int32_t size = taosArrayGetSize(pInfo->pChildren); + addPullWindow(pInfo->pPullDataMap, &winRes, size); + qDebug("===stream===prepare retrive %" PRId64 ", size:%d", winRes.ts, size); + } else { + int32_t index = -1; + SArray* chArray = NULL; + int32_t chId = 0; + if (chIds) { + chArray = *(void**)chIds; + chId = getChildIndex(pSDataBlock); + index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ); + } + if (index == -1 || pSDataBlock->info.type == STREAM_PULL_DATA) { + ignore = false; + } + } + + if (ignore) { + startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin); + if (startPos < 0) { + break; + } + continue; + } + } + + int32_t code = setOutputBuf(pInfo->pState, &nextWin, &pResult, groupId, pSup->pCtx, numOfOutput, + pSup->rowEntryInfoOffset, &pInfo->aggSup); + if (code != TSDB_CODE_SUCCESS || pResult == NULL) { + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL, + TSDB_ORDER_ASC); + if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) { + saveWinResultInfo(pResult->win.skey, groupId, pUpdatedMap); + } + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); + doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, + pSDataBlock->info.rows, numOfOutput); + SWinKey key = { + .ts = nextWin.skey, + .groupId = groupId, + }; + saveOutputBuf(pInfo->pState, &key, pResult, pInfo->aggSup.resultRowSize); + releaseOutputBuf(pInfo->pState, &key, pResult); + if (pInfo->delKey.ts > key.ts) { + pInfo->delKey = key; + } + int32_t prevEndPos = (forwardRows - 1) * step + startPos; + ASSERT(pSDataBlock->info.window.skey > 0 && pSDataBlock->info.window.ekey > 0); + startPos = + getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, TSDB_ORDER_ASC); + if (startPos < 0) { + break; + } + } +} + static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { - SStreamFinalIntervalOperatorInfo* pInfo = pOperator->info; + SStreamIntervalOperatorInfo* pInfo = pOperator->info; SOperatorInfo* downstream = pOperator->pDownstream[0]; TSKEY maxTs = INT64_MIN; @@ -3169,7 +3178,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { return pInfo->pDelRes; } - doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); + doBuildResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo); if (pInfo->binfo.pRes->info.rows != 0) { printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi"); return pInfo->binfo.pRes; @@ -3182,12 +3191,13 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { clearStreamIntervalOperator(pInfo); qDebug("===stream===clear semi operator"); } else { - freeAllPages(pInfo->pRecycledPages, pInfo->aggSup.pResultBuf); + deleteIntervalDiscBuf(pInfo->pState, pInfo->pPullDataMap, pInfo->twAggSup.maxTs - pInfo->twAggSup.deleteMark, + &pInfo->interval, &pInfo->delKey); } return NULL; } else { if (!IS_FINAL_OP(pInfo)) { - doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); + doBuildResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo); if (pInfo->binfo.pRes->info.rows != 0) { printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi"); return pInfo->binfo.pRes; @@ -3227,16 +3237,15 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { pInfo->binfo.pRes->info.type = pBlock->info.type; } else if (pBlock->info.type == STREAM_CLEAR) { SArray* pUpWins = taosArrayInit(8, sizeof(SWinKey)); - doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, pUpWins); + doDeleteWindows(pOperator, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, pUpWins, NULL); if (IS_FINAL_OP(pInfo)) { - int32_t childIndex = getChildIndex(pBlock); - SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); - SStreamFinalIntervalOperatorInfo* pChildInfo = pChildOp->info; - SExprSupp* pChildSup = &pChildOp->exprSupp; + int32_t childIndex = getChildIndex(pBlock); + SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); + SStreamIntervalOperatorInfo* pChildInfo = pChildOp->info; + SExprSupp* pChildSup = &pChildOp->exprSupp; - doClearWindows(&pChildInfo->aggSup, pChildSup, &pChildInfo->interval, pChildSup->numOfExprs, pBlock, NULL); - rebuildIntervalWindow(pInfo, pSup, pUpWins, pInfo->binfo.pRes->info.groupId, pOperator->exprSupp.numOfExprs, - pOperator->pTaskInfo, NULL); + doDeleteWindows(pChildOp, &pChildInfo->interval, pChildOp->exprSupp.numOfExprs, pBlock, NULL, NULL); + rebuildIntervalWindow(pOperator, pSup, pUpWins, pUpdatedMap); taosArrayDestroy(pUpWins); continue; } @@ -3247,15 +3256,14 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { break; } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) { SArray* delWins = taosArrayInit(8, sizeof(SWinKey)); - doDeleteSpecifyIntervalWindow(&pInfo->aggSup, &pInfo->twAggSup, pBlock, delWins, &pInfo->interval, pUpdatedMap); + doDeleteWindows(pOperator, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, delWins, pUpdatedMap); if (IS_FINAL_OP(pInfo)) { - int32_t childIndex = getChildIndex(pBlock); - SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); - SStreamFinalIntervalOperatorInfo* pChildInfo = pChildOp->info; - SExprSupp* pChildSup = &pChildOp->exprSupp; - doDeleteSpecifyIntervalWindow(&pChildInfo->aggSup, &pInfo->twAggSup, pBlock, NULL, &pChildInfo->interval, NULL); - rebuildIntervalWindow(pInfo, pSup, delWins, pInfo->binfo.pRes->info.groupId, pOperator->exprSupp.numOfExprs, - pOperator->pTaskInfo, pUpdatedMap); + int32_t childIndex = getChildIndex(pBlock); + SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); + SStreamIntervalOperatorInfo* pChildInfo = pChildOp->info; + SExprSupp* pChildSup = &pChildOp->exprSupp; + doDeleteWindows(pChildOp, &pChildInfo->interval, pChildOp->exprSupp.numOfExprs, pBlock, NULL, NULL); + rebuildIntervalWindow(pOperator, pSup, delWins, pUpdatedMap); addRetriveWindow(delWins, pInfo); taosArrayAddAll(pInfo->pDelWins, delWins); taosArrayDestroy(delWins); @@ -3269,10 +3277,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdatedMap); continue; } else if (pBlock->info.type == STREAM_RETRIEVE && !IS_FINAL_OP(pInfo)) { - SArray* pUpWins = taosArrayInit(8, sizeof(SWinKey)); - doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, pUpWins); - removeResults(pUpWins, pUpdatedMap); - taosArrayDestroy(pUpWins); + doDeleteWindows(pOperator, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, NULL, pUpdatedMap); if (taosArrayGetSize(pUpdated) > 0) { break; } @@ -3286,8 +3291,8 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SExprSupp* pExprSup = &pInfo->scalarSupp; projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); } - setInputDataBlock(pOperator, pSup->pCtx, pBlock, pInfo->order, MAIN_SCAN, true); - doHashIntervalAgg(pOperator, pBlock, pBlock->info.groupId, pUpdatedMap); + setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); + doStreamIntervalAggImpl(pOperator, pBlock, pBlock->info.groupId, pUpdatedMap); if (IS_FINAL_OP(pInfo)) { int32_t chIndex = getChildIndex(pBlock); int32_t size = taosArrayGetSize(pInfo->pChildren); @@ -3297,15 +3302,15 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { if (!pChildOp) { T_LONG_JMP(pOperator->pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - SStreamFinalIntervalOperatorInfo* pTmpInfo = pChildOp->info; + SStreamIntervalOperatorInfo* pTmpInfo = pChildOp->info; pTmpInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE; taosArrayPush(pInfo->pChildren, &pChildOp); qDebug("===stream===add child, id:%d", chIndex); } - SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, chIndex); - SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info; - setInputDataBlock(pChildOp, pChildOp->exprSupp.pCtx, pBlock, pChInfo->order, MAIN_SCAN, true); - doHashIntervalAgg(pChildOp, pBlock, pBlock->info.groupId, NULL); + SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, chIndex); + SStreamIntervalOperatorInfo* pChInfo = pChildOp->info; + setInputDataBlock(pChildOp, pChildOp->exprSupp.pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); + doStreamIntervalAggImpl(pChildOp, pBlock, pBlock->info.groupId, NULL); } maxTs = TMAX(maxTs, pBlock->info.window.ekey); maxTs = TMAX(maxTs, pBlock->info.watermark); @@ -3315,12 +3320,11 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, minTs); if (IS_FINAL_OP(pInfo)) { - closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pInfo->pPullDataMap, - pUpdatedMap, pInfo->pRecycledPages, pInfo->aggSup.pResultBuf); - closeChildIntervalWindow(pInfo->pChildren, pInfo->twAggSup.maxTs); - } else { - pInfo->binfo.pRes->info.watermark = pInfo->twAggSup.maxTs; + closeStreamIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, + pInfo->pPullDataMap, pUpdatedMap, pOperator); + closeChildIntervalWindow(pOperator, pInfo->pChildren, pInfo->twAggSup.maxTs); } + pInfo->binfo.pRes->info.watermark = pInfo->twAggSup.maxTs; void* pIte = NULL; while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) { @@ -3329,7 +3333,6 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { taosHashCleanup(pUpdatedMap); taosArraySort(pUpdated, resultrowComparAsc); - finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset); initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); @@ -3348,7 +3351,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { return pInfo->pDelRes; } - doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); + doBuildResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo); if (pInfo->binfo.pRes->info.rows != 0) { printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi"); return pInfo->binfo.pRes; @@ -3366,15 +3369,14 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild) { - SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode; - SStreamFinalIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamFinalIntervalOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode; + SStreamIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamIntervalOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } pOperator->pTaskInfo = pTaskInfo; - pInfo->order = TSDB_ORDER_ASC; pInfo->interval = (SInterval){.interval = pIntervalPhyNode->interval, .sliding = pIntervalPhyNode->sliding, .intervalUnit = pIntervalPhyNode->intervalUnit, @@ -3386,6 +3388,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, .calTrigger = pIntervalPhyNode->window.triggerType, .maxTs = INT64_MIN, .minTs = INT64_MAX, + .deleteMark = INT64_MAX, }; ASSERT(pInfo->twAggSup.calTrigger != STREAM_TRIGGER_MAX_DELAY); pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; @@ -3415,6 +3418,10 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, ASSERT(numOfCols > 0); initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); + pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState)); + *(pInfo->pState) = *(pTaskInfo->streamInfo.pState); + streamStateSetNumber(pInfo->pState, -1); + initResultRowInfo(&pInfo->binfo.resultRowInfo); pInfo->pChildren = NULL; if (numOfChild > 0) { @@ -3422,9 +3429,10 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, for (int32_t i = 0; i < numOfChild; i++) { SOperatorInfo* pChildOp = createStreamFinalIntervalOperatorInfo(NULL, pPhyNode, pTaskInfo, 0); if (pChildOp) { - SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info; + SStreamIntervalOperatorInfo* pChInfo = pChildOp->info; pChInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE; taosArrayPush(pInfo->pChildren, &pChildOp); + streamStateSetNumber(pChInfo->pState, i); continue; } goto _error; @@ -3458,7 +3466,8 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT); pInfo->delIndex = 0; pInfo->pDelWins = taosArrayInit(4, sizeof(SWinKey)); - pInfo->pRecycledPages = taosArrayInit(4, sizeof(int32_t)); + pInfo->delKey.ts = INT64_MAX; + pInfo->delKey.groupId = 0; pOperator->operatorType = pPhyNode->type; pOperator->blocking = true; @@ -5600,160 +5609,6 @@ _error: return NULL; } -static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock, - int32_t scanFlag, SHashObj* pUpdatedMap) { - SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperatorInfo->info; - - SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; - SExprSupp* pSup = &pOperatorInfo->exprSupp; - - int32_t startPos = 0; - int32_t numOfOutput = pSup->numOfExprs; - SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex); - TSKEY* tsCols = (TSKEY*)pColDataInfo->pData; - uint64_t tableGroupId = pBlock->info.groupId; - bool ascScan = true; - TSKEY ts = getStartTsKey(&pBlock->info.window, tsCols); - SResultRow* pResult = NULL; - - STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, TSDB_ORDER_ASC); - int32_t ret = TSDB_CODE_SUCCESS; - if ((!pInfo->ignoreExpiredData || !isCloseWindow(&win, &pInfo->twAggSup)) && - inSlidingWindow(&pInfo->interval, &win, &pBlock->info)) { - ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx, - numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); - if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { - saveWinResultRow(pResult, tableGroupId, pUpdatedMap); - setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pResultRowInfo->cur); - } - } - - TSKEY ekey = ascScan ? win.ekey : win.skey; - int32_t forwardRows = - getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); - ASSERT(forwardRows > 0); - - if ((!pInfo->ignoreExpiredData || !isCloseWindow(&win, &pInfo->twAggSup)) && - inSlidingWindow(&pInfo->interval, &win, &pBlock->info)) { - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true); - doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows, - numOfOutput); - } - - STimeWindow nextWin = win; - while (1) { - int32_t prevEndPos = forwardRows - 1 + startPos; - startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pBlock->info, tsCols, prevEndPos, TSDB_ORDER_ASC); - if (startPos < 0) { - break; - } - if (pInfo->ignoreExpiredData && isCloseWindow(&nextWin, &pInfo->twAggSup)) { - ekey = ascScan ? nextWin.ekey : nextWin.skey; - forwardRows = - getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); - continue; - } - - // null data, failed to allocate more memory buffer - int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, - pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); - if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { - saveWinResultRow(pResult, tableGroupId, pUpdatedMap); - setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pResultRowInfo->cur); - } - - ekey = ascScan ? nextWin.ekey : nextWin.skey; - forwardRows = - getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); - doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows, - numOfOutput); - } -} - -static void doStreamIntervalAggImpl2(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t tableGroupId, - SHashObj* pUpdatedMap) { - SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperatorInfo->info; - - SResultRowInfo* pResultRowInfo = &(pInfo->binfo.resultRowInfo); - SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; - SExprSupp* pSup = &pOperatorInfo->exprSupp; - int32_t numOfOutput = pSup->numOfExprs; - int32_t step = 1; - TSKEY* tsCols = NULL; - SResultRow* pResult = NULL; - int32_t forwardRows = 0; - - ASSERT(pSDataBlock->pDataBlock != NULL); - SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); - tsCols = (int64_t*)pColDataInfo->pData; - - int32_t startPos = 0; - TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols); - STimeWindow nextWin = - getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, TSDB_ORDER_ASC); - while (1) { - bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup); - if ((pInfo->ignoreExpiredData && isClosed) || !inSlidingWindow(&pInfo->interval, &nextWin, &pSDataBlock->info)) { - startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin); - if (startPos < 0) { - break; - } - continue; - } - - int32_t code = setOutputBuf(&nextWin, &pResult, tableGroupId, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, - &pInfo->aggSup, pTaskInfo); - if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL, - TSDB_ORDER_ASC); - if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) { - saveWinResultInfo(pResult->win.skey, tableGroupId, pUpdatedMap); - } - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); - doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, - pSDataBlock->info.rows, numOfOutput); - SWinKey key = { - .ts = nextWin.skey, - .groupId = tableGroupId, - }; - saveOutputBuf(pTaskInfo, &key, pResult, pInfo->aggSup.resultRowSize); - releaseOutputBuf(pTaskInfo, &key, pResult); - int32_t prevEndPos = (forwardRows - 1) * step + startPos; - ASSERT(pSDataBlock->info.window.skey > 0 && pSDataBlock->info.window.ekey > 0); - startPos = - getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, TSDB_ORDER_ASC); - if (startPos < 0) { - break; - } - } -} - -void doBuildResult(SOperatorInfo* pOperator, SSDataBlock* pBlock, SGroupResInfo* pGroupResInfo) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - // set output datablock version - pBlock->info.version = pTaskInfo->version; - - blockDataCleanup(pBlock); - if (!hasRemainResults(pGroupResInfo)) { - return; - } - - // clear the existed group id - pBlock->info.groupId = 0; - buildDataBlockFromGroupRes(pTaskInfo, pBlock, &pOperator->exprSupp, pGroupResInfo); -} - static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -5772,12 +5627,13 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { return pInfo->pDelRes; } - doBuildResult(pOperator, pInfo->binfo.pRes, &pInfo->groupResInfo); + doBuildResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo); if (pInfo->binfo.pRes->info.rows > 0) { printDataBlock(pInfo->binfo.pRes, "single interval"); return pInfo->binfo.pRes; } - + deleteIntervalDiscBuf(pInfo->pState, NULL, pInfo->twAggSup.maxTs - pInfo->twAggSup.deleteMark, &pInfo->interval, + &pInfo->delKey); doSetOperatorCompleted(pOperator); return NULL; } @@ -5800,8 +5656,6 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { qDebug("%s clear existed time window results for updates checked", GET_TASKID(pTaskInfo)); continue; } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) { - // doDeleteSpecifyIntervalWindow(&pInfo->aggSup, &pInfo->twAggSup, pBlock, pInfo->pDelWins, &pInfo->interval, - // pUpdatedMap); doDeleteWindows(pOperator, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, pInfo->pDelWins, pUpdatedMap); continue; @@ -5830,9 +5684,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { maxTs = TMAX(maxTs, pBlock->info.window.ekey); minTs = TMIN(minTs, pBlock->info.window.skey); - // doStreamIntervalAggImpl(pOperator, &pInfo->binfo.resultRowInfo, pBlock, MAIN_SCAN, pUpdatedMap); - // new disc buf - doStreamIntervalAggImpl2(pOperator, pBlock, pBlock->info.groupId, pUpdatedMap); + doStreamIntervalAggImpl(pOperator, pBlock, pBlock->info.groupId, pUpdatedMap); } pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, minTs); @@ -5857,7 +5709,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { return pInfo->pDelRes; } - doBuildResult(pOperator, pInfo->binfo.pRes, &pInfo->groupResInfo); + doBuildResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo); if (pInfo->binfo.pRes->info.rows > 0) { printDataBlock(pInfo->binfo.pRes, "single interval"); return pInfo->binfo.pRes; @@ -5866,20 +5718,6 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { return NULL; } -void destroyStreamIntervalOperatorInfo(void* param) { - SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)param; - cleanupBasicInfo(&pInfo->binfo); - cleanupAggSup(&pInfo->aggSup); - pInfo->pRecycledPages = taosArrayDestroy(pInfo->pRecycledPages); - - pInfo->pDelWins = taosArrayDestroy(pInfo->pDelWins); - pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes); - - cleanupGroupResInfo(&pInfo->groupResInfo); - colDataDestroy(&pInfo->twAggSup.timeWindowData); - taosMemoryFreeClear(param); -} - SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo) { SStreamIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamIntervalOperatorInfo)); @@ -5938,20 +5776,35 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); pInfo->invertible = allInvertible(pSup->pCtx, numOfCols); - pInfo->invertible = false; // Todo(liuyao): Dependent TSDB API - pInfo->pRecycledPages = taosArrayInit(4, sizeof(int32_t)); + pInfo->invertible = false; pInfo->pDelWins = taosArrayInit(4, sizeof(SWinKey)); pInfo->delIndex = 0; pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT); initResultRowInfo(&pInfo->binfo.resultRowInfo); + pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState)); + *(pInfo->pState) = *(pTaskInfo->streamInfo.pState); + streamStateSetNumber(pInfo->pState, -1); + + pInfo->pUpdateRes = NULL; + pInfo->returnUpdate = false; + pInfo->pPhyNode = NULL; // create new child + pInfo->pPullDataMap = NULL; + pInfo->pPullWins = NULL; // SPullWindowInfo + pInfo->pullIndex = 0; + pInfo->pPullDataRes = NULL; + pInfo->isFinal = false; + pInfo->pChildren = NULL; + pInfo->delKey.ts = INT64_MAX; + pInfo->delKey.groupId = 0; + pOperator->name = "StreamIntervalOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL; pOperator->blocking = true; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doStreamIntervalAgg, NULL, NULL, destroyStreamIntervalOperatorInfo, + createOperatorFpSet(operatorDummyOpenFn, doStreamIntervalAgg, NULL, NULL, destroyStreamFinalIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, &pInfo->twAggSup); @@ -5963,7 +5816,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys return pOperator; _error: - destroyStreamIntervalOperatorInfo(pInfo); + destroyStreamFinalIntervalOperatorInfo(pInfo); taosMemoryFreeClear(pOperator); pTaskInfo->code = code; return NULL; diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index f507e1b3be..cb671cfff9 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -54,11 +54,17 @@ void* indexQhandle = NULL; int32_t indexRefMgt; +int32_t indexThreads = 5; + static void indexDestroy(void* sIdx); -void indexInit() { +void indexInit(int32_t threadNum) { + indexThreads = threadNum; + if (indexThreads <= 1) indexThreads = INDEX_NUM_OF_THREADS; +} +void indexEnvInit() { // refactor later - indexQhandle = taosInitScheduler(INDEX_QUEUE_SIZE, INDEX_NUM_OF_THREADS, "index", NULL); + indexQhandle = taosInitScheduler(INDEX_QUEUE_SIZE, indexThreads, "index", NULL); indexRefMgt = taosOpenRef(1000, indexDestroy); } void indexCleanup() { @@ -99,7 +105,7 @@ static void indexWait(void* idx) { int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { int ret = TSDB_CODE_SUCCESS; - taosThreadOnce(&isInit, indexInit); + taosThreadOnce(&isInit, indexEnvInit); SIndex* idx = taosMemoryCalloc(1, sizeof(SIndex)); if (idx == NULL) { return TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index 39bba4e269..72f693f7e5 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -22,7 +22,7 @@ #define MAX_INDEX_KEY_LEN 256 // test only, change later #define MEM_TERM_LIMIT 10 * 10000 -#define MEM_THRESHOLD 512 * 1024 +#define MEM_THRESHOLD 8 * 512 * 1024 // 8M #define MEM_SIGNAL_QUIT MEM_THRESHOLD * 20 #define MEM_ESTIMATE_RADIO 1.5 diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index 1fc631e9f3..f3009c051b 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -555,6 +555,7 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) { taosArraySort(v->tableId, idxUidCompare); taosArrayRemoveDuplicate(v->tableId, idxUidCompare, NULL); int32_t tbsz = taosArrayGetSize(v->tableId); + if (tbsz == 0) continue; fstOffset += TF_TABLE_TATOAL_SIZE(tbsz); } tfileWriteFstOffset(tw, fstOffset); @@ -566,6 +567,7 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) { TFileValue* v = taosArrayGetP((SArray*)data, i); int32_t tbsz = taosArrayGetSize(v->tableId); + if (tbsz == 0) continue; // check buf has enough space or not int32_t ttsz = TF_TABLE_TATOAL_SIZE(tbsz); @@ -592,6 +594,10 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) { for (size_t i = 0; i < sz; i++) { // TODO, fst batch write later TFileValue* v = taosArrayGetP((SArray*)data, i); + + int32_t tbsz = taosArrayGetSize(v->tableId); + if (tbsz == 0) continue; + if (tfileWriteData(tw, v) != 0) { indexError("failed to write data: %s, offset: %d len: %d", v->colVal, v->offset, (int)taosArrayGetSize(v->tableId)); diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 3428a85823..da0d0fbd6d 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -18,6 +18,37 @@ #include "tcommon.h" #include "ttimer.h" +// todo refactor +typedef struct SStateKey { + SWinKey key; + int64_t opNum; +} SStateKey; + +static inline int SStateKeyCmpr(const void* pKey1, int kLen1, const void* pKey2, int kLen2) { + SStateKey* pWin1 = (SStateKey*)pKey1; + SStateKey* pWin2 = (SStateKey*)pKey2; + + if (pWin1->opNum > pWin2->opNum) { + return 1; + } else if (pWin1->opNum < pWin2->opNum) { + return -1; + } + + if (pWin1->key.ts > pWin2->key.ts) { + return 1; + } else if (pWin1->key.ts < pWin2->key.ts) { + return -1; + } + + if (pWin1->key.groupId > pWin2->key.groupId) { + return 1; + } else if (pWin1->key.groupId < pWin2->key.groupId) { + return -1; + } + + return 0; +} + SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath) { SStreamState* pState = taosMemoryCalloc(1, sizeof(SStreamState)); if (pState == NULL) { @@ -36,7 +67,7 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath) { } // open state storage backend - if (tdbTbOpen("state.db", sizeof(SWinKey), -1, SWinKeyCmpr, pState->db, &pState->pStateDb) < 0) { + if (tdbTbOpen("state.db", sizeof(SStateKey), -1, SStateKeyCmpr, pState->db, &pState->pStateDb) < 0) { goto _err; } @@ -130,8 +161,10 @@ int32_t streamStateFuncDel(SStreamState* pState, const STupleKey* key) { return tdbTbDelete(pState->pFuncStateDb, key, sizeof(STupleKey), &pState->txn); } +// todo refactor int32_t streamStatePut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen) { - return tdbTbUpsert(pState->pStateDb, key, sizeof(SWinKey), value, vLen, &pState->txn); + SStateKey sKey = {.key = *key, .opNum = pState->number}; + return tdbTbUpsert(pState->pStateDb, &sKey, sizeof(SStateKey), value, vLen, &pState->txn); } // todo refactor @@ -139,8 +172,10 @@ int32_t streamStateFillPut(SStreamState* pState, const SWinKey* key, const void* return tdbTbUpsert(pState->pFillStateDb, key, sizeof(SWinKey), value, vLen, &pState->txn); } +// todo refactor int32_t streamStateGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen) { - return tdbTbGet(pState->pStateDb, key, sizeof(SWinKey), pVal, pVLen); + SStateKey sKey = {.key = *key, .opNum = pState->number}; + return tdbTbGet(pState->pStateDb, &sKey, sizeof(SStateKey), pVal, pVLen); } // todo refactor @@ -148,10 +183,30 @@ int32_t streamStateFillGet(SStreamState* pState, const SWinKey* key, void** pVal return tdbTbGet(pState->pFillStateDb, key, sizeof(SWinKey), pVal, pVLen); } +// todo refactor int32_t streamStateDel(SStreamState* pState, const SWinKey* key) { - return tdbTbDelete(pState->pStateDb, key, sizeof(SWinKey), &pState->txn); + SStateKey sKey = {.key = *key, .opNum = pState->number}; + return tdbTbDelete(pState->pStateDb, &sKey, sizeof(SStateKey), &pState->txn); } +int32_t streamStateClear(SStreamState* pState) { + SWinKey key = {.ts = 0, .groupId = 0}; + streamStatePut(pState, &key, NULL, 0); + while (1) { + SStreamStateCur* pCur = streamStateSeekKeyNext(pState, &key); + SWinKey delKey = {0}; + int32_t code = streamStateGetKVByCur(pCur, &delKey, NULL, 0); + if (code == 0) { + streamStateDel(pState, &delKey); + } else { + break; + } + } + return 0; +} + +void streamStateSetNumber(SStreamState* pState, int32_t number) { pState->number = number; } + // todo refactor int32_t streamStateFillDel(SStreamState* pState, const SWinKey* key) { return tdbTbDelete(pState->pFillStateDb, key, sizeof(SWinKey), &pState->txn); @@ -179,12 +234,14 @@ SStreamStateCur* streamStateGetCur(SStreamState* pState, const SWinKey* key) { if (pCur == NULL) return NULL; tdbTbcOpen(pState->pStateDb, &pCur->pCur, NULL); - int32_t c; + int32_t c; + SStateKey sKey = {.key = *key, .opNum = pState->number}; tdbTbcMoveTo(pCur->pCur, key, sizeof(SWinKey), &c); if (c != 0) { taosMemoryFree(pCur); return NULL; } + pCur->number = pState->number; return pCur; } @@ -214,6 +271,25 @@ SStreamStateCur* streamStateGetAndCheckCur(SStreamState* pState, SWinKey* key) { } int32_t streamStateGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen) { + if (!pCur) { + return -1; + } + const SStateKey* pKTmp = NULL; + int32_t kLen; + if (tdbTbcGet(pCur->pCur, (const void**)&pKTmp, &kLen, pVal, pVLen) < 0) { + return -1; + } + if (pKTmp->opNum != pCur->number) { + return -1; + } + *pKey = pKTmp->key; + return 0; +} + +int32_t streamStateFillGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen) { + if (!pCur) { + return -1; + } const SWinKey* pKTmp = NULL; int32_t kLen; if (tdbTbcGet(pCur->pCur, (const void**)&pKTmp, &kLen, pVal, pVLen) < 0) { @@ -225,7 +301,7 @@ int32_t streamStateGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** int32_t streamStateGetGroupKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen) { uint64_t groupId = pKey->groupId; - int32_t code = streamStateGetKVByCur(pCur, pKey, pVal, pVLen); + int32_t code = streamStateFillGetKVByCur(pCur, pKey, pVal, pVLen); if (code == 0) { if (pKey->groupId == groupId) { return 0; @@ -234,6 +310,16 @@ int32_t streamStateGetGroupKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const v return -1; } +int32_t streamStateGetFirst(SStreamState* pState, SWinKey* key) { + // todo refactor + SWinKey tmp = {.ts = 0, .groupId = 0}; + streamStatePut(pState, &tmp, NULL, 0); + SStreamStateCur* pCur = streamStateSeekKeyNext(pState, &tmp); + int32_t code = streamStateGetKVByCur(pCur, key, NULL, 0); + streamStateDel(pState, &tmp); + return code; +} + int32_t streamStateSeekFirst(SStreamState* pState, SStreamStateCur* pCur) { // return tdbTbcMoveToFirst(pCur->pCur); @@ -244,6 +330,34 @@ int32_t streamStateSeekLast(SStreamState* pState, SStreamStateCur* pCur) { return tdbTbcMoveToLast(pCur->pCur); } +SStreamStateCur* streamStateSeekKeyNext(SStreamState* pState, const SWinKey* key) { + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + if (pCur == NULL) { + return NULL; + } + pCur->number = pState->number; + if (tdbTbcOpen(pState->pStateDb, &pCur->pCur, NULL) < 0) { + taosMemoryFree(pCur); + return NULL; + } + + SStateKey sKey = {.key = *key, .opNum = pState->number}; + int32_t c; + if (tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateKey), &c) < 0) { + tdbTbcClose(pCur->pCur); + taosMemoryFree(pCur); + return NULL; + } + if (c > 0) return pCur; + + if (tdbTbcMoveToNext(pCur->pCur) < 0) { + taosMemoryFree(pCur); + return NULL; + } + + return pCur; +} + SStreamStateCur* streamStateFillSeekKeyNext(SStreamState* pState, const SWinKey* key) { SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); if (pCur == NULL) { @@ -303,9 +417,15 @@ int32_t streamStateCurNext(SStreamState* pState, SStreamStateCur* pCur) { int32_t streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur) { // + if (!pCur) { + return -1; + } return tdbTbcMoveToPrev(pCur->pCur); } void streamStateFreeCur(SStreamStateCur* pCur) { + if (!pCur) { + return; + } tdbTbcClose(pCur->pCur); taosMemoryFree(pCur); } diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 5f3171ee0e..ae9f3155b4 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -193,7 +193,7 @@ bool transReadComplete(SConnBuffer* connBuf) { int transSetConnOption(uv_tcp_t* stream) { uv_tcp_nodelay(stream, 1); - int ret = uv_tcp_keepalive(stream, 5, 5); + int ret = uv_tcp_keepalive(stream, 5, 60); return ret; } diff --git a/tests/script/tsim/stream/distributeInterval0.sim b/tests/script/tsim/stream/distributeInterval0.sim index 9b2e940556..1d58922928 100644 --- a/tests/script/tsim/stream/distributeInterval0.sim +++ b/tests/script/tsim/stream/distributeInterval0.sim @@ -33,7 +33,8 @@ if $data(2)[4] != ready then endi print ===== step2 - +sql drop stream if exists stream_t1; +sql drop database if exists test; sql create database test vgroups 4; sql use test; sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int);