xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'nnn' into dv3

This commit is contained in:
yihaoDeng 2022-09-30 19:25:21 +08:00
parent 738fddb814 7674fa972c
commit 4a3436b15c
71 changed files with 4009 additions and 2394 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
docs/en/07-develop/03-insert-data/03-opentsdb-telnet.mdx
View File

@ -64,10 +64,10 @@ taos> use test;
Database changed. Database changed.
taos> show stables; taos> show stables;
name | created_time | columns | tags | tables | name |
============================================================================================ =================================
meters.current | 2022-03-30 17:04:10.877 | 2 | 2 | 2 | meters.current |
meters.voltage | 2022-03-30 17:04:10.882 | 2 | 2 | 2 | meters.voltage |
Query OK, 2 row(s) in set (0.002544s) Query OK, 2 row(s) in set (0.002544s)
taos> select tbname, * from `meters.current`; taos> select tbname, * from `meters.current`;

8
docs/en/07-develop/03-insert-data/04-opentsdb-json.mdx
View File

@ -81,10 +81,10 @@ taos> use test;
Database changed. Database changed.
taos> show stables; taos> show stables;
name | created_time | columns | tags | tables | name |
============================================================================================ =================================
meters.current | 2022-03-29 16:05:25.193 | 2 | 2 | 1 | meters.current |
meters.voltage | 2022-03-29 16:05:25.200 | 2 | 2 | 1 | meters.voltage |
Query OK, 2 row(s) in set (0.001954s) Query OK, 2 row(s) in set (0.001954s)
taos> select * from `meters.current`; taos> select * from `meters.current`;

10
docs/en/14-reference/03-connector/_verify_linux.mdx
View File

@ -4,11 +4,11 @@ Execute TDengine CLI program `taos` directly from the Linux shell to connect to
$ taos $ taos
taos> show databases; taos> show databases;
name | create_time | vgroups | ntables | replica | strict | duration | keep | buffer | pagesize | pages | minrows | maxrows | comp | precision | status | retention | single_stable | cachemodel | cachesize | wal_level | wal_fsync_period | wal_retention_period | wal_retention_size | wal_roll_period | wal_seg_size | name |
========================================================================================================================================================================================================================================================================================================================================================================================================================================================================= =================================
information_schema | NULL | NULL | 14 | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | ready | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | information_schema |
performance_schema | NULL | NULL | 3 | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | ready | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | performance_schema |
db | 2022-08-04 14:14:49.385 | 2 | 4 | 1 | off | 14400m | 5254560m,5254560m,5254560m | 96 | 4 | 256 | 100 | 4096 | 2 | ms | ready | NULL | false | none | 1 | 1 | 3000 | 0 | 0 | 0 | 0 | db |
Query OK, 3 rows in database (0.019154s) Query OK, 3 rows in database (0.019154s)
taos> taos>

11
docs/en/14-reference/03-connector/_verify_windows.mdx
View File

@ -2,12 +2,11 @@ Go to the `C:\TDengine` directory from `cmd` and execute TDengine CLI program `t
```text ```text
taos> show databases; taos> show databases;
name | create_time | vgroups | ntables | replica | strict | duration | keep | buffer | pagesize | pages | minrows | maxrows | comp | precision | status | retention | single_stable | cachemodel | cachesize | wal_level | wal_fsync_period | wal_retention_period | wal_retention_size | wal_roll_period | wal_seg_size | name |
========================================================================================================================================================================================================================================================================================================================================================================================================================================================================= =================================
information_schema | NULL | NULL | 14 | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | ready | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | information_schema |
performance_schema | NULL | NULL | 3 | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | ready | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | performance_schema |
test | 2022-08-04 16:46:40.506 | 2 | 0 | 1 | off | 14400m | 5256000m,5256000m,5256000m | 96 | 4 | 256 | test |
100 | 4096 | 2 | ms | ready | NULL | false | none | 1 | 1 | 3000 | 0 | 0 | 0 | 0 |
Query OK, 3 rows in database (0.123000s) Query OK, 3 rows in database (0.123000s)
taos> taos>

9
docs/en/14-reference/11-docker/index.md
View File

@ -25,10 +25,11 @@ The TDengine client taos can be executed in this container to access TDengine us
$ docker exec -it tdengine taos $ docker exec -it tdengine taos
taos> show databases; taos> show databases;
name | created_time | ntables | vgroups | replica | quorum | days | keep | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | cachelast | precision | update | status | name |
==================================================================================================================================================================================================================================================================================== =================================
log | 2022-01-17 13:57:22.270 | 10 | 1 | 1 | 1 | 10 | 30 | 1 | 3 | 100 | 4096 | 1 | 3000 | 2 | 0 | us | 0 | ready | information_schema |
Query OK, 1 row(s) in set (0.002843s) performance_schema |
Query OK, 2 row(s) in set (0.002843s)
``` ```
The TDengine server running in the container uses the container's hostname to establish a connection. Using TDengine CLI or various connectors (such as JDBC-JNI) to access the TDengine inside the container from outside the container is more complicated. So the above is the simplest way to access the TDengine service in the container and is suitable for some simple scenarios. Please refer to the next section if you want to access the TDengine service in the container from outside the container using TDengine CLI or various connectors for complex scenarios. The TDengine server running in the container uses the container's hostname to establish a connection. Using TDengine CLI or various connectors (such as JDBC-JNI) to access the TDengine inside the container from outside the container is more complicated. So the above is the simplest way to access the TDengine service in the container and is suitable for some simple scenarios. Please refer to the next section if you want to access the TDengine service in the container from outside the container using TDengine CLI or various connectors for complex scenarios.

1
docs/en/14-reference/_statsd.mdx
View File

@ -51,5 +51,6 @@ port: 8125
Start StatsD after adding the following (assuming the config file is modified to config.js) Start StatsD after adding the following (assuming the config file is modified to config.js)
``` ```
npm install
node stats.js config.js & node stats.js config.js &
``` ```

19
docs/en/20-third-party/02-prometheus.md vendored
View File

@ -30,21 +30,20 @@ After restarting Prometheus, you can refer to the following example to verify th
``` ```
taos> show databases; taos> show databases;
name | created_time | ntables | vgroups | replica | quorum | days | keep | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | cachelast | precision | update | status | name |
==================================================================================================================================================================================================================================================================================== =================================
test | 2022-04-12 08:07:58.756 | 1 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ms | 0 | ready | information_schema |
log | 2022-04-20 07:19:50.260 | 2 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ms | 0 | ready | performance_schema |
prometheus_data | 2022-04-20 07:21:09.202 | 158 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ns | 2 | ready | prometheus_data |
db | 2022-04-15 06:37:08.512 | 1 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ms | 0 | ready | Query OK, 3 row(s) in set (0.000585s)
Query OK, 4 row(s) in set (0.000585s)
taos> use prometheus_data; taos> use prometheus_data;
Database changed. Database changed.
taos> show stables; taos> show stables;
name | created_time | columns | tags | tables | name |
============================================================================================ =================================
metrics | 2022-04-20 07:21:09.209 | 2 | 1 | 1389 | metrics |
Query OK, 1 row(s) in set (0.000487s) Query OK, 1 row(s) in set (0.000487s)
taos> select * from metrics limit 10; taos> select * from metrics limit 10;

20
docs/en/20-third-party/03-telegraf.md vendored
View File

@ -43,16 +43,16 @@ taos> use telegraf;
Database changed. Database changed.
taos> show stables; taos> show stables;
name | created_time | columns | tags | tables | name |
============================================================================================ =================================
swap | 2022-04-20 08:47:53.532 | 7 | 1 | 1 | swap |
cpu | 2022-04-20 08:48:03.488 | 11 | 2 | 5 | cpu |
system | 2022-04-20 08:47:53.512 | 8 | 1 | 1 | system |
diskio | 2022-04-20 08:47:53.550 | 12 | 2 | 15 | diskio |
kernel | 2022-04-20 08:47:53.503 | 6 | 1 | 1 | kernel |
mem | 2022-04-20 08:47:53.521 | 35 | 1 | 1 | mem |
processes | 2022-04-20 08:47:53.555 | 12 | 1 | 1 | processes |
disk | 2022-04-20 08:47:53.541 | 8 | 5 | 2 | disk |
Query OK, 8 row(s) in set (0.000521s) Query OK, 8 row(s) in set (0.000521s)
taos> select * from telegraf.system limit 10; taos> select * from telegraf.system limit 10;

35
docs/en/20-third-party/05-collectd.md vendored
View File

@ -32,28 +32,29 @@ Use the TDengine CLI to verify that collectd's data is written to TDengine and c
``` ```
taos> show databases; taos> show databases;
name | created_time | ntables | vgroups | replica | quorum | days | keep | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | cachelast | precision | update | status | name |
==================================================================================================================================================================================================================================================================================== =================================
collectd | 2022-04-20 09:27:45.460 | 95 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ns | 2 | ready | information_schema |
log | 2022-04-20 07:19:50.260 | 11 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ms | 0 | ready | performance_schema |
Query OK, 2 row(s) in set (0.003266s) collectd |
Query OK, 3 row(s) in set (0.003266s)
taos> use collectd; taos> use collectd;
Database changed. Database changed.
taos> show stables; taos> show stables;
name | created_time | columns | tags | tables | name |
============================================================================================ =================================
load_1 | 2022-04-20 09:27:45.492 | 2 | 2 | 1 | load_1 |
memory_value | 2022-04-20 09:27:45.463 | 2 | 3 | 6 | memory_value |
df_value | 2022-04-20 09:27:45.463 | 2 | 4 | 25 | df_value |
load_2 | 2022-04-20 09:27:45.501 | 2 | 2 | 1 | load_2 |
load_0 | 2022-04-20 09:27:45.485 | 2 | 2 | 1 | load_0 |
interface_1 | 2022-04-20 09:27:45.488 | 2 | 3 | 12 | interface_1 |
irq_value | 2022-04-20 09:27:45.476 | 2 | 3 | 31 | irq_value |
interface_0 | 2022-04-20 09:27:45.480 | 2 | 3 | 12 | interface_0 |
entropy_value | 2022-04-20 09:27:45.473 | 2 | 2 | 1 | entropy_value |
swap_value | 2022-04-20 09:27:45.477 | 2 | 3 | 5 | swap_value |
Query OK, 10 row(s) in set (0.002236s) Query OK, 10 row(s) in set (0.002236s)
taos> select * from collectd.memory_value limit 10; taos> select * from collectd.memory_value limit 10;

24
docs/en/20-third-party/06-statsd.md vendored
View File

@ -26,7 +26,7 @@ Start StatsD:
``` ```
$ node stats.js config.js & $ node stats.js config.js &
[1] 8546 [1] 8546
$ 20 Apr 09:54:41 - [8546] reading config file: exampleConfig.js $ 20 Apr 09:54:41 - [8546] reading config file: config.js
20 Apr 09:54:41 - server is up INFO 20 Apr 09:54:41 - server is up INFO
``` ```
@ -40,19 +40,20 @@ Use the TDengine CLI to verify that StatsD data is written to TDengine and can r
``` ```
taos> show databases; taos> show databases;
name | created_time | ntables | vgroups | replica | quorum | days | keep | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | cachelast | precision | update | status | name |
==================================================================================================================================================================================================================================================================================== =================================
log | 2022-04-20 07:19:50.260 | 11 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ms | 0 | ready | information_schema |
statsd | 2022-04-20 09:54:51.220 | 1 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ns | 2 | ready | performance_schema |
Query OK, 2 row(s) in set (0.003142s) statsd |
Query OK, 3 row(s) in set (0.003142s)
taos> use statsd; taos> use statsd;
Database changed. Database changed.
taos> show stables; taos> show stables;
name | created_time | columns | tags | tables | name |
============================================================================================ =================================
foo | 2022-04-20 09:54:51.234 | 2 | 1 | 1 | foo |
Query OK, 1 row(s) in set (0.002161s) Query OK, 1 row(s) in set (0.002161s)
taos> select * from foo; taos> select * from foo;
@ -63,3 +64,8 @@ Query OK, 1 row(s) in set (0.004179s)
taos> taos>
``` ```
:::note
- TDengine will automatically create unique IDs for sub-table names by the rule.
:::

48
docs/en/20-third-party/07-icinga2.md vendored
View File

@ -47,30 +47,30 @@ taos> use icinga2;
Database changed. Database changed.
taos> show stables; taos> show stables;
name | created_time | columns | tags | tables | name |
============================================================================================ =================================
icinga.service.users.state_... | 2022-04-20 12:11:39.726 | 2 | 1 | 1 | icinga.service.users.state_... |
icinga.service.users.acknow... | 2022-04-20 12:11:39.756 | 2 | 1 | 1 | icinga.service.users.acknow... |
icinga.service.procs.downti... | 2022-04-20 12:11:44.541 | 2 | 1 | 1 | icinga.service.procs.downti... |
icinga.service.users.users | 2022-04-20 12:11:39.770 | 2 | 1 | 1 | icinga.service.users.users |
icinga.service.procs.procs_min | 2022-04-20 12:11:44.599 | 2 | 1 | 1 | icinga.service.procs.procs_min |
icinga.service.users.users_min | 2022-04-20 12:11:39.809 | 2 | 1 | 1 | icinga.service.users.users_min |
icinga.check.max_check_atte... | 2022-04-20 12:11:39.847 | 2 | 3 | 2 | icinga.check.max_check_atte... |
icinga.service.procs.state_... | 2022-04-20 12:11:44.522 | 2 | 1 | 1 | icinga.service.procs.state_... |
icinga.service.procs.procs_... | 2022-04-20 12:11:44.576 | 2 | 1 | 1 | icinga.service.procs.procs_... |
icinga.service.users.users_... | 2022-04-20 12:11:39.796 | 2 | 1 | 1 | icinga.service.users.users_... |
icinga.check.latency | 2022-04-20 12:11:39.869 | 2 | 3 | 2 | icinga.check.latency |
icinga.service.procs.procs_... | 2022-04-20 12:11:44.588 | 2 | 1 | 1 | icinga.service.procs.procs_... |
icinga.service.users.downti... | 2022-04-20 12:11:39.746 | 2 | 1 | 1 | icinga.service.users.downti... |
icinga.service.users.users_... | 2022-04-20 12:11:39.783 | 2 | 1 | 1 | icinga.service.users.users_... |
icinga.service.users.reachable | 2022-04-20 12:11:39.736 | 2 | 1 | 1 | icinga.service.users.reachable |
icinga.service.procs.procs | 2022-04-20 12:11:44.565 | 2 | 1 | 1 | icinga.service.procs.procs |
icinga.service.procs.acknow... | 2022-04-20 12:11:44.554 | 2 | 1 | 1 | icinga.service.procs.acknow... |
icinga.service.procs.state | 2022-04-20 12:11:44.509 | 2 | 1 | 1 | icinga.service.procs.state |
icinga.service.procs.reachable | 2022-04-20 12:11:44.532 | 2 | 1 | 1 | icinga.service.procs.reachable |
icinga.check.current_attempt | 2022-04-20 12:11:39.825 | 2 | 3 | 2 | icinga.check.current_attempt |
icinga.check.execution_time | 2022-04-20 12:11:39.898 | 2 | 3 | 2 | icinga.check.execution_time |
icinga.service.users.state | 2022-04-20 12:11:39.704 | 2 | 1 | 1 | icinga.service.users.state |
Query OK, 22 row(s) in set (0.002317s) Query OK, 22 row(s) in set (0.002317s)
``` ```

56
docs/en/20-third-party/08-tcollector.md vendored
View File

@ -33,35 +33,41 @@ Wait for a few seconds and then use the TDengine CLI to query whether the corres
``` ```
taos> show databases; taos> show databases;
name | created_time | ntables | vgroups | replica | quorum | days | keep | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | cachelast | precision | update | status | name |
==================================================================================================================================================================================================================================================================================== =================================
tcollector | 2022-04-20 12:44:49.604 | 88 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ns | 2 | ready | information_schema |
log | 2022-04-20 07:19:50.260 | 11 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ms | 0 | ready | performance_schema |
Query OK, 2 row(s) in set (0.002679s) tcollector |
Query OK, 3 rows in database (0.001647s)
taos> use tcollector; taos> use tcollector;
Database changed. Database changed.
taos> show stables; taos> show stables;
name | created_time | columns | tags | tables | name |
============================================================================================ =================================
proc.meminfo.hugepages_rsvd | 2022-04-20 12:44:53.945 | 2 | 1 | 1 | proc.meminfo.hugepages_rsvd |
proc.meminfo.directmap1g | 2022-04-20 12:44:54.110 | 2 | 1 | 1 | proc.meminfo.directmap1g |
proc.meminfo.vmallocchunk | 2022-04-20 12:44:53.724 | 2 | 1 | 1 | proc.meminfo.vmallocchunk |
proc.meminfo.hugepagesize | 2022-04-20 12:44:54.004 | 2 | 1 | 1 | proc.meminfo.hugepagesize |
tcollector.reader.lines_dro... | 2022-04-20 12:44:49.675 | 2 | 1 | 1 | tcollector.reader.lines_dro... |
proc.meminfo.sunreclaim | 2022-04-20 12:44:53.437 | 2 | 1 | 1 | proc.meminfo.sunreclaim |
proc.stat.ctxt | 2022-04-20 12:44:55.363 | 2 | 1 | 1 | proc.stat.ctxt |
proc.meminfo.swaptotal | 2022-04-20 12:44:53.158 | 2 | 1 | 1 | proc.meminfo.swaptotal |
proc.uptime.total | 2022-04-20 12:44:52.813 | 2 | 1 | 1 | proc.uptime.total |
tcollector.collector.lines_... | 2022-04-20 12:44:49.895 | 2 | 2 | 51 | tcollector.collector.lines_... |
proc.meminfo.vmallocused | 2022-04-20 12:44:53.704 | 2 | 1 | 1 | proc.meminfo.vmallocused |
proc.meminfo.memavailable | 2022-04-20 12:44:52.939 | 2 | 1 | 1 | proc.meminfo.memavailable |
sys.numa.foreign_allocs | 2022-04-20 12:44:57.929 | 2 | 2 | 1 | sys.numa.foreign_allocs |
proc.meminfo.committed_as | 2022-04-20 12:44:53.639 | 2 | 1 | 1 | proc.meminfo.committed_as |
proc.vmstat.pswpin | 2022-04-20 12:44:54.177 | 2 | 1 | 1 | proc.vmstat.pswpin |
proc.meminfo.cmafree | 2022-04-20 12:44:53.865 | 2 | 1 | 1 | proc.meminfo.cmafree |
proc.meminfo.mapped | 2022-04-20 12:44:53.349 | 2 | 1 | 1 | proc.meminfo.mapped |
proc.vmstat.pgmajfault | 2022-04-20 12:44:54.251 | 2 | 1 | 1 | proc.vmstat.pgmajfault |
... ...
``` ```
:::note
- TDengine will automatically create unique IDs for sub-table names by the rule.
:::

8
docs/zh/07-develop/03-insert-data/03-opentsdb-telnet.mdx
View File

@ -66,10 +66,10 @@ taos> use test;
Database changed. Database changed.
taos> show stables; taos> show stables;
name | created_time | columns | tags | tables | name |
============================================================================================ =================================
meters.current | 2022-03-30 17:04:10.877 | 2 | 2 | 2 | meters.current |
meters.voltage | 2022-03-30 17:04:10.882 | 2 | 2 | 2 | meters.voltage |
Query OK, 2 row(s) in set (0.002544s) Query OK, 2 row(s) in set (0.002544s)
taos> select tbname, * from `meters.current`; taos> select tbname, * from `meters.current`;

8
docs/zh/07-develop/03-insert-data/04-opentsdb-json.mdx
View File

@ -82,10 +82,10 @@ taos> use test;
Database changed. Database changed.
taos> show stables; taos> show stables;
name | created_time | columns | tags | tables | name |
============================================================================================ =================================
meters.current | 2022-03-29 16:05:25.193 | 2 | 2 | 1 | meters.current |
meters.voltage | 2022-03-29 16:05:25.200 | 2 | 2 | 1 | meters.voltage |
Query OK, 2 row(s) in set (0.001954s) Query OK, 2 row(s) in set (0.001954s)
taos> select * from `meters.current`; taos> select * from `meters.current`;

10
docs/zh/08-connector/_verify_linux.mdx
View File

@ -4,11 +4,11 @@
$ taos $ taos
taos> show databases; taos> show databases;
name | create_time | vgroups | ntables | replica | strict | duration | keep | buffer | pagesize | pages | minrows | maxrows | comp | precision | status | retention | single_stable | cachemodel | cachesize | wal_level | wal_fsync_period | wal_retention_period | wal_retention_size | wal_roll_period | wal_seg_size | name |
========================================================================================================================================================================================================================================================================================================================================================================================================================================================================= =================================
information_schema | NULL | NULL | 14 | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | ready | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | information_schema |
performance_schema | NULL | NULL | 3 | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | ready | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | performance_schema |
db | 2022-08-04 14:14:49.385 | 2 | 4 | 1 | off | 14400m | 5254560m,5254560m,5254560m | 96 | 4 | 256 | 100 | 4096 | 2 | ms | ready | NULL | false | none | 1 | 1 | 3000 | 0 | 0 | 0 | 0 | db |
Query OK, 3 rows in database (0.019154s) Query OK, 3 rows in database (0.019154s)
taos> taos>

1
docs/zh/14-reference/_statsd.mdx
View File

@ -51,5 +51,6 @@ port: 8125
增加如下内容后启动 StatsD（假设配置文件修改为 config.js）。 增加如下内容后启动 StatsD（假设配置文件修改为 config.js）。
``` ```
npm install
node stats.js config.js & node stats.js config.js &
``` ```

19
docs/zh/20-third-party/02-prometheus.md vendored
View File

@ -29,21 +29,20 @@ Prometheus 提供了 `remote_write` 和 `remote_read` 接口来利用其它数
### 使用 TDengine CLI 查询写入数据 ### 使用 TDengine CLI 查询写入数据
``` ```
taos> show databases; taos> show databases;
name | created_time | ntables | vgroups | replica | quorum | days | keep | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | cachelast | precision | update | status | name |
==================================================================================================================================================================================================================================================================================== =================================
test | 2022-04-12 08:07:58.756 | 1 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ms | 0 | ready | information_schema |
log | 2022-04-20 07:19:50.260 | 2 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ms | 0 | ready | performance_schema |
prometheus_data | 2022-04-20 07:21:09.202 | 158 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ns | 2 | ready | prometheus_data |
db | 2022-04-15 06:37:08.512 | 1 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ms | 0 | ready | Query OK, 3 row(s) in set (0.000585s)
Query OK, 4 row(s) in set (0.000585s)
taos> use prometheus_data; taos> use prometheus_data;
Database changed. Database changed.
taos> show stables; taos> show stables;
name | created_time | columns | tags | tables | name |
============================================================================================ =================================
metrics | 2022-04-20 07:21:09.209 | 2 | 1 | 1389 | metrics |
Query OK, 1 row(s) in set (0.000487s) Query OK, 1 row(s) in set (0.000487s)
taos> select * from metrics limit 10; taos> select * from metrics limit 10;

20
docs/zh/20-third-party/03-telegraf.md vendored
View File

@ -44,16 +44,16 @@ taos> use telegraf;
Database changed. Database changed.
taos> show stables; taos> show stables;
name | created_time | columns | tags | tables | name |
============================================================================================ =================================
swap | 2022-04-20 08:47:53.532 | 7 | 1 | 1 | swap |
cpu | 2022-04-20 08:48:03.488 | 11 | 2 | 5 | cpu |
system | 2022-04-20 08:47:53.512 | 8 | 1 | 1 | system |
diskio | 2022-04-20 08:47:53.550 | 12 | 2 | 15 | diskio |
kernel | 2022-04-20 08:47:53.503 | 6 | 1 | 1 | kernel |
mem | 2022-04-20 08:47:53.521 | 35 | 1 | 1 | mem |
processes | 2022-04-20 08:47:53.555 | 12 | 1 | 1 | processes |
disk | 2022-04-20 08:47:53.541 | 8 | 5 | 2 | disk |
Query OK, 8 row(s) in set (0.000521s) Query OK, 8 row(s) in set (0.000521s)
taos> select * from telegraf.system limit 10; taos> select * from telegraf.system limit 10;

35
docs/zh/20-third-party/05-collectd.md vendored
View File

@ -32,28 +32,29 @@ sudo systemctl restart collectd
``` ```
taos> show databases; taos> show databases;
name | created_time | ntables | vgroups | replica | quorum | days | keep | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | cachelast | precision | update | status | name |
==================================================================================================================================================================================================================================================================================== =================================
collectd | 2022-04-20 09:27:45.460 | 95 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ns | 2 | ready | information_schema |
log | 2022-04-20 07:19:50.260 | 11 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ms | 0 | ready | performance_schema |
Query OK, 2 row(s) in set (0.003266s) collectd |
Query OK, 3 row(s) in set (0.003266s)
taos> use collectd; taos> use collectd;
Database changed. Database changed.
taos> show stables; taos> show stables;
name | created_time | columns | tags | tables | name |
============================================================================================ =================================
load_1 | 2022-04-20 09:27:45.492 | 2 | 2 | 1 | load_1 |
memory_value | 2022-04-20 09:27:45.463 | 2 | 3 | 6 | memory_value |
df_value | 2022-04-20 09:27:45.463 | 2 | 4 | 25 | df_value |
load_2 | 2022-04-20 09:27:45.501 | 2 | 2 | 1 | load_2 |
load_0 | 2022-04-20 09:27:45.485 | 2 | 2 | 1 | load_0 |
interface_1 | 2022-04-20 09:27:45.488 | 2 | 3 | 12 | interface_1 |
irq_value | 2022-04-20 09:27:45.476 | 2 | 3 | 31 | irq_value |
interface_0 | 2022-04-20 09:27:45.480 | 2 | 3 | 12 | interface_0 |
entropy_value | 2022-04-20 09:27:45.473 | 2 | 2 | 1 | entropy_value |
swap_value | 2022-04-20 09:27:45.477 | 2 | 3 | 5 | swap_value |
Query OK, 10 row(s) in set (0.002236s) Query OK, 10 row(s) in set (0.002236s)
taos> select * from collectd.memory_value limit 10; taos> select * from collectd.memory_value limit 10;

24
docs/zh/20-third-party/06-statsd.md vendored
View File

@ -27,7 +27,7 @@ StatsD 是汇总和总结应用指标的一个简单的守护进程，近些年
``` ```
$ node stats.js config.js & $ node stats.js config.js &
[1] 8546 [1] 8546
$ 20 Apr 09:54:41 - [8546] reading config file: exampleConfig.js $ 20 Apr 09:54:41 - [8546] reading config file: config.js
20 Apr 09:54:41 - server is up INFO 20 Apr 09:54:41 - server is up INFO
``` ```
@ -41,19 +41,20 @@ $ echo "foo:1|c" | nc -u -w0 127.0.0.1 8125
``` ```
taos> show databases; taos> show databases;
name | created_time | ntables | vgroups | replica | quorum | days | keep | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | cachelast | precision | update | status | name |
==================================================================================================================================================================================================================================================================================== =================================
log | 2022-04-20 07:19:50.260 | 11 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ms | 0 | ready | information_schema |
statsd | 2022-04-20 09:54:51.220 | 1 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ns | 2 | ready | performance_schema |
Query OK, 2 row(s) in set (0.003142s) statsd |
Query OK, 3 row(s) in set (0.003142s)
taos> use statsd; taos> use statsd;
Database changed. Database changed.
taos> show stables; taos> show stables;
name | created_time | columns | tags | tables | name |
============================================================================================ =================================
foo | 2022-04-20 09:54:51.234 | 2 | 1 | 1 | foo |
Query OK, 1 row(s) in set (0.002161s) Query OK, 1 row(s) in set (0.002161s)
taos> select * from foo; taos> select * from foo;
@ -64,3 +65,8 @@ Query OK, 1 row(s) in set (0.004179s)
taos> taos>
``` ```
:::note
- TDengine will automatically create unique IDs for sub-table names by the rule.
:::

48
docs/zh/20-third-party/07-icinga2.md vendored
View File

@ -48,30 +48,30 @@ taos> use icinga2;
Database changed. Database changed.
taos> show stables; taos> show stables;
name | created_time | columns | tags | tables | name |
============================================================================================ =================================
icinga.service.users.state_... | 2022-04-20 12:11:39.726 | 2 | 1 | 1 | icinga.service.users.state_... |
icinga.service.users.acknow... | 2022-04-20 12:11:39.756 | 2 | 1 | 1 | icinga.service.users.acknow... |
icinga.service.procs.downti... | 2022-04-20 12:11:44.541 | 2 | 1 | 1 | icinga.service.procs.downti... |
icinga.service.users.users | 2022-04-20 12:11:39.770 | 2 | 1 | 1 | icinga.service.users.users |
icinga.service.procs.procs_min | 2022-04-20 12:11:44.599 | 2 | 1 | 1 | icinga.service.procs.procs_min |
icinga.service.users.users_min | 2022-04-20 12:11:39.809 | 2 | 1 | 1 | icinga.service.users.users_min |
icinga.check.max_check_atte... | 2022-04-20 12:11:39.847 | 2 | 3 | 2 | icinga.check.max_check_atte... |
icinga.service.procs.state_... | 2022-04-20 12:11:44.522 | 2 | 1 | 1 | icinga.service.procs.state_... |
icinga.service.procs.procs_... | 2022-04-20 12:11:44.576 | 2 | 1 | 1 | icinga.service.procs.procs_... |
icinga.service.users.users_... | 2022-04-20 12:11:39.796 | 2 | 1 | 1 | icinga.service.users.users_... |
icinga.check.latency | 2022-04-20 12:11:39.869 | 2 | 3 | 2 | icinga.check.latency |
icinga.service.procs.procs_... | 2022-04-20 12:11:44.588 | 2 | 1 | 1 | icinga.service.procs.procs_... |
icinga.service.users.downti... | 2022-04-20 12:11:39.746 | 2 | 1 | 1 | icinga.service.users.downti... |
icinga.service.users.users_... | 2022-04-20 12:11:39.783 | 2 | 1 | 1 | icinga.service.users.users_... |
icinga.service.users.reachable | 2022-04-20 12:11:39.736 | 2 | 1 | 1 | icinga.service.users.reachable |
icinga.service.procs.procs | 2022-04-20 12:11:44.565 | 2 | 1 | 1 | icinga.service.procs.procs |
icinga.service.procs.acknow... | 2022-04-20 12:11:44.554 | 2 | 1 | 1 | icinga.service.procs.acknow... |
icinga.service.procs.state | 2022-04-20 12:11:44.509 | 2 | 1 | 1 | icinga.service.procs.state |
icinga.service.procs.reachable | 2022-04-20 12:11:44.532 | 2 | 1 | 1 | icinga.service.procs.reachable |
icinga.check.current_attempt | 2022-04-20 12:11:39.825 | 2 | 3 | 2 | icinga.check.current_attempt |
icinga.check.execution_time | 2022-04-20 12:11:39.898 | 2 | 3 | 2 | icinga.check.execution_time |
icinga.service.users.state | 2022-04-20 12:11:39.704 | 2 | 1 | 1 | icinga.service.users.state |
Query OK, 22 row(s) in set (0.002317s) Query OK, 22 row(s) in set (0.002317s)
``` ```

57
docs/zh/20-third-party/08-tcollector.md vendored
View File

@ -34,35 +34,42 @@ sudo systemctl restart taosadapter
``` ```
taos> show databases; taos> show databases;
name | created_time | ntables | vgroups | replica | quorum | days | keep | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | cachelast | precision | update | status | name |
==================================================================================================================================================================================================================================================================================== =================================
tcollector | 2022-04-20 12:44:49.604 | 88 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ns | 2 | ready | information_schema |
log | 2022-04-20 07:19:50.260 | 11 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ms | 0 | ready | performance_schema |
Query OK, 2 row(s) in set (0.002679s) tcollector |
Query OK, 3 rows in database (0.001647s)
taos> use tcollector; taos> use tcollector;
Database changed. Database changed.
taos> show stables; taos> show stables;
name | created_time | columns | tags | tables | name |
============================================================================================ =================================
proc.meminfo.hugepages_rsvd | 2022-04-20 12:44:53.945 | 2 | 1 | 1 | proc.meminfo.hugepages_rsvd |
proc.meminfo.directmap1g | 2022-04-20 12:44:54.110 | 2 | 1 | 1 | proc.meminfo.directmap1g |
proc.meminfo.vmallocchunk | 2022-04-20 12:44:53.724 | 2 | 1 | 1 | proc.meminfo.vmallocchunk |
proc.meminfo.hugepagesize | 2022-04-20 12:44:54.004 | 2 | 1 | 1 | proc.meminfo.hugepagesize |
tcollector.reader.lines_dro... | 2022-04-20 12:44:49.675 | 2 | 1 | 1 | tcollector.reader.lines_dro... |
proc.meminfo.sunreclaim | 2022-04-20 12:44:53.437 | 2 | 1 | 1 | proc.meminfo.sunreclaim |
proc.stat.ctxt | 2022-04-20 12:44:55.363 | 2 | 1 | 1 | proc.stat.ctxt |
proc.meminfo.swaptotal | 2022-04-20 12:44:53.158 | 2 | 1 | 1 | proc.meminfo.swaptotal |
proc.uptime.total | 2022-04-20 12:44:52.813 | 2 | 1 | 1 | proc.uptime.total |
tcollector.collector.lines_... | 2022-04-20 12:44:49.895 | 2 | 2 | 51 | tcollector.collector.lines_... |
proc.meminfo.vmallocused | 2022-04-20 12:44:53.704 | 2 | 1 | 1 | proc.meminfo.vmallocused |
proc.meminfo.memavailable | 2022-04-20 12:44:52.939 | 2 | 1 | 1 | proc.meminfo.memavailable |
sys.numa.foreign_allocs | 2022-04-20 12:44:57.929 | 2 | 2 | 1 | sys.numa.foreign_allocs |
proc.meminfo.committed_as | 2022-04-20 12:44:53.639 | 2 | 1 | 1 | proc.meminfo.committed_as |
proc.vmstat.pswpin | 2022-04-20 12:44:54.177 | 2 | 1 | 1 | proc.vmstat.pswpin |
proc.meminfo.cmafree | 2022-04-20 12:44:53.865 | 2 | 1 | 1 | proc.meminfo.cmafree |
proc.meminfo.mapped | 2022-04-20 12:44:53.349 | 2 | 1 | 1 | proc.meminfo.mapped |
proc.vmstat.pgmajfault | 2022-04-20 12:44:54.251 | 2 | 1 | 1 | proc.vmstat.pgmajfault |
... ...
``` ```
:::note
- TDengine 默认生成的子表名是根据规则生成的唯一 ID 值。
:::

12
include/common/tdataformat.h
View File

@ -162,17 +162,7 @@ struct STSRowBuilder {
struct SValue { struct SValue {
union { union {
int8_t i8; // TSDB_DATA_TYPE_BOOL||TSDB_DATA_TYPE_TINYINT int64_t val;
uint8_t u8; // TSDB_DATA_TYPE_UTINYINT
int16_t i16; // TSDB_DATA_TYPE_SMALLINT
uint16_t u16; // TSDB_DATA_TYPE_USMALLINT
int32_t i32; // TSDB_DATA_TYPE_INT
uint32_t u32; // TSDB_DATA_TYPE_UINT
int64_t i64; // TSDB_DATA_TYPE_BIGINT
uint64_t u64; // TSDB_DATA_TYPE_UBIGINT
TSKEY ts; // TSDB_DATA_TYPE_TIMESTAMP
float f; // TSDB_DATA_TYPE_FLOAT
double d; // TSDB_DATA_TYPE_DOUBLE
struct { struct {
uint32_t nData; uint32_t nData;
uint8_t *pData; uint8_t *pData;

9
include/common/ttypes.h
View File

@ -333,10 +333,10 @@ typedef struct tDataTypeDescriptor {
char *name; char *name;
int64_t minValue; int64_t minValue;
int64_t maxValue; int64_t maxValue;
int32_t (*compFunc)(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, int32_t (*compFunc)(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); int32_t nBuf);
int32_t (*decompFunc)(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, int32_t (*decompFunc)(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); int32_t nBuf);
void (*statisFunc)(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numofrow, int64_t *min, void (*statisFunc)(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numofrow, int64_t *min,
int64_t *max, int64_t *sum, int16_t *minindex, int16_t *maxindex, int16_t *numofnull); int64_t *max, int64_t *sum, int16_t *minindex, int16_t *maxindex, int16_t *numofnull);
} tDataTypeDescriptor; } tDataTypeDescriptor;
@ -356,7 +356,6 @@ void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type);
void *getDataMin(int32_t type); void *getDataMin(int32_t type);
void *getDataMax(int32_t type); void *getDataMax(int32_t type);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

8
include/libs/stream/streamState.h
View File

@ -33,6 +33,7 @@ typedef struct {
TTB* pFuncStateDb; TTB* pFuncStateDb;
TTB* pFillStateDb; // todo refactor TTB* pFillStateDb; // todo refactor
TXN txn; TXN txn;
int32_t number;
} SStreamState; } SStreamState;
SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath); SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath);
@ -42,7 +43,8 @@ int32_t streamStateCommit(SStreamState* pState);
int32_t streamStateAbort(SStreamState* pState); int32_t streamStateAbort(SStreamState* pState);
typedef struct { typedef struct {
TBC* pCur; TBC* pCur;
int64_t number;
} SStreamStateCur; } SStreamStateCur;
int32_t streamStateFuncPut(SStreamState* pState, const STupleKey* key, const void* value, int32_t vLen); int32_t streamStateFuncPut(SStreamState* pState, const STupleKey* key, const void* value, int32_t vLen);
@ -52,6 +54,8 @@ int32_t streamStateFuncDel(SStreamState* pState, const STupleKey* key);
int32_t streamStatePut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); int32_t streamStatePut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
int32_t streamStateGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); int32_t streamStateGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen);
int32_t streamStateDel(SStreamState* pState, const SWinKey* key); int32_t streamStateDel(SStreamState* pState, const SWinKey* key);
int32_t streamStateClear(SStreamState* pState);
void streamStateSetNumber(SStreamState* pState, int32_t number);
int32_t streamStateFillPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); int32_t streamStateFillPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
int32_t streamStateFillGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); int32_t streamStateFillGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen);
@ -63,6 +67,7 @@ void streamFreeVal(void* val);
SStreamStateCur* streamStateGetCur(SStreamState* pState, const SWinKey* key); SStreamStateCur* streamStateGetCur(SStreamState* pState, const SWinKey* key);
SStreamStateCur* streamStateGetAndCheckCur(SStreamState* pState, SWinKey* key); SStreamStateCur* streamStateGetAndCheckCur(SStreamState* pState, SWinKey* key);
SStreamStateCur* streamStateSeekKeyNext(SStreamState* pState, const SWinKey* key);
SStreamStateCur* streamStateFillSeekKeyNext(SStreamState* pState, const SWinKey* key); SStreamStateCur* streamStateFillSeekKeyNext(SStreamState* pState, const SWinKey* key);
SStreamStateCur* streamStateFillSeekKeyPrev(SStreamState* pState, const SWinKey* key); SStreamStateCur* streamStateFillSeekKeyPrev(SStreamState* pState, const SWinKey* key);
void streamStateFreeCur(SStreamStateCur* pCur); void streamStateFreeCur(SStreamStateCur* pCur);
@ -70,6 +75,7 @@ void streamStateFreeCur(SStreamStateCur* pCur);
int32_t streamStateGetGroupKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); int32_t streamStateGetGroupKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen);
int32_t streamStateGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); int32_t streamStateGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen);
int32_t streamStateGetFirst(SStreamState* pState, SWinKey* key);
int32_t streamStateSeekFirst(SStreamState* pState, SStreamStateCur* pCur); int32_t streamStateSeekFirst(SStreamState* pState, SStreamStateCur* pCur);
int32_t streamStateSeekLast(SStreamState* pState, SStreamStateCur* pCur); int32_t streamStateSeekLast(SStreamState* pState, SStreamStateCur* pCur);

6
include/util/tcoding.h
View File

@ -23,8 +23,8 @@ extern "C" {
#endif #endif
#define ENCODE_LIMIT (((uint8_t)1) << 7) #define ENCODE_LIMIT (((uint8_t)1) << 7)
#define ZIGZAGE(T, v) ((u##T)((v) >> (sizeof(T) * 8 - 1))) ^ (((u##T)(v)) << 1) // zigzag encode #define ZIGZAGE(T, v) (((u##T)((v) >> (sizeof(T) * 8 - 1))) ^ (((u##T)(v)) << 1)) // zigzag encode
#define ZIGZAGD(T, v) ((v) >> 1) ^ -((T)((v)&1)) // zigzag decode #define ZIGZAGD(T, v) (((v) >> 1) ^ -((T)((v)&1))) // zigzag decode
/* ------------------------ LEGACY CODES ------------------------ */ /* ------------------------ LEGACY CODES ------------------------ */
#if 1 #if 1
@ -70,7 +70,7 @@ static FORCE_INLINE int32_t taosEncodeFixedBool(void **buf, bool value) {
} }
static FORCE_INLINE void *taosDecodeFixedBool(const void *buf, bool *value) { static FORCE_INLINE void *taosDecodeFixedBool(const void *buf, bool *value) {
*value = ( (((int8_t *)buf)[0] == 0) ? false : true ); *value = ((((int8_t *)buf)[0] == 0) ? false : true);
return POINTER_SHIFT(buf, sizeof(int8_t)); return POINTER_SHIFT(buf, sizeof(int8_t));
} }

350
include/util/tcompression.h
View File

@ -51,287 +51,12 @@ extern "C" {
#define HEAD_MODE(x) x % 2 #define HEAD_MODE(x) x % 2
#define HEAD_ALGO(x) x / 2 #define HEAD_ALGO(x) x / 2
extern int32_t tsCompressINTImp(const char *const input, const int32_t nelements, char *const output, const char type);
extern int32_t tsDecompressINTImp(const char *const input, const int32_t nelements, char *const output,
const char type);
extern int32_t tsCompressBoolImp(const char *const input, const int32_t nelements, char *const output);
extern int32_t tsDecompressBoolImp(const char *const input, const int32_t nelements, char *const output);
extern int32_t tsCompressStringImp(const char *const input, int32_t inputSize, char *const output, int32_t outputSize);
extern int32_t tsDecompressStringImp(const char *const input, int32_t compressedSize, char *const output,
int32_t outputSize);
extern int32_t tsCompressTimestampImp(const char *const input, const int32_t nelements, char *const output);
extern int32_t tsDecompressTimestampImp(const char *const input, const int32_t nelements, char *const output);
extern int32_t tsCompressDoubleImp(const char *const input, const int32_t nelements, char *const output);
extern int32_t tsDecompressDoubleImp(const char *const input, const int32_t nelements, char *const output);
extern int32_t tsCompressFloatImp(const char *const input, const int32_t nelements, char *const output);
extern int32_t tsDecompressFloatImp(const char *const input, const int32_t nelements, char *const output);
// lossy
extern int32_t tsCompressFloatLossyImp(const char *input, const int32_t nelements, char *const output);
extern int32_t tsDecompressFloatLossyImp(const char *input, int32_t compressedSize, const int32_t nelements,
char *const output);
extern int32_t tsCompressDoubleLossyImp(const char *input, const int32_t nelements, char *const output);
extern int32_t tsDecompressDoubleLossyImp(const char *input, int32_t compressedSize, const int32_t nelements,
char *const output);
#ifdef TD_TSZ #ifdef TD_TSZ
extern bool lossyFloat; extern bool lossyFloat;
extern bool lossyDouble; extern bool lossyDouble;
int32_t tsCompressInit(); int32_t tsCompressInit();
void tsCompressExit(); void tsCompressExit();
#endif
static FORCE_INLINE int32_t tsCompressTinyint(const char *const input, int32_t inputSize, const int32_t nelements,
char *const output, int32_t outputSize, char algorithm,
char *const buffer, int32_t bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_TINYINT);
} else if (algorithm == TWO_STAGE_COMP) {
int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_TINYINT);
return tsCompressStringImp(buffer, len, output, outputSize);
} else {
assert(0);
return -1;
}
}
static FORCE_INLINE int32_t tsDecompressTinyint(const char *const input, int32_t compressedSize,
const int32_t nelements, char *const output, int32_t outputSize,
char algorithm, char *const buffer, int32_t bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_TINYINT);
} else if (algorithm == TWO_STAGE_COMP) {
if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1;
return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_TINYINT);
} else {
assert(0);
return -1;
}
}
static FORCE_INLINE int32_t tsCompressSmallint(const char *const input, int32_t inputSize, const int32_t nelements,
char *const output, int32_t outputSize, char algorithm,
char *const buffer, int32_t bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_SMALLINT);
} else if (algorithm == TWO_STAGE_COMP) {
int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_SMALLINT);
return tsCompressStringImp(buffer, len, output, outputSize);
} else {
assert(0);
return -1;
}
}
static FORCE_INLINE int32_t tsDecompressSmallint(const char *const input, int32_t compressedSize,
const int32_t nelements, char *const output, int32_t outputSize,
char algorithm, char *const buffer, int32_t bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_SMALLINT);
} else if (algorithm == TWO_STAGE_COMP) {
if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1;
return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_SMALLINT);
} else {
assert(0);
return -1;
}
}
static FORCE_INLINE int32_t tsCompressInt(const char *const input, int32_t inputSize, const int32_t nelements,
char *const output, int32_t outputSize, char algorithm, char *const buffer,
int32_t bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_INT);
} else if (algorithm == TWO_STAGE_COMP) {
int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_INT);
return tsCompressStringImp(buffer, len, output, outputSize);
} else {
assert(0);
return -1;
}
}
static FORCE_INLINE int32_t tsDecompressInt(const char *const input, int32_t compressedSize, const int32_t nelements,
char *const output, int32_t outputSize, char algorithm, char *const buffer,
int32_t bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_INT);
} else if (algorithm == TWO_STAGE_COMP) {
if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1;
return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_INT);
} else {
assert(0);
return -1;
}
}
static FORCE_INLINE int32_t tsCompressBigint(const char *const input, int32_t inputSize, const int32_t nelements,
char *const output, int32_t outputSize, char algorithm, char *const buffer,
int32_t bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_BIGINT);
} else if (algorithm == TWO_STAGE_COMP) {
int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_BIGINT);
return tsCompressStringImp(buffer, len, output, outputSize);
} else {
assert(0);
return -1;
}
}
static FORCE_INLINE int32_t tsDecompressBigint(const char *const input, int32_t compressedSize, const int32_t nelements,
char *const output, int32_t outputSize, char algorithm,
char *const buffer, int32_t bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_BIGINT);
} else if (algorithm == TWO_STAGE_COMP) {
if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1;
return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_BIGINT);
} else {
assert(0);
return -1;
}
}
static FORCE_INLINE int32_t tsCompressBool(const char *const input, int32_t inputSize, const int32_t nelements,
char *const output, int32_t outputSize, char algorithm, char *const buffer,
int32_t bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsCompressBoolImp(input, nelements, output);
} else if (algorithm == TWO_STAGE_COMP) {
int32_t len = tsCompressBoolImp(input, nelements, buffer);
return tsCompressStringImp(buffer, len, output, outputSize);
} else {
assert(0);
return -1;
}
}
static FORCE_INLINE int32_t tsDecompressBool(const char *const input, int32_t compressedSize, const int32_t nelements,
char *const output, int32_t outputSize, char algorithm, char *const buffer,
int32_t bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsDecompressBoolImp(input, nelements, output);
} else if (algorithm == TWO_STAGE_COMP) {
if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1;
return tsDecompressBoolImp(buffer, nelements, output);
} else {
assert(0);
return -1;
}
}
static FORCE_INLINE int32_t tsCompressString(const char *const input, int32_t inputSize, const int32_t nelements,
char *const output, int32_t outputSize, char algorithm, char *const buffer,
int32_t bufferSize) {
return tsCompressStringImp(input, inputSize, output, outputSize);
}
static FORCE_INLINE int32_t tsDecompressString(const char *const input, int32_t compressedSize, const int32_t nelements,
char *const output, int32_t outputSize, char algorithm,
char *const buffer, int32_t bufferSize) {
return tsDecompressStringImp(input, compressedSize, output, outputSize);
}
static FORCE_INLINE int32_t tsCompressFloat(const char *const input, int32_t inputSize, const int32_t nelements,
char *const output, int32_t outputSize, char algorithm, char *const buffer,
int32_t bufferSize) {
#ifdef TD_TSZ
// lossy mode
if (lossyFloat) {
return tsCompressFloatLossyImp(input, nelements, output);
// lossless mode
} else {
#endif
if (algorithm == ONE_STAGE_COMP) {
return tsCompressFloatImp(input, nelements, output);
} else if (algorithm == TWO_STAGE_COMP) {
int32_t len = tsCompressFloatImp(input, nelements, buffer);
return tsCompressStringImp(buffer, len, output, outputSize);
} else {
assert(0);
return -1;
}
#ifdef TD_TSZ
}
#endif
}
static FORCE_INLINE int32_t tsDecompressFloat(const char *const input, int32_t compressedSize, const int32_t nelements,
char *const output, int32_t outputSize, char algorithm,
char *const buffer, int32_t bufferSize) {
#ifdef TD_TSZ
if (HEAD_ALGO(input[0]) == ALGO_SZ_LOSSY) {
// decompress lossy
return tsDecompressFloatLossyImp(input, compressedSize, nelements, output);
} else {
#endif
// decompress lossless
if (algorithm == ONE_STAGE_COMP) {
return tsDecompressFloatImp(input, nelements, output);
} else if (algorithm == TWO_STAGE_COMP) {
if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1;
return tsDecompressFloatImp(buffer, nelements, output);
} else {
assert(0);
return -1;
}
#ifdef TD_TSZ
}
#endif
}
static FORCE_INLINE int32_t tsCompressDouble(const char *const input, int32_t inputSize, const int32_t nelements,
char *const output, int32_t outputSize, char algorithm, char *const buffer,
int32_t bufferSize) {
#ifdef TD_TSZ
if (lossyDouble) {
// lossy mode
return tsCompressDoubleLossyImp(input, nelements, output);
} else {
#endif
// lossless mode
if (algorithm == ONE_STAGE_COMP) {
return tsCompressDoubleImp(input, nelements, output);
} else if (algorithm == TWO_STAGE_COMP) {
int32_t len = tsCompressDoubleImp(input, nelements, buffer);
return tsCompressStringImp(buffer, len, output, outputSize);
} else {
assert(0);
return -1;
}
#ifdef TD_TSZ
}
#endif
}
static FORCE_INLINE int32_t tsDecompressDouble(const char *const input, int32_t compressedSize, const int32_t nelements,
char *const output, int32_t outputSize, char algorithm,
char *const buffer, int32_t bufferSize) {
#ifdef TD_TSZ
if (HEAD_ALGO(input[0]) == ALGO_SZ_LOSSY) {
// decompress lossy
return tsDecompressDoubleLossyImp(input, compressedSize, nelements, output);
} else {
#endif
// decompress lossless
if (algorithm == ONE_STAGE_COMP) {
return tsDecompressDoubleImp(input, nelements, output);
} else if (algorithm == TWO_STAGE_COMP) {
if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1;
return tsDecompressDoubleImp(buffer, nelements, output);
} else {
assert(0);
return -1;
}
#ifdef TD_TSZ
}
#endif
}
#ifdef TD_TSZ
//
// lossy float double
//
static FORCE_INLINE int32_t tsCompressFloatLossy(const char *const input, int32_t inputSize, const int32_t nelements, static FORCE_INLINE int32_t tsCompressFloatLossy(const char *const input, int32_t inputSize, const int32_t nelements,
char *const output, int32_t outputSize, char algorithm, char *const output, int32_t outputSize, char algorithm,
char *const buffer, int32_t bufferSize) { char *const buffer, int32_t bufferSize) {
@ -358,33 +83,56 @@ static FORCE_INLINE int32_t tsDecompressDoubleLossy(const char *const input, int
#endif #endif
static FORCE_INLINE int32_t tsCompressTimestamp(const char *const input, int32_t inputSize, const int32_t nelements, /*************************************************************************
char *const output, int32_t outputSize, char algorithm, * REGULAR COMPRESSION
char *const buffer, int32_t bufferSize) { *************************************************************************/
if (algorithm == ONE_STAGE_COMP) { int32_t tsCompressTimestamp(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
return tsCompressTimestampImp(input, nelements, output); int32_t nBuf);
} else if (algorithm == TWO_STAGE_COMP) { int32_t tsDecompressTimestamp(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg,
int32_t len = tsCompressTimestampImp(input, nelements, buffer); void *pBuf, int32_t nBuf);
return tsCompressStringImp(buffer, len, output, outputSize); int32_t tsCompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
} else { int32_t nBuf);
assert(0); int32_t tsDecompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
return -1; int32_t nBuf);
} int32_t tsCompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
} int32_t nBuf);
int32_t tsDecompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t nBuf);
int32_t tsCompressString(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t nBuf);
int32_t tsDecompressString(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t nBuf);
int32_t tsCompressBool(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t nBuf);
int32_t tsDecompressBool(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t nBuf);
int32_t tsCompressTinyint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t nBuf);
int32_t tsDecompressTinyint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t nBuf);
int32_t tsCompressSmallint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t nBuf);
int32_t tsDecompressSmallint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg,
void *pBuf, int32_t nBuf);
int32_t tsCompressInt(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t nBuf);
int32_t tsDecompressInt(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t nBuf);
int32_t tsCompressBigint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t nBuf);
int32_t tsDecompressBigint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t nBuf);
static FORCE_INLINE int32_t tsDecompressTimestamp(const char *const input, int32_t compressedSize, /*************************************************************************
const int32_t nelements, char *const output, int32_t outputSize, * STREAM COMPRESSION
char algorithm, char *const buffer, int32_t bufferSize) { *************************************************************************/
if (algorithm == ONE_STAGE_COMP) { typedef struct SCompressor SCompressor;
return tsDecompressTimestampImp(input, nelements, output);
} else if (algorithm == TWO_STAGE_COMP) { int32_t tCompressorCreate(SCompressor **ppCmprsor);
if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; int32_t tCompressorDestroy(SCompressor *pCmprsor);
return tsDecompressTimestampImp(buffer, nelements, output); int32_t tCompressStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg);
} else { int32_t tCompressEnd(SCompressor *pCmprsor, const uint8_t **ppOut, int32_t *nOut, int32_t *nOrigin);
assert(0); int32_t tCompress(SCompressor *pCmprsor, const void *pData, int64_t nData);
return -1;
}
}
#ifdef __cplusplus #ifdef __cplusplus
} }

705
source/common/src/tdataformat.c
View File

@ -33,105 +33,21 @@ typedef struct {
// SValue // SValue
int32_t tPutValue(uint8_t *p, SValue *pValue, int8_t type) { int32_t tPutValue(uint8_t *p, SValue *pValue, int8_t type) {
int32_t n = 0;
if (IS_VAR_DATA_TYPE(type)) { if (IS_VAR_DATA_TYPE(type)) {
n += tPutBinary(p ? p + n : p, pValue->pData, pValue->nData); return tPutBinary(p, pValue->pData, pValue->nData);
} else { } else {
switch (type) { if (p) memcpy(p, &pValue->val, tDataTypes[type].bytes);
case TSDB_DATA_TYPE_BOOL: return tDataTypes[type].bytes;
n += tPutI8(p ? p + n : p, pValue->i8 ? 1 : 0);
break;
case TSDB_DATA_TYPE_TINYINT:
n += tPutI8(p ? p + n : p, pValue->i8);
break;
case TSDB_DATA_TYPE_SMALLINT:
n += tPutI16(p ? p + n : p, pValue->i16);
break;
case TSDB_DATA_TYPE_INT:
n += tPutI32(p ? p + n : p, pValue->i32);
break;
case TSDB_DATA_TYPE_BIGINT:
n += tPutI64(p ? p + n : p, pValue->i64);
break;
case TSDB_DATA_TYPE_FLOAT:
n += tPutFloat(p ? p + n : p, pValue->f);
break;
case TSDB_DATA_TYPE_DOUBLE:
n += tPutDouble(p ? p + n : p, pValue->d);
break;
case TSDB_DATA_TYPE_TIMESTAMP:
n += tPutI64(p ? p + n : p, pValue->ts);
break;
case TSDB_DATA_TYPE_UTINYINT:
n += tPutU8(p ? p + n : p, pValue->u8);
break;
case TSDB_DATA_TYPE_USMALLINT:
n += tPutU16(p ? p + n : p, pValue->u16);
break;
case TSDB_DATA_TYPE_UINT:
n += tPutU32(p ? p + n : p, pValue->u32);
break;
case TSDB_DATA_TYPE_UBIGINT:
n += tPutU64(p ? p + n : p, pValue->u64);
break;
default:
ASSERT(0);
}
} }
return n;
} }
int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type) { int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type) {
int32_t n = 0;
if (IS_VAR_DATA_TYPE(type)) { if (IS_VAR_DATA_TYPE(type)) {
n += tGetBinary(p, &pValue->pData, pValue ? &pValue->nData : NULL); return tGetBinary(p, &pValue->pData, pValue ? &pValue->nData : NULL);
} else { } else {
switch (type) { memcpy(&pValue->val, p, tDataTypes[type].bytes);
case TSDB_DATA_TYPE_BOOL: return tDataTypes[type].bytes;
n += tGetI8(p, &pValue->i8);
break;
case TSDB_DATA_TYPE_TINYINT:
n += tGetI8(p, &pValue->i8);
break;
case TSDB_DATA_TYPE_SMALLINT:
n += tGetI16(p, &pValue->i16);
break;
case TSDB_DATA_TYPE_INT:
n += tGetI32(p, &pValue->i32);
break;
case TSDB_DATA_TYPE_BIGINT:
n += tGetI64(p, &pValue->i64);
break;
case TSDB_DATA_TYPE_FLOAT:
n += tGetFloat(p, &pValue->f);
break;
case TSDB_DATA_TYPE_DOUBLE:
n += tGetDouble(p, &pValue->d);
break;
case TSDB_DATA_TYPE_TIMESTAMP:
n += tGetI64(p, &pValue->ts);
break;
case TSDB_DATA_TYPE_UTINYINT:
n += tGetU8(p, &pValue->u8);
break;
case TSDB_DATA_TYPE_USMALLINT:
n += tGetU16(p, &pValue->u16);
break;
case TSDB_DATA_TYPE_UINT:
n += tGetU32(p, &pValue->u32);
break;
case TSDB_DATA_TYPE_UBIGINT:
n += tGetU64(p, &pValue->u64);
break;
default:
ASSERT(0);
}
} }
return n;
} }
int tValueCmprFn(const SValue *pValue1, const SValue *pValue2, int8_t type) { int tValueCmprFn(const SValue *pValue1, const SValue *pValue2, int8_t type) {
@ -1219,288 +1135,397 @@ static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, SColVal *pColVa
_exit: _exit:
return code; return code;
} }
static FORCE_INLINE int32_t tColDataAppendValue0(SColData *pColData, SColVal *pColVal) { // 0 static FORCE_INLINE int32_t tColDataAppendValue00(SColData *pColData, SColVal *pColVal) {
int32_t code = 0; int32_t code = 0;
pColData->flag = HAS_VALUE;
if (COL_VAL_IS_NONE(pColVal)) {
pColData->flag = HAS_NONE;
} else if (COL_VAL_IS_NULL(pColVal)) {
pColData->flag = HAS_NULL;
} else {
pColData->flag = HAS_VALUE;
code = tColDataPutValue(pColData, pColVal);
if (code) goto _exit;
}
pColData->nVal++;
_exit:
return code;
}
static FORCE_INLINE int32_t tColDataAppendValue1(SColData *pColData, SColVal *pColVal) { // HAS_NONE
int32_t code = 0;
if (!COL_VAL_IS_NONE(pColVal)) {
int32_t nBit = BIT1_SIZE(pColData->nVal + 1);
code = tRealloc(&pColData->pBitMap, nBit);
if (code) goto _exit;
memset(pColData->pBitMap, 0, nBit);
SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
if (COL_VAL_IS_NULL(pColVal)) {
pColData->flag |= HAS_NULL;
} else {
pColData->flag |= HAS_VALUE;
if (pColData->nVal) {
if (IS_VAR_DATA_TYPE(pColData->type)) {
int32_t nOffset = sizeof(int32_t) * pColData->nVal;
code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset);
if (code) goto _exit;
memset(pColData->aOffset, 0, nOffset);
} else {
pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal;
code = tRealloc(&pColData->pData, pColData->nData);
if (code) goto _exit;
memset(pColData->pData, 0, pColData->nData);
}
}
code = tColDataPutValue(pColData, pColVal);
if (code) goto _exit;
}
}
pColData->nVal++;
_exit:
return code;
}
static FORCE_INLINE int32_t tColDataAppendValue2(SColData *pColData, SColVal *pColVal) { // HAS_NULL
int32_t code = 0;
if (!COL_VAL_IS_NULL(pColVal)) {
int32_t nBit = BIT1_SIZE(pColData->nVal + 1);
code = tRealloc(&pColData->pBitMap, nBit);
if (code) goto _exit;
if (COL_VAL_IS_NONE(pColVal)) {
pColData->flag |= HAS_NONE;
memset(pColData->pBitMap, 255, nBit);
SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
} else {
pColData->flag |= HAS_VALUE;
memset(pColData->pBitMap, 0, nBit);
SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
if (pColData->nVal) {
if (IS_VAR_DATA_TYPE(pColData->type)) {
int32_t nOffset = sizeof(int32_t) * pColData->nVal;
code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset);
if (code) goto _exit;
memset(pColData->aOffset, 0, nOffset);
} else {
pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal;
code = tRealloc(&pColData->pData, pColData->nData);
if (code) goto _exit;
memset(pColData->pData, 0, pColData->nData);
}
}
code = tColDataPutValue(pColData, pColVal);
if (code) goto _exit;
}
}
pColData->nVal++;
_exit:
return code;
}
static FORCE_INLINE int32_t tColDataAppendValue3(SColData *pColData, SColVal *pColVal) { // HAS_NULL|HAS_NONE
int32_t code = 0;
if (COL_VAL_IS_NONE(pColVal)) {
code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
if (code) goto _exit;
SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
} else if (COL_VAL_IS_NULL(pColVal)) {
code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
if (code) goto _exit;
SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
} else {
pColData->flag |= HAS_VALUE;
uint8_t *pBitMap = NULL;
code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1));
if (code) goto _exit;
for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal));
}
SET_BIT2(pBitMap, pColData->nVal, 2);
tFree(pColData->pBitMap);
pColData->pBitMap = pBitMap;
if (pColData->nVal) {
if (IS_VAR_DATA_TYPE(pColData->type)) {
int32_t nOffset = sizeof(int32_t) * pColData->nVal;
code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset);
if (code) goto _exit;
memset(pColData->aOffset, 0, nOffset);
} else {
pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal;
code = tRealloc(&pColData->pData, pColData->nData);
if (code) goto _exit;
memset(pColData->pData, 0, pColData->nData);
}
}
code = tColDataPutValue(pColData, pColVal);
if (code) goto _exit;
}
pColData->nVal++;
_exit:
return code;
}
static FORCE_INLINE int32_t tColDataAppendValue4(SColData *pColData, SColVal *pColVal) { // HAS_VALUE
int32_t code = 0;
if (!COL_VAL_IS_VALUE(pColVal)) {
if (COL_VAL_IS_NONE(pColVal)) {
pColData->flag |= HAS_NONE;
} else {
pColData->flag |= HAS_NULL;
}
int32_t nBit = BIT1_SIZE(pColData->nVal + 1);
code = tRealloc(&pColData->pBitMap, nBit);
if (code) goto _exit;
memset(pColData->pBitMap, 255, nBit);
SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
code = tColDataPutValue(pColData, pColVal);
if (code) goto _exit;
} else {
code = tColDataPutValue(pColData, pColVal);
if (code) goto _exit;
}
pColData->nVal++;
_exit:
return code;
}
static FORCE_INLINE int32_t tColDataAppendValue5(SColData *pColData, SColVal *pColVal) { // HAS_VALUE|HAS_NONE
int32_t code = 0;
if (COL_VAL_IS_NULL(pColVal)) {
pColData->flag |= HAS_NULL;
uint8_t *pBitMap = NULL;
code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1));
if (code) goto _exit;
for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 0);
}
SET_BIT2(pBitMap, pColData->nVal, 1);
tFree(pColData->pBitMap);
pColData->pBitMap = pBitMap;
} else {
code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
if (code) goto _exit;
if (COL_VAL_IS_NONE(pColVal)) {
SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
} else {
SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
}
}
code = tColDataPutValue(pColData, pColVal); code = tColDataPutValue(pColData, pColVal);
if (code) goto _exit; if (code) return code;
pColData->nVal++; pColData->nVal++;
_exit:
return code; return code;
} }
static FORCE_INLINE int32_t tColDataAppendValue6(SColData *pColData, SColVal *pColVal) { // HAS_VALUE|HAS_NULL static FORCE_INLINE int32_t tColDataAppendValue01(SColData *pColData, SColVal *pColVal) {
int32_t code = 0;
pColData->flag = HAS_NONE;
pColData->nVal++;
return code;
}
static FORCE_INLINE int32_t tColDataAppendValue02(SColData *pColData, SColVal *pColVal) {
int32_t code = 0;
pColData->flag = HAS_NULL;
pColData->nVal++;
return code;
}
static FORCE_INLINE int32_t tColDataAppendValue10(SColData *pColData, SColVal *pColVal) {
int32_t code = 0; int32_t code = 0;
if (COL_VAL_IS_NONE(pColVal)) { int32_t nBit = BIT1_SIZE(pColData->nVal + 1);
pColData->flag |= HAS_NONE; code = tRealloc(&pColData->pBitMap, nBit);
if (code) return code;
uint8_t *pBitMap = NULL; memset(pColData->pBitMap, 0, nBit);
code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1)); SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
if (code) goto _exit;
for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { pColData->flag |= HAS_VALUE;
SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 1);
}
SET_BIT2(pBitMap, pColData->nVal, 0);
tFree(pColData->pBitMap); if (pColData->nVal) {
pColData->pBitMap = pBitMap; if (IS_VAR_DATA_TYPE(pColData->type)) {
} else { int32_t nOffset = sizeof(int32_t) * pColData->nVal;
code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset);
if (code) goto _exit; if (code) return code;
memset(pColData->aOffset, 0, nOffset);
if (COL_VAL_IS_NULL(pColVal)) {
SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
} else { } else {
SET_BIT1(pColData->pBitMap, pColData->nVal, 1); pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal;
code = tRealloc(&pColData->pData, pColData->nData);
if (code) return code;
memset(pColData->pData, 0, pColData->nData);
} }
} }
code = tColDataPutValue(pColData, pColVal); code = tColDataPutValue(pColData, pColVal);
if (code) goto _exit; if (code) return code;
pColData->nVal++;
return code;
}
static FORCE_INLINE int32_t tColDataAppendValue11(SColData *pColData, SColVal *pColVal) {
int32_t code = 0;
pColData->nVal++;
return code;
}
static FORCE_INLINE int32_t tColDataAppendValue12(SColData *pColData, SColVal *pColVal) {
int32_t code = 0;
int32_t nBit = BIT1_SIZE(pColData->nVal + 1);
code = tRealloc(&pColData->pBitMap, nBit);
if (code) return code;
memset(pColData->pBitMap, 0, nBit);
SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
pColData->flag |= HAS_NULL;
pColData->nVal++;
return code;
}
static FORCE_INLINE int32_t tColDataAppendValue20(SColData *pColData, SColVal *pColVal) {
int32_t code = 0;
int32_t nBit = BIT1_SIZE(pColData->nVal + 1);
code = tRealloc(&pColData->pBitMap, nBit);
if (code) return code;
memset(pColData->pBitMap, 0, nBit);
SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
pColData->flag |= HAS_VALUE;
if (pColData->nVal) {
if (IS_VAR_DATA_TYPE(pColData->type)) {
int32_t nOffset = sizeof(int32_t) * pColData->nVal;
code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset);
if (code) return code;
memset(pColData->aOffset, 0, nOffset);
} else {
pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal;
code = tRealloc(&pColData->pData, pColData->nData);
if (code) return code;
memset(pColData->pData, 0, pColData->nData);
}
}
code = tColDataPutValue(pColData, pColVal);
if (code) return code;
pColData->nVal++;
return code;
}
static FORCE_INLINE int32_t tColDataAppendValue21(SColData *pColData, SColVal *pColVal) {
int32_t code = 0;
int32_t nBit = BIT1_SIZE(pColData->nVal + 1);
code = tRealloc(&pColData->pBitMap, nBit);
if (code) return code;
memset(pColData->pBitMap, 255, nBit);
SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
pColData->flag |= HAS_NONE;
pColData->nVal++;
return code;
}
static FORCE_INLINE int32_t tColDataAppendValue22(SColData *pColData, SColVal *pColVal) {
int32_t code = 0;
pColData->nVal++;
return code;
}
static FORCE_INLINE int32_t tColDataAppendValue30(SColData *pColData, SColVal *pColVal) {
int32_t code = 0;
pColData->flag |= HAS_VALUE;
uint8_t *pBitMap = NULL;
code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1));
if (code) return code;
for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal));
}
SET_BIT2(pBitMap, pColData->nVal, 2);
tFree(pColData->pBitMap);
pColData->pBitMap = pBitMap;
if (pColData->nVal) {
if (IS_VAR_DATA_TYPE(pColData->type)) {
int32_t nOffset = sizeof(int32_t) * pColData->nVal;
code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset);
if (code) return code;
memset(pColData->aOffset, 0, nOffset);
} else {
pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal;
code = tRealloc(&pColData->pData, pColData->nData);
if (code) return code;
memset(pColData->pData, 0, pColData->nData);
}
}
code = tColDataPutValue(pColData, pColVal);
if (code) return code;
pColData->nVal++;
return code;
}
static FORCE_INLINE int32_t tColDataAppendValue31(SColData *pColData, SColVal *pColVal) {
int32_t code = 0;
code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
if (code) return code;
SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
pColData->nVal++;
return code;
}
static FORCE_INLINE int32_t tColDataAppendValue32(SColData *pColData, SColVal *pColVal) {
int32_t code = 0;
code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
if (code) return code;
SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
pColData->nVal++;
return code;
}
static FORCE_INLINE int32_t tColDataAppendValue40(SColData *pColData, SColVal *pColVal) {
int32_t code = 0;
code = tColDataPutValue(pColData, pColVal);
if (code) return code;
pColData->nVal++;
return code;
}
static FORCE_INLINE int32_t tColDataAppendValue41(SColData *pColData, SColVal *pColVal) {
int32_t code = 0;
pColData->flag |= HAS_NONE;
int32_t nBit = BIT1_SIZE(pColData->nVal + 1);
code = tRealloc(&pColData->pBitMap, nBit);
if (code) return code;
memset(pColData->pBitMap, 255, nBit);
SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
code = tColDataPutValue(pColData, pColVal);
if (code) return code;
pColData->nVal++; pColData->nVal++;
_exit:
return code; return code;
} }
static FORCE_INLINE int32_t tColDataAppendValue7(SColData *pColData, static FORCE_INLINE int32_t tColDataAppendValue42(SColData *pColData, SColVal *pColVal) {
SColVal *pColVal) { // HAS_VALUE|HAS_NULL|HAS_NONE int32_t code = 0;
pColData->flag |= HAS_NULL;
int32_t nBit = BIT1_SIZE(pColData->nVal + 1);
code = tRealloc(&pColData->pBitMap, nBit);
if (code) return code;
memset(pColData->pBitMap, 255, nBit);
SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
code = tColDataPutValue(pColData, pColVal);
if (code) return code;
pColData->nVal++;
return code;
}
static FORCE_INLINE int32_t tColDataAppendValue50(SColData *pColData, SColVal *pColVal) {
int32_t code = 0;
code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
if (code) return code;
SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
code = tColDataPutValue(pColData, pColVal);
if (code) return code;
pColData->nVal++;
return code;
}
static FORCE_INLINE int32_t tColDataAppendValue51(SColData *pColData, SColVal *pColVal) {
int32_t code = 0;
code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
if (code) return code;
SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
code = tColDataPutValue(pColData, pColVal);
if (code) return code;
pColData->nVal++;
return code;
}
static FORCE_INLINE int32_t tColDataAppendValue52(SColData *pColData, SColVal *pColVal) {
int32_t code = 0;
pColData->flag |= HAS_NULL;
uint8_t *pBitMap = NULL;
code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1));
if (code) return code;
for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 0);
}
SET_BIT2(pBitMap, pColData->nVal, 1);
tFree(pColData->pBitMap);
pColData->pBitMap = pBitMap;
code = tColDataPutValue(pColData, pColVal);
if (code) return code;
pColData->nVal++;
return code;
}
static FORCE_INLINE int32_t tColDataAppendValue60(SColData *pColData, SColVal *pColVal) {
int32_t code = 0;
code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
if (code) return code;
SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
code = tColDataPutValue(pColData, pColVal);
if (code) return code;
pColData->nVal++;
return code;
}
static FORCE_INLINE int32_t tColDataAppendValue61(SColData *pColData, SColVal *pColVal) {
int32_t code = 0;
pColData->flag |= HAS_NONE;
uint8_t *pBitMap = NULL;
code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1));
if (code) return code;
for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 1);
}
SET_BIT2(pBitMap, pColData->nVal, 0);
tFree(pColData->pBitMap);
pColData->pBitMap = pBitMap;
code = tColDataPutValue(pColData, pColVal);
if (code) return code;
pColData->nVal++;
return code;
}
static FORCE_INLINE int32_t tColDataAppendValue62(SColData *pColData, SColVal *pColVal) {
int32_t code = 0;
code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
if (code) return code;
SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
code = tColDataPutValue(pColData, pColVal);
if (code) return code;
pColData->nVal++;
return code;
}
static FORCE_INLINE int32_t tColDataAppendValue70(SColData *pColData, SColVal *pColVal) {
int32_t code = 0; int32_t code = 0;
code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1)); code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1));
if (code) goto _exit; if (code) return code;
SET_BIT2(pColData->pBitMap, pColData->nVal, 2);
if (COL_VAL_IS_NONE(pColVal)) {
SET_BIT2(pColData->pBitMap, pColData->nVal, 0);
} else if (COL_VAL_IS_NULL(pColVal)) {
SET_BIT2(pColData->pBitMap, pColData->nVal, 1);
} else {
SET_BIT2(pColData->pBitMap, pColData->nVal, 2);
}
code = tColDataPutValue(pColData, pColVal); code = tColDataPutValue(pColData, pColVal);
if (code) goto _exit; if (code) return code;
pColData->nVal++; pColData->nVal++;
_exit:
return code; return code;
} }
static int32_t (*tColDataAppendValueImpl[])(SColData *pColData, SColVal *pColVal) = { static FORCE_INLINE int32_t tColDataAppendValue71(SColData *pColData, SColVal *pColVal) {
tColDataAppendValue0, // 0 int32_t code = 0;
tColDataAppendValue1, // HAS_NONE
tColDataAppendValue2, // HAS_NULL code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1));
tColDataAppendValue3, // HAS_NULL|HAS_NONE if (code) return code;
tColDataAppendValue4, // HAS_VALUE SET_BIT2(pColData->pBitMap, pColData->nVal, 0);
tColDataAppendValue5, // HAS_VALUE|HAS_NONE
tColDataAppendValue6, // HAS_VALUE|HAS_NULL code = tColDataPutValue(pColData, pColVal);
tColDataAppendValue7 // HAS_VALUE|HAS_NULL|HAS_NONE if (code) return code;
pColData->nVal++;
return code;
}
static FORCE_INLINE int32_t tColDataAppendValue72(SColData *pColData, SColVal *pColVal) {
int32_t code = 0;
code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1));
if (code) return code;
SET_BIT2(pColData->pBitMap, pColData->nVal, 1);
code = tColDataPutValue(pColData, pColVal);
if (code) return code;
pColData->nVal++;
return code;
}
static int32_t (*tColDataAppendValueImpl[8][3])(SColData *pColData, SColVal *pColVal) = {
{tColDataAppendValue00, tColDataAppendValue01, tColDataAppendValue02}, // 0
{tColDataAppendValue10, tColDataAppendValue11, tColDataAppendValue12}, // HAS_NONE
{tColDataAppendValue20, tColDataAppendValue21, tColDataAppendValue22}, // HAS_NULL
{tColDataAppendValue30, tColDataAppendValue31, tColDataAppendValue32}, // HAS_NULL|HAS_NONE
{tColDataAppendValue40, tColDataAppendValue41, tColDataAppendValue42}, // HAS_VALUE
{tColDataAppendValue50, tColDataAppendValue51, tColDataAppendValue52}, // HAS_VALUE|HAS_NONE
{tColDataAppendValue60, tColDataAppendValue61, tColDataAppendValue62}, // HAS_VALUE|HAS_NULL
{tColDataAppendValue70, tColDataAppendValue71, tColDataAppendValue72}, // HAS_VALUE|HAS_NULL|HAS_NONE
}; };
int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal) { int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal) {
ASSERT(pColData->cid == pColVal->cid && pColData->type == pColVal->type); ASSERT(pColData->cid == pColVal->cid && pColData->type == pColVal->type);
return tColDataAppendValueImpl[pColData->flag](pColData, pColVal); return tColDataAppendValueImpl[pColData->flag][pColVal->flag](pColData, pColVal);
} }
static FORCE_INLINE void tColDataGetValue1(SColData *pColData, int32_t iVal, SColVal *pColVal) { // HAS_NONE static FORCE_INLINE void tColDataGetValue1(SColData *pColData, int32_t iVal, SColVal *pColVal) { // HAS_NONE

2
source/common/src/trow.c
View File

@ -689,7 +689,7 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) {
memcpy(varDataVal(varBuf), pColVal->value.pData, pColVal->value.nData); memcpy(varDataVal(varBuf), pColVal->value.pData, pColVal->value.nData);
val = varBuf; val = varBuf;
} else { } else {
val = (const void *)&pColVal->value.i64; val = (const void *)&pColVal->value.val;
} }
} else { } else {
pColVal = NULL; pColVal = NULL;

2
source/common/test/dataformatTest.cpp
View File

@ -13,6 +13,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#if 0
#include <gtest/gtest.h> #include <gtest/gtest.h>
#include <taoserror.h> #include <taoserror.h>
@ -477,3 +478,4 @@ TEST(testCase, NoneTest) {
taosMemoryFree(pTSchema); taosMemoryFree(pTSchema);
} }
#endif #endif
#endif

1
source/dnode/mnode/impl/src/mndCluster.c
View File

@ -237,6 +237,7 @@ static int32_t mndCreateDefaultCluster(SMnode *pMnode) {
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, NULL, "create-cluster"); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, NULL, "create-cluster");
if (pTrans == NULL) { if (pTrans == NULL) {
sdbFreeRaw(pRaw);
mError("cluster:%" PRId64 ", failed to create since %s", clusterObj.id, terrstr()); mError("cluster:%" PRId64 ", failed to create since %s", clusterObj.id, terrstr());
return -1; return -1;
} }

8
source/dnode/mnode/impl/src/mndDb.c
View File

@ -1594,7 +1594,7 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb,
break; break;
} }
char precVstr[10] = {0}; char precVstr[10] = {0};
STR_WITH_SIZE_TO_VARSTR(precVstr, precStr, 2); STR_WITH_MAXSIZE_TO_VARSTR(precVstr, precStr, 10);
char *statusStr = "ready"; char *statusStr = "ready";
if (objStatus == SDB_STATUS_CREATING) { if (objStatus == SDB_STATUS_CREATING) {
@ -1607,7 +1607,7 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb,
} }
} }
char statusVstr[24] = {0}; char statusVstr[24] = {0};
STR_WITH_SIZE_TO_VARSTR(statusVstr, statusStr, strlen(statusStr)); STR_WITH_MAXSIZE_TO_VARSTR(statusVstr, statusStr, 24);
if (sysDb || !sysinfo) { if (sysDb || !sysinfo) {
for (int32_t i = 0; i < pShow->numOfColumns; ++i) { for (int32_t i = 0; i < pShow->numOfColumns; ++i) {
@ -1644,7 +1644,7 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb,
const char *strictStr = pDb->cfg.strict ? "on" : "off"; const char *strictStr = pDb->cfg.strict ? "on" : "off";
char strictVstr[24] = {0}; char strictVstr[24] = {0};
STR_WITH_SIZE_TO_VARSTR(strictVstr, strictStr, strlen(strictStr)); STR_WITH_MAXSIZE_TO_VARSTR(strictVstr, strictStr, 24);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, rows, (const char *)strictVstr, false); colDataAppend(pColInfo, rows, (const char *)strictVstr, false);
@ -1704,7 +1704,7 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb,
const char *cacheModelStr = getCacheModelStr(pDb->cfg.cacheLast); const char *cacheModelStr = getCacheModelStr(pDb->cfg.cacheLast);
char cacheModelVstr[24] = {0}; char cacheModelVstr[24] = {0};
STR_WITH_SIZE_TO_VARSTR(cacheModelVstr, cacheModelStr, strlen(cacheModelStr)); STR_WITH_MAXSIZE_TO_VARSTR(cacheModelVstr, cacheModelStr, 24);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, rows, (const char *)cacheModelVstr, false); colDataAppend(pColInfo, rows, (const char *)cacheModelVstr, false);

20
source/dnode/mnode/impl/src/mndDef.c
View File

@ -42,9 +42,23 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) {
if (tEncodeI64(pEncoder, pObj->targetStbUid) < 0) return -1; if (tEncodeI64(pEncoder, pObj->targetStbUid) < 0) return -1;
if (tEncodeI32(pEncoder, pObj->fixedSinkVgId) < 0) return -1; if (tEncodeI32(pEncoder, pObj->fixedSinkVgId) < 0) return -1;
if (tEncodeCStr(pEncoder, pObj->sql) < 0) return -1; if (pObj->sql != NULL) {
if (tEncodeCStr(pEncoder, pObj->ast) < 0) return -1; if (tEncodeCStr(pEncoder, pObj->sql) < 0) return -1;
if (tEncodeCStr(pEncoder, pObj->physicalPlan) < 0) return -1; } else {
if (tEncodeCStr(pEncoder, "") < 0) return -1;
}
if (pObj->ast != NULL) {
if (tEncodeCStr(pEncoder, pObj->ast) < 0) return -1;
} else {
if (tEncodeCStr(pEncoder, "") < 0) return -1;
}
if (pObj->physicalPlan != NULL) {
if (tEncodeCStr(pEncoder, pObj->physicalPlan) < 0) return -1;
} else {
if (tEncodeCStr(pEncoder, "") < 0) return -1;
}
int32_t sz = taosArrayGetSize(pObj->tasks); int32_t sz = taosArrayGetSize(pObj->tasks);
if (tEncodeI32(pEncoder, sz) < 0) return -1; if (tEncodeI32(pEncoder, sz) < 0) return -1;

14
source/dnode/mnode/impl/src/mndDnode.c
View File

@ -102,7 +102,7 @@ static int32_t mndCreateDefaultDnode(SMnode *pMnode) {
dnodeObj.updateTime = dnodeObj.createdTime; dnodeObj.updateTime = dnodeObj.createdTime;
dnodeObj.port = tsServerPort; dnodeObj.port = tsServerPort;
memcpy(&dnodeObj.fqdn, tsLocalFqdn, TSDB_FQDN_LEN); memcpy(&dnodeObj.fqdn, tsLocalFqdn, TSDB_FQDN_LEN);
snprintf(dnodeObj.ep, TSDB_EP_LEN, "%s:%u", dnodeObj.fqdn, dnodeObj.port); snprintf(dnodeObj.ep, TSDB_EP_LEN - 1, "%s:%u", dnodeObj.fqdn, dnodeObj.port);
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, NULL, "create-dnode"); pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, NULL, "create-dnode");
if (pTrans == NULL) goto _OVER; if (pTrans == NULL) goto _OVER;
@ -190,7 +190,7 @@ _OVER:
static int32_t mndDnodeActionInsert(SSdb *pSdb, SDnodeObj *pDnode) { static int32_t mndDnodeActionInsert(SSdb *pSdb, SDnodeObj *pDnode) {
mTrace("dnode:%d, perform insert action, row:%p", pDnode->id, pDnode); mTrace("dnode:%d, perform insert action, row:%p", pDnode->id, pDnode);
pDnode->offlineReason = DND_REASON_STATUS_NOT_RECEIVED; pDnode->offlineReason = DND_REASON_STATUS_NOT_RECEIVED;
snprintf(pDnode->ep, TSDB_EP_LEN, "%s:%u", pDnode->fqdn, pDnode->port); snprintf(pDnode->ep, TSDB_EP_LEN - 1, "%s:%u", pDnode->fqdn, pDnode->port);
return 0; return 0;
} }
@ -253,7 +253,7 @@ int32_t mndGetDnodeSize(SMnode *pMnode) {
bool mndIsDnodeOnline(SDnodeObj *pDnode, int64_t curMs) { bool mndIsDnodeOnline(SDnodeObj *pDnode, int64_t curMs) {
int64_t interval = TABS(pDnode->lastAccessTime - curMs); int64_t interval = TABS(pDnode->lastAccessTime - curMs);
if (interval > 5000 * tsStatusInterval) { if (interval > 5000 * (int64_t)tsStatusInterval) {
if (pDnode->rebootTime > 0) { if (pDnode->rebootTime > 0) {
pDnode->offlineReason = DND_REASON_STATUS_MSG_TIMEOUT; pDnode->offlineReason = DND_REASON_STATUS_MSG_TIMEOUT;
} }
@ -486,7 +486,7 @@ static int32_t mndCreateDnode(SMnode *pMnode, SRpcMsg *pReq, SCreateDnodeReq *pC
dnodeObj.updateTime = dnodeObj.createdTime; dnodeObj.updateTime = dnodeObj.createdTime;
dnodeObj.port = pCreate->port; dnodeObj.port = pCreate->port;
memcpy(dnodeObj.fqdn, pCreate->fqdn, TSDB_FQDN_LEN); memcpy(dnodeObj.fqdn, pCreate->fqdn, TSDB_FQDN_LEN);
snprintf(dnodeObj.ep, TSDB_EP_LEN, "%s:%u", dnodeObj.fqdn, dnodeObj.port); snprintf(dnodeObj.ep, TSDB_EP_LEN - 1, "%s:%u", dnodeObj.fqdn, dnodeObj.port);
pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_GLOBAL, pReq, "create-dnode"); pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_GLOBAL, pReq, "create-dnode");
if (pTrans == NULL) goto _OVER; if (pTrans == NULL) goto _OVER;
@ -673,12 +673,14 @@ static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SM
mInfo("trans:%d, used to drop dnode:%d", pTrans->id, pDnode->id); mInfo("trans:%d, used to drop dnode:%d", pTrans->id, pDnode->id);
pRaw = mndDnodeActionEncode(pDnode); pRaw = mndDnodeActionEncode(pDnode);
if (pRaw == NULL || mndTransAppendRedolog(pTrans, pRaw) != 0) goto _OVER; if (pRaw == NULL) goto _OVER;
if (mndTransAppendRedolog(pTrans, pRaw) != 0) goto _OVER;
sdbSetRawStatus(pRaw, SDB_STATUS_DROPPING); sdbSetRawStatus(pRaw, SDB_STATUS_DROPPING);
pRaw = NULL; pRaw = NULL;
pRaw = mndDnodeActionEncode(pDnode); pRaw = mndDnodeActionEncode(pDnode);
if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER; if (pRaw == NULL) goto _OVER;
if (mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER;
sdbSetRawStatus(pRaw, SDB_STATUS_DROPPED); sdbSetRawStatus(pRaw, SDB_STATUS_DROPPED);
pRaw = NULL; pRaw = NULL;

9
source/dnode/mnode/impl/src/mndFunc.c
View File

@ -257,15 +257,18 @@ static int32_t mndDropFunc(SMnode *pMnode, SRpcMsg *pReq, SFuncObj *pFunc) {
mInfo("trans:%d, used to drop user:%s", pTrans->id, pFunc->name); mInfo("trans:%d, used to drop user:%s", pTrans->id, pFunc->name);
SSdbRaw *pRedoRaw = mndFuncActionEncode(pFunc); SSdbRaw *pRedoRaw = mndFuncActionEncode(pFunc);
if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) goto _OVER; if (pRedoRaw == NULL) goto _OVER;
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) goto _OVER;
sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING); sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING);
SSdbRaw *pUndoRaw = mndFuncActionEncode(pFunc); SSdbRaw *pUndoRaw = mndFuncActionEncode(pFunc);
if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) goto _OVER; if (pUndoRaw == NULL) goto _OVER;
if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) goto _OVER;
sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY); sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY);
SSdbRaw *pCommitRaw = mndFuncActionEncode(pFunc); SSdbRaw *pCommitRaw = mndFuncActionEncode(pFunc);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) goto _OVER; if (pCommitRaw == NULL) goto _OVER;
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) goto _OVER;
sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED);
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;

28
source/dnode/mnode/impl/src/mndGrant.c
View File

@ -30,85 +30,85 @@ static int32_t mndRetrieveGrant(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl
cols = 0; cols = 0;
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols); SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
const char *src = "community"; const char *src = "community";
STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32);
colDataAppend(pColInfo, numOfRows, tmp, false); colDataAppend(pColInfo, numOfRows, tmp, false);
cols++; cols++;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols); pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
src = "unlimited"; src = "unlimited";
STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32);
colDataAppend(pColInfo, numOfRows, tmp, false); colDataAppend(pColInfo, numOfRows, tmp, false);
cols++; cols++;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols); pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
src = "false"; src = "false";
STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32);
colDataAppend(pColInfo, numOfRows, tmp, false); colDataAppend(pColInfo, numOfRows, tmp, false);
cols++; cols++;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols); pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
src = "unlimited"; src = "unlimited";
STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32);
colDataAppend(pColInfo, numOfRows, tmp, false); colDataAppend(pColInfo, numOfRows, tmp, false);
cols++; cols++;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols); pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
src = "unlimited"; src = "unlimited";
STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32);
colDataAppend(pColInfo, numOfRows, tmp, false); colDataAppend(pColInfo, numOfRows, tmp, false);
cols++; cols++;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols); pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
src = "unlimited"; src = "unlimited";
STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32);
colDataAppend(pColInfo, numOfRows, tmp, false); colDataAppend(pColInfo, numOfRows, tmp, false);
cols++; cols++;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols); pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
src = "unlimited"; src = "unlimited";
STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32);
colDataAppend(pColInfo, numOfRows, tmp, false); colDataAppend(pColInfo, numOfRows, tmp, false);
cols++; cols++;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols); pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
src = "unlimited"; src = "unlimited";
STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32);
colDataAppend(pColInfo, numOfRows, tmp, false); colDataAppend(pColInfo, numOfRows, tmp, false);
cols++; cols++;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols); pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
src = "unlimited"; src = "unlimited";
STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32);
colDataAppend(pColInfo, numOfRows, tmp, false); colDataAppend(pColInfo, numOfRows, tmp, false);
cols++; cols++;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols); pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
src = "unlimited"; src = "unlimited";
STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32);
colDataAppend(pColInfo, numOfRows, tmp, false); colDataAppend(pColInfo, numOfRows, tmp, false);
cols++; cols++;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols); pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
src = "unlimited"; src = "unlimited";
STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32);
colDataAppend(pColInfo, numOfRows, tmp, false); colDataAppend(pColInfo, numOfRows, tmp, false);
cols++; cols++;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols); pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
src = "unlimited"; src = "unlimited";
STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32);
colDataAppend(pColInfo, numOfRows, tmp, false); colDataAppend(pColInfo, numOfRows, tmp, false);
cols++; cols++;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols); pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
src = "unlimited"; src = "unlimited";
STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32);
colDataAppend(pColInfo, numOfRows, tmp, false); colDataAppend(pColInfo, numOfRows, tmp, false);
cols++; cols++;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols); pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
src = "unlimited"; src = "unlimited";
STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32);
colDataAppend(pColInfo, numOfRows, tmp, false); colDataAppend(pColInfo, numOfRows, tmp, false);
numOfRows++; numOfRows++;

2
source/dnode/mnode/impl/src/mndMain.c
View File

@ -649,7 +649,7 @@ int32_t mndProcessRpcMsg(SRpcMsg *pMsg) {
void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp) { void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp) {
tmsg_t type = TMSG_INDEX(msgType); tmsg_t type = TMSG_INDEX(msgType);
if (type >= 0 && type < TDMT_MAX) { if (type < TDMT_MAX) {
pMnode->msgFp[type] = fp; pMnode->msgFp[type] = fp;
} }
} }

11
source/dnode/mnode/impl/src/mndMnode.c
View File

@ -93,6 +93,7 @@ static int32_t mndCreateDefaultMnode(SMnode *pMnode) {
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, NULL, "create-mnode"); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, NULL, "create-mnode");
if (pTrans == NULL) { if (pTrans == NULL) {
sdbFreeRaw(pRaw);
mError("mnode:%d, failed to create since %s", mnodeObj.id, terrstr()); mError("mnode:%d, failed to create since %s", mnodeObj.id, terrstr());
return -1; return -1;
} }
@ -220,8 +221,12 @@ bool mndIsMnode(SMnode *pMnode, int32_t dnodeId) {
void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) { void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) {
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
int32_t totalMnodes = sdbGetSize(pSdb, SDB_MNODE); int32_t totalMnodes = sdbGetSize(pSdb, SDB_MNODE);
void *pIter = NULL; if (totalMnodes == 0) {
syncGetRetryEpSet(pMnode->syncMgmt.sync, pEpSet);
return;
}
void *pIter = NULL;
while (1) { while (1) {
SMnodeObj *pObj = NULL; SMnodeObj *pObj = NULL;
pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pObj); pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pObj);
@ -658,7 +663,7 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
colDataAppend(pColInfo, numOfRows, (const char *)&pObj->id, false); colDataAppend(pColInfo, numOfRows, (const char *)&pObj->id, false);
char b1[TSDB_EP_LEN + VARSTR_HEADER_SIZE] = {0}; char b1[TSDB_EP_LEN + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(b1, pObj->pDnode->ep, pShow->pMeta->pSchemas[cols].bytes); STR_WITH_MAXSIZE_TO_VARSTR(b1, pObj->pDnode->ep, TSDB_EP_LEN + VARSTR_HEADER_SIZE);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, b1, false); colDataAppend(pColInfo, numOfRows, b1, false);
@ -667,7 +672,7 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
if (pObj->id == pMnode->selfDnodeId) { if (pObj->id == pMnode->selfDnodeId) {
roles = syncStr(TAOS_SYNC_STATE_LEADER); roles = syncStr(TAOS_SYNC_STATE_LEADER);
} }
if (pObj->pDnode && mndIsDnodeOnline(pObj->pDnode, curMs)) { if (mndIsDnodeOnline(pObj->pDnode, curMs)) {
roles = syncStr(pObj->state); roles = syncStr(pObj->state);
if (pObj->state == TAOS_SYNC_STATE_LEADER && pObj->id != pMnode->selfDnodeId) { if (pObj->state == TAOS_SYNC_STATE_LEADER && pObj->id != pMnode->selfDnodeId) {
roles = syncStr(TAOS_SYNC_STATE_ERROR); roles = syncStr(TAOS_SYNC_STATE_ERROR);

6
source/dnode/mnode/impl/src/mndPerfSchema.c
View File

@ -26,7 +26,7 @@ int32_t mndInitPerfsTableSchema(const SSysDbTableSchema *pSrc, int32_t colNum, S
} }
for (int32_t i = 0; i < colNum; ++i) { for (int32_t i = 0; i < colNum; ++i) {
strcpy(schema[i].name, pSrc[i].name); tstrncpy(schema[i].name, pSrc[i].name, sizeof(schema[i].name));
schema[i].type = pSrc[i].type; schema[i].type = pSrc[i].type;
schema[i].colId = i + 1; schema[i].colId = i + 1;
@ -40,7 +40,7 @@ int32_t mndInitPerfsTableSchema(const SSysDbTableSchema *pSrc, int32_t colNum, S
int32_t mndPerfsInitMeta(SHashObj *hash) { int32_t mndPerfsInitMeta(SHashObj *hash) {
STableMetaRsp meta = {0}; STableMetaRsp meta = {0};
strcpy(meta.dbFName, TSDB_INFORMATION_SCHEMA_DB); tstrncpy(meta.dbFName, TSDB_INFORMATION_SCHEMA_DB, sizeof(meta.dbFName));
meta.tableType = TSDB_SYSTEM_TABLE; meta.tableType = TSDB_SYSTEM_TABLE;
meta.sversion = 1; meta.sversion = 1;
meta.tversion = 1; meta.tversion = 1;
@ -50,7 +50,7 @@ int32_t mndPerfsInitMeta(SHashObj *hash) {
getPerfDbMeta(&pSysDbTableMeta, &size); getPerfDbMeta(&pSysDbTableMeta, &size);
for (int32_t i = 0; i < size; ++i) { for (int32_t i = 0; i < size; ++i) {
strcpy(meta.tbName, pSysDbTableMeta[i].name); tstrncpy(meta.tbName, pSysDbTableMeta[i].name, sizeof(meta.tbName));
meta.numOfColumns = pSysDbTableMeta[i].colNum; meta.numOfColumns = pSysDbTableMeta[i].colNum;
if (mndInitPerfsTableSchema(pSysDbTableMeta[i].schema, pSysDbTableMeta[i].colNum, &meta.pSchemas)) { if (mndInitPerfsTableSchema(pSysDbTableMeta[i].schema, pSysDbTableMeta[i].colNum, &meta.pSchemas)) {

4
source/dnode/mnode/impl/src/mndProfile.c
View File

@ -664,7 +664,7 @@ static int32_t mndProcessKillConnReq(SRpcMsg *pReq) {
static int32_t mndProcessSvrVerReq(SRpcMsg *pReq) { static int32_t mndProcessSvrVerReq(SRpcMsg *pReq) {
int32_t code = -1; int32_t code = -1;
SServerVerRsp rsp = {0}; SServerVerRsp rsp = {0};
strcpy(rsp.ver, version); tstrncpy(rsp.ver, version, sizeof(rsp.ver));
int32_t contLen = tSerializeSServerVerRsp(NULL, 0, &rsp); int32_t contLen = tSerializeSServerVerRsp(NULL, 0, &rsp);
if (contLen < 0) goto _over; if (contLen < 0) goto _over;
@ -702,7 +702,7 @@ static int32_t mndRetrieveConns(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl
break; break;
} }
if ((taosGetTimestampMs() - pConn->lastAccessTimeMs) > (keepTime * 1000)) { if ((taosGetTimestampMs() - pConn->lastAccessTimeMs) > ((int64_t)keepTime * 1000)) {
continue; continue;
} }

22
source/dnode/mnode/impl/src/mndStb.c
View File

@ -554,7 +554,7 @@ int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate) {
for (int32_t i = 0; i < pCreate->numOfColumns; ++i) { for (int32_t i = 0; i < pCreate->numOfColumns; ++i) {
SField *pField1 = taosArrayGet(pCreate->pColumns, i); SField *pField1 = taosArrayGet(pCreate->pColumns, i);
if (pField1->type < 0) { if (pField1->type >= TSDB_DATA_TYPE_MAX) {
terrno = TSDB_CODE_MND_INVALID_STB_OPTION; terrno = TSDB_CODE_MND_INVALID_STB_OPTION;
return -1; return -1;
} }
@ -570,7 +570,7 @@ int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate) {
for (int32_t i = 0; i < pCreate->numOfTags; ++i) { for (int32_t i = 0; i < pCreate->numOfTags; ++i) {
SField *pField1 = taosArrayGet(pCreate->pTags, i); SField *pField1 = taosArrayGet(pCreate->pTags, i);
if (pField1->type < 0) { if (pField1->type >= TSDB_DATA_TYPE_MAX) {
terrno = TSDB_CODE_MND_INVALID_STB_OPTION; terrno = TSDB_CODE_MND_INVALID_STB_OPTION;
return -1; return -1;
} }
@ -982,8 +982,8 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) {
goto _OVER; goto _OVER;
} }
} else { } else {
mError("stb:%s, already exist while create, input tagVer:%d colVer:%d is invalid", createReq.name, mError("stb:%s, already exist while create, input tagVer:%d colVer:%d is invalid, origin tagVer:%d colVer:%d",
createReq.tagVer, createReq.colVer, pStb->tagVer, pStb->colVer); createReq.name, createReq.tagVer, createReq.colVer, pStb->tagVer, pStb->colVer);
terrno = TSDB_CODE_MND_INVALID_SCHEMA_VER; terrno = TSDB_CODE_MND_INVALID_SCHEMA_VER;
goto _OVER; goto _OVER;
} }
@ -1603,9 +1603,9 @@ static int32_t mndBuildStbSchemaImp(SDbObj *pDb, SStbObj *pStb, const char *tbNa
return -1; return -1;
} }
strcpy(pRsp->dbFName, pStb->db); tstrncpy(pRsp->dbFName, pStb->db, sizeof(pRsp->dbFName));
strcpy(pRsp->tbName, tbName); tstrncpy(pRsp->tbName, tbName, sizeof(pRsp->tbName));
strcpy(pRsp->stbName, tbName); tstrncpy(pRsp->stbName, tbName, sizeof(pRsp->stbName));
pRsp->dbId = pDb->uid; pRsp->dbId = pDb->uid;
pRsp->numOfTags = pStb->numOfTags; pRsp->numOfTags = pStb->numOfTags;
pRsp->numOfColumns = pStb->numOfColumns; pRsp->numOfColumns = pStb->numOfColumns;
@ -1649,9 +1649,9 @@ static int32_t mndBuildStbCfgImp(SDbObj *pDb, SStbObj *pStb, const char *tbName,
return -1; return -1;
} }
strcpy(pRsp->dbFName, pStb->db); tstrncpy(pRsp->dbFName, pStb->db, sizeof(pRsp->dbFName));
strcpy(pRsp->tbName, tbName); tstrncpy(pRsp->tbName, tbName, sizeof(pRsp->tbName));
strcpy(pRsp->stbName, tbName); tstrncpy(pRsp->stbName, tbName, sizeof(pRsp->stbName));
pRsp->numOfTags = pStb->numOfTags; pRsp->numOfTags = pStb->numOfTags;
pRsp->numOfColumns = pStb->numOfColumns; pRsp->numOfColumns = pStb->numOfColumns;
pRsp->tableType = TSDB_SUPER_TABLE; pRsp->tableType = TSDB_SUPER_TABLE;
@ -2551,7 +2551,7 @@ static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)maxDelay, false); colDataAppend(pColInfo, numOfRows, (const char *)maxDelay, false);
char rollup[128 + VARSTR_HEADER_SIZE] = {0}; char rollup[160 + VARSTR_HEADER_SIZE] = {0};
int32_t rollupNum = (int32_t)taosArrayGetSize(pStb->pFuncs); int32_t rollupNum = (int32_t)taosArrayGetSize(pStb->pFuncs);
for (int32_t i = 0; i < rollupNum; ++i) { for (int32_t i = 0; i < rollupNum; ++i) {
char *funcName = taosArrayGet(pStb->pFuncs, i); char *funcName = taosArrayGet(pStb->pFuncs, i);

37
source/dnode/mnode/impl/src/mndStream.c
View File

@ -425,8 +425,10 @@ static int32_t mndSetStreamRecover(SMnode *pMnode, STrans *pTrans, const SStream
SStreamObj streamObj = {0}; SStreamObj streamObj = {0};
memcpy(streamObj.name, pStream->name, TSDB_STREAM_FNAME_LEN); memcpy(streamObj.name, pStream->name, TSDB_STREAM_FNAME_LEN);
streamObj.status = STREAM_STATUS__RECOVER; streamObj.status = STREAM_STATUS__RECOVER;
SSdbRaw *pCommitRaw = mndStreamActionEncode(&streamObj); SSdbRaw *pCommitRaw = mndStreamActionEncode(&streamObj);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { if (pCommitRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("stream trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); mError("stream trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return -1;
@ -771,12 +773,14 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
if (mndDropStreamTasks(pMnode, pTrans, pStream) < 0) { if (mndDropStreamTasks(pMnode, pTrans, pStream) < 0) {
mError("stream:%s, failed to drop task since %s", dropReq.name, terrstr()); mError("stream:%s, failed to drop task since %s", dropReq.name, terrstr());
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans);
return -1; return -1;
} }
// drop stream // drop stream
if (mndPersistDropStreamLog(pMnode, pTrans, pStream) < 0) { if (mndPersistDropStreamLog(pMnode, pTrans, pStream) < 0) {
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans);
return -1; return -1;
} }
@ -945,10 +949,8 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
SName n; SName n;
int32_t cols = 0; int32_t cols = 0;
char streamName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; char streamName[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
tNameFromString(&n, pStream->name, T_NAME_ACCT | T_NAME_DB); STR_WITH_MAXSIZE_TO_VARSTR(streamName, mndGetDbStr(pStream->name), sizeof(streamName));
tNameGetDbName(&n, varDataVal(streamName));
varDataSetLen(streamName, strlen(varDataVal(streamName)));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)streamName, false); colDataAppend(pColInfo, numOfRows, (const char *)streamName, false);
@ -956,28 +958,24 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
colDataAppend(pColInfo, numOfRows, (const char *)&pStream->createTime, false); colDataAppend(pColInfo, numOfRows, (const char *)&pStream->createTime, false);
char sql[TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE] = {0}; char sql[TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE] = {0};
tstrncpy(&sql[VARSTR_HEADER_SIZE], pStream->sql, TSDB_SHOW_SQL_LEN); STR_WITH_MAXSIZE_TO_VARSTR(sql, pStream->sql, sizeof(sql));
varDataSetLen(sql, strlen(&sql[VARSTR_HEADER_SIZE]));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)sql, false); colDataAppend(pColInfo, numOfRows, (const char *)sql, false);
char status[20 + VARSTR_HEADER_SIZE] = {0}; char status[20 + VARSTR_HEADER_SIZE] = {0};
mndShowStreamStatus(&status[VARSTR_HEADER_SIZE], pStream); char status2[20] = {0};
varDataSetLen(status, strlen(varDataVal(status))); mndShowStreamStatus(status2, pStream);
STR_WITH_MAXSIZE_TO_VARSTR(status, status2, sizeof(status));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)&status, false); colDataAppend(pColInfo, numOfRows, (const char *)&status, false);
char sourceDB[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; char sourceDB[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
tNameFromString(&n, pStream->sourceDb, T_NAME_ACCT | T_NAME_DB); STR_WITH_MAXSIZE_TO_VARSTR(sourceDB, mndGetDbStr(pStream->sourceDb), sizeof(sourceDB));
tNameGetDbName(&n, varDataVal(sourceDB));
varDataSetLen(sourceDB, strlen(varDataVal(sourceDB)));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)&sourceDB, false); colDataAppend(pColInfo, numOfRows, (const char *)&sourceDB, false);
char targetDB[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; char targetDB[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
tNameFromString(&n, pStream->targetDb, T_NAME_ACCT | T_NAME_DB); STR_WITH_MAXSIZE_TO_VARSTR(targetDB, mndGetDbStr(pStream->targetDb), sizeof(targetDB));
tNameGetDbName(&n, varDataVal(targetDB));
varDataSetLen(targetDB, strlen(varDataVal(targetDB)));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)&targetDB, false); colDataAppend(pColInfo, numOfRows, (const char *)&targetDB, false);
@ -986,9 +984,7 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
colDataAppend(pColInfo, numOfRows, NULL, true); colDataAppend(pColInfo, numOfRows, NULL, true);
} else { } else {
char targetSTB[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; char targetSTB[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
tNameFromString(&n, pStream->targetSTbName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); STR_WITH_MAXSIZE_TO_VARSTR(targetSTB, mndGetStbStr(pStream->targetSTbName), sizeof(targetSTB));
strcpy(&targetSTB[VARSTR_HEADER_SIZE], tNameGetTableName(&n));
varDataSetLen(targetSTB, strlen(varDataVal(targetSTB)));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)&targetSTB, false); colDataAppend(pColInfo, numOfRows, (const char *)&targetSTB, false);
} }
@ -997,8 +993,9 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
colDataAppend(pColInfo, numOfRows, (const char *)&pStream->watermark, false); colDataAppend(pColInfo, numOfRows, (const char *)&pStream->watermark, false);
char trigger[20 + VARSTR_HEADER_SIZE] = {0}; char trigger[20 + VARSTR_HEADER_SIZE] = {0};
mndShowStreamTrigger(&trigger[VARSTR_HEADER_SIZE], pStream); char trigger2[20] = {0};
varDataSetLen(trigger, strlen(varDataVal(trigger))); mndShowStreamTrigger(trigger2, pStream);
STR_WITH_MAXSIZE_TO_VARSTR(trigger, trigger2, sizeof(trigger));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)&trigger, false); colDataAppend(pColInfo, numOfRows, (const char *)&trigger, false);

2
source/dnode/mnode/impl/src/mndSubscribe.c
View File

@ -123,7 +123,7 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const SMqSubscri
req.subType = pSub->subType; req.subType = pSub->subType;
req.withMeta = pSub->withMeta; req.withMeta = pSub->withMeta;
req.suid = pSub->stbUid; req.suid = pSub->stbUid;
strncpy(req.subKey, pSub->key, TSDB_SUBSCRIBE_KEY_LEN); tstrncpy(req.subKey, pSub->key, TSDB_SUBSCRIBE_KEY_LEN);
int32_t tlen = sizeof(SMsgHead) + tEncodeSMqRebVgReq(NULL, &req); int32_t tlen = sizeof(SMsgHead) + tEncodeSMqRebVgReq(NULL, &req);
void *buf = taosMemoryMalloc(tlen); void *buf = taosMemoryMalloc(tlen);

2
source/dnode/vnode/inc/vnode.h
View File

@ -159,7 +159,7 @@ uint64_t getReaderMaxVersion(STsdbReader *pReader);
int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, SArray *pTableIdList, int32_t numOfCols, void **pReader); int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, SArray *pTableIdList, int32_t numOfCols, void **pReader);
int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, SArray *pTableUids); int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, SArray *pTableUids);
int32_t tsdbCacherowsReaderClose(void *pReader); void* tsdbCacherowsReaderClose(void *pReader);
int32_t tsdbGetTableSchema(SVnode *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid); int32_t tsdbGetTableSchema(SVnode *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid);
void tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity); void tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity);

127
source/dnode/vnode/src/inc/tsdb.h
View File

@ -38,39 +38,43 @@ extern "C" {
goto LABEL; \ goto LABEL; \
} }
typedef struct TSDBROW TSDBROW; typedef struct TSDBROW TSDBROW;
typedef struct TABLEID TABLEID; typedef struct TABLEID TABLEID;
typedef struct TSDBKEY TSDBKEY; typedef struct TSDBKEY TSDBKEY;
typedef struct SDelData SDelData; typedef struct SDelData SDelData;
typedef struct SDelIdx SDelIdx; typedef struct SDelIdx SDelIdx;
typedef struct STbData STbData; typedef struct STbData STbData;
typedef struct SMemTable SMemTable; typedef struct SMemTable SMemTable;
typedef struct STbDataIter STbDataIter; typedef struct STbDataIter STbDataIter;
typedef struct SMapData SMapData; typedef struct SMapData SMapData;
typedef struct SBlockIdx SBlockIdx; typedef struct SBlockIdx SBlockIdx;
typedef struct SDataBlk SDataBlk; typedef struct SDataBlk SDataBlk;
typedef struct SSttBlk SSttBlk; typedef struct SSttBlk SSttBlk;
typedef struct SDiskDataHdr SDiskDataHdr; typedef struct SDiskDataHdr SDiskDataHdr;
typedef struct SBlockData SBlockData; typedef struct SBlockData SBlockData;
typedef struct SDelFile SDelFile; typedef struct SDelFile SDelFile;
typedef struct SHeadFile SHeadFile; typedef struct SHeadFile SHeadFile;
typedef struct SDataFile SDataFile; typedef struct SDataFile SDataFile;
typedef struct SSttFile SSttFile; typedef struct SSttFile SSttFile;
typedef struct SSmaFile SSmaFile; typedef struct SSmaFile SSmaFile;
typedef struct SDFileSet SDFileSet; typedef struct SDFileSet SDFileSet;
typedef struct SDataFWriter SDataFWriter; typedef struct SDataFWriter SDataFWriter;
typedef struct SDataFReader SDataFReader; typedef struct SDataFReader SDataFReader;
typedef struct SDelFWriter SDelFWriter; typedef struct SDelFWriter SDelFWriter;
typedef struct SDelFReader SDelFReader; typedef struct SDelFReader SDelFReader;
typedef struct SRowIter SRowIter; typedef struct SRowIter SRowIter;
typedef struct STsdbFS STsdbFS; typedef struct STsdbFS STsdbFS;
typedef struct SRowMerger SRowMerger; typedef struct SRowMerger SRowMerger;
typedef struct STsdbReadSnap STsdbReadSnap; typedef struct STsdbReadSnap STsdbReadSnap;
typedef struct SBlockInfo SBlockInfo; typedef struct SBlockInfo SBlockInfo;
typedef struct SSmaInfo SSmaInfo; typedef struct SSmaInfo SSmaInfo;
typedef struct SBlockCol SBlockCol; typedef struct SBlockCol SBlockCol;
typedef struct SVersionRange SVersionRange; typedef struct SVersionRange SVersionRange;
typedef struct SLDataIter SLDataIter; typedef struct SLDataIter SLDataIter;
typedef struct SDiskCol SDiskCol;
typedef struct SDiskData SDiskData;
typedef struct SDiskDataBuilder SDiskDataBuilder;
typedef struct SBlkInfo SBlkInfo;
#define TSDB_FILE_DLMT ((uint32_t)0xF00AFA0F) #define TSDB_FILE_DLMT ((uint32_t)0xF00AFA0F)
#define TSDB_MAX_SUBBLOCKS 8 #define TSDB_MAX_SUBBLOCKS 8
@ -170,7 +174,7 @@ int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut
int32_t aBufN[]); int32_t aBufN[]);
int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uint8_t *aBuf[]); int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uint8_t *aBuf[]);
// SDiskDataHdr // SDiskDataHdr
int32_t tPutDiskDataHdr(uint8_t *p, void *ph); int32_t tPutDiskDataHdr(uint8_t *p, const SDiskDataHdr *pHdr);
int32_t tGetDiskDataHdr(uint8_t *p, void *ph); int32_t tGetDiskDataHdr(uint8_t *p, void *ph);
// SDelIdx // SDelIdx
int32_t tPutDelIdx(uint8_t *p, void *ph); int32_t tPutDelIdx(uint8_t *p, void *ph);
@ -267,6 +271,7 @@ int32_t tsdbWriteDataBlk(SDataFWriter *pWriter, SMapData *mDataBlk, SBlockIdx *p
int32_t tsdbWriteSttBlk(SDataFWriter *pWriter, SArray *aSttBlk); int32_t tsdbWriteSttBlk(SDataFWriter *pWriter, SArray *aSttBlk);
int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo, int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo,
int8_t cmprAlg, int8_t toLast); int8_t cmprAlg, int8_t toLast);
int32_t tsdbWriteDiskData(SDataFWriter *pWriter, const SDiskData *pDiskData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo);
int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo); int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo);
// SDataFReader // SDataFReader
@ -300,7 +305,7 @@ int32_t tsdbMerge(STsdb *pTsdb);
#define TSDB_CACHE_LAST_ROW(c) (((c).cacheLast & 1) > 0) #define TSDB_CACHE_LAST_ROW(c) (((c).cacheLast & 1) > 0)
#define TSDB_CACHE_LAST(c) (((c).cacheLast & 2) > 0) #define TSDB_CACHE_LAST(c) (((c).cacheLast & 2) > 0)
// tsdbCache // tsdbCache ==============================================================================================
int32_t tsdbOpenCache(STsdb *pTsdb); int32_t tsdbOpenCache(STsdb *pTsdb);
void tsdbCloseCache(STsdb *pTsdb); void tsdbCloseCache(STsdb *pTsdb);
int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb *pTsdb); int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb *pTsdb);
@ -318,6 +323,15 @@ size_t tsdbCacheGetCapacity(SVnode *pVnode);
int32_t tsdbCacheLastArray2Row(SArray *pLastArray, STSRow **ppRow, STSchema *pSchema); int32_t tsdbCacheLastArray2Row(SArray *pLastArray, STSRow **ppRow, STSchema *pSchema);
// tsdbDiskData ==============================================================================================
int32_t tDiskDataBuilderCreate(SDiskDataBuilder **ppBuilder);
void *tDiskDataBuilderDestroy(SDiskDataBuilder *pBuilder);
int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TABLEID *pId, uint8_t cmprAlg,
uint8_t calcSma);
int32_t tDiskDataBuilderClear(SDiskDataBuilder *pBuilder);
int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTSchema, TABLEID *pId);
int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, const SDiskData **ppDiskData, const SBlkInfo **ppBlkInfo);
// structs ======================= // structs =======================
struct STsdbFS { struct STsdbFS {
SDelFile *pDelFile; SDelFile *pDelFile;
@ -438,6 +452,17 @@ struct SSmaInfo {
int32_t size; int32_t size;
}; };
struct SBlkInfo {
int64_t minUid;
int64_t maxUid;
TSKEY minKey;
TSKEY maxKey;
int64_t minVer;
int64_t maxVer;
TSDBKEY minTKey;
TSDBKEY maxTKey;
};
struct SDataBlk { struct SDataBlk {
TSDBKEY minKey; TSDBKEY minKey;
TSDBKEY maxKey; TSDBKEY maxKey;
@ -661,6 +686,38 @@ typedef struct {
STSchema *pTSchema; STSchema *pTSchema;
} SSkmInfo; } SSkmInfo;
struct SDiskCol {
SBlockCol bCol;
const uint8_t *pBit;
const uint8_t *pOff;
const uint8_t *pVal;
SColumnDataAgg agg;
};
struct SDiskData {
SDiskDataHdr hdr;
const uint8_t *pUid;
const uint8_t *pVer;
const uint8_t *pKey;
SArray *aDiskCol; // SArray<SDiskCol>
};
struct SDiskDataBuilder {
int64_t suid;
int64_t uid;
int32_t nRow;
uint8_t cmprAlg;
uint8_t calcSma;
SCompressor *pUidC;
SCompressor *pVerC;
SCompressor *pKeyC;
int32_t nBuilder;
SArray *aBuilder; // SArray<SDiskColBuilder>
uint8_t *aBuf[2];
SDiskData dd;
SBlkInfo bi;
};
int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid, int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid,
STimeWindow *pTimeWindow, SVersionRange *pVerRange, SSttBlockLoadInfo *pBlockLoadInfo, STimeWindow *pTimeWindow, SVersionRange *pVerRange, SSttBlockLoadInfo *pBlockLoadInfo,
bool destroyLoadInfo, const char *idStr); bool destroyLoadInfo, const char *idStr);

19
source/dnode/vnode/src/tsdb/tsdbCache.c
View File

@ -260,7 +260,7 @@ int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb
SLastCol *tTsVal = (SLastCol *)taosArrayGet(pLast, iCol); SLastCol *tTsVal = (SLastCol *)taosArrayGet(pLast, iCol);
if (keyTs > tTsVal->ts) { if (keyTs > tTsVal->ts) {
STColumn *pTColumn = &pTSchema->columns[0]; STColumn *pTColumn = &pTSchema->columns[0];
SColVal tColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = keyTs}); SColVal tColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = keyTs});
taosArraySet(pLast, iCol, &(SLastCol){.ts = keyTs, .colVal = tColVal}); taosArraySet(pLast, iCol, &(SLastCol){.ts = keyTs, .colVal = tColVal});
} }
@ -447,7 +447,7 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) {
if (--state->iFileSet >= 0) { if (--state->iFileSet >= 0) {
pFileSet = (SDFileSet *)taosArrayGet(state->aDFileSet, state->iFileSet); pFileSet = (SDFileSet *)taosArrayGet(state->aDFileSet, state->iFileSet);
} else { } else {
// tMergeTreeClose(&state->mergeTree); tMergeTreeClose(&state->mergeTree);
*ppRow = NULL; *ppRow = NULL;
return code; return code;
@ -463,7 +463,7 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) {
bool hasVal = tMergeTreeNext(&state->mergeTree); bool hasVal = tMergeTreeNext(&state->mergeTree);
if (!hasVal) { if (!hasVal) {
state->state = SFSLASTNEXTROW_FILESET; state->state = SFSLASTNEXTROW_FILESET;
// tMergeTreeClose(&state->mergeTree); tMergeTreeClose(&state->mergeTree);
goto _next_fileset; goto _next_fileset;
} }
state->state = SFSLASTNEXTROW_BLOCKROW; state->state = SFSLASTNEXTROW_BLOCKROW;
@ -590,7 +590,10 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) {
goto _next_fileset; goto _next_fileset;
} }
tMapDataReset(&state->blockMap); if (state->blockMap.pData != NULL) {
tMapDataClear(&state->blockMap);
}
code = tsdbReadDataBlk(state->pDataFReader, state->pBlockIdx, &state->blockMap); code = tsdbReadDataBlk(state->pDataFReader, state->pBlockIdx, &state->blockMap);
if (code) goto _err; if (code) goto _err;
@ -695,6 +698,10 @@ int32_t clearNextRowFromFS(void *iter) {
state->pBlockData = NULL; state->pBlockData = NULL;
} }
if (state->blockMap.pData != NULL) {
tMapDataClear(&state->blockMap);
}
return code; return code;
} }
@ -1052,7 +1059,7 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, STSRow **ppRo
lastRowTs = TSDBROW_TS(pRow); lastRowTs = TSDBROW_TS(pRow);
STColumn *pTColumn = &pTSchema->columns[0]; STColumn *pTColumn = &pTSchema->columns[0];
*pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = lastRowTs}); *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = lastRowTs});
if (taosArrayPush(pColArray, pColVal) == NULL) { if (taosArrayPush(pColArray, pColVal) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _err;
@ -1151,7 +1158,7 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray) {
lastRowTs = rowTs; lastRowTs = rowTs;
STColumn *pTColumn = &pTSchema->columns[0]; STColumn *pTColumn = &pTSchema->columns[0];
*pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = lastRowTs}); *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = lastRowTs});
if (taosArrayPush(pColArray, &(SLastCol){.ts = lastRowTs, .colVal = *pColVal}) == NULL) { if (taosArrayPush(pColArray, &(SLastCol){.ts = lastRowTs, .colVal = *pColVal}) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _err;

4
source/dnode/vnode/src/tsdb/tsdbCacheRead.c
View File

@ -101,7 +101,7 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, SArray* pTableIdList
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t tsdbCacherowsReaderClose(void* pReader) { void* tsdbCacherowsReaderClose(void* pReader) {
SCacheRowsReader* p = pReader; SCacheRowsReader* p = pReader;
if (p->pSchema != NULL) { if (p->pSchema != NULL) {
@ -114,7 +114,7 @@ int32_t tsdbCacherowsReaderClose(void* pReader) {
} }
taosMemoryFree(pReader); taosMemoryFree(pReader);
return TSDB_CODE_SUCCESS; return NULL;
} }
static int32_t doExtractCacheRow(SCacheRowsReader* pr, SLRUCache* lruCache, uint64_t uid, STSRow** pRow, static int32_t doExtractCacheRow(SCacheRowsReader* pr, SLRUCache* lruCache, uint64_t uid, STSRow** pRow,

709
source/dnode/vnode/src/tsdb/tsdbCommit.c
View File

File diff suppressed because it is too large Load Diff

709
source/dnode/vnode/src/tsdb/tsdbDiskData.c
View File

@ -16,69 +16,682 @@
#include "tsdb.h" #include "tsdb.h"
typedef struct SDiskColBuilder SDiskColBuilder; typedef struct SDiskColBuilder SDiskColBuilder;
struct SDiskColBuilder { struct SDiskColBuilder {
uint8_t flags; int16_t cid;
uint8_t *pBitMap; int8_t type;
int32_t *aOffset; uint8_t cmprAlg;
int32_t nData; uint8_t calcSma;
uint8_t *pData; int8_t flag;
int32_t nVal;
uint8_t *pBitMap;
int32_t offset;
SCompressor *pOffC;
SCompressor *pValC;
SColumnDataAgg sma;
uint8_t minSet;
uint8_t maxSet;
uint8_t *aBuf[2];
}; };
int32_t tDiskColAddVal(SDiskColBuilder *pBuilder, SColVal *pColVal) { // SDiskData ================================================
static int32_t tDiskDataDestroy(SDiskData *pDiskData) {
int32_t code = 0; int32_t code = 0;
// TODO pDiskData->aDiskCol = taosArrayDestroy(pDiskData->aDiskCol);
return code; return code;
} }
// ================================================================ // SDiskColBuilder ================================================
typedef struct SDiskDataBuilder SDiskDataBuilder; #define tDiskColBuilderCreate() \
struct SDiskDataBuilder { (SDiskColBuilder) { 0 }
SDiskDataHdr hdr;
SArray *aBlockCol; // SArray<SBlockCol>
};
int32_t tDiskDataBuilderCreate(SDiskDataBuilder **ppBuilder) { static int32_t tDiskColBuilderDestroy(SDiskColBuilder *pBuilder) {
int32_t code = 0;
// TODO
return code;
}
void tDiskDataBuilderDestroy(SDiskDataBuilder *pBuilder) {
// TODO
}
void tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, int64_t suid, int64_t uid, STSchema *pTSchema, int8_t cmprAlg) {
pBuilder->hdr = (SDiskDataHdr){.delimiter = TSDB_FILE_DLMT, //
.fmtVer = 0,
.suid = suid,
.uid = uid,
.cmprAlg = cmprAlg};
}
void tDiskDataBuilderReset(SDiskDataBuilder *pBuilder) {
// TODO
}
int32_t tDiskDataBuilderAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTSchema, int64_t uid) {
int32_t code = 0; int32_t code = 0;
// uid (todo) tFree(pBuilder->pBitMap);
if (pBuilder->pOffC) tCompressorDestroy(pBuilder->pOffC);
// version (todo) if (pBuilder->pValC) tCompressorDestroy(pBuilder->pValC);
for (int32_t iBuf = 0; iBuf < sizeof(pBuilder->aBuf) / sizeof(pBuilder->aBuf[0]); iBuf++) {
// TSKEY (todo) tFree(pBuilder->aBuf[iBuf]);
SRowIter iter = {0};
tRowIterInit(&iter, pRow, pTSchema);
for (int32_t iDiskCol = 0; iDiskCol < 0; iDiskCol++) {
} }
return code; return code;
} }
int32_t tDiskDataBuilderGet(SDiskDataBuilder *pBuilder, uint8_t **ppData) { static int32_t tDiskColBuilderInit(SDiskColBuilder *pBuilder, int16_t cid, int8_t type, uint8_t cmprAlg,
uint8_t calcSma) {
int32_t code = 0; int32_t code = 0;
// TODO
pBuilder->cid = cid;
pBuilder->type = type;
pBuilder->cmprAlg = cmprAlg;
pBuilder->calcSma = IS_VAR_DATA_TYPE(type) ? 0 : calcSma;
pBuilder->flag = 0;
pBuilder->nVal = 0;
pBuilder->offset = 0;
if (IS_VAR_DATA_TYPE(type)) {
if (pBuilder->pOffC == NULL && (code = tCompressorCreate(&pBuilder->pOffC))) return code;
code = tCompressStart(pBuilder->pOffC, TSDB_DATA_TYPE_INT, cmprAlg);
if (code) return code;
}
if (pBuilder->pValC == NULL && (code = tCompressorCreate(&pBuilder->pValC))) return code;
code = tCompressStart(pBuilder->pValC, type, cmprAlg);
if (code) return code;
if (pBuilder->calcSma) {
pBuilder->sma = (SColumnDataAgg){.colId = cid};
pBuilder->minSet = 0;
pBuilder->maxSet = 0;
}
return code;
}
static int32_t tGnrtDiskCol(SDiskColBuilder *pBuilder, SDiskCol *pDiskCol) {
int32_t code = 0;
ASSERT(pBuilder->flag && pBuilder->flag != HAS_NONE);
*pDiskCol = (SDiskCol){(SBlockCol){.cid = pBuilder->cid,
.type = pBuilder->type,
.smaOn = pBuilder->calcSma,
.flag = pBuilder->flag,
.szOrigin = 0,
.szBitmap = 0,
.szOffset = 0,
.szValue = 0,
.offset = 0},
.pBit = NULL, .pOff = NULL, .pVal = NULL, .agg = pBuilder->sma};
if (pBuilder->flag == HAS_NULL) return code;
// BITMAP
if (pBuilder->flag != HAS_VALUE) {
int32_t nBit;
if (pBuilder->flag == (HAS_VALUE | HAS_NULL | HAS_NONE)) {
nBit = BIT2_SIZE(pBuilder->nVal);
} else {
nBit = BIT1_SIZE(pBuilder->nVal);
}
code = tRealloc(&pBuilder->aBuf[0], nBit + COMP_OVERFLOW_BYTES);
if (code) return code;
code = tRealloc(&pBuilder->aBuf[1], nBit + COMP_OVERFLOW_BYTES);
if (code) return code;
pDiskCol->bCol.szBitmap =
tsCompressTinyint(pBuilder->pBitMap, nBit, nBit, pBuilder->aBuf[0], nBit + COMP_OVERFLOW_BYTES,
pBuilder->cmprAlg, pBuilder->aBuf[1], nBit + COMP_OVERFLOW_BYTES);
pDiskCol->pBit = pBuilder->aBuf[0];
}
// OFFSET
if (IS_VAR_DATA_TYPE(pBuilder->type)) {
code = tCompressEnd(pBuilder->pOffC, &pDiskCol->pOff, &pDiskCol->bCol.szOffset, NULL);
if (code) return code;
}
// VALUE
if (pBuilder->flag != (HAS_NULL | HAS_NONE)) {
code = tCompressEnd(pBuilder->pValC, &pDiskCol->pVal, &pDiskCol->bCol.szValue, &pDiskCol->bCol.szOrigin);
if (code) return code;
}
return code;
}
static FORCE_INLINE int32_t tDiskColPutValue(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
if (IS_VAR_DATA_TYPE(pColVal->type)) {
code = tCompress(pBuilder->pOffC, &pBuilder->offset, sizeof(int32_t));
if (code) return code;
pBuilder->offset += pColVal->value.nData;
code = tCompress(pBuilder->pValC, pColVal->value.pData, pColVal->value.nData);
if (code) return code;
} else {
code = tCompress(pBuilder->pValC, &pColVal->value.val, tDataTypes[pColVal->type].bytes);
if (code) return code;
}
return code;
}
static FORCE_INLINE int32_t tDiskColAddVal00(SDiskColBuilder *pBuilder, SColVal *pColVal) {
pBuilder->flag = HAS_VALUE;
return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal01(SDiskColBuilder *pBuilder, SColVal *pColVal) {
pBuilder->flag = HAS_NONE;
return 0;
}
static FORCE_INLINE int32_t tDiskColAddVal02(SDiskColBuilder *pBuilder, SColVal *pColVal) {
pBuilder->flag = HAS_NULL;
return 0;
}
static FORCE_INLINE int32_t tDiskColAddVal10(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
// bit map
int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1);
code = tRealloc(&pBuilder->pBitMap, nBit);
if (code) return code;
memset(pBuilder->pBitMap, 0, nBit);
SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1);
// value
pBuilder->flag |= HAS_VALUE;
SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0});
for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) {
code = tDiskColPutValue(pBuilder, &cv);
if (code) return code;
}
return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal12(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1);
code = tRealloc(&pBuilder->pBitMap, nBit);
if (code) return code;
memset(pBuilder->pBitMap, 0, nBit);
SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1);
pBuilder->flag |= HAS_NULL;
return code;
}
static FORCE_INLINE int32_t tDiskColAddVal20(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1);
code = tRealloc(&pBuilder->pBitMap, nBit);
if (code) return code;
pBuilder->flag |= HAS_VALUE;
memset(pBuilder->pBitMap, 0, nBit);
SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1);
SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0});
for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) {
code = tDiskColPutValue(pBuilder, &cv);
if (code) return code;
}
return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal21(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1);
code = tRealloc(&pBuilder->pBitMap, nBit);
if (code) return code;
pBuilder->flag |= HAS_NONE;
memset(pBuilder->pBitMap, 255, nBit);
SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0);
return code;
}
static FORCE_INLINE int32_t tDiskColAddVal30(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
pBuilder->flag |= HAS_VALUE;
uint8_t *pBitMap = NULL;
code = tRealloc(&pBitMap, BIT2_SIZE(pBuilder->nVal + 1));
if (code) return code;
for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) {
SET_BIT2(pBitMap, iVal, GET_BIT1(pBuilder->pBitMap, iVal));
}
SET_BIT2(pBitMap, pBuilder->nVal, 2);
tFree(pBuilder->pBitMap);
pBuilder->pBitMap = pBitMap;
SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0});
for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) {
code = tDiskColPutValue(pBuilder, &cv);
if (code) return code;
}
return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal31(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1));
if (code) return code;
SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0);
return code;
}
static FORCE_INLINE int32_t tDiskColAddVal32(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1));
if (code) return code;
SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1);
return code;
}
static FORCE_INLINE int32_t tDiskColAddVal40(SDiskColBuilder *pBuilder, SColVal *pColVal) {
return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal41(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
pBuilder->flag |= HAS_NONE;
int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1);
code = tRealloc(&pBuilder->pBitMap, nBit);
if (code) return code;
memset(pBuilder->pBitMap, 255, nBit);
SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0);
return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal42(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
pBuilder->flag |= HAS_NULL;
int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1);
code = tRealloc(&pBuilder->pBitMap, nBit);
if (code) return code;
memset(pBuilder->pBitMap, 255, nBit);
SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0);
return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal50(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1));
if (code) return code;
SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1);
return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal51(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1));
if (code) return code;
SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0);
return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal52(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
pBuilder->flag |= HAS_NULL;
uint8_t *pBitMap = NULL;
code = tRealloc(&pBitMap, BIT2_SIZE(pBuilder->nVal + 1));
if (code) return code;
for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) {
SET_BIT2(pBitMap, iVal, GET_BIT1(pBuilder->pBitMap, iVal) ? 2 : 0);
}
SET_BIT2(pBitMap, pBuilder->nVal, 1);
tFree(pBuilder->pBitMap);
pBuilder->pBitMap = pBitMap;
return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal60(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1));
if (code) return code;
SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1);
return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal61(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
pBuilder->flag |= HAS_NONE;
uint8_t *pBitMap = NULL;
code = tRealloc(&pBitMap, BIT2_SIZE(pBuilder->nVal + 1));
if (code) return code;
for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) {
SET_BIT2(pBitMap, iVal, GET_BIT1(pBuilder->pBitMap, iVal) ? 2 : 1);
}
SET_BIT2(pBitMap, pBuilder->nVal, 0);
tFree(pBuilder->pBitMap);
pBuilder->pBitMap = pBitMap;
return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal62(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1));
if (code) return code;
SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0);
return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal70(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
code = tRealloc(&pBuilder->pBitMap, BIT2_SIZE(pBuilder->nVal + 1));
if (code) return code;
SET_BIT2(pBuilder->pBitMap, pBuilder->nVal, 2);
return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal71(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
code = tRealloc(&pBuilder->pBitMap, BIT2_SIZE(pBuilder->nVal + 1));
if (code) return code;
SET_BIT2(pBuilder->pBitMap, pBuilder->nVal, 0);
return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal72(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
code = tRealloc(&pBuilder->pBitMap, BIT2_SIZE(pBuilder->nVal + 1));
if (code) return code;
SET_BIT2(pBuilder->pBitMap, pBuilder->nVal, 1);
return tDiskColPutValue(pBuilder, pColVal);
}
static int32_t (*tDiskColAddValImpl[8][3])(SDiskColBuilder *pBuilder, SColVal *pColVal) = {
{tDiskColAddVal00, tDiskColAddVal01, tDiskColAddVal02}, // 0
{tDiskColAddVal10, NULL, tDiskColAddVal12}, // HAS_NONE
{tDiskColAddVal20, tDiskColAddVal21, NULL}, // HAS_NULL
{tDiskColAddVal30, tDiskColAddVal31, tDiskColAddVal32}, // HAS_NULL|HAS_NONE
{tDiskColAddVal40, tDiskColAddVal41, tDiskColAddVal42}, // HAS_VALUE
{tDiskColAddVal50, tDiskColAddVal51, tDiskColAddVal52}, // HAS_VALUE|HAS_NONE
{tDiskColAddVal60, tDiskColAddVal61, tDiskColAddVal62}, // HAS_VALUE|HAS_NULL
{tDiskColAddVal70, tDiskColAddVal71, tDiskColAddVal72} // HAS_VALUE|HAS_NULL|HAS_NONE
};
extern void (*tSmaUpdateImpl[])(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet);
static int32_t tDiskColAddVal(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
if (pBuilder->calcSma) {
if (COL_VAL_IS_VALUE(pColVal)) {
tSmaUpdateImpl[pBuilder->type](&pBuilder->sma, pColVal, &pBuilder->minSet, &pBuilder->maxSet);
} else {
pBuilder->sma.numOfNull++;
}
}
if (tDiskColAddValImpl[pBuilder->flag][pColVal->flag]) {
code = tDiskColAddValImpl[pBuilder->flag][pColVal->flag](pBuilder, pColVal);
if (code) return code;
}
pBuilder->nVal++;
return code;
}
// SDiskDataBuilder ================================================
int32_t tDiskDataBuilderCreate(SDiskDataBuilder **ppBuilder) {
int32_t code = 0;
*ppBuilder = (SDiskDataBuilder *)taosMemoryCalloc(1, sizeof(SDiskDataBuilder));
if (*ppBuilder == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
return code;
}
return code;
}
void *tDiskDataBuilderDestroy(SDiskDataBuilder *pBuilder) {
if (pBuilder == NULL) return NULL;
if (pBuilder->pUidC) tCompressorDestroy(pBuilder->pUidC);
if (pBuilder->pVerC) tCompressorDestroy(pBuilder->pVerC);
if (pBuilder->pKeyC) tCompressorDestroy(pBuilder->pKeyC);
if (pBuilder->aBuilder) {
for (int32_t iBuilder = 0; iBuilder < taosArrayGetSize(pBuilder->aBuilder); iBuilder++) {
SDiskColBuilder *pDCBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, iBuilder);
tDiskColBuilderDestroy(pDCBuilder);
}
taosArrayDestroy(pBuilder->aBuilder);
}
for (int32_t iBuf = 0; iBuf < sizeof(pBuilder->aBuf) / sizeof(pBuilder->aBuf[0]); iBuf++) {
tFree(pBuilder->aBuf[iBuf]);
}
tDiskDataDestroy(&pBuilder->dd);
taosMemoryFree(pBuilder);
return NULL;
}
int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TABLEID *pId, uint8_t cmprAlg,
uint8_t calcSma) {
int32_t code = 0;
ASSERT(pId->suid || pId->uid);
pBuilder->suid = pId->suid;
pBuilder->uid = pId->uid;
pBuilder->nRow = 0;
pBuilder->cmprAlg = cmprAlg;
pBuilder->calcSma = calcSma;
pBuilder->bi = (SBlkInfo){.minUid = INT64_MAX,
.maxUid = INT64_MIN,
.minKey = TSKEY_MAX,
.maxKey = TSKEY_MIN,
.minVer = VERSION_MAX,
.maxVer = VERSION_MIN,
.minTKey = TSDBKEY_MAX,
.maxTKey = TSDBKEY_MIN};
if (pBuilder->pUidC == NULL && (code = tCompressorCreate(&pBuilder->pUidC))) return code;
code = tCompressStart(pBuilder->pUidC, TSDB_DATA_TYPE_BIGINT, cmprAlg);
if (code) return code;
if (pBuilder->pVerC == NULL && (code = tCompressorCreate(&pBuilder->pVerC))) return code;
code = tCompressStart(pBuilder->pVerC, TSDB_DATA_TYPE_BIGINT, cmprAlg);
if (code) return code;
if (pBuilder->pKeyC == NULL && (code = tCompressorCreate(&pBuilder->pKeyC))) return code;
code = tCompressStart(pBuilder->pKeyC, TSDB_DATA_TYPE_TIMESTAMP, cmprAlg);
if (code) return code;
if (pBuilder->aBuilder == NULL) {
pBuilder->aBuilder = taosArrayInit(pTSchema->numOfCols - 1, sizeof(SDiskColBuilder));
if (pBuilder->aBuilder == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
return code;
}
}
pBuilder->nBuilder = 0;
for (int32_t iCol = 1; iCol < pTSchema->numOfCols; iCol++) {
STColumn *pTColumn = &pTSchema->columns[iCol];
if (pBuilder->nBuilder >= taosArrayGetSize(pBuilder->aBuilder)) {
SDiskColBuilder dc = tDiskColBuilderCreate();
if (taosArrayPush(pBuilder->aBuilder, &dc) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
return code;
}
}
SDiskColBuilder *pDCBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, pBuilder->nBuilder);
code = tDiskColBuilderInit(pDCBuilder, pTColumn->colId, pTColumn->type, cmprAlg,
(calcSma && (pTColumn->flags & COL_SMA_ON)));
if (code) return code;
pBuilder->nBuilder++;
}
return code;
}
int32_t tDiskDataBuilderClear(SDiskDataBuilder *pBuilder) {
int32_t code = 0;
pBuilder->suid = 0;
pBuilder->uid = 0;
pBuilder->nRow = 0;
return code;
}
int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTSchema, TABLEID *pId) {
int32_t code = 0;
ASSERT(pBuilder->suid || pBuilder->uid);
ASSERT(pId->suid == pBuilder->suid);
TSDBKEY kRow = TSDBROW_KEY(pRow);
if (tsdbKeyCmprFn(&pBuilder->bi.minTKey, &kRow) > 0) pBuilder->bi.minTKey = kRow;
if (tsdbKeyCmprFn(&pBuilder->bi.maxTKey, &kRow) < 0) pBuilder->bi.maxTKey = kRow;
// uid
if (pBuilder->uid && pBuilder->uid != pId->uid) {
ASSERT(pBuilder->suid);
for (int32_t iRow = 0; iRow < pBuilder->nRow; iRow++) {
code = tCompress(pBuilder->pUidC, &pBuilder->uid, sizeof(int64_t));
if (code) return code;
}
pBuilder->uid = 0;
}
if (pBuilder->uid == 0) {
code = tCompress(pBuilder->pUidC, &pId->uid, sizeof(int64_t));
if (code) return code;
}
if (pBuilder->bi.minUid > pId->uid) pBuilder->bi.minUid = pId->uid;
if (pBuilder->bi.maxUid < pId->uid) pBuilder->bi.maxUid = pId->uid;
// version
code = tCompress(pBuilder->pVerC, &kRow.version, sizeof(int64_t));
if (code) return code;
if (pBuilder->bi.minVer > kRow.version) pBuilder->bi.minVer = kRow.version;
if (pBuilder->bi.maxVer < kRow.version) pBuilder->bi.maxVer = kRow.version;
// TSKEY
code = tCompress(pBuilder->pKeyC, &kRow.ts, sizeof(int64_t));
if (code) return code;
if (pBuilder->bi.minKey > kRow.ts) pBuilder->bi.minKey = kRow.ts;
if (pBuilder->bi.maxKey < kRow.ts) pBuilder->bi.maxKey = kRow.ts;
SRowIter iter = {0};
tRowIterInit(&iter, pRow, pTSchema);
SColVal *pColVal = tRowIterNext(&iter);
for (int32_t iBuilder = 0; iBuilder < pBuilder->nBuilder; iBuilder++) {
SDiskColBuilder *pDCBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, iBuilder);
while (pColVal && pColVal->cid < pDCBuilder->cid) {
pColVal = tRowIterNext(&iter);
}
if (pColVal && pColVal->cid == pDCBuilder->cid) {
code = tDiskColAddVal(pDCBuilder, pColVal);
if (code) return code;
pColVal = tRowIterNext(&iter);
} else {
code = tDiskColAddVal(pDCBuilder, &COL_VAL_NONE(pDCBuilder->cid, pDCBuilder->type));
if (code) return code;
}
}
pBuilder->nRow++;
return code;
}
int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, const SDiskData **ppDiskData, const SBlkInfo **ppBlkInfo) {
int32_t code = 0;
ASSERT(pBuilder->nRow);
*ppDiskData = NULL;
*ppBlkInfo = NULL;
SDiskData *pDiskData = &pBuilder->dd;
// reset SDiskData
pDiskData->hdr = (SDiskDataHdr){.delimiter = TSDB_FILE_DLMT,
.fmtVer = 0,
.suid = pBuilder->suid,
.uid = pBuilder->uid,
.szUid = 0,
.szVer = 0,
.szKey = 0,
.szBlkCol = 0,
.nRow = pBuilder->nRow,
.cmprAlg = pBuilder->cmprAlg};
pDiskData->pUid = NULL;
pDiskData->pVer = NULL;
pDiskData->pKey = NULL;
// UID
if (pBuilder->uid == 0) {
code = tCompressEnd(pBuilder->pUidC, &pDiskData->pUid, &pDiskData->hdr.szUid, NULL);
if (code) return code;
}
// VERSION
code = tCompressEnd(pBuilder->pVerC, &pDiskData->pVer, &pDiskData->hdr.szVer, NULL);
if (code) return code;
// TSKEY
code = tCompressEnd(pBuilder->pKeyC, &pDiskData->pKey, &pDiskData->hdr.szKey, NULL);
if (code) return code;
// aDiskCol
if (pDiskData->aDiskCol) {
taosArrayClear(pDiskData->aDiskCol);
} else {
pDiskData->aDiskCol = taosArrayInit(pBuilder->nBuilder, sizeof(SDiskCol));
if (pDiskData->aDiskCol == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
return code;
}
}
int32_t offset = 0;
for (int32_t iBuilder = 0; iBuilder < pBuilder->nBuilder; iBuilder++) {
SDiskColBuilder *pDCBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, iBuilder);
if (pDCBuilder->flag == HAS_NONE) continue;
SDiskCol dCol;
code = tGnrtDiskCol(pDCBuilder, &dCol);
if (code) return code;
dCol.bCol.offset = offset;
offset = offset + dCol.bCol.szBitmap + dCol.bCol.szOffset + dCol.bCol.szValue;
if (taosArrayPush(pDiskData->aDiskCol, &dCol) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
return code;
}
pDiskData->hdr.szBlkCol += tPutBlockCol(NULL, &dCol.bCol);
}
*ppDiskData = pDiskData;
*ppBlkInfo = &pBuilder->bi;
return code; return code;
} }

12
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

@ -926,6 +926,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
} }
} }
// fill the mis-matched columns with null value
while (i < numOfOutputCols) { while (i < numOfOutputCols) {
pColData = taosArrayGet(pResBlock->pDataBlock, i); pColData = taosArrayGet(pResBlock->pDataBlock, i);
colDataAppendNNULL(pColData, 0, remain); colDataAppendNNULL(pColData, 0, remain);
@ -935,12 +936,15 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
pResBlock->info.rows = remain; pResBlock->info.rows = remain;
pDumpInfo->rowIndex += step * remain; pDumpInfo->rowIndex += step * remain;
// check if current block are all handled
if (pDumpInfo->rowIndex >= 0 && pDumpInfo->rowIndex < pBlock->nRow) { if (pDumpInfo->rowIndex >= 0 && pDumpInfo->rowIndex < pBlock->nRow) {
// int64_t ts = pBlockData->aTSKEY[pDumpInfo->rowIndex]; int64_t ts = pBlockData->aTSKEY[pDumpInfo->rowIndex];
// setBlockAllDumped(pDumpInfo, ts, pReader->order); if (outOfTimeWindow(ts, &pReader->window)) { // the remain data has out of query time window, ignore current block
setBlockAllDumped(pDumpInfo, ts, pReader->order);
}
} else { } else {
int64_t k = asc ? pBlock->maxKey.ts : pBlock->minKey.ts; int64_t ts = asc ? pBlock->maxKey.ts : pBlock->minKey.ts;
setBlockAllDumped(pDumpInfo, k, pReader->order); setBlockAllDumped(pDumpInfo, ts, pReader->order);
} }
double elapsedTime = (taosGetTimestampUs() - st) / 1000.0; double elapsedTime = (taosGetTimestampUs() - st) / 1000.0;

133
source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
View File

@ -128,7 +128,7 @@ _exit:
return code; return code;
} }
static int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size) { static int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size) {
int32_t code = 0; int32_t code = 0;
int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage); int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage);
int64_t pgno = OFFSET_PGNO(fOffset, pFD->szPage); int64_t pgno = OFFSET_PGNO(fOffset, pFD->szPage);
@ -522,9 +522,6 @@ static int32_t tsdbWriteBlockSma(SDataFWriter *pWriter, SBlockData *pBlockData,
// write // write
if (pSmaInfo->size) { if (pSmaInfo->size) {
code = tRealloc(&pWriter->aBuf[0], pSmaInfo->size);
if (code) goto _err;
code = tsdbWriteFile(pWriter->pSmaFD, pWriter->fSma.size, pWriter->aBuf[0], pSmaInfo->size); code = tsdbWriteFile(pWriter->pSmaFD, pWriter->fSma.size, pWriter->aBuf[0], pSmaInfo->size);
if (code) goto _err; if (code) goto _err;
@ -607,6 +604,132 @@ _err:
return code; return code;
} }
int32_t tsdbWriteDiskData(SDataFWriter *pWriter, const SDiskData *pDiskData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo) {
int32_t code = 0;
int32_t lino = 0;
STsdbFD *pFD = NULL;
if (pSmaInfo) {
pFD = pWriter->pDataFD;
pBlkInfo->offset = pWriter->fData.size;
} else {
pFD = pWriter->pSttFD;
pBlkInfo->offset = pWriter->fStt[pWriter->wSet.nSttF - 1].size;
}
pBlkInfo->szBlock = 0;
pBlkInfo->szKey = 0;
// hdr
int32_t n = tPutDiskDataHdr(NULL, &pDiskData->hdr);
code = tRealloc(&pWriter->aBuf[0], n);
TSDB_CHECK_CODE(code, lino, _exit);
tPutDiskDataHdr(pWriter->aBuf[0], &pDiskData->hdr);
code = tsdbWriteFile(pFD, pBlkInfo->offset, pWriter->aBuf[0], n);
TSDB_CHECK_CODE(code, lino, _exit);
pBlkInfo->szKey += n;
pBlkInfo->szBlock += n;
// uid + ver + key
if (pDiskData->pUid) {
code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskData->pUid, pDiskData->hdr.szUid);
TSDB_CHECK_CODE(code, lino, _exit);
pBlkInfo->szKey += pDiskData->hdr.szUid;
pBlkInfo->szBlock += pDiskData->hdr.szUid;
}
code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskData->pVer, pDiskData->hdr.szVer);
TSDB_CHECK_CODE(code, lino, _exit);
pBlkInfo->szKey += pDiskData->hdr.szVer;
pBlkInfo->szBlock += pDiskData->hdr.szVer;
code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskData->pKey, pDiskData->hdr.szKey);
TSDB_CHECK_CODE(code, lino, _exit);
pBlkInfo->szKey += pDiskData->hdr.szKey;
pBlkInfo->szBlock += pDiskData->hdr.szKey;
// aBlockCol
if (pDiskData->hdr.szBlkCol) {
code = tRealloc(&pWriter->aBuf[0], pDiskData->hdr.szBlkCol);
TSDB_CHECK_CODE(code, lino, _exit);
n = 0;
for (int32_t iDiskCol = 0; iDiskCol < taosArrayGetSize(pDiskData->aDiskCol); iDiskCol++) {
SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol);
n += tPutBlockCol(pWriter->aBuf[0] + n, pDiskCol);
}
ASSERT(n == pDiskData->hdr.szBlkCol);
code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pWriter->aBuf[0], pDiskData->hdr.szBlkCol);
TSDB_CHECK_CODE(code, lino, _exit);
pBlkInfo->szBlock += pDiskData->hdr.szBlkCol;
}
// aDiskCol
for (int32_t iDiskCol = 0; iDiskCol < taosArrayGetSize(pDiskData->aDiskCol); iDiskCol++) {
SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol);
if (pDiskCol->pBit) {
code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskCol->pBit, pDiskCol->bCol.szBitmap);
TSDB_CHECK_CODE(code, lino, _exit);
pBlkInfo->szBlock += pDiskCol->bCol.szBitmap;
}
if (pDiskCol->pOff) {
code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskCol->pOff, pDiskCol->bCol.szOffset);
TSDB_CHECK_CODE(code, lino, _exit);
pBlkInfo->szBlock += pDiskCol->bCol.szOffset;
}
if (pDiskCol->pVal) {
code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskCol->pVal, pDiskCol->bCol.szValue);
TSDB_CHECK_CODE(code, lino, _exit);
pBlkInfo->szBlock += pDiskCol->bCol.szValue;
}
}
if (pSmaInfo) {
pWriter->fData.size += pBlkInfo->szBlock;
} else {
pWriter->fStt[pWriter->wSet.nSttF - 1].size += pBlkInfo->szBlock;
goto _exit;
}
pSmaInfo->offset = 0;
pSmaInfo->size = 0;
for (int32_t iDiskCol = 0; iDiskCol < taosArrayGetSize(pDiskData->aDiskCol); iDiskCol++) {
SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol);
if (IS_VAR_DATA_TYPE(pDiskCol->bCol.type)) continue;
if (pDiskCol->bCol.flag == HAS_NULL || pDiskCol->bCol.flag == (HAS_NULL | HAS_NONE)) continue;
if (!pDiskCol->bCol.smaOn) continue;
code = tRealloc(&pWriter->aBuf[0], pSmaInfo->size + tPutColumnDataAgg(NULL, &pDiskCol->agg));
TSDB_CHECK_CODE(code, lino, _exit);
pSmaInfo->size += tPutColumnDataAgg(pWriter->aBuf[0] + pSmaInfo->size, &pDiskCol->agg);
}
if (pSmaInfo->size) {
pSmaInfo->offset = pWriter->fSma.size;
code = tsdbWriteFile(pWriter->pSmaFD, pSmaInfo->offset, pWriter->aBuf[0], pSmaInfo->size);
TSDB_CHECK_CODE(code, lino, _exit);
pWriter->fSma.size += pSmaInfo->size;
}
_exit:
if (code) {
tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code));
}
return code;
}
int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) {
int32_t code = 0; int32_t code = 0;
int64_t n; int64_t n;
@ -946,8 +1069,8 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo
ASSERT(hdr.delimiter == TSDB_FILE_DLMT); ASSERT(hdr.delimiter == TSDB_FILE_DLMT);
ASSERT(pBlockData->suid == hdr.suid); ASSERT(pBlockData->suid == hdr.suid);
ASSERT(pBlockData->uid == hdr.uid);
pBlockData->uid = hdr.uid;
pBlockData->nRow = hdr.nRow; pBlockData->nRow = hdr.nRow;
// uid // uid

277
source/dnode/vnode/src/tsdb/tsdbUtil.c
View File

@ -649,7 +649,7 @@ int32_t tRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRo
ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP); ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP);
*pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = key.ts}); *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = key.ts});
if (taosArrayPush(pMerger->pArray, pColVal) == NULL) { if (taosArrayPush(pMerger->pArray, pColVal) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit; goto _exit;
@ -690,7 +690,7 @@ int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) {
STColumn *pTColumn; STColumn *pTColumn;
int32_t iCol, jCol = 1; int32_t iCol, jCol = 1;
ASSERT(((SColVal *)pMerger->pArray->pData)->value.ts == key.ts); ASSERT(((SColVal *)pMerger->pArray->pData)->value.val == key.ts);
for (iCol = 1; iCol < pMerger->pTSchema->numOfCols && jCol < pTSchema->numOfCols; ++iCol) { for (iCol = 1; iCol < pMerger->pTSchema->numOfCols && jCol < pTSchema->numOfCols; ++iCol) {
pTColumn = &pMerger->pTSchema->columns[iCol]; pTColumn = &pMerger->pTSchema->columns[iCol];
@ -744,7 +744,7 @@ int32_t tRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) {
ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP); ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP);
*pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = key.ts}); *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = key.ts});
if (taosArrayPush(pMerger->pArray, pColVal) == NULL) { if (taosArrayPush(pMerger->pArray, pColVal) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit; goto _exit;
@ -770,7 +770,7 @@ int32_t tRowMerge(SRowMerger *pMerger, TSDBROW *pRow) {
TSDBKEY key = TSDBROW_KEY(pRow); TSDBKEY key = TSDBROW_KEY(pRow);
SColVal *pColVal = &(SColVal){0}; SColVal *pColVal = &(SColVal){0};
ASSERT(((SColVal *)pMerger->pArray->pData)->value.ts == key.ts); ASSERT(((SColVal *)pMerger->pArray->pData)->value.val == key.ts);
for (int32_t iCol = 1; iCol < pMerger->pTSchema->numOfCols; iCol++) { for (int32_t iCol = 1; iCol < pMerger->pTSchema->numOfCols; iCol++) {
tsdbRowGetColVal(pRow, pMerger->pTSchema, iCol, pColVal); tsdbRowGetColVal(pRow, pMerger->pTSchema, iCol, pColVal);
@ -1105,9 +1105,8 @@ int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTS
pColVal = tRowIterNext(&rIter); pColVal = tRowIterNext(&rIter);
} }
} }
_exit:
pBlockData->nRow++; pBlockData->nRow++;
return code; return code;
_err: _err:
@ -1455,9 +1454,8 @@ _exit:
} }
// SDiskDataHdr ============================== // SDiskDataHdr ==============================
int32_t tPutDiskDataHdr(uint8_t *p, void *ph) { int32_t tPutDiskDataHdr(uint8_t *p, const SDiskDataHdr *pHdr) {
int32_t n = 0; int32_t n = 0;
SDiskDataHdr *pHdr = (SDiskDataHdr *)ph;
n += tPutU32(p ? p + n : p, pHdr->delimiter); n += tPutU32(p ? p + n : p, pHdr->delimiter);
n += tPutU32v(p ? p + n : p, pHdr->fmtVer); n += tPutU32v(p ? p + n : p, pHdr->fmtVer);
@ -1516,174 +1514,107 @@ int32_t tGetColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg) {
return n; return n;
} }
#define SMA_UPDATE(SUM_V, MIN_V, MAX_V, VAL, MINSET, MAXSET) \
do { \
(SUM_V) += (VAL); \
if (!(MINSET)) { \
(MIN_V) = (VAL); \
(MINSET) = 1; \
} else if ((MIN_V) > (VAL)) { \
(MIN_V) = (VAL); \
} \
if (!(MAXSET)) { \
(MAX_V) = (VAL); \
(MAXSET) = 1; \
} else if ((MAX_V) < (VAL)) { \
(MAX_V) = (VAL); \
} \
} while (0)
static FORCE_INLINE void tSmaUpdateBool(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) {
int8_t val = *(int8_t *)&pColVal->value.val ? 1 : 0;
SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet);
}
static FORCE_INLINE void tSmaUpdateTinyint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet,
uint8_t *maxSet) {
int8_t val = *(int8_t *)&pColVal->value.val;
SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet);
}
static FORCE_INLINE void tSmaUpdateSmallint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet,
uint8_t *maxSet) {
int16_t val = *(int16_t *)&pColVal->value.val;
SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet);
}
static FORCE_INLINE void tSmaUpdateInt(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) {
int32_t val = *(int32_t *)&pColVal->value.val;
SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet);
}
static FORCE_INLINE void tSmaUpdateBigint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) {
int64_t val = *(int64_t *)&pColVal->value.val;
SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet);
}
static FORCE_INLINE void tSmaUpdateFloat(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) {
float val = *(float *)&pColVal->value.val;
SMA_UPDATE(*(double *)&pColAgg->sum, *(double *)&pColAgg->min, *(double *)&pColAgg->max, val, *minSet, *maxSet);
}
static FORCE_INLINE void tSmaUpdateDouble(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) {
double val = *(double *)&pColVal->value.val;
SMA_UPDATE(*(double *)&pColAgg->sum, *(double *)&pColAgg->min, *(double *)&pColAgg->max, val, *minSet, *maxSet);
}
static FORCE_INLINE void tSmaUpdateUTinyint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet,
uint8_t *maxSet) {
uint8_t val = *(uint8_t *)&pColVal->value.val;
SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet);
}
static FORCE_INLINE void tSmaUpdateUSmallint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet,
uint8_t *maxSet) {
uint16_t val = *(uint16_t *)&pColVal->value.val;
SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet);
}
static FORCE_INLINE void tSmaUpdateUInt(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) {
uint32_t val = *(uint32_t *)&pColVal->value.val;
SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet);
}
static FORCE_INLINE void tSmaUpdateUBigint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet,
uint8_t *maxSet) {
uint64_t val = *(uint64_t *)&pColVal->value.val;
SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet);
}
void (*tSmaUpdateImpl[])(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) = {
NULL,
tSmaUpdateBool, // TSDB_DATA_TYPE_BOOL
tSmaUpdateTinyint, // TSDB_DATA_TYPE_TINYINT
tSmaUpdateSmallint, // TSDB_DATA_TYPE_SMALLINT
tSmaUpdateInt, // TSDB_DATA_TYPE_INT
tSmaUpdateBigint, // TSDB_DATA_TYPE_BIGINT
tSmaUpdateFloat, // TSDB_DATA_TYPE_FLOAT
tSmaUpdateDouble, // TSDB_DATA_TYPE_DOUBLE
NULL, // TSDB_DATA_TYPE_VARCHAR
tSmaUpdateBigint, // TSDB_DATA_TYPE_TIMESTAMP
NULL, // TSDB_DATA_TYPE_NCHAR
tSmaUpdateUTinyint, // TSDB_DATA_TYPE_UTINYINT
tSmaUpdateUSmallint, // TSDB_DATA_TYPE_USMALLINT
tSmaUpdateUInt, // TSDB_DATA_TYPE_UINT
tSmaUpdateUBigint, // TSDB_DATA_TYPE_UBIGINT
NULL, // TSDB_DATA_TYPE_JSON
NULL, // TSDB_DATA_TYPE_VARBINARY
NULL, // TSDB_DATA_TYPE_DECIMAL
NULL, // TSDB_DATA_TYPE_BLOB
NULL, // TSDB_DATA_TYPE_MEDIUMBLOB
};
void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg) { void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg) {
SColVal colVal;
SColVal *pColVal = &colVal;
memset(pColAgg, 0, sizeof(*pColAgg));
bool minAssigned = false;
bool maxAssigned = false;
*pColAgg = (SColumnDataAgg){.colId = pColData->cid}; *pColAgg = (SColumnDataAgg){.colId = pColData->cid};
for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { uint8_t minSet = 0;
tColDataGetValue(pColData, iVal, pColVal); uint8_t maxSet = 0;
if (!COL_VAL_IS_VALUE(pColVal)) { SColVal cv;
pColAgg->numOfNull++; for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
tColDataGetValue(pColData, iVal, &cv);
if (COL_VAL_IS_VALUE(&cv)) {
tSmaUpdateImpl[pColData->type](pColAgg, &cv, &minSet, &maxSet);
} else { } else {
switch (pColData->type) { pColAgg->numOfNull++;
case TSDB_DATA_TYPE_NULL:
break;
case TSDB_DATA_TYPE_BOOL:
break;
case TSDB_DATA_TYPE_TINYINT: {
pColAgg->sum += colVal.value.i8;
if (!minAssigned || pColAgg->min > colVal.value.i8) {
pColAgg->min = colVal.value.i8;
minAssigned = true;
}
if (!maxAssigned || pColAgg->max < colVal.value.i8) {
pColAgg->max = colVal.value.i8;
maxAssigned = true;
}
break;
}
case TSDB_DATA_TYPE_SMALLINT: {
pColAgg->sum += colVal.value.i16;
if (!minAssigned || pColAgg->min > colVal.value.i16) {
pColAgg->min = colVal.value.i16;
minAssigned = true;
}
if (!maxAssigned || pColAgg->max < colVal.value.i16) {
pColAgg->max = colVal.value.i16;
maxAssigned = true;
}
break;
}
case TSDB_DATA_TYPE_INT: {
pColAgg->sum += colVal.value.i32;
if (!minAssigned || pColAgg->min > colVal.value.i32) {
pColAgg->min = colVal.value.i32;
minAssigned = true;
}
if (!maxAssigned || pColAgg->max < colVal.value.i32) {
pColAgg->max = colVal.value.i32;
maxAssigned = true;
}
break;
}
case TSDB_DATA_TYPE_BIGINT: {
pColAgg->sum += colVal.value.i64;
if (!minAssigned || pColAgg->min > colVal.value.i64) {
pColAgg->min = colVal.value.i64;
minAssigned = true;
}
if (!maxAssigned || pColAgg->max < colVal.value.i64) {
pColAgg->max = colVal.value.i64;
maxAssigned = true;
}
break;
}
case TSDB_DATA_TYPE_FLOAT: {
*(double *)(&pColAgg->sum) += colVal.value.f;
if (!minAssigned || *(double *)(&pColAgg->min) > colVal.value.f) {
*(double *)(&pColAgg->min) = colVal.value.f;
minAssigned = true;
}
if (!maxAssigned || *(double *)(&pColAgg->max) < colVal.value.f) {
*(double *)(&pColAgg->max) = colVal.value.f;
maxAssigned = true;
}
break;
}
case TSDB_DATA_TYPE_DOUBLE: {
*(double *)(&pColAgg->sum) += colVal.value.d;
if (!minAssigned || *(double *)(&pColAgg->min) > colVal.value.d) {
*(double *)(&pColAgg->min) = colVal.value.d;
minAssigned = true;
}
if (!maxAssigned || *(double *)(&pColAgg->max) < colVal.value.d) {
*(double *)(&pColAgg->max) = colVal.value.d;
maxAssigned = true;
}
break;
}
case TSDB_DATA_TYPE_VARCHAR:
break;
case TSDB_DATA_TYPE_TIMESTAMP: {
if (!minAssigned || pColAgg->min > colVal.value.i64) {
pColAgg->min = colVal.value.i64;
minAssigned = true;
}
if (!maxAssigned || pColAgg->max < colVal.value.i64) {
pColAgg->max = colVal.value.i64;
maxAssigned = true;
}
break;
}
case TSDB_DATA_TYPE_NCHAR:
break;
case TSDB_DATA_TYPE_UTINYINT: {
pColAgg->sum += colVal.value.u8;
if (!minAssigned || pColAgg->min > colVal.value.u8) {
pColAgg->min = colVal.value.u8;
minAssigned = true;
}
if (!maxAssigned || pColAgg->max < colVal.value.u8) {
pColAgg->max = colVal.value.u8;
maxAssigned = true;
}
break;
}
case TSDB_DATA_TYPE_USMALLINT: {
pColAgg->sum += colVal.value.u16;
if (!minAssigned || pColAgg->min > colVal.value.u16) {
pColAgg->min = colVal.value.u16;
minAssigned = true;
}
if (!maxAssigned || pColAgg->max < colVal.value.u16) {
pColAgg->max = colVal.value.u16;
maxAssigned = true;
}
break;
}
case TSDB_DATA_TYPE_UINT: {
pColAgg->sum += colVal.value.u32;
if (!minAssigned || pColAgg->min > colVal.value.u32) {
pColAgg->min = colVal.value.u32;
minAssigned = true;
}
if (!minAssigned || pColAgg->max < colVal.value.u32) {
pColAgg->max = colVal.value.u32;
maxAssigned = true;
}
break;
}
case TSDB_DATA_TYPE_UBIGINT: {
pColAgg->sum += colVal.value.u64;
if (!minAssigned || pColAgg->min > colVal.value.u64) {
pColAgg->min = colVal.value.u64;
minAssigned = true;
}
if (!maxAssigned || pColAgg->max < colVal.value.u64) {
pColAgg->max = colVal.value.u64;
maxAssigned = true;
}
break;
}
case TSDB_DATA_TYPE_JSON:
break;
case TSDB_DATA_TYPE_VARBINARY:
break;
case TSDB_DATA_TYPE_DECIMAL:
break;
case TSDB_DATA_TYPE_BLOB:
break;
case TSDB_DATA_TYPE_MEDIUMBLOB:
break;
default:
ASSERT(0);
}
} }
} }
} }

12
source/dnode/vnode/src/vnd/vnodeSync.c
View File

@ -328,14 +328,14 @@ int32_t vnodeProcessSyncMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
const STraceId *trace = &pMsg->info.traceId; const STraceId *trace = &pMsg->info.traceId;
if (!syncEnvIsStart()) { if (!syncEnvIsStart()) {
vGError("vgId:%d, msg:%p failed to process since sync env not start", pVnode->config.vgId); vGError("vgId:%d, msg:%p failed to process since sync env not start", pVnode->config.vgId, pMsg);
terrno = TSDB_CODE_APP_ERROR; terrno = TSDB_CODE_APP_ERROR;
return -1; return -1;
} }
SSyncNode *pSyncNode = syncNodeAcquire(pVnode->sync); SSyncNode *pSyncNode = syncNodeAcquire(pVnode->sync);
if (pSyncNode == NULL) { if (pSyncNode == NULL) {
vGError("vgId:%d, msg:%p failed to process since invalid sync node", pVnode->config.vgId); vGError("vgId:%d, msg:%p failed to process since invalid sync node", pVnode->config.vgId, pMsg);
terrno = TSDB_CODE_SYN_INTERNAL_ERROR; terrno = TSDB_CODE_SYN_INTERNAL_ERROR;
return -1; return -1;
} }
@ -394,7 +394,7 @@ int32_t vnodeProcessSyncMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
SRpcMsg rsp = {.code = code, .info = pMsg->info}; SRpcMsg rsp = {.code = code, .info = pMsg->info};
tmsgSendRsp(&rsp); tmsgSendRsp(&rsp);
} else { } else {
vGError("vgId:%d, msg:%p failed to process since error msg type:%d", pVnode->config.vgId, pMsg->msgType); vGError("vgId:%d, msg:%p failed to process since error msg type:%d", pVnode->config.vgId, pMsg, pMsg->msgType);
code = -1; code = -1;
} }
@ -459,7 +459,7 @@ int32_t vnodeProcessSyncMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
SRpcMsg rsp = {.code = code, .info = pMsg->info}; SRpcMsg rsp = {.code = code, .info = pMsg->info};
tmsgSendRsp(&rsp); tmsgSendRsp(&rsp);
} else { } else {
vGError("vgId:%d, msg:%p failed to process since error msg type:%d", pVnode->config.vgId, pMsg->msgType); vGError("vgId:%d, msg:%p failed to process since error msg type:%d", pVnode->config.vgId, pMsg, pMsg->msgType);
code = -1; code = -1;
} }
} }
@ -630,7 +630,7 @@ static int32_t vnodeSnapshotStartWrite(struct SSyncFSM *pFsm, void *pParam, void
vInfo("vgId:%d, start write vnode snapshot since apply queue is empty", pVnode->config.vgId); vInfo("vgId:%d, start write vnode snapshot since apply queue is empty", pVnode->config.vgId);
break; break;
} else { } else {
vInfo("vgId:%d, write vnode snapshot later since %d items in apply queue", pVnode->config.vgId); vInfo("vgId:%d, write vnode snapshot later since %d items in apply queue", pVnode->config.vgId, itemSize);
taosMsleep(10); taosMsleep(10);
} }
} while (true); } while (true);
@ -683,7 +683,7 @@ static void vnodeRestoreFinish(struct SSyncFSM *pFsm) {
vInfo("vgId:%d, apply queue is empty, restore finish", pVnode->config.vgId); vInfo("vgId:%d, apply queue is empty, restore finish", pVnode->config.vgId);
break; break;
} else { } else {
vInfo("vgId:%d, restore not finish since %d items in apply queue", pVnode->config.vgId); vInfo("vgId:%d, restore not finish since %d items in apply queue", pVnode->config.vgId, itemSize);
taosMsleep(10); taosMsleep(10);
} }
} while (true); } while (true);

28
source/libs/executor/inc/executil.h
View File

@ -15,7 +15,6 @@
#ifndef TDENGINE_QUERYUTIL_H #ifndef TDENGINE_QUERYUTIL_H
#define TDENGINE_QUERYUTIL_H #define TDENGINE_QUERYUTIL_H
#include "vnode.h"
#include "function.h" #include "function.h"
#include "nodes.h" #include "nodes.h"
#include "plannodes.h" #include "plannodes.h"
@ -23,6 +22,7 @@
#include "tcommon.h" #include "tcommon.h"
#include "tpagedbuf.h" #include "tpagedbuf.h"
#include "tsimplehash.h" #include "tsimplehash.h"
#include "vnode.h"
#define T_LONG_JMP(_obj, _c) \ #define T_LONG_JMP(_obj, _c) \
do { \ do { \
@ -93,7 +93,7 @@ void resetResultRow(SResultRow* pResultRow, size_t entrySize);
struct SResultRowEntryInfo* getResultEntryInfo(const SResultRow* pRow, int32_t index, const int32_t* offset); struct SResultRowEntryInfo* getResultEntryInfo(const SResultRow* pRow, int32_t index, const int32_t* offset);
static FORCE_INLINE SResultRow* getResultRowByPos(SDiskbasedBuf* pBuf, SResultRowPosition* pos, bool forUpdate) { static FORCE_INLINE SResultRow* getResultRowByPos(SDiskbasedBuf* pBuf, SResultRowPosition* pos, bool forUpdate) {
SFilePage* bufPage = (SFilePage*)getBufPage(pBuf, pos->pageId); SFilePage* bufPage = (SFilePage*)getBufPage(pBuf, pos->pageId);
if (forUpdate) { if (forUpdate) {
setBufPageDirty(bufPage, true); setBufPageDirty(bufPage, true);
} }
@ -101,11 +101,6 @@ static FORCE_INLINE SResultRow* getResultRowByPos(SDiskbasedBuf* pBuf, SResultRo
return pRow; return pRow;
} }
static FORCE_INLINE void setResultBufPageDirty(SDiskbasedBuf* pBuf, SResultRowPosition* pos) {
void* pPage = getBufPage(pBuf, pos->pageId);
setBufPageDirty(pPage, true);
}
void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, int32_t order); void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, int32_t order);
void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo); void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo);
@ -117,17 +112,18 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo);
SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode); SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode);
EDealRes doTranslateTagExpr(SNode** pNode, void* pContext); EDealRes doTranslateTagExpr(SNode** pNode, void* pContext);
int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond, STableListInfo* pListInfo); int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond,
int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId); STableListInfo* pListInfo);
int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableListInfo* pTableListInfo); int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId);
size_t getTableTagsBufLen(const SNodeList* pGroups); int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableListInfo* pTableListInfo);
size_t getTableTagsBufLen(const SNodeList* pGroups);
SArray* createSortInfo(SNodeList* pNodeList); SArray* createSortInfo(SNodeList* pNodeList);
SArray* extractPartitionColInfo(SNodeList* pNodeList); SArray* extractPartitionColInfo(SNodeList* pNodeList);
SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols, SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols,
int32_t type); int32_t type);
void createExprFromTargetNode(SExprInfo* pExp, STargetNode* pTargetNode); void createExprFromTargetNode(SExprInfo* pExp, STargetNode* pTargetNode);
SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* numOfExprs); SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* numOfExprs);
SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowEntryInfoOffset); SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowEntryInfoOffset);

50
source/libs/executor/inc/executorimpl.h
View File

@ -577,13 +577,7 @@ typedef struct SIntervalAggOperatorInfo {
int32_t inputOrder; // input data ts order int32_t inputOrder; // input data ts order
EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model] EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model]
STimeWindowAggSupp twAggSup; STimeWindowAggSupp twAggSup;
bool invertible;
SArray* pPrevValues; // SArray<SGroupKeys> used to keep the previous not null value for interpolation. SArray* pPrevValues; // SArray<SGroupKeys> used to keep the previous not null value for interpolation.
bool ignoreExpiredData;
SArray* pRecycledPages;
SArray* pDelWins; // SWinRes
int32_t delIndex;
SSDataBlock* pDelRes;
SNode* pCondition; SNode* pCondition;
} SIntervalAggOperatorInfo; } SIntervalAggOperatorInfo;
@ -609,38 +603,21 @@ typedef struct SStreamIntervalOperatorInfo {
STimeWindowAggSupp twAggSup; STimeWindowAggSupp twAggSup;
bool invertible; bool invertible;
bool ignoreExpiredData; bool ignoreExpiredData;
SArray* pRecycledPages;
SArray* pDelWins; // SWinRes SArray* pDelWins; // SWinRes
int32_t delIndex; int32_t delIndex;
SSDataBlock* pDelRes; SSDataBlock* pDelRes;
bool isFinal;
} SStreamIntervalOperatorInfo;
typedef struct SStreamFinalIntervalOperatorInfo {
// SOptrBasicInfo should be first, SAggSupporter should be second for stream encode
SOptrBasicInfo binfo; // basic info
SAggSupporter aggSup; // aggregate supporter
SExprSupp scalarSupp; // supporter for perform scalar function
SGroupResInfo groupResInfo; // multiple results build supporter
SInterval interval; // interval info
int32_t primaryTsIndex; // primary time stamp slot id from result of downstream operator.
int32_t order; // current SSDataBlock scan order
STimeWindowAggSupp twAggSup;
SArray* pChildren;
SSDataBlock* pUpdateRes; SSDataBlock* pUpdateRes;
bool returnUpdate; bool returnUpdate;
SPhysiNode* pPhyNode; // create new child SPhysiNode* pPhyNode; // create new child
bool isFinal;
SHashObj* pPullDataMap; SHashObj* pPullDataMap;
SArray* pPullWins; // SPullWindowInfo SArray* pPullWins; // SPullWindowInfo
int32_t pullIndex; int32_t pullIndex;
SSDataBlock* pPullDataRes; SSDataBlock* pPullDataRes;
bool ignoreExpiredData; bool isFinal;
SArray* pRecycledPages; SArray* pChildren;
SArray* pDelWins; // SWinRes SStreamState* pState;
int32_t delIndex; SWinKey delKey;
SSDataBlock* pDelRes; } SStreamIntervalOperatorInfo;
} SStreamFinalIntervalOperatorInfo;
typedef struct SAggOperatorInfo { typedef struct SAggOperatorInfo {
// SOptrBasicInfo should be first, SAggSupporter should be second for stream encode // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode
@ -1086,7 +1063,7 @@ bool functionNeedToExecute(SqlFunctionCtx* pCtx);
bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup); bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup);
bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup); bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup);
bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup); bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup);
bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SOperatorInfo* pOperator, STimeWindowAggSupp* pTwSup); bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SStreamState* pState, STimeWindowAggSupp* pTwSup);
void appendOneRow(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid, uint64_t* pGp); void appendOneRow(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid, uint64_t* pGp);
void printDataBlock(SSDataBlock* pBlock, const char* flag); void printDataBlock(SSDataBlock* pBlock, const char* flag);
uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId); uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId);
@ -1108,13 +1085,12 @@ void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsCol
bool groupbyTbname(SNodeList* pGroupList); bool groupbyTbname(SNodeList* pGroupList);
int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* groupKey); int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* groupKey);
void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput); void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput);
int32_t buildDataBlockFromGroupRes(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, int32_t buildDataBlockFromGroupRes(SExecTaskInfo* pTaskInfo, SStreamState* pState, SSDataBlock* pBlock, SExprSupp* pSup,
SGroupResInfo* pGroupResInfo); SGroupResInfo* pGroupResInfo);
int32_t setOutputBuf(STimeWindow* win, SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx, int32_t setOutputBuf(SStreamState* pState, STimeWindow* win, SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx,
int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup, int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup);
SExecTaskInfo* pTaskInfo); int32_t releaseOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult);
int32_t releaseOutputBuf(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult); int32_t saveOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult, int32_t resSize);
int32_t saveOutputBuf(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult, int32_t resSize);
void getNextIntervalWindow(SInterval* pInterval, STimeWindow* tw, int32_t order); void getNextIntervalWindow(SInterval* pInterval, STimeWindow* tw, int32_t order);
#ifdef __cplusplus #ifdef __cplusplus

11
source/libs/executor/src/cachescanoperator.c
View File

@ -207,7 +207,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
} }
} }
tsdbCacherowsReaderClose(pInfo->pLastrowReader); pInfo->pLastrowReader = tsdbCacherowsReaderClose(pInfo->pLastrowReader);
return pInfo->pRes; return pInfo->pRes;
} }
} }
@ -220,6 +220,15 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
void destroyLastrowScanOperator(void* param) { void destroyLastrowScanOperator(void* param) {
SLastrowScanInfo* pInfo = (SLastrowScanInfo*)param; SLastrowScanInfo* pInfo = (SLastrowScanInfo*)param;
blockDataDestroy(pInfo->pRes); blockDataDestroy(pInfo->pRes);
blockDataDestroy(pInfo->pBufferredRes);
taosMemoryFree(pInfo->pSlotIds);
taosArrayDestroy(pInfo->pUidList);
taosArrayDestroy(pInfo->pColMatchInfo);
if (pInfo->pLastrowReader != NULL) {
pInfo->pLastrowReader = tsdbCacherowsReaderClose(pInfo->pLastrowReader);
}
taosMemoryFreeClear(param); taosMemoryFreeClear(param);
} }

28
source/libs/executor/src/executorimpl.c
View File

@ -4183,9 +4183,8 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlF
return code; return code;
} }
int32_t setOutputBuf(STimeWindow* win, SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx, int32_t setOutputBuf(SStreamState* pState, STimeWindow* win, SResultRow** pResult, int64_t tableGroupId,
int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup) {
SExecTaskInfo* pTaskInfo) {
SWinKey key = { SWinKey key = {
.ts = win->skey, .ts = win->skey,
.groupId = tableGroupId, .groupId = tableGroupId,
@ -4194,7 +4193,7 @@ int32_t setOutputBuf(STimeWindow* win, SResultRow** pResult, int64_t tableGroupI
int32_t size = pAggSup->resultRowSize; int32_t size = pAggSup->resultRowSize;
tSimpleHashPut(pAggSup->pResultRowHashTable, &key, sizeof(SWinKey), NULL, 0); tSimpleHashPut(pAggSup->pResultRowHashTable, &key, sizeof(SWinKey), NULL, 0);
if (streamStateAddIfNotExist(pTaskInfo->streamInfo.pState, &key, (void**)&value, &size) < 0) { if (streamStateAddIfNotExist(pState, &key, (void**)&value, &size) < 0) {
return TSDB_CODE_QRY_OUT_OF_MEMORY; return TSDB_CODE_QRY_OUT_OF_MEMORY;
} }
*pResult = (SResultRow*)value; *pResult = (SResultRow*)value;
@ -4205,18 +4204,17 @@ int32_t setOutputBuf(STimeWindow* win, SResultRow** pResult, int64_t tableGroupI
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t releaseOutputBuf(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult) { int32_t releaseOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult) {
streamStateReleaseBuf(pTaskInfo->streamInfo.pState, pKey, pResult); streamStateReleaseBuf(pState, pKey, pResult);
/*taosMemoryFree((*(void**)pResult));*/
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t saveOutputBuf(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult, int32_t resSize) { int32_t saveOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult, int32_t resSize) {
streamStatePut(pTaskInfo->streamInfo.pState, pKey, pResult, resSize); streamStatePut(pState, pKey, pResult, resSize);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t buildDataBlockFromGroupRes(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, int32_t buildDataBlockFromGroupRes(SExecTaskInfo* pTaskInfo, SStreamState* pState, SSDataBlock* pBlock, SExprSupp* pSup,
SGroupResInfo* pGroupResInfo) { SGroupResInfo* pGroupResInfo) {
SExprInfo* pExprInfo = pSup->pExprInfo; SExprInfo* pExprInfo = pSup->pExprInfo;
int32_t numOfExprs = pSup->numOfExprs; int32_t numOfExprs = pSup->numOfExprs;
@ -4233,14 +4231,14 @@ int32_t buildDataBlockFromGroupRes(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock
.ts = *(TSKEY*)pPos->key, .ts = *(TSKEY*)pPos->key,
.groupId = pPos->groupId, .groupId = pPos->groupId,
}; };
int32_t code = streamStateGet(pTaskInfo->streamInfo.pState, &key, &pVal, &size); int32_t code = streamStateGet(pState, &key, &pVal, &size);
ASSERT(code == 0); ASSERT(code == 0);
SResultRow* pRow = (SResultRow*)pVal; SResultRow* pRow = (SResultRow*)pVal;
doUpdateNumOfRows(pCtx, pRow, numOfExprs, rowEntryOffset); doUpdateNumOfRows(pCtx, pRow, numOfExprs, rowEntryOffset);
// no results, continue to check the next one // no results, continue to check the next one
if (pRow->numOfRows == 0) { if (pRow->numOfRows == 0) {
pGroupResInfo->index += 1; pGroupResInfo->index += 1;
releaseOutputBuf(pTaskInfo, &key, pRow); releaseOutputBuf(pState, &key, pRow);
continue; continue;
} }
@ -4249,14 +4247,14 @@ int32_t buildDataBlockFromGroupRes(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock
} else { } else {
// current value belongs to different group, it can't be packed into one datablock // current value belongs to different group, it can't be packed into one datablock
if (pBlock->info.groupId != pPos->groupId) { if (pBlock->info.groupId != pPos->groupId) {
releaseOutputBuf(pTaskInfo, &key, pRow); releaseOutputBuf(pState, &key, pRow);
break; break;
} }
} }
if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) {
ASSERT(pBlock->info.rows > 0); ASSERT(pBlock->info.rows > 0);
releaseOutputBuf(pTaskInfo, &key, pRow); releaseOutputBuf(pState, &key, pRow);
break; break;
} }
@ -4286,7 +4284,7 @@ int32_t buildDataBlockFromGroupRes(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock
} }
pBlock->info.rows += pRow->numOfRows; pBlock->info.rows += pRow->numOfRows;
releaseOutputBuf(pTaskInfo, &key, pRow); releaseOutputBuf(pState, &key, pRow);
} }
blockDataUpdateTsWindow(pBlock, 0); blockDataUpdateTsWindow(pBlock, 0);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;

84
source/libs/executor/src/scanoperator.c
View File

@ -41,9 +41,9 @@ static int32_t buildDbTableInfoBlock(bool sysInfo, const SSDataBlock* p, const S
static bool processBlockWithProbability(const SSampleExecInfo* pInfo); static bool processBlockWithProbability(const SSampleExecInfo* pInfo);
static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, SMetaReader* smr, const char* dbname, static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, SMetaReader* smrSuperTable,
const char* tableName, int32_t* pNumOfRows, SMetaReader* smrChildTable, const char* dbname, const char* tableName,
const SSDataBlock* dataBlock); int32_t* pNumOfRows, const SSDataBlock* dataBlock);
static void relocateAndFilterSysTagsScanResult(SSysTableScanInfo* pInfo, int32_t numOfRows, SSDataBlock* dataBlock); static void relocateAndFilterSysTagsScanResult(SSysTableScanInfo* pInfo, int32_t numOfRows, SSDataBlock* dataBlock);
bool processBlockWithProbability(const SSampleExecInfo* pInfo) { bool processBlockWithProbability(const SSampleExecInfo* pInfo) {
@ -1357,7 +1357,8 @@ static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock
// must check update info first. // must check update info first.
bool update = updateInfoIsUpdated(pInfo->pUpdateInfo, pBlock->info.uid, tsCol[rowId]); bool update = updateInfoIsUpdated(pInfo->pUpdateInfo, pBlock->info.uid, tsCol[rowId]);
bool closedWin = isClosed && isSignleIntervalWindow(pInfo) && bool closedWin = isClosed && isSignleIntervalWindow(pInfo) &&
isDeletedStreamWindow(&win, pBlock->info.groupId, pInfo->pTableScanOp, &pInfo->twAggSup); isDeletedStreamWindow(&win, pBlock->info.groupId,
pInfo->pTableScanOp->pTaskInfo->streamInfo.pState, &pInfo->twAggSup);
if ((update || closedWin) && out) { if ((update || closedWin) && out) {
qDebug("stream update check not pass, update %d, closedWin %d", update, closedWin); qDebug("stream update check not pass, update %d, closedWin %d", update, closedWin);
uint64_t gpId = 0; uint64_t gpId = 0;
@ -2135,6 +2136,9 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
pInfo->pUpdateInfo = NULL; pInfo->pUpdateInfo = NULL;
pInfo->pTableScanOp = pTableScanOp; pInfo->pTableScanOp = pTableScanOp;
if (pInfo->pTableScanOp->pTaskInfo->streamInfo.pState) {
streamStateSetNumber(pInfo->pTableScanOp->pTaskInfo->streamInfo.pState, -1);
}
pInfo->readHandle = *pHandle; pInfo->readHandle = *pHandle;
pInfo->tableUid = pScanPhyNode->uid; pInfo->tableUid = pScanPhyNode->uid;
@ -2411,11 +2415,10 @@ static bool sysTableIsOperatorCondOnOneTable(SNode* pCond, char* condTable) {
strcasecmp(nodesGetNameFromColumnNode(node->pLeft), "table_name") == 0 && strcasecmp(nodesGetNameFromColumnNode(node->pLeft), "table_name") == 0 &&
nodeType(node->pRight) == QUERY_NODE_VALUE) { nodeType(node->pRight) == QUERY_NODE_VALUE) {
SValueNode* pValue = (SValueNode*)node->pRight; SValueNode* pValue = (SValueNode*)node->pRight;
if (pValue->node.type == TSDB_DATA_TYPE_NCHAR || pValue->node.type == TSDB_DATA_TYPE_VARCHAR || if (pValue->node.resType.type == TSDB_DATA_TYPE_NCHAR || pValue->node.resType.type == TSDB_DATA_TYPE_VARCHAR ||
pValue->node.type == TSDB_DATA_TYPE_BINARY) { pValue->node.resType.type == TSDB_DATA_TYPE_BINARY) {
char* value = nodesGetStrValueFromNode(pValue); char* value = nodesGetValueFromNode(pValue);
strncpy(condTable, value, TSDB_TABLE_NAME_LEN); strncpy(condTable, varDataVal(value), TSDB_TABLE_NAME_LEN);
taosMemoryFree(value);
return true; return true;
} }
} }
@ -2476,18 +2479,28 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) {
char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
STR_TO_VARSTR(tableName, condTableName); STR_TO_VARSTR(tableName, condTableName);
SMetaReader smr = {0}; SMetaReader smrChildTable = {0};
metaReaderInit(&smr, pInfo->readHandle.meta, 0); metaReaderInit(&smrChildTable, pInfo->readHandle.meta, 0);
metaGetTableEntryByName(&smr, condTableName); metaGetTableEntryByName(&smrChildTable, condTableName);
sysTableUserTagsFillOneTableTags(pInfo, &smr, dbname, tableName, &numOfRows, dataBlock); if (smrChildTable.me.type != TSDB_CHILD_TABLE) {
metaReaderClear(&smr); metaReaderClear(&smrChildTable);
blockDataDestroy(dataBlock);
pInfo->loadInfo.totalRows = 0;
return NULL;
}
SMetaReader smrSuperTable = {0};
metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, 0);
metaGetTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid);
sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &smrChildTable, dbname, tableName, &numOfRows, dataBlock);
metaReaderClear(&smrSuperTable);
metaReaderClear(&smrChildTable);
if (numOfRows > 0) { if (numOfRows > 0) {
relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock); relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock);
numOfRows = 0; numOfRows = 0;
} }
blockDataDestroy(dataBlock); blockDataDestroy(dataBlock);
pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; pInfo->loadInfo.totalRows += pInfo->pRes->info.rows;
doSetOperatorCompleted(pOperator);
return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes;
} }
@ -2504,23 +2517,22 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) {
char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
STR_TO_VARSTR(tableName, pInfo->pCur->mr.me.name); STR_TO_VARSTR(tableName, pInfo->pCur->mr.me.name);
SMetaReader smr = {0}; SMetaReader smrSuperTable = {0};
metaReaderInit(&smr, pInfo->readHandle.meta, 0); metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, 0);
uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid;
int32_t code = metaGetTableEntryByUid(&smr, suid); int32_t code = metaGetTableEntryByUid(&smrSuperTable, suid);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
qError("failed to get super table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno), qError("failed to get super table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno),
GET_TASKID(pTaskInfo)); GET_TASKID(pTaskInfo));
metaReaderClear(&smr); metaReaderClear(&smrSuperTable);
metaCloseTbCursor(pInfo->pCur); metaCloseTbCursor(pInfo->pCur);
pInfo->pCur = NULL; pInfo->pCur = NULL;
T_LONG_JMP(pTaskInfo->env, terrno); T_LONG_JMP(pTaskInfo->env, terrno);
} }
sysTableUserTagsFillOneTableTags(pInfo, &smr, dbname, tableName, &numOfRows, dataBlock); sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &pInfo->pCur->mr, dbname, tableName, &numOfRows, dataBlock);
metaReaderClear(&smr); metaReaderClear(&smrSuperTable);
if (numOfRows >= pOperator->resultInfo.capacity) { if (numOfRows >= pOperator->resultInfo.capacity) {
relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock); relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock);
@ -2558,15 +2570,15 @@ static void relocateAndFilterSysTagsScanResult(SSysTableScanInfo* pInfo, int32_t
blockDataCleanup(dataBlock); blockDataCleanup(dataBlock);
} }
static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, SMetaReader* smr, const char* dbname, static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, SMetaReader* smrSuperTable,
const char* tableName, int32_t* pNumOfRows, SMetaReader* smrChildTable, const char* dbname, const char* tableName,
const SSDataBlock* dataBlock) { int32_t* pNumOfRows, const SSDataBlock* dataBlock) {
char stableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; char stableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
STR_TO_VARSTR(stableName, (*smr).me.name); STR_TO_VARSTR(stableName, (*smrSuperTable).me.name);
int32_t numOfRows = *pNumOfRows; int32_t numOfRows = *pNumOfRows;
int32_t numOfTags = (*smr).me.stbEntry.schemaTag.nCols; int32_t numOfTags = (*smrSuperTable).me.stbEntry.schemaTag.nCols;
for (int32_t i = 0; i < numOfTags; ++i) { for (int32_t i = 0; i < numOfTags; ++i) {
SColumnInfoData* pColInfoData = NULL; SColumnInfoData* pColInfoData = NULL;
@ -2584,35 +2596,35 @@ static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo,
// tag name // tag name
char tagName[TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; char tagName[TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
STR_TO_VARSTR(tagName, (*smr).me.stbEntry.schemaTag.pSchema[i].name); STR_TO_VARSTR(tagName, (*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].name);
pColInfoData = taosArrayGet(dataBlock->pDataBlock, 3); pColInfoData = taosArrayGet(dataBlock->pDataBlock, 3);
colDataAppend(pColInfoData, numOfRows, tagName, false); colDataAppend(pColInfoData, numOfRows, tagName, false);
// tag type // tag type
int8_t tagType = (*smr).me.stbEntry.schemaTag.pSchema[i].type; int8_t tagType = (*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].type;
pColInfoData = taosArrayGet(dataBlock->pDataBlock, 4); pColInfoData = taosArrayGet(dataBlock->pDataBlock, 4);
char tagTypeStr[VARSTR_HEADER_SIZE + 32]; char tagTypeStr[VARSTR_HEADER_SIZE + 32];
int tagTypeLen = sprintf(varDataVal(tagTypeStr), "%s", tDataTypes[tagType].name); int tagTypeLen = sprintf(varDataVal(tagTypeStr), "%s", tDataTypes[tagType].name);
if (tagType == TSDB_DATA_TYPE_VARCHAR) { if (tagType == TSDB_DATA_TYPE_VARCHAR) {
tagTypeLen += sprintf(varDataVal(tagTypeStr) + tagTypeLen, "(%d)", tagTypeLen += sprintf(varDataVal(tagTypeStr) + tagTypeLen, "(%d)",
(int32_t)((*smr).me.stbEntry.schemaTag.pSchema[i].bytes - VARSTR_HEADER_SIZE)); (int32_t)((*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].bytes - VARSTR_HEADER_SIZE));
} else if (tagType == TSDB_DATA_TYPE_NCHAR) { } else if (tagType == TSDB_DATA_TYPE_NCHAR) {
tagTypeLen += tagTypeLen += sprintf(
sprintf(varDataVal(tagTypeStr) + tagTypeLen, "(%d)", varDataVal(tagTypeStr) + tagTypeLen, "(%d)",
(int32_t)(((*smr).me.stbEntry.schemaTag.pSchema[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); (int32_t)(((*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE));
} }
varDataSetLen(tagTypeStr, tagTypeLen); varDataSetLen(tagTypeStr, tagTypeLen);
colDataAppend(pColInfoData, numOfRows, (char*)tagTypeStr, false); colDataAppend(pColInfoData, numOfRows, (char*)tagTypeStr, false);
STagVal tagVal = {0}; STagVal tagVal = {0};
tagVal.cid = (*smr).me.stbEntry.schemaTag.pSchema[i].colId; tagVal.cid = (*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].colId;
char* tagData = NULL; char* tagData = NULL;
uint32_t tagLen = 0; uint32_t tagLen = 0;
if (tagType == TSDB_DATA_TYPE_JSON) { if (tagType == TSDB_DATA_TYPE_JSON) {
tagData = (char*)pInfo->pCur->mr.me.ctbEntry.pTags; tagData = (char*)smrChildTable->me.ctbEntry.pTags;
} else { } else {
bool exist = tTagGet((STag*)pInfo->pCur->mr.me.ctbEntry.pTags, &tagVal); bool exist = tTagGet((STag*)smrChildTable->me.ctbEntry.pTags, &tagVal);
if (exist) { if (exist) {
if (IS_VAR_DATA_TYPE(tagType)) { if (IS_VAR_DATA_TYPE(tagType)) {
tagData = (char*)tagVal.pData; tagData = (char*)tagVal.pData;

4
source/libs/executor/src/tfill.c
View File

@ -1620,9 +1620,7 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi
goto _error; goto _error;
} }
SInterval* pInterval = QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL == downstream->operatorType SInterval* pInterval = &((SStreamIntervalOperatorInfo*)downstream->info)->interval;
? &((SStreamFinalIntervalOperatorInfo*)downstream->info)->interval
: &((SStreamIntervalOperatorInfo*)downstream->info)->interval;
int32_t numOfFillCols = 0; int32_t numOfFillCols = 0;
SExprInfo* pFillExprInfo = createExprInfo(pPhyFillNode->pFillExprs, NULL, &numOfFillCols); SExprInfo* pFillExprInfo = createExprInfo(pPhyFillNode->pFillExprs, NULL, &numOfFillCols);
pInfo->pFillSup = initStreamFillSup(pPhyFillNode, pInterval, pFillExprInfo, numOfFillCols); pInfo->pFillSup = initStreamFillSup(pPhyFillNode, pInterval, pFillExprInfo, numOfFillCols);

755
source/libs/executor/src/timewindowoperator.c
View File

File diff suppressed because it is too large Load Diff

1
source/libs/parser/inc/parUtil.h
View File

@ -91,6 +91,7 @@ STableComInfo getTableInfo(const STableMeta* pTableMeta);
STableMeta* tableMetaDup(const STableMeta* pTableMeta); STableMeta* tableMetaDup(const STableMeta* pTableMeta);
int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen); int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen);
int32_t getInsTagsTableTargetName(int32_t acctId, SNode* pWhere, SName* pName);
int32_t buildCatalogReq(SParseContext* pCxt, const SParseMetaCache* pMetaCache, SCatalogReq* pCatalogReq); int32_t buildCatalogReq(SParseContext* pCxt, const SParseMetaCache* pMetaCache, SCatalogReq* pCatalogReq);
int32_t putMetaDataToCache(const SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SParseMetaCache* pMetaCache, int32_t putMetaDataToCache(const SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SParseMetaCache* pMetaCache,

21
source/libs/parser/src/parAstParser.c
View File

@ -125,6 +125,23 @@ static bool needGetTableIndex(SNode* pStmt) {
return false; return false;
} }
static int32_t collectMetaKeyFromInsTagsImpl(SCollectMetaKeyCxt* pCxt, SName* pName) {
if (TSDB_DB_NAME_T == pName->type) {
return reserveDbVgInfoInCache(pName->acctId, pName->dbname, pCxt->pMetaCache);
}
return reserveTableVgroupInCacheExt(pName, pCxt->pMetaCache);
}
static int32_t collectMetaKeyFromInsTags(SCollectMetaKeyCxt* pCxt) {
SSelectStmt* pSelect = (SSelectStmt*)pCxt->pStmt;
SName name = {0};
int32_t code = getInsTagsTableTargetName(pCxt->pParseCxt->acctId, pSelect->pWhere, &name);
if (TSDB_CODE_SUCCESS == code) {
code = collectMetaKeyFromInsTagsImpl(pCxt, &name);
}
return code;
}
static int32_t collectMetaKeyFromRealTableImpl(SCollectMetaKeyCxt* pCxt, const char* pDb, const char* pTable, static int32_t collectMetaKeyFromRealTableImpl(SCollectMetaKeyCxt* pCxt, const char* pDb, const char* pTable,
AUTH_TYPE authType) { AUTH_TYPE authType) {
int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, pDb, pTable, pCxt->pMetaCache); int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, pDb, pTable, pCxt->pMetaCache);
@ -143,6 +160,10 @@ static int32_t collectMetaKeyFromRealTableImpl(SCollectMetaKeyCxt* pCxt, const c
if (TSDB_CODE_SUCCESS == code && (0 == strcmp(pTable, TSDB_INS_TABLE_DNODE_VARIABLES))) { if (TSDB_CODE_SUCCESS == code && (0 == strcmp(pTable, TSDB_INS_TABLE_DNODE_VARIABLES))) {
code = reserveDnodeRequiredInCache(pCxt->pMetaCache); code = reserveDnodeRequiredInCache(pCxt->pMetaCache);
} }
if (TSDB_CODE_SUCCESS == code && (0 == strcmp(pTable, TSDB_INS_TABLE_TAGS)) &&
QUERY_NODE_SELECT_STMT == nodeType(pCxt->pStmt)) {
code = collectMetaKeyFromInsTags(pCxt);
}
return code; return code;
} }

110
source/libs/parser/src/parTranslater.c
View File

@ -2198,70 +2198,70 @@ static int32_t getTagsTableVgroupListImpl(STranslateContext* pCxt, SName* pTarge
return code; return code;
} }
static int32_t getTagsTableTargetNameFromOp(STranslateContext* pCxt, SOperatorNode* pOper, SName* pName) { // static int32_t getTagsTableTargetNameFromOp(STranslateContext* pCxt, SOperatorNode* pOper, SName* pName) {
if (OP_TYPE_EQUAL != pOper->opType) { // if (OP_TYPE_EQUAL != pOper->opType) {
return TSDB_CODE_SUCCESS; // return TSDB_CODE_SUCCESS;
} // }
SColumnNode* pCol = NULL; // SColumnNode* pCol = NULL;
SValueNode* pVal = NULL; // SValueNode* pVal = NULL;
if (QUERY_NODE_COLUMN == nodeType(pOper->pLeft)) { // if (QUERY_NODE_COLUMN == nodeType(pOper->pLeft)) {
pCol = (SColumnNode*)pOper->pLeft; // pCol = (SColumnNode*)pOper->pLeft;
} else if (QUERY_NODE_VALUE == nodeType(pOper->pLeft)) { // } else if (QUERY_NODE_VALUE == nodeType(pOper->pLeft)) {
pVal = (SValueNode*)pOper->pLeft; // pVal = (SValueNode*)pOper->pLeft;
} // }
if (QUERY_NODE_COLUMN == nodeType(pOper->pRight)) { // if (QUERY_NODE_COLUMN == nodeType(pOper->pRight)) {
pCol = (SColumnNode*)pOper->pRight; // pCol = (SColumnNode*)pOper->pRight;
} else if (QUERY_NODE_VALUE == nodeType(pOper->pRight)) { // } else if (QUERY_NODE_VALUE == nodeType(pOper->pRight)) {
pVal = (SValueNode*)pOper->pRight; // pVal = (SValueNode*)pOper->pRight;
} // }
if (NULL == pCol || NULL == pVal) { // if (NULL == pCol || NULL == pVal) {
return TSDB_CODE_SUCCESS; // return TSDB_CODE_SUCCESS;
} // }
if (0 == strcmp(pCol->colName, "db_name")) { // if (0 == strcmp(pCol->colName, "db_name")) {
return tNameSetDbName(pName, pCxt->pParseCxt->acctId, pVal->literal, strlen(pVal->literal)); // return tNameSetDbName(pName, pCxt->pParseCxt->acctId, pVal->literal, strlen(pVal->literal));
} else if (0 == strcmp(pCol->colName, "table_name")) { // } else if (0 == strcmp(pCol->colName, "table_name")) {
return tNameAddTbName(pName, pVal->literal, strlen(pVal->literal)); // return tNameAddTbName(pName, pVal->literal, strlen(pVal->literal));
} // }
return TSDB_CODE_SUCCESS; // return TSDB_CODE_SUCCESS;
} // }
static void getTagsTableTargetObjName(STranslateContext* pCxt, SNode* pNode, SName* pName) { // static void getTagsTableTargetObjName(STranslateContext* pCxt, SNode* pNode, SName* pName) {
if (QUERY_NODE_OPERATOR == nodeType(pNode)) { // if (QUERY_NODE_OPERATOR == nodeType(pNode)) {
getTagsTableTargetNameFromOp(pCxt, (SOperatorNode*)pNode, pName); // getTagsTableTargetNameFromOp(pCxt, (SOperatorNode*)pNode, pName);
} // }
} // }
static int32_t getTagsTableTargetNameFromCond(STranslateContext* pCxt, SLogicConditionNode* pCond, SName* pName) { // static int32_t getTagsTableTargetNameFromCond(STranslateContext* pCxt, SLogicConditionNode* pCond, SName* pName) {
if (LOGIC_COND_TYPE_AND != pCond->condType) { // if (LOGIC_COND_TYPE_AND != pCond->condType) {
return TSDB_CODE_SUCCESS; // return TSDB_CODE_SUCCESS;
} // }
SNode* pNode = NULL; // SNode* pNode = NULL;
FOREACH(pNode, pCond->pParameterList) { getTagsTableTargetObjName(pCxt, pNode, pName); } // FOREACH(pNode, pCond->pParameterList) { getTagsTableTargetObjName(pCxt, pNode, pName); }
if ('\0' == pName->dbname[0]) { // if ('\0' == pName->dbname[0]) {
pName->type = 0; // pName->type = 0;
} // }
return TSDB_CODE_SUCCESS; // return TSDB_CODE_SUCCESS;
} // }
static int32_t getTagsTableTargetName(STranslateContext* pCxt, SNode* pWhere, SName* pName) { // static int32_t getTagsTableTargetName(STranslateContext* pCxt, SNode* pWhere, SName* pName) {
if (NULL == pWhere) { // if (NULL == pWhere) {
return TSDB_CODE_SUCCESS; // return TSDB_CODE_SUCCESS;
} // }
if (QUERY_NODE_OPERATOR == nodeType(pWhere)) { // if (QUERY_NODE_OPERATOR == nodeType(pWhere)) {
return getTagsTableTargetNameFromOp(pCxt, (SOperatorNode*)pWhere, pName); // return getTagsTableTargetNameFromOp(pCxt, (SOperatorNode*)pWhere, pName);
} // }
if (QUERY_NODE_LOGIC_CONDITION == nodeType(pWhere)) { // if (QUERY_NODE_LOGIC_CONDITION == nodeType(pWhere)) {
return getTagsTableTargetNameFromCond(pCxt, (SLogicConditionNode*)pWhere, pName); // return getTagsTableTargetNameFromCond(pCxt, (SLogicConditionNode*)pWhere, pName);
} // }
return TSDB_CODE_SUCCESS; // return TSDB_CODE_SUCCESS;
} // }
static int32_t getTagsTableVgroupList(STranslateContext* pCxt, SName* pName, SArray** pVgroupList) { static int32_t getTagsTableVgroupList(STranslateContext* pCxt, SName* pName, SArray** pVgroupList) {
if (!isSelectStmt(pCxt->pCurrStmt)) { if (!isSelectStmt(pCxt->pCurrStmt)) {
@ -2269,7 +2269,7 @@ static int32_t getTagsTableVgroupList(STranslateContext* pCxt, SName* pName, SAr
} }
SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt; SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt;
SName targetName = {0}; SName targetName = {0};
int32_t code = getTagsTableTargetName(pCxt, pSelect->pWhere, &targetName); int32_t code = getInsTagsTableTargetName(pCxt->pParseCxt->acctId, pSelect->pWhere, &targetName);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = getTagsTableVgroupListImpl(pCxt, &targetName, pName, pVgroupList); code = getTagsTableVgroupListImpl(pCxt, &targetName, pName, pVgroupList);
} }

65
source/libs/parser/src/parUtil.c
View File

@ -420,6 +420,71 @@ end:
return retCode; return retCode;
} }
static int32_t getInsTagsTableTargetNameFromOp(int32_t acctId, SOperatorNode* pOper, SName* pName) {
if (OP_TYPE_EQUAL != pOper->opType) {
return TSDB_CODE_SUCCESS;
}
SColumnNode* pCol = NULL;
SValueNode* pVal = NULL;
if (QUERY_NODE_COLUMN == nodeType(pOper->pLeft)) {
pCol = (SColumnNode*)pOper->pLeft;
} else if (QUERY_NODE_VALUE == nodeType(pOper->pLeft)) {
pVal = (SValueNode*)pOper->pLeft;
}
if (QUERY_NODE_COLUMN == nodeType(pOper->pRight)) {
pCol = (SColumnNode*)pOper->pRight;
} else if (QUERY_NODE_VALUE == nodeType(pOper->pRight)) {
pVal = (SValueNode*)pOper->pRight;
}
if (NULL == pCol || NULL == pVal) {
return TSDB_CODE_SUCCESS;
}
if (0 == strcmp(pCol->colName, "db_name")) {
return tNameSetDbName(pName, acctId, pVal->literal, strlen(pVal->literal));
} else if (0 == strcmp(pCol->colName, "table_name")) {
return tNameAddTbName(pName, pVal->literal, strlen(pVal->literal));
}
return TSDB_CODE_SUCCESS;
}
static void getInsTagsTableTargetObjName(int32_t acctId, SNode* pNode, SName* pName) {
if (QUERY_NODE_OPERATOR == nodeType(pNode)) {
getInsTagsTableTargetNameFromOp(acctId, (SOperatorNode*)pNode, pName);
}
}
static int32_t getInsTagsTableTargetNameFromCond(int32_t acctId, SLogicConditionNode* pCond, SName* pName) {
if (LOGIC_COND_TYPE_AND != pCond->condType) {
return TSDB_CODE_SUCCESS;
}
SNode* pNode = NULL;
FOREACH(pNode, pCond->pParameterList) { getInsTagsTableTargetObjName(acctId, pNode, pName); }
if ('\0' == pName->dbname[0]) {
pName->type = 0;
}
return TSDB_CODE_SUCCESS;
}
int32_t getInsTagsTableTargetName(int32_t acctId, SNode* pWhere, SName* pName) {
if (NULL == pWhere) {
return TSDB_CODE_SUCCESS;
}
if (QUERY_NODE_OPERATOR == nodeType(pWhere)) {
return getInsTagsTableTargetNameFromOp(acctId, (SOperatorNode*)pWhere, pName);
}
if (QUERY_NODE_LOGIC_CONDITION == nodeType(pWhere)) {
return getInsTagsTableTargetNameFromCond(acctId, (SLogicConditionNode*)pWhere, pName);
}
return TSDB_CODE_SUCCESS;
}
static int32_t userAuthToString(int32_t acctId, const char* pUser, const char* pDb, AUTH_TYPE type, char* pStr) { static int32_t userAuthToString(int32_t acctId, const char* pUser, const char* pDb, AUTH_TYPE type, char* pStr) {
return sprintf(pStr, "%s*%d.%s*%d", pUser, acctId, pDb, type); return sprintf(pStr, "%s*%d.%s*%d", pUser, acctId, pDb, type);
} }

6
source/libs/parser/test/parSelectTest.cpp
View File

@ -420,9 +420,11 @@ TEST_F(ParserSelectTest, setOperatorSemanticCheck) {
} }
TEST_F(ParserSelectTest, informationSchema) { TEST_F(ParserSelectTest, informationSchema) {
useDb("root", "test"); useDb("root", "information_schema");
run("SELECT * FROM information_schema.ins_databases WHERE name = 'information_schema'"); run("SELECT * FROM ins_databases WHERE name = 'information_schema'");
run("SELECT * FROM ins_tags WHERE db_name = 'test' and table_name = 'st1'");
} }
TEST_F(ParserSelectTest, withoutFrom) { TEST_F(ParserSelectTest, withoutFrom) {

2
source/libs/parser/test/parTestMain.cpp
View File

@ -53,7 +53,7 @@ class ParserEnv : public testing::Environment {
private: private:
void initLog(const char* path) { void initLog(const char* path) {
int32_t logLevel = getLogLevel(); int32_t logLevel = getLogLevel() | DEBUG_SCREEN;
dDebugFlag = logLevel; dDebugFlag = logLevel;
vDebugFlag = logLevel; vDebugFlag = logLevel;
mDebugFlag = logLevel; mDebugFlag = logLevel;

2
source/libs/planner/test/planTestMain.cpp
View File

@ -48,7 +48,7 @@ class PlannerEnv : public testing::Environment {
private: private:
void initLog(const char* path) { void initLog(const char* path) {
int32_t logLevel = getLogLevel(); int32_t logLevel = getLogLevel() | DEBUG_SCREEN;
dDebugFlag = logLevel; dDebugFlag = logLevel;
vDebugFlag = logLevel; vDebugFlag = logLevel;
mDebugFlag = logLevel; mDebugFlag = logLevel;

132
source/libs/stream/src/streamState.c
View File

@ -18,6 +18,37 @@
#include "tcommon.h" #include "tcommon.h"
#include "ttimer.h" #include "ttimer.h"
// todo refactor
typedef struct SStateKey {
SWinKey key;
int64_t opNum;
} SStateKey;
static inline int SStateKeyCmpr(const void* pKey1, int kLen1, const void* pKey2, int kLen2) {
SStateKey* pWin1 = (SStateKey*)pKey1;
SStateKey* pWin2 = (SStateKey*)pKey2;
if (pWin1->opNum > pWin2->opNum) {
return 1;
} else if (pWin1->opNum < pWin2->opNum) {
return -1;
}
if (pWin1->key.ts > pWin2->key.ts) {
return 1;
} else if (pWin1->key.ts < pWin2->key.ts) {
return -1;
}
if (pWin1->key.groupId > pWin2->key.groupId) {
return 1;
} else if (pWin1->key.groupId < pWin2->key.groupId) {
return -1;
}
return 0;
}
SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath) { SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath) {
SStreamState* pState = taosMemoryCalloc(1, sizeof(SStreamState)); SStreamState* pState = taosMemoryCalloc(1, sizeof(SStreamState));
if (pState == NULL) { if (pState == NULL) {
@ -36,7 +67,7 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath) {
} }
// open state storage backend // open state storage backend
if (tdbTbOpen("state.db", sizeof(SWinKey), -1, SWinKeyCmpr, pState->db, &pState->pStateDb) < 0) { if (tdbTbOpen("state.db", sizeof(SStateKey), -1, SStateKeyCmpr, pState->db, &pState->pStateDb) < 0) {
goto _err; goto _err;
} }
@ -130,8 +161,10 @@ int32_t streamStateFuncDel(SStreamState* pState, const STupleKey* key) {
return tdbTbDelete(pState->pFuncStateDb, key, sizeof(STupleKey), &pState->txn); return tdbTbDelete(pState->pFuncStateDb, key, sizeof(STupleKey), &pState->txn);
} }
// todo refactor
int32_t streamStatePut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen) { int32_t streamStatePut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen) {
return tdbTbUpsert(pState->pStateDb, key, sizeof(SWinKey), value, vLen, &pState->txn); SStateKey sKey = {.key = *key, .opNum = pState->number};
return tdbTbUpsert(pState->pStateDb, &sKey, sizeof(SStateKey), value, vLen, &pState->txn);
} }
// todo refactor // todo refactor
@ -139,8 +172,10 @@ int32_t streamStateFillPut(SStreamState* pState, const SWinKey* key, const void*
return tdbTbUpsert(pState->pFillStateDb, key, sizeof(SWinKey), value, vLen, &pState->txn); return tdbTbUpsert(pState->pFillStateDb, key, sizeof(SWinKey), value, vLen, &pState->txn);
} }
// todo refactor
int32_t streamStateGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen) { int32_t streamStateGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen) {
return tdbTbGet(pState->pStateDb, key, sizeof(SWinKey), pVal, pVLen); SStateKey sKey = {.key = *key, .opNum = pState->number};
return tdbTbGet(pState->pStateDb, &sKey, sizeof(SStateKey), pVal, pVLen);
} }
// todo refactor // todo refactor
@ -148,10 +183,30 @@ int32_t streamStateFillGet(SStreamState* pState, const SWinKey* key, void** pVal
return tdbTbGet(pState->pFillStateDb, key, sizeof(SWinKey), pVal, pVLen); return tdbTbGet(pState->pFillStateDb, key, sizeof(SWinKey), pVal, pVLen);
} }
// todo refactor
int32_t streamStateDel(SStreamState* pState, const SWinKey* key) { int32_t streamStateDel(SStreamState* pState, const SWinKey* key) {
return tdbTbDelete(pState->pStateDb, key, sizeof(SWinKey), &pState->txn); SStateKey sKey = {.key = *key, .opNum = pState->number};
return tdbTbDelete(pState->pStateDb, &sKey, sizeof(SStateKey), &pState->txn);
} }
int32_t streamStateClear(SStreamState* pState) {
SWinKey key = {.ts = 0, .groupId = 0};
streamStatePut(pState, &key, NULL, 0);
while (1) {
SStreamStateCur* pCur = streamStateSeekKeyNext(pState, &key);
SWinKey delKey = {0};
int32_t code = streamStateGetKVByCur(pCur, &delKey, NULL, 0);
if (code == 0) {
streamStateDel(pState, &delKey);
} else {
break;
}
}
return 0;
}
void streamStateSetNumber(SStreamState* pState, int32_t number) { pState->number = number; }
// todo refactor // todo refactor
int32_t streamStateFillDel(SStreamState* pState, const SWinKey* key) { int32_t streamStateFillDel(SStreamState* pState, const SWinKey* key) {
return tdbTbDelete(pState->pFillStateDb, key, sizeof(SWinKey), &pState->txn); return tdbTbDelete(pState->pFillStateDb, key, sizeof(SWinKey), &pState->txn);
@ -179,12 +234,14 @@ SStreamStateCur* streamStateGetCur(SStreamState* pState, const SWinKey* key) {
if (pCur == NULL) return NULL; if (pCur == NULL) return NULL;
tdbTbcOpen(pState->pStateDb, &pCur->pCur, NULL); tdbTbcOpen(pState->pStateDb, &pCur->pCur, NULL);
int32_t c; int32_t c;
SStateKey sKey = {.key = *key, .opNum = pState->number};
tdbTbcMoveTo(pCur->pCur, key, sizeof(SWinKey), &c); tdbTbcMoveTo(pCur->pCur, key, sizeof(SWinKey), &c);
if (c != 0) { if (c != 0) {
taosMemoryFree(pCur); taosMemoryFree(pCur);
return NULL; return NULL;
} }
pCur->number = pState->number;
return pCur; return pCur;
} }
@ -214,6 +271,25 @@ SStreamStateCur* streamStateGetAndCheckCur(SStreamState* pState, SWinKey* key) {
} }
int32_t streamStateGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen) { int32_t streamStateGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen) {
if (!pCur) {
return -1;
}
const SStateKey* pKTmp = NULL;
int32_t kLen;
if (tdbTbcGet(pCur->pCur, (const void**)&pKTmp, &kLen, pVal, pVLen) < 0) {
return -1;
}
if (pKTmp->opNum != pCur->number) {
return -1;
}
*pKey = pKTmp->key;
return 0;
}
int32_t streamStateFillGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen) {
if (!pCur) {
return -1;
}
const SWinKey* pKTmp = NULL; const SWinKey* pKTmp = NULL;
int32_t kLen; int32_t kLen;
if (tdbTbcGet(pCur->pCur, (const void**)&pKTmp, &kLen, pVal, pVLen) < 0) { if (tdbTbcGet(pCur->pCur, (const void**)&pKTmp, &kLen, pVal, pVLen) < 0) {
@ -225,7 +301,7 @@ int32_t streamStateGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void**
int32_t streamStateGetGroupKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen) { int32_t streamStateGetGroupKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen) {
uint64_t groupId = pKey->groupId; uint64_t groupId = pKey->groupId;
int32_t code = streamStateGetKVByCur(pCur, pKey, pVal, pVLen); int32_t code = streamStateFillGetKVByCur(pCur, pKey, pVal, pVLen);
if (code == 0) { if (code == 0) {
if (pKey->groupId == groupId) { if (pKey->groupId == groupId) {
return 0; return 0;
@ -234,6 +310,16 @@ int32_t streamStateGetGroupKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const v
return -1; return -1;
} }
int32_t streamStateGetFirst(SStreamState* pState, SWinKey* key) {
// todo refactor
SWinKey tmp = {.ts = 0, .groupId = 0};
streamStatePut(pState, &tmp, NULL, 0);
SStreamStateCur* pCur = streamStateSeekKeyNext(pState, &tmp);
int32_t code = streamStateGetKVByCur(pCur, key, NULL, 0);
streamStateDel(pState, &tmp);
return code;
}
int32_t streamStateSeekFirst(SStreamState* pState, SStreamStateCur* pCur) { int32_t streamStateSeekFirst(SStreamState* pState, SStreamStateCur* pCur) {
// //
return tdbTbcMoveToFirst(pCur->pCur); return tdbTbcMoveToFirst(pCur->pCur);
@ -244,6 +330,34 @@ int32_t streamStateSeekLast(SStreamState* pState, SStreamStateCur* pCur) {
return tdbTbcMoveToLast(pCur->pCur); return tdbTbcMoveToLast(pCur->pCur);
} }
SStreamStateCur* streamStateSeekKeyNext(SStreamState* pState, const SWinKey* key) {
SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur));
if (pCur == NULL) {
return NULL;
}
pCur->number = pState->number;
if (tdbTbcOpen(pState->pStateDb, &pCur->pCur, NULL) < 0) {
taosMemoryFree(pCur);
return NULL;
}
SStateKey sKey = {.key = *key, .opNum = pState->number};
int32_t c;
if (tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateKey), &c) < 0) {
tdbTbcClose(pCur->pCur);
taosMemoryFree(pCur);
return NULL;
}
if (c > 0) return pCur;
if (tdbTbcMoveToNext(pCur->pCur) < 0) {
taosMemoryFree(pCur);
return NULL;
}
return pCur;
}
SStreamStateCur* streamStateFillSeekKeyNext(SStreamState* pState, const SWinKey* key) { SStreamStateCur* streamStateFillSeekKeyNext(SStreamState* pState, const SWinKey* key) {
SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur));
if (pCur == NULL) { if (pCur == NULL) {
@ -303,9 +417,15 @@ int32_t streamStateCurNext(SStreamState* pState, SStreamStateCur* pCur) {
int32_t streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur) { int32_t streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur) {
// //
if (!pCur) {
return -1;
}
return tdbTbcMoveToPrev(pCur->pCur); return tdbTbcMoveToPrev(pCur->pCur);
} }
void streamStateFreeCur(SStreamStateCur* pCur) { void streamStateFreeCur(SStreamStateCur* pCur) {
if (!pCur) {
return;
}
tdbTbcClose(pCur->pCur); tdbTbcClose(pCur->pCur);
taosMemoryFree(pCur); taosMemoryFree(pCur);
} }

1306
source/util/src/tcompression.c
View File

File diff suppressed because it is too large Load Diff

3
tests/script/tsim/stream/distributeInterval0.sim
View File

@ -33,7 +33,8 @@ if $data(2)[4] != ready then
endi endi
print ===== step2 print ===== step2
sql drop stream if exists stream_t1;
sql drop database if exists test;
sql create database test vgroups 4; sql create database test vgroups 4;
sql use test; sql use test;
sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int); sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int);

48
tests/script/tsim/stream/sliding.sim
View File

@ -369,9 +369,10 @@ endi
#$loop_all = 0 #$loop_all = 0
#=looptest: #=looptest:
sql drop database IF EXISTS test2;
sql drop stream IF EXISTS streams21; sql drop stream IF EXISTS streams21;
sql drop stream IF EXISTS streams22; sql drop stream IF EXISTS streams22;
sql drop stream IF EXISTS streams23;
sql drop database IF EXISTS test2;
sql create database test2 vgroups 6; sql create database test2 vgroups 6;
sql use test2; sql use test2;
@ -381,6 +382,7 @@ sql create table t2 using st tags(2,2,2);
sql create stream streams21 trigger at_once into streamt as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s, 5s); sql create stream streams21 trigger at_once into streamt as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s, 5s);
sql create stream streams22 trigger at_once into streamt2 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s, 5s); sql create stream streams22 trigger at_once into streamt2 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s, 5s);
sql create stream streams23 trigger at_once into streamt3 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(20s) sliding(10s);
sql insert into t1 values(1648791213000,1,1,1,1.0); sql insert into t1 values(1648791213000,1,1,1,1.0);
sql insert into t1 values(1648791223001,2,2,2,1.1); sql insert into t1 values(1648791223001,2,2,2,1.1);
@ -508,6 +510,50 @@ if $data32 != 8 then
goto loop3 goto loop3
endi endi
$loop_count = 0
loop4:
sleep 100
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
sql select * from streamt3;
# row 0
if $rows != 5 then
print =====rows=$rows
goto loop4
endi
if $data01 != 4 then
print =====data01=$data01
goto loop4
endi
if $data11 != 6 then
print =====data11=$data11
goto loop4
endi
if $data21 != 4 then
print =====data21=$data21
goto loop4
endi
if $data31 != 4 then
print =====data31=$data31
goto loop4
endi
if $data41 != 2 then
print =====data41=$data41
goto loop4
endi
$loop_all = $loop_all + 1 $loop_all = $loop_all + 1
print ============loop_all=$loop_all print ============loop_all=$loop_all