From a3f7cb3e27f613d9c3cbdb78fe6e893fdb522420 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 10 Jan 2023 15:44:05 +0800 Subject: [PATCH 01/25] docs: add description of multi-tier storage --- docs/en/12-taos-sql/02-database.md | 2 +- docs/zh/12-taos-sql/02-database.md | 2 +- docs/zh/14-reference/12-config/index.md | 19 +++++++++++++++++++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/docs/en/12-taos-sql/02-database.md b/docs/en/12-taos-sql/02-database.md index c3c7e5928b..4fd3fb7982 100644 --- a/docs/en/12-taos-sql/02-database.md +++ b/docs/en/12-taos-sql/02-database.md @@ -56,7 +56,7 @@ database_option: { - WAL_FSYNC_PERIOD: specifies the interval (in milliseconds) at which data is written from the WAL to disk. This parameter takes effect only when the WAL parameter is set to 2. The default value is 3000. Enter a value between 0 and 180000. The value 0 indicates that incoming data is immediately written to disk. - MAXROWS: specifies the maximum number of rows recorded in a block. The default value is 4096. - MINROWS: specifies the minimum number of rows recorded in a block. The default value is 100. -- KEEP: specifies the time for which data is retained. Enter a value between 1 and 365000. The default value is 3650. The value of the KEEP parameter must be greater than or equal to the value of the DURATION parameter. TDengine automatically deletes data that is older than the value of the KEEP parameter. You can use m (minutes), h (hours), and d (days) as the unit, for example KEEP 100h or KEEP 10d. If you do not include a unit, d is used by default. +- KEEP: specifies the time for which data is retained. Enter a value between 1 and 365000. The default value is 3650. The value of the KEEP parameter must be greater than or equal to the value of the DURATION parameter. TDengine automatically deletes data that is older than the value of the KEEP parameter. You can use m (minutes), h (hours), and d (days) as the unit, for example KEEP 100h or KEEP 10d. If you do not include a unit, d is used by default. The official version supports multi-tier storage, corresponding to multiple KEEP values(comma separated and up to 3 values keep0,keep1,keep2, and meet keep0 <= keep1 <= keep2, e.g. KEEP 100h,100d,3650d); the community version does not support multi-tier storage(although multiple keep values are configured, they do not take effect, only the maximum keep value is used as KEEP). - PAGES: specifies the number of pages in the metadata storage engine cache on each vnode. Enter a value greater than or equal to 64. The default value is 256. The space occupied by metadata storage on each vnode is equal to the product of the values of the PAGESIZE and PAGES parameters. The space occupied by default is 1 MB. - PAGESIZE: specifies the size (in KB) of each page in the metadata storage engine cache on each vnode. The default value is 4. Enter a value between 1 and 16384. - PRECISION: specifies the precision at which a database records timestamps. Enter ms for milliseconds, us for microseconds, or ns for nanoseconds. The default value is ms. diff --git a/docs/zh/12-taos-sql/02-database.md b/docs/zh/12-taos-sql/02-database.md index df52a0890b..e29b62f69c 100644 --- a/docs/zh/12-taos-sql/02-database.md +++ b/docs/zh/12-taos-sql/02-database.md @@ -56,7 +56,7 @@ database_option: { - WAL_FSYNC_PERIOD：当 WAL 参数设置为 2 时，落盘的周期。默认为 3000，单位毫秒。最小为 0，表示每次写入立即落盘；最大为 180000，即三分钟。 - MAXROWS：文件块中记录的最大条数，默认为 4096 条。 - MINROWS：文件块中记录的最小条数，默认为 100 条。 -- KEEP：表示数据文件保存的天数，缺省值为 3650，取值范围 [1, 365000]，且必须大于或等于 DURATION 参数值。数据库会自动删除保存时间超过 KEEP 值的数据。KEEP 可以使用加单位的表示形式，如 KEEP 100h、KEEP 10d 等，支持 m（分钟）、h（小时）和 d（天）三个单位。也可以不写单位，如 KEEP 50，此时默认单位为天。 +- KEEP：表示数据文件保存的天数，缺省值为 3650，取值范围 [1, 365000]，且必须大于或等于 DURATION 参数值。数据库会自动删除保存时间超过 KEEP 值的数据。KEEP 可以使用加单位的表示形式，如 KEEP 100h、KEEP 10d 等，支持 m（分钟）、h（小时）和 d（天）三个单位。也可以不写单位，如 KEEP 50，此时默认单位为天。企业版支持多级存储功能, 对应多个保存时间(多个以逗号分隔, 最多 3 级, 如 KEEP 100h,100d,3650d, 其中, keep0 <= keep1 <= keep2); 社区版不支持多级存储(即使配置了多个保存时间, 也不会生效, KEEP 会取最大的 keep 值)。 - PAGES：一个 VNODE 中元数据存储引擎的缓存页个数，默认为 256，最小 64。一个 VNODE 元数据存储占用 PAGESIZE \* PAGES，默认情况下为 1MB 内存。 - PAGESIZE：一个 VNODE 中元数据存储引擎的页大小，单位为 KB，默认为 4 KB。范围为 1 到 16384，即 1 KB 到 16 MB。 - PRECISION：数据库的时间戳精度。ms 表示毫秒，us 表示微秒，ns 表示纳秒，默认 ms 毫秒。 diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index c42130901f..8f0df3a0fe 100644 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -305,6 +305,25 @@ charset 的有效值是 UTF-8。 | 适用范围 | 仅服务端适用 | | 含义 | 数据文件目录，所有的数据文件都将写入该目录 | | 缺省值 | /var/lib/taos | +``` +多级存储目录通过 dataDir 配置. 格式如下： +dataDir [path] +dataDir: 配置文件( taos.cfg )中的配置项 +path: 需要挂载的目录（不可省略） +level: 挂载等级，取值为 0，1 或 2，省略默认为 0。 最多支持 3 级, 每一级最多支持 16 个挂载点, 因此, 最多支持 48 个挂载点. +primary: 是否为主挂载点，0（否）或 1（是），默认为 1. 主挂载点只允许在 level 0, 且只允许一个主挂载点存在（level=0，primary=1）. +如下，为一个正常的配置： +dataDir /mnt/data01 0 1 +dataDir /mnt/data02 0 0 +dataDir /mnt/data03 0 0 +dataDir /mnt/data10 1 0 +dataDir /mnt/data11 1 0 +dataDir /mnt/data12 1 0 +dataDir /mnt/data20 2 0 +dataDir /mnt/data21 2 0 +dataDir /mnt/data22 2 0 +``` + ### tempDir From ea13d2bc99146ed11e825e74a0da2e9f6daa3776 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 10 Jan 2023 17:03:57 +0800 Subject: [PATCH 02/25] docs: add description of keep/multi-tier storage --- docs/en/12-taos-sql/02-database.md | 2 +- docs/en/14-reference/12-config/index.md | 26 +++++++++++++++++++++++++ docs/zh/12-taos-sql/02-database.md | 2 +- docs/zh/14-reference/12-config/index.md | 21 +++++++++++++------- 4 files changed, 42 insertions(+), 9 deletions(-) diff --git a/docs/en/12-taos-sql/02-database.md b/docs/en/12-taos-sql/02-database.md index 4fd3fb7982..ce95b0a165 100644 --- a/docs/en/12-taos-sql/02-database.md +++ b/docs/en/12-taos-sql/02-database.md @@ -56,7 +56,7 @@ database_option: { - WAL_FSYNC_PERIOD: specifies the interval (in milliseconds) at which data is written from the WAL to disk. This parameter takes effect only when the WAL parameter is set to 2. The default value is 3000. Enter a value between 0 and 180000. The value 0 indicates that incoming data is immediately written to disk. - MAXROWS: specifies the maximum number of rows recorded in a block. The default value is 4096. - MINROWS: specifies the minimum number of rows recorded in a block. The default value is 100. -- KEEP: specifies the time for which data is retained. Enter a value between 1 and 365000. The default value is 3650. The value of the KEEP parameter must be greater than or equal to the value of the DURATION parameter. TDengine automatically deletes data that is older than the value of the KEEP parameter. You can use m (minutes), h (hours), and d (days) as the unit, for example KEEP 100h or KEEP 10d. If you do not include a unit, d is used by default. The official version supports multi-tier storage, corresponding to multiple KEEP values(comma separated and up to 3 values keep0,keep1,keep2, and meet keep0 <= keep1 <= keep2, e.g. KEEP 100h,100d,3650d); the community version does not support multi-tier storage(although multiple keep values are configured, they do not take effect, only the maximum keep value is used as KEEP). +- KEEP: specifies the time for which data is retained. Enter a value between 1 and 365000. The default value is 3650. The value of the KEEP parameter must be greater than or equal to the value of the DURATION parameter. TDengine automatically deletes data that is older than the value of the KEEP parameter. You can use m (minutes), h (hours), and d (days) as the unit, for example KEEP 100h or KEEP 10d. If you do not include a unit, d is used by default. The official version supports [multi-tier storage](https://docs.tdengine.com/reference/config/#datadir), corresponding to multiple KEEP values(comma separated and up to 3 values keep0,keep1,keep2 supported, and meet keep0 <= keep1 <= keep2, e.g. KEEP 100h,100d,3650d); the community version does not support multi-tier storage(although multiple keep values are configured, they do not take effect, only the maximum keep value is used as KEEP). - PAGES: specifies the number of pages in the metadata storage engine cache on each vnode. Enter a value greater than or equal to 64. The default value is 256. The space occupied by metadata storage on each vnode is equal to the product of the values of the PAGESIZE and PAGES parameters. The space occupied by default is 1 MB. - PAGESIZE: specifies the size (in KB) of each page in the metadata storage engine cache on each vnode. The default value is 4. Enter a value between 1 and 16384. - PRECISION: specifies the precision at which a database records timestamps. Enter ms for milliseconds, us for microseconds, or ns for nanoseconds. The default value is ms. diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index b6bfa4bc7d..b6e11de952 100644 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -314,6 +314,32 @@ The charset that takes effect is UTF-8. | Applicable | Server Only | | Meaning | All data files are stored in this directory | | Default Value | /var/lib/taos | +``` +The official version supports multi-tier storage (need to cooperate with [KEEP parameter](https://docs.tdengine.com/taos-sql/database/#create-a-database)). The configuration method is as follows. + +The multi-tier storage directory is configured through dataDir, the format is: dataDir [path] + +dataDir: Configuration items in the configuration file (taos.cfg) +path: the directory to mount (cannot be omitted) +level: mount level, the value is 0, 1 or 2, if omitted, the default is 0. Up to 3 levels are supported, and each level supports up to 16 mount points, so a maximum of 48 mount points are supported. +primary: whether it is the primary mount point, 0 (no) or 1 (yes), the default is 1. The primary mount point is only allowed at level 0, and only one primary mount point is allowed (level=0, primary=1). + +Here is a normal configuration (3 storage tiers, each with 3 mount points): +Data directory /mnt/data01 0 1 +Data directory /mnt/data02 0 0 +Data directory /mnt/data03 0 0 +Data directory /mnt/data10 1 0 +Data directory /mnt/data11 1 0 +Data directory /mnt/data12 1 0 +Data directory /mnt/data20 2 0 +Data directory /mnt/data21 2 0 +Data directory /mnt/data22 2 0 + +The community version does not support multi-tier storage, and the following three configurations are acceptable: +Data directory /var/lib/taos +or dataDir /var/lib/taos 0 +or dataDir /var/lib/taos 0 1 +``` ### tempDir diff --git a/docs/zh/12-taos-sql/02-database.md b/docs/zh/12-taos-sql/02-database.md index e29b62f69c..0b0167a306 100644 --- a/docs/zh/12-taos-sql/02-database.md +++ b/docs/zh/12-taos-sql/02-database.md @@ -56,7 +56,7 @@ database_option: { - WAL_FSYNC_PERIOD：当 WAL 参数设置为 2 时，落盘的周期。默认为 3000，单位毫秒。最小为 0，表示每次写入立即落盘；最大为 180000，即三分钟。 - MAXROWS：文件块中记录的最大条数，默认为 4096 条。 - MINROWS：文件块中记录的最小条数，默认为 100 条。 -- KEEP：表示数据文件保存的天数，缺省值为 3650，取值范围 [1, 365000]，且必须大于或等于 DURATION 参数值。数据库会自动删除保存时间超过 KEEP 值的数据。KEEP 可以使用加单位的表示形式，如 KEEP 100h、KEEP 10d 等，支持 m（分钟）、h（小时）和 d（天）三个单位。也可以不写单位，如 KEEP 50，此时默认单位为天。企业版支持多级存储功能, 对应多个保存时间(多个以逗号分隔, 最多 3 级, 如 KEEP 100h,100d,3650d, 其中, keep0 <= keep1 <= keep2); 社区版不支持多级存储(即使配置了多个保存时间, 也不会生效, KEEP 会取最大的 keep 值)。 +- KEEP：表示数据文件保存的天数，缺省值为 3650，取值范围 [1, 365000]，且必须大于或等于 DURATION 参数值。数据库会自动删除保存时间超过 KEEP 值的数据。KEEP 可以使用加单位的表示形式，如 KEEP 100h、KEEP 10d 等，支持 m（分钟）、h（小时）和 d（天）三个单位。也可以不写单位，如 KEEP 50，此时默认单位为天。企业版支持[多级存储](https://docs.taosdata.com/reference/config/#datadir)功能, 对应多个保存时间（多个以英文逗号分隔，最多 3 级：keep0,keep1,keep2，满足 keep0 <= keep1 <= keep2，如 KEEP 100h,100d,3650d）; 社区版不支持多级存储（即使配置了多个保存时间, 也不会生效, KEEP 会取最大的 keep 值）。 - PAGES：一个 VNODE 中元数据存储引擎的缓存页个数，默认为 256，最小 64。一个 VNODE 元数据存储占用 PAGESIZE \* PAGES，默认情况下为 1MB 内存。 - PAGESIZE：一个 VNODE 中元数据存储引擎的页大小，单位为 KB，默认为 4 KB。范围为 1 到 16384，即 1 KB 到 16 MB。 - PRECISION：数据库的时间戳精度。ms 表示毫秒，us 表示微秒，ns 表示纳秒，默认 ms 毫秒。 diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index 8f0df3a0fe..e3101830a1 100644 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -306,13 +306,16 @@ charset 的有效值是 UTF-8。 | 含义 | 数据文件目录，所有的数据文件都将写入该目录 | | 缺省值 | /var/lib/taos | ``` -多级存储目录通过 dataDir 配置. 格式如下： -dataDir [path] -dataDir: 配置文件( taos.cfg )中的配置项 +企业版支持多级存储 (需要与[KEEP 参数](https://docs.taosdata.com/taos-sql/database/#%E5%88%9B%E5%BB%BA%E6%95%B0%E6%8D%AE%E5%BA%93)配合使用). 配置方法如下。 + +多级存储目录通过 dataDir 配置。配置项格式： dataDir [path] + +dataDir: 配置文件（taos.cfg）中的配置项 path: 需要挂载的目录（不可省略） -level: 挂载等级，取值为 0，1 或 2，省略默认为 0。 最多支持 3 级, 每一级最多支持 16 个挂载点, 因此, 最多支持 48 个挂载点. -primary: 是否为主挂载点，0（否）或 1（是），默认为 1. 主挂载点只允许在 level 0, 且只允许一个主挂载点存在（level=0，primary=1）. -如下，为一个正常的配置： +level: 挂载等级，取值为 0，1 或 2，省略默认为 0。最多支持 3 级, 每一级最多支持 16 个挂载点, 因此, 最多支持 48 个挂载点。 +primary: 是否为主挂载点，0（否）或 1（是），默认为 1。主挂载点只允许在 level 0，且只允许一个主挂载点存在（level = 0，primary = 1）。 + +如下，为正常的配置 ( 3 个存储层级，每个层级 3 个挂载点)： dataDir /mnt/data01 0 1 dataDir /mnt/data02 0 0 dataDir /mnt/data03 0 0 @@ -322,8 +325,12 @@ dataDir /mnt/data12 1 0 dataDir /mnt/data20 2 0 dataDir /mnt/data21 2 0 dataDir /mnt/data22 2 0 -``` +社区版不支持多级存储, 以下 3 种均为正常配置: +dataDir /var/lib/taos +或 dataDir /var/lib/taos 0 +或 dataDir /var/lib/taos 0 1 +``` ### tempDir From ae1a82b0ed29afbc10446607cffa4c6aaa7e47b3 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 10 Jan 2023 17:10:33 +0800 Subject: [PATCH 03/25] docs: format optimization for dataDir --- docs/en/14-reference/12-config/index.md | 4 ++-- docs/zh/14-reference/12-config/index.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index b6e11de952..9b06263073 100644 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -314,7 +314,7 @@ The charset that takes effect is UTF-8. | Applicable | Server Only | | Meaning | All data files are stored in this directory | | Default Value | /var/lib/taos | -``` +:::info The official version supports multi-tier storage (need to cooperate with [KEEP parameter](https://docs.tdengine.com/taos-sql/database/#create-a-database)). The configuration method is as follows. The multi-tier storage directory is configured through dataDir, the format is: dataDir [path] @@ -339,7 +339,7 @@ The community version does not support multi-tier storage, and the following thr Data directory /var/lib/taos or dataDir /var/lib/taos 0 or dataDir /var/lib/taos 0 1 -``` +::: ### tempDir diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index e3101830a1..2e3449e731 100644 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -305,7 +305,7 @@ charset 的有效值是 UTF-8。 | 适用范围 | 仅服务端适用 | | 含义 | 数据文件目录，所有的数据文件都将写入该目录 | | 缺省值 | /var/lib/taos | -``` +:::info 企业版支持多级存储 (需要与[KEEP 参数](https://docs.taosdata.com/taos-sql/database/#%E5%88%9B%E5%BB%BA%E6%95%B0%E6%8D%AE%E5%BA%93)配合使用). 配置方法如下。 多级存储目录通过 dataDir 配置。配置项格式： dataDir [path] @@ -330,7 +330,7 @@ dataDir /mnt/data22 2 0 dataDir /var/lib/taos 或 dataDir /var/lib/taos 0 或 dataDir /var/lib/taos 0 1 -``` +::: ### tempDir From cae3532e58cedde9a01d03eccccd389d0a5bbca0 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 10 Jan 2023 17:11:55 +0800 Subject: [PATCH 04/25] docs: format optimization for dataDir --- docs/en/14-reference/12-config/index.md | 2 ++ docs/zh/14-reference/12-config/index.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index 9b06263073..51bac10f8d 100644 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -314,6 +314,7 @@ The charset that takes effect is UTF-8. | Applicable | Server Only | | Meaning | All data files are stored in this directory | | Default Value | /var/lib/taos | + :::info The official version supports multi-tier storage (need to cooperate with [KEEP parameter](https://docs.tdengine.com/taos-sql/database/#create-a-database)). The configuration method is as follows. @@ -339,6 +340,7 @@ The community version does not support multi-tier storage, and the following thr Data directory /var/lib/taos or dataDir /var/lib/taos 0 or dataDir /var/lib/taos 0 1 + ::: ### tempDir diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index 2e3449e731..96612bf82f 100644 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -305,6 +305,7 @@ charset 的有效值是 UTF-8。 | 适用范围 | 仅服务端适用 | | 含义 | 数据文件目录，所有的数据文件都将写入该目录 | | 缺省值 | /var/lib/taos | + :::info 企业版支持多级存储 (需要与[KEEP 参数](https://docs.taosdata.com/taos-sql/database/#%E5%88%9B%E5%BB%BA%E6%95%B0%E6%8D%AE%E5%BA%93)配合使用). 配置方法如下。 @@ -330,6 +331,7 @@ dataDir /mnt/data22 2 0 dataDir /var/lib/taos 或 dataDir /var/lib/taos 0 或 dataDir /var/lib/taos 0 1 + ::: ### tempDir From 90fd9496066ddc5992cc798d6691583710221764 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 10 Jan 2023 17:15:09 +0800 Subject: [PATCH 05/25] docs: format optimization for dataDir --- docs/en/14-reference/12-config/index.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index 51bac10f8d..1488219bcf 100644 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -318,14 +318,15 @@ The charset that takes effect is UTF-8. :::info The official version supports multi-tier storage (need to cooperate with [KEEP parameter](https://docs.tdengine.com/taos-sql/database/#create-a-database)). The configuration method is as follows. -The multi-tier storage directory is configured through dataDir, the format is: dataDir [path] - -dataDir: Configuration items in the configuration file (taos.cfg) +The multi-tier storage directory is configured through `dataDir`, the format is: `dataDir [path] ` +``` +dataDir: Configuration items in the configuration file (`taos.cfg`) path: the directory to mount (cannot be omitted) level: mount level, the value is 0, 1 or 2, if omitted, the default is 0. Up to 3 levels are supported, and each level supports up to 16 mount points, so a maximum of 48 mount points are supported. primary: whether it is the primary mount point, 0 (no) or 1 (yes), the default is 1. The primary mount point is only allowed at level 0, and only one primary mount point is allowed (level=0, primary=1). - +``` Here is a normal configuration (3 storage tiers, each with 3 mount points): +``` Data directory /mnt/data01 0 1 Data directory /mnt/data02 0 0 Data directory /mnt/data03 0 0 @@ -335,11 +336,13 @@ Data directory /mnt/data12 1 0 Data directory /mnt/data20 2 0 Data directory /mnt/data21 2 0 Data directory /mnt/data22 2 0 - +``` The community version does not support multi-tier storage, and the following three configurations are acceptable: +``` Data directory /var/lib/taos or dataDir /var/lib/taos 0 or dataDir /var/lib/taos 0 1 +``` ::: From 3915fa6c293c3073b49dbe6ab4190c9cc3fa43dc Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 10 Jan 2023 17:53:18 +0800 Subject: [PATCH 06/25] chore: docs update for dataDir --- docs/en/14-reference/12-config/index.md | 40 ++++++------------------- docs/zh/14-reference/12-config/index.md | 2 +- 2 files changed, 10 insertions(+), 32 deletions(-) diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index 1488219bcf..c3d5de0213 100644 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -142,6 +142,15 @@ The parameters described in this document by the effect that they have on the sy | Meaning | Switch for allowing TDengine to collect and report service usage information | | Value Range | 0: Not allowed; 1: Allowed | | Default Value | 1 | +### crashReporting + +| Attribute | Description | +| -------- | -------------------------------------------- | +| Applicable | Server Only | +| Meaning |Switch for allowing TDengine to collect and report crash related information | +| Value Range | 0,1 0: Not allowed；1：allowed | +| Default Value | 1 | + ## Query Parameters @@ -315,37 +324,6 @@ The charset that takes effect is UTF-8. | Meaning | All data files are stored in this directory | | Default Value | /var/lib/taos | -:::info -The official version supports multi-tier storage (need to cooperate with [KEEP parameter](https://docs.tdengine.com/taos-sql/database/#create-a-database)). The configuration method is as follows. - -The multi-tier storage directory is configured through `dataDir`, the format is: `dataDir [path] ` -``` -dataDir: Configuration items in the configuration file (`taos.cfg`) -path: the directory to mount (cannot be omitted) -level: mount level, the value is 0, 1 or 2, if omitted, the default is 0. Up to 3 levels are supported, and each level supports up to 16 mount points, so a maximum of 48 mount points are supported. -primary: whether it is the primary mount point, 0 (no) or 1 (yes), the default is 1. The primary mount point is only allowed at level 0, and only one primary mount point is allowed (level=0, primary=1). -``` -Here is a normal configuration (3 storage tiers, each with 3 mount points): -``` -Data directory /mnt/data01 0 1 -Data directory /mnt/data02 0 0 -Data directory /mnt/data03 0 0 -Data directory /mnt/data10 1 0 -Data directory /mnt/data11 1 0 -Data directory /mnt/data12 1 0 -Data directory /mnt/data20 2 0 -Data directory /mnt/data21 2 0 -Data directory /mnt/data22 2 0 -``` -The community version does not support multi-tier storage, and the following three configurations are acceptable: -``` -Data directory /var/lib/taos -or dataDir /var/lib/taos 0 -or dataDir /var/lib/taos 0 1 -``` - -::: - ### tempDir | Attribute | Description | diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index 96612bf82f..8197a664ca 100644 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -306,7 +306,7 @@ charset 的有效值是 UTF-8。 | 含义 | 数据文件目录，所有的数据文件都将写入该目录 | | 缺省值 | /var/lib/taos | -:::info + 企业版支持多级存储 (需要与[KEEP 参数](https://docs.taosdata.com/taos-sql/database/#%E5%88%9B%E5%BB%BA%E6%95%B0%E6%8D%AE%E5%BA%93)配合使用). 配置方法如下。 多级存储目录通过 dataDir 配置。配置项格式： dataDir [path] From c1474e37464cd3db9dcbbb32926abc2c8c154cb7 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 10 Jan 2023 17:58:50 +0800 Subject: [PATCH 07/25] chore: docs update for dataDir --- docs/en/14-reference/12-config/index.md | 42 +++++++++++++++++++------ docs/zh/14-reference/12-config/index.md | 16 ++++++---- 2 files changed, 43 insertions(+), 15 deletions(-) diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index c3d5de0213..5febda4daf 100644 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -142,15 +142,6 @@ The parameters described in this document by the effect that they have on the sy | Meaning | Switch for allowing TDengine to collect and report service usage information | | Value Range | 0: Not allowed; 1: Allowed | | Default Value | 1 | -### crashReporting - -| Attribute | Description | -| -------- | -------------------------------------------- | -| Applicable | Server Only | -| Meaning |Switch for allowing TDengine to collect and report crash related information | -| Value Range | 0,1 0: Not allowed；1：allowed | -| Default Value | 1 | - ## Query Parameters @@ -324,6 +315,39 @@ The charset that takes effect is UTF-8. | Meaning | All data files are stored in this directory | | Default Value | /var/lib/taos | +:::info +`The official version` supports multi-tier storage (need to cooperate with [KEEP parameter](https://docs.tdengine.com/taos-sql/database/#create-a-database)). The configuration method is as follows. + +The multi-tier storage directory is configured through `dataDir`, the format is: +`dataDir [path] ` + +dataDir: Configuration items in the configuration file (`taos.cfg`) +path: the directory to mount (cannot be omitted) +level: mount level, the value is 0, 1 or 2, if omitted, the default is 0. Up to 3 levels are supported, and each level supports up to 16 mount points, so a maximum of 48 mount points are supported. +primary: whether it is the primary mount point, 0 (no) or 1 (yes), the default is 1. The primary mount point is only allowed at level 0, and only one primary mount point is allowed (level=0, primary=1). + +Here is a normal configuration (3 storage tiers, each with 3 mount points): +``` +dataDir /mnt/data01 0 1 +dataDir /mnt/data02 0 0 +dataDir /mnt/data03 0 0 +dataDir /mnt/data10 1 0 +dataDir /mnt/data11 1 0 +dataDir /mnt/data12 1 0 +dataDir /mnt/data20 2 0 +dataDir /mnt/data21 2 0 +dataDir /mnt/data22 2 0 +``` + +`The community version` does not support multi-tier storage, and the following three configurations are acceptable: +``` +dataDir /var/lib/taos +or dataDir /var/lib/taos 0 +or dataDir /var/lib/taos 0 1 +``` + +::: + ### tempDir | Attribute | Description | diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index 8197a664ca..ae1c928304 100644 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -306,17 +306,18 @@ charset 的有效值是 UTF-8。 | 含义 | 数据文件目录，所有的数据文件都将写入该目录 | | 缺省值 | /var/lib/taos | +:::info +`企业版`支持多级存储 (需要与[KEEP 参数](https://docs.taosdata.com/taos-sql/database/#%E5%88%9B%E5%BB%BA%E6%95%B0%E6%8D%AE%E5%BA%93)配合使用). 配置方法如下。 -企业版支持多级存储 (需要与[KEEP 参数](https://docs.taosdata.com/taos-sql/database/#%E5%88%9B%E5%BB%BA%E6%95%B0%E6%8D%AE%E5%BA%93)配合使用). 配置方法如下。 +多级存储目录通过 `dataDir` 配置。配置项格式： dataDir [path] -多级存储目录通过 dataDir 配置。配置项格式： dataDir [path] - -dataDir: 配置文件（taos.cfg）中的配置项 +dataDir: 配置文件（`taos.cfg`）中的配置项 path: 需要挂载的目录（不可省略） level: 挂载等级，取值为 0，1 或 2，省略默认为 0。最多支持 3 级, 每一级最多支持 16 个挂载点, 因此, 最多支持 48 个挂载点。 primary: 是否为主挂载点，0（否）或 1（是），默认为 1。主挂载点只允许在 level 0，且只允许一个主挂载点存在（level = 0，primary = 1）。 -如下，为正常的配置 ( 3 个存储层级，每个层级 3 个挂载点)： +如下，为正常配置 ( 3 个存储层级，每个层级 3 个挂载点)： +``` dataDir /mnt/data01 0 1 dataDir /mnt/data02 0 0 dataDir /mnt/data03 0 0 @@ -326,11 +327,14 @@ dataDir /mnt/data12 1 0 dataDir /mnt/data20 2 0 dataDir /mnt/data21 2 0 dataDir /mnt/data22 2 0 +``` -社区版不支持多级存储, 以下 3 种均为正常配置: +`社区版`不支持多级存储, 以下 3 种均为正常配置: +``` dataDir /var/lib/taos 或 dataDir /var/lib/taos 0 或 dataDir /var/lib/taos 0 1 +``` ::: From 258fbcfd7d322bacffe8c3809c12cc6c116b44c5 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 10 Jan 2023 18:07:09 +0800 Subject: [PATCH 08/25] chore: dataDir docs update --- docs/en/14-reference/12-config/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index 5febda4daf..54e4139645 100644 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -339,7 +339,7 @@ dataDir /mnt/data21 2 0 dataDir /mnt/data22 2 0 ``` -`The community version` does not support multi-tier storage, and the following three configurations are acceptable: +`The community version` does not support multi-tier storage, and the following three configurations are all normal: ``` dataDir /var/lib/taos or dataDir /var/lib/taos 0 From e22271d2b9edb8d271ad25594167e4379fef790d Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 10 Jan 2023 18:33:23 +0800 Subject: [PATCH 09/25] Update index.md --- docs/zh/14-reference/12-config/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index ae1c928304..c7ce07facf 100644 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -316,7 +316,7 @@ path: 需要挂载的目录（不可省略） level: 挂载等级，取值为 0，1 或 2，省略默认为 0。最多支持 3 级, 每一级最多支持 16 个挂载点, 因此, 最多支持 48 个挂载点。 primary: 是否为主挂载点，0（否）或 1（是），默认为 1。主挂载点只允许在 level 0，且只允许一个主挂载点存在（level = 0，primary = 1）。 -如下，为正常配置 ( 3 个存储层级，每个层级 3 个挂载点)： +如下，为正常配置 (3 个存储层级，每个层级 3 个挂载点)： ``` dataDir /mnt/data01 0 1 dataDir /mnt/data02 0 0 From a266f374b5efdaf06925088d14be1cc523004826 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 13 Jan 2023 14:40:36 +0800 Subject: [PATCH 10/25] Update 02-database.md --- docs/zh/12-taos-sql/02-database.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/12-taos-sql/02-database.md b/docs/zh/12-taos-sql/02-database.md index 0b0167a306..d4988e7d85 100644 --- a/docs/zh/12-taos-sql/02-database.md +++ b/docs/zh/12-taos-sql/02-database.md @@ -56,7 +56,7 @@ database_option: { - WAL_FSYNC_PERIOD：当 WAL 参数设置为 2 时，落盘的周期。默认为 3000，单位毫秒。最小为 0，表示每次写入立即落盘；最大为 180000，即三分钟。 - MAXROWS：文件块中记录的最大条数，默认为 4096 条。 - MINROWS：文件块中记录的最小条数，默认为 100 条。 -- KEEP：表示数据文件保存的天数，缺省值为 3650，取值范围 [1, 365000]，且必须大于或等于 DURATION 参数值。数据库会自动删除保存时间超过 KEEP 值的数据。KEEP 可以使用加单位的表示形式，如 KEEP 100h、KEEP 10d 等，支持 m（分钟）、h（小时）和 d（天）三个单位。也可以不写单位，如 KEEP 50，此时默认单位为天。企业版支持[多级存储](https://docs.taosdata.com/reference/config/#datadir)功能, 对应多个保存时间（多个以英文逗号分隔，最多 3 级：keep0,keep1,keep2，满足 keep0 <= keep1 <= keep2，如 KEEP 100h,100d,3650d）; 社区版不支持多级存储（即使配置了多个保存时间, 也不会生效, KEEP 会取最大的 keep 值）。 +- KEEP：表示数据文件保存的天数，缺省值为 3650，取值范围 [1, 365000]，且必须大于或等于 DURATION 参数值。数据库会自动删除保存时间超过 KEEP 值的数据。KEEP 可以使用加单位的表示形式，如 KEEP 100h、KEEP 10d 等，支持 m（分钟）、h（小时）和 d（天）三个单位。也可以不写单位，如 KEEP 50，此时默认单位为天。企业版支持[多级存储](https://docs.taosdata.com/tdinternal/arch/#%E5%A4%9A%E7%BA%A7%E5%AD%98%E5%82%A8)功能, 因此, 可以设置多个保存时间（多个以英文逗号分隔，最多 3 个，满足 keep 0 <= keep 1 <= keep 2，如 KEEP 100h,100d,3650d）; 社区版不支持多级存储功能（即使配置了多个保存时间, 也不会生效, KEEP 会取最大的保存时间）。 - PAGES：一个 VNODE 中元数据存储引擎的缓存页个数，默认为 256，最小 64。一个 VNODE 元数据存储占用 PAGESIZE \* PAGES，默认情况下为 1MB 内存。 - PAGESIZE：一个 VNODE 中元数据存储引擎的页大小，单位为 KB，默认为 4 KB。范围为 1 到 16384，即 1 KB 到 16 MB。 - PRECISION：数据库的时间戳精度。ms 表示毫秒，us 表示微秒，ns 表示纳秒，默认 ms 毫秒。 From d5b904e4468bbb8d95ce40483ffff6e030a5d72b Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 13 Jan 2023 14:44:15 +0800 Subject: [PATCH 11/25] Update 02-database.md --- docs/en/12-taos-sql/02-database.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/12-taos-sql/02-database.md b/docs/en/12-taos-sql/02-database.md index ce95b0a165..648719f078 100644 --- a/docs/en/12-taos-sql/02-database.md +++ b/docs/en/12-taos-sql/02-database.md @@ -56,7 +56,7 @@ database_option: { - WAL_FSYNC_PERIOD: specifies the interval (in milliseconds) at which data is written from the WAL to disk. This parameter takes effect only when the WAL parameter is set to 2. The default value is 3000. Enter a value between 0 and 180000. The value 0 indicates that incoming data is immediately written to disk. - MAXROWS: specifies the maximum number of rows recorded in a block. The default value is 4096. - MINROWS: specifies the minimum number of rows recorded in a block. The default value is 100. -- KEEP: specifies the time for which data is retained. Enter a value between 1 and 365000. The default value is 3650. The value of the KEEP parameter must be greater than or equal to the value of the DURATION parameter. TDengine automatically deletes data that is older than the value of the KEEP parameter. You can use m (minutes), h (hours), and d (days) as the unit, for example KEEP 100h or KEEP 10d. If you do not include a unit, d is used by default. The official version supports [multi-tier storage](https://docs.tdengine.com/reference/config/#datadir), corresponding to multiple KEEP values(comma separated and up to 3 values keep0,keep1,keep2 supported, and meet keep0 <= keep1 <= keep2, e.g. KEEP 100h,100d,3650d); the community version does not support multi-tier storage(although multiple keep values are configured, they do not take effect, only the maximum keep value is used as KEEP). +- KEEP: specifies the time for which data is retained. Enter a value between 1 and 365000. The default value is 3650. The value of the KEEP parameter must be greater than or equal to the value of the DURATION parameter. TDengine automatically deletes data that is older than the value of the KEEP parameter. You can use m (minutes), h (hours), and d (days) as the unit, for example KEEP 100h or KEEP 10d. If you do not include a unit, d is used by default. The Enterprise Edition supports [tiered storage](https://docs.tdengine.com/tdinternal/arch/#tiered-storage) function, thus multiple KEEP values (comma separated and up to 3 values supported, and meet keep 0 <= keep 1 <= keep 2, e.g. KEEP 100h,100d,3650d) are supported; the Community Edition does not support tiered storage (although multiple keep values are configured, they do not take effect, only the maximum keep value is used as KEEP). - PAGES: specifies the number of pages in the metadata storage engine cache on each vnode. Enter a value greater than or equal to 64. The default value is 256. The space occupied by metadata storage on each vnode is equal to the product of the values of the PAGESIZE and PAGES parameters. The space occupied by default is 1 MB. - PAGESIZE: specifies the size (in KB) of each page in the metadata storage engine cache on each vnode. The default value is 4. Enter a value between 1 and 16384. - PRECISION: specifies the precision at which a database records timestamps. Enter ms for milliseconds, us for microseconds, or ns for nanoseconds. The default value is ms. From 91e11faf8a2590a27888beef893c96cc75772f9e Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 13 Jan 2023 14:44:57 +0800 Subject: [PATCH 12/25] Update index.md --- docs/en/14-reference/12-config/index.md | 33 ------------------------- 1 file changed, 33 deletions(-) diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index 54e4139645..b6bfa4bc7d 100644 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -315,39 +315,6 @@ The charset that takes effect is UTF-8. | Meaning | All data files are stored in this directory | | Default Value | /var/lib/taos | -:::info -`The official version` supports multi-tier storage (need to cooperate with [KEEP parameter](https://docs.tdengine.com/taos-sql/database/#create-a-database)). The configuration method is as follows. - -The multi-tier storage directory is configured through `dataDir`, the format is: -`dataDir [path] ` - -dataDir: Configuration items in the configuration file (`taos.cfg`) -path: the directory to mount (cannot be omitted) -level: mount level, the value is 0, 1 or 2, if omitted, the default is 0. Up to 3 levels are supported, and each level supports up to 16 mount points, so a maximum of 48 mount points are supported. -primary: whether it is the primary mount point, 0 (no) or 1 (yes), the default is 1. The primary mount point is only allowed at level 0, and only one primary mount point is allowed (level=0, primary=1). - -Here is a normal configuration (3 storage tiers, each with 3 mount points): -``` -dataDir /mnt/data01 0 1 -dataDir /mnt/data02 0 0 -dataDir /mnt/data03 0 0 -dataDir /mnt/data10 1 0 -dataDir /mnt/data11 1 0 -dataDir /mnt/data12 1 0 -dataDir /mnt/data20 2 0 -dataDir /mnt/data21 2 0 -dataDir /mnt/data22 2 0 -``` - -`The community version` does not support multi-tier storage, and the following three configurations are all normal: -``` -dataDir /var/lib/taos -or dataDir /var/lib/taos 0 -or dataDir /var/lib/taos 0 1 -``` - -::: - ### tempDir | Attribute | Description | From 2f4c9e51641be25b8e2007433ab42031dfe2f600 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 13 Jan 2023 14:49:37 +0800 Subject: [PATCH 13/25] Update index.md --- docs/zh/14-reference/12-config/index.md | 33 +------------------------ 1 file changed, 1 insertion(+), 32 deletions(-) diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index c7ce07facf..701d7f6647 100644 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -305,38 +305,7 @@ charset 的有效值是 UTF-8。 | 适用范围 | 仅服务端适用 | | 含义 | 数据文件目录，所有的数据文件都将写入该目录 | | 缺省值 | /var/lib/taos | - -:::info -`企业版`支持多级存储 (需要与[KEEP 参数](https://docs.taosdata.com/taos-sql/database/#%E5%88%9B%E5%BB%BA%E6%95%B0%E6%8D%AE%E5%BA%93)配合使用). 配置方法如下。 - -多级存储目录通过 `dataDir` 配置。配置项格式： dataDir [path] - -dataDir: 配置文件（`taos.cfg`）中的配置项 -path: 需要挂载的目录（不可省略） -level: 挂载等级，取值为 0，1 或 2，省略默认为 0。最多支持 3 级, 每一级最多支持 16 个挂载点, 因此, 最多支持 48 个挂载点。 -primary: 是否为主挂载点，0（否）或 1（是），默认为 1。主挂载点只允许在 level 0，且只允许一个主挂载点存在（level = 0，primary = 1）。 - -如下，为正常配置 (3 个存储层级，每个层级 3 个挂载点)： -``` -dataDir /mnt/data01 0 1 -dataDir /mnt/data02 0 0 -dataDir /mnt/data03 0 0 -dataDir /mnt/data10 1 0 -dataDir /mnt/data11 1 0 -dataDir /mnt/data12 1 0 -dataDir /mnt/data20 2 0 -dataDir /mnt/data21 2 0 -dataDir /mnt/data22 2 0 -``` - -`社区版`不支持多级存储, 以下 3 种均为正常配置: -``` -dataDir /var/lib/taos -或 dataDir /var/lib/taos 0 -或 dataDir /var/lib/taos 0 1 -``` - -::: +| 补充说明 | [KEEP](https://docs.taosdata.com/taos-sql/database/#%E5%8F%82%E6%95%B0%E8%AF%B4%E6%98%8E), [多级存储](https://docs.taosdata.com/tdinternal/arch/#%E5%A4%9A%E7%BA%A7%E5%AD%98%E5%82%A8) | ### tempDir From 97e97eb3fc283e9074995a6f455a81499de8f074 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 13 Jan 2023 14:56:45 +0800 Subject: [PATCH 14/25] docs: note for dataDir --- docs/en/14-reference/12-config/index.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index b6bfa4bc7d..1113c59baf 100644 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -314,6 +314,7 @@ The charset that takes effect is UTF-8. | Applicable | Server Only | | Meaning | All data files are stored in this directory | | Default Value | /var/lib/taos | +| Note | [KEEP](https://docs.tdengine.com/taos-sql/database/#parameters) [Tiered Storage](https://docs.tdengine.com/tdinternal/arch/#tiered-storage) | ### tempDir From 40d8bc1480bc0e9436925efb87e70eadf78a8147 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 13 Jan 2023 14:59:21 +0800 Subject: [PATCH 15/25] Update 02-database.md --- docs/en/12-taos-sql/02-database.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/12-taos-sql/02-database.md b/docs/en/12-taos-sql/02-database.md index 648719f078..2e2bb1e012 100644 --- a/docs/en/12-taos-sql/02-database.md +++ b/docs/en/12-taos-sql/02-database.md @@ -56,7 +56,7 @@ database_option: { - WAL_FSYNC_PERIOD: specifies the interval (in milliseconds) at which data is written from the WAL to disk. This parameter takes effect only when the WAL parameter is set to 2. The default value is 3000. Enter a value between 0 and 180000. The value 0 indicates that incoming data is immediately written to disk. - MAXROWS: specifies the maximum number of rows recorded in a block. The default value is 4096. - MINROWS: specifies the minimum number of rows recorded in a block. The default value is 100. -- KEEP: specifies the time for which data is retained. Enter a value between 1 and 365000. The default value is 3650. The value of the KEEP parameter must be greater than or equal to the value of the DURATION parameter. TDengine automatically deletes data that is older than the value of the KEEP parameter. You can use m (minutes), h (hours), and d (days) as the unit, for example KEEP 100h or KEEP 10d. If you do not include a unit, d is used by default. The Enterprise Edition supports [tiered storage](https://docs.tdengine.com/tdinternal/arch/#tiered-storage) function, thus multiple KEEP values (comma separated and up to 3 values supported, and meet keep 0 <= keep 1 <= keep 2, e.g. KEEP 100h,100d,3650d) are supported; the Community Edition does not support tiered storage (although multiple keep values are configured, they do not take effect, only the maximum keep value is used as KEEP). +- KEEP: specifies the time for which data is retained. Enter a value between 1 and 365000. The default value is 3650. The value of the KEEP parameter must be greater than or equal to the value of the DURATION parameter. TDengine automatically deletes data that is older than the value of the KEEP parameter. You can use m (minutes), h (hours), and d (days) as the unit, for example KEEP 100h or KEEP 10d. If you do not include a unit, d is used by default. The Enterprise Edition supports [Tiered Storage](https://docs.tdengine.com/tdinternal/arch/#tiered-storage) function, thus multiple KEEP values (comma separated and up to 3 values supported, and meet keep 0 <= keep 1 <= keep 2, e.g. KEEP 100h,100d,3650d) are supported; the Community Edition does not support [Tiered Storage](https://docs.tdengine.com/tdinternal/arch/#tiered-storage) function (although multiple keep values are configured, they do not take effect, only the maximum keep value is used as KEEP). - PAGES: specifies the number of pages in the metadata storage engine cache on each vnode. Enter a value greater than or equal to 64. The default value is 256. The space occupied by metadata storage on each vnode is equal to the product of the values of the PAGESIZE and PAGES parameters. The space occupied by default is 1 MB. - PAGESIZE: specifies the size (in KB) of each page in the metadata storage engine cache on each vnode. The default value is 4. Enter a value between 1 and 16384. - PRECISION: specifies the precision at which a database records timestamps. Enter ms for milliseconds, us for microseconds, or ns for nanoseconds. The default value is ms. From f6ecd5ea9b6700b400759c3e1b9c7483ea3c33b3 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 13 Jan 2023 15:00:55 +0800 Subject: [PATCH 16/25] Update 02-database.md --- docs/zh/12-taos-sql/02-database.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/12-taos-sql/02-database.md b/docs/zh/12-taos-sql/02-database.md index d4988e7d85..c3a9ef4213 100644 --- a/docs/zh/12-taos-sql/02-database.md +++ b/docs/zh/12-taos-sql/02-database.md @@ -56,7 +56,7 @@ database_option: { - WAL_FSYNC_PERIOD：当 WAL 参数设置为 2 时，落盘的周期。默认为 3000，单位毫秒。最小为 0，表示每次写入立即落盘；最大为 180000，即三分钟。 - MAXROWS：文件块中记录的最大条数，默认为 4096 条。 - MINROWS：文件块中记录的最小条数，默认为 100 条。 -- KEEP：表示数据文件保存的天数，缺省值为 3650，取值范围 [1, 365000]，且必须大于或等于 DURATION 参数值。数据库会自动删除保存时间超过 KEEP 值的数据。KEEP 可以使用加单位的表示形式，如 KEEP 100h、KEEP 10d 等，支持 m（分钟）、h（小时）和 d（天）三个单位。也可以不写单位，如 KEEP 50，此时默认单位为天。企业版支持[多级存储](https://docs.taosdata.com/tdinternal/arch/#%E5%A4%9A%E7%BA%A7%E5%AD%98%E5%82%A8)功能, 因此, 可以设置多个保存时间（多个以英文逗号分隔，最多 3 个，满足 keep 0 <= keep 1 <= keep 2，如 KEEP 100h,100d,3650d）; 社区版不支持多级存储功能（即使配置了多个保存时间, 也不会生效, KEEP 会取最大的保存时间）。 +- KEEP：表示数据文件保存的天数，缺省值为 3650，取值范围 [1, 365000]，且必须大于或等于 DURATION 参数值。数据库会自动删除保存时间超过 KEEP 值的数据。KEEP 可以使用加单位的表示形式，如 KEEP 100h、KEEP 10d 等，支持 m（分钟）、h（小时）和 d（天）三个单位。也可以不写单位，如 KEEP 50，此时默认单位为天。企业版支持[多级存储](https://docs.taosdata.com/tdinternal/arch/#%E5%A4%9A%E7%BA%A7%E5%AD%98%E5%82%A8)功能, 因此, 可以设置多个保存时间（多个以英文逗号分隔，最多 3 个，满足 keep 0 <= keep 1 <= keep 2，如 KEEP 100h,100d,3650d）; 社区版不支持[多级存储](https://docs.taosdata.com/tdinternal/arch/#%E5%A4%9A%E7%BA%A7%E5%AD%98%E5%82%A8)功能（即使配置了多个保存时间, 也不会生效, KEEP 会取最大的保存时间）。 - PAGES：一个 VNODE 中元数据存储引擎的缓存页个数，默认为 256，最小 64。一个 VNODE 元数据存储占用 PAGESIZE \* PAGES，默认情况下为 1MB 内存。 - PAGESIZE：一个 VNODE 中元数据存储引擎的页大小，单位为 KB，默认为 4 KB。范围为 1 到 16384，即 1 KB 到 16 MB。 - PRECISION：数据库的时间戳精度。ms 表示毫秒，us 表示微秒，ns 表示纳秒，默认 ms 毫秒。 From 9c17ba1ca0b041791c06a38f2444fcc1a78e64b2 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 13 Jan 2023 15:01:36 +0800 Subject: [PATCH 17/25] Update 02-database.md --- docs/zh/12-taos-sql/02-database.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/12-taos-sql/02-database.md b/docs/zh/12-taos-sql/02-database.md index c3a9ef4213..d4988e7d85 100644 --- a/docs/zh/12-taos-sql/02-database.md +++ b/docs/zh/12-taos-sql/02-database.md @@ -56,7 +56,7 @@ database_option: { - WAL_FSYNC_PERIOD：当 WAL 参数设置为 2 时，落盘的周期。默认为 3000，单位毫秒。最小为 0，表示每次写入立即落盘；最大为 180000，即三分钟。 - MAXROWS：文件块中记录的最大条数，默认为 4096 条。 - MINROWS：文件块中记录的最小条数，默认为 100 条。 -- KEEP：表示数据文件保存的天数，缺省值为 3650，取值范围 [1, 365000]，且必须大于或等于 DURATION 参数值。数据库会自动删除保存时间超过 KEEP 值的数据。KEEP 可以使用加单位的表示形式，如 KEEP 100h、KEEP 10d 等，支持 m（分钟）、h（小时）和 d（天）三个单位。也可以不写单位，如 KEEP 50，此时默认单位为天。企业版支持[多级存储](https://docs.taosdata.com/tdinternal/arch/#%E5%A4%9A%E7%BA%A7%E5%AD%98%E5%82%A8)功能, 因此, 可以设置多个保存时间（多个以英文逗号分隔，最多 3 个，满足 keep 0 <= keep 1 <= keep 2，如 KEEP 100h,100d,3650d）; 社区版不支持[多级存储](https://docs.taosdata.com/tdinternal/arch/#%E5%A4%9A%E7%BA%A7%E5%AD%98%E5%82%A8)功能（即使配置了多个保存时间, 也不会生效, KEEP 会取最大的保存时间）。 +- KEEP：表示数据文件保存的天数，缺省值为 3650，取值范围 [1, 365000]，且必须大于或等于 DURATION 参数值。数据库会自动删除保存时间超过 KEEP 值的数据。KEEP 可以使用加单位的表示形式，如 KEEP 100h、KEEP 10d 等，支持 m（分钟）、h（小时）和 d（天）三个单位。也可以不写单位，如 KEEP 50，此时默认单位为天。企业版支持[多级存储](https://docs.taosdata.com/tdinternal/arch/#%E5%A4%9A%E7%BA%A7%E5%AD%98%E5%82%A8)功能, 因此, 可以设置多个保存时间（多个以英文逗号分隔，最多 3 个，满足 keep 0 <= keep 1 <= keep 2，如 KEEP 100h,100d,3650d）; 社区版不支持多级存储功能（即使配置了多个保存时间, 也不会生效, KEEP 会取最大的保存时间）。 - PAGES：一个 VNODE 中元数据存储引擎的缓存页个数，默认为 256，最小 64。一个 VNODE 元数据存储占用 PAGESIZE \* PAGES，默认情况下为 1MB 内存。 - PAGESIZE：一个 VNODE 中元数据存储引擎的页大小，单位为 KB，默认为 4 KB。范围为 1 到 16384，即 1 KB 到 16 MB。 - PRECISION：数据库的时间戳精度。ms 表示毫秒，us 表示微秒，ns 表示纳秒，默认 ms 毫秒。 From 20e50d00b70b97f44c0951071426d4fba2e473a5 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 13 Jan 2023 15:02:00 +0800 Subject: [PATCH 18/25] Update 02-database.md --- docs/en/12-taos-sql/02-database.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/12-taos-sql/02-database.md b/docs/en/12-taos-sql/02-database.md index 2e2bb1e012..cdd53c803f 100644 --- a/docs/en/12-taos-sql/02-database.md +++ b/docs/en/12-taos-sql/02-database.md @@ -56,7 +56,7 @@ database_option: { - WAL_FSYNC_PERIOD: specifies the interval (in milliseconds) at which data is written from the WAL to disk. This parameter takes effect only when the WAL parameter is set to 2. The default value is 3000. Enter a value between 0 and 180000. The value 0 indicates that incoming data is immediately written to disk. - MAXROWS: specifies the maximum number of rows recorded in a block. The default value is 4096. - MINROWS: specifies the minimum number of rows recorded in a block. The default value is 100. -- KEEP: specifies the time for which data is retained. Enter a value between 1 and 365000. The default value is 3650. The value of the KEEP parameter must be greater than or equal to the value of the DURATION parameter. TDengine automatically deletes data that is older than the value of the KEEP parameter. You can use m (minutes), h (hours), and d (days) as the unit, for example KEEP 100h or KEEP 10d. If you do not include a unit, d is used by default. The Enterprise Edition supports [Tiered Storage](https://docs.tdengine.com/tdinternal/arch/#tiered-storage) function, thus multiple KEEP values (comma separated and up to 3 values supported, and meet keep 0 <= keep 1 <= keep 2, e.g. KEEP 100h,100d,3650d) are supported; the Community Edition does not support [Tiered Storage](https://docs.tdengine.com/tdinternal/arch/#tiered-storage) function (although multiple keep values are configured, they do not take effect, only the maximum keep value is used as KEEP). +- KEEP: specifies the time for which data is retained. Enter a value between 1 and 365000. The default value is 3650. The value of the KEEP parameter must be greater than or equal to the value of the DURATION parameter. TDengine automatically deletes data that is older than the value of the KEEP parameter. You can use m (minutes), h (hours), and d (days) as the unit, for example KEEP 100h or KEEP 10d. If you do not include a unit, d is used by default. The Enterprise Edition supports [Tiered Storage](https://docs.tdengine.com/tdinternal/arch/#tiered-storage) function, thus multiple KEEP values (comma separated and up to 3 values supported, and meet keep 0 <= keep 1 <= keep 2, e.g. KEEP 100h,100d,3650d) are supported; the Community Edition does not support Tiered Storage function (although multiple keep values are configured, they do not take effect, only the maximum keep value is used as KEEP). - PAGES: specifies the number of pages in the metadata storage engine cache on each vnode. Enter a value greater than or equal to 64. The default value is 256. The space occupied by metadata storage on each vnode is equal to the product of the values of the PAGESIZE and PAGES parameters. The space occupied by default is 1 MB. - PAGESIZE: specifies the size (in KB) of each page in the metadata storage engine cache on each vnode. The default value is 4. Enter a value between 1 and 16384. - PRECISION: specifies the precision at which a database records timestamps. Enter ms for milliseconds, us for microseconds, or ns for nanoseconds. The default value is ms. From 18378e513acea737f7aa2a42322b0dc7cc37827f Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 13 Jan 2023 15:04:03 +0800 Subject: [PATCH 19/25] Update index.md --- docs/zh/14-reference/12-config/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index 701d7f6647..580b995184 100644 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -305,7 +305,7 @@ charset 的有效值是 UTF-8。 | 适用范围 | 仅服务端适用 | | 含义 | 数据文件目录，所有的数据文件都将写入该目录 | | 缺省值 | /var/lib/taos | -| 补充说明 | [KEEP](https://docs.taosdata.com/taos-sql/database/#%E5%8F%82%E6%95%B0%E8%AF%B4%E6%98%8E), [多级存储](https://docs.taosdata.com/tdinternal/arch/#%E5%A4%9A%E7%BA%A7%E5%AD%98%E5%82%A8) | +| 补充说明 | [KEEP](https://docs.taosdata.com/taos-sql/database/#%E5%8F%82%E6%95%B0%E8%AF%B4%E6%98%8E) [多级存储](https://docs.taosdata.com/tdinternal/arch/#%E5%A4%9A%E7%BA%A7%E5%AD%98%E5%82%A8) | ### tempDir From 7fffebcd41f52dc70960c84c2542518e25bc685e Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sat, 14 Jan 2023 12:05:07 +0800 Subject: [PATCH 20/25] Update index.md --- docs/zh/14-reference/12-config/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index 580b995184..0159442692 100644 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -305,7 +305,7 @@ charset 的有效值是 UTF-8。 | 适用范围 | 仅服务端适用 | | 含义 | 数据文件目录，所有的数据文件都将写入该目录 | | 缺省值 | /var/lib/taos | -| 补充说明 | [KEEP](https://docs.taosdata.com/taos-sql/database/#%E5%8F%82%E6%95%B0%E8%AF%B4%E6%98%8E) [多级存储](https://docs.taosdata.com/tdinternal/arch/#%E5%A4%9A%E7%BA%A7%E5%AD%98%E5%82%A8) | +| 补充说明 | [多级存储](https://docs.taosdata.com/tdinternal/arch/#%E5%A4%9A%E7%BA%A7%E5%AD%98%E5%82%A8) 功能需要与 [KEEP](https://docs.taosdata.com/taos-sql/database/#%E5%8F%82%E6%95%B0%E8%AF%B4%E6%98%8E) 参数配合使用 | ### tempDir From 2d64dfa1310fdc9fb88e8175ead37d5a9a6f0659 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sat, 14 Jan 2023 12:08:31 +0800 Subject: [PATCH 21/25] Update index.md --- docs/en/14-reference/12-config/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index 1113c59baf..dd9158c80a 100644 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -314,7 +314,7 @@ The charset that takes effect is UTF-8. | Applicable | Server Only | | Meaning | All data files are stored in this directory | | Default Value | /var/lib/taos | -| Note | [KEEP](https://docs.tdengine.com/taos-sql/database/#parameters) [Tiered Storage](https://docs.tdengine.com/tdinternal/arch/#tiered-storage) | +| Note | The [Tiered Storage](https://docs.tdengine.com/tdinternal/arch/#tiered-storage) function needs to be used in conjunction with the [KEEP](https://docs.tdengine.com/taos-sql/database/#parameters) parameter | ### tempDir From 2f9218b79ca366df1d804c9bb2e179292351023c Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Sun, 29 Jan 2023 15:14:26 +0800 Subject: [PATCH 22/25] fix: tqOffsetSnapRead success on tq/offset-ver not existing --- source/dnode/vnode/src/tq/tqOffsetSnapshot.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tqOffsetSnapshot.c b/source/dnode/vnode/src/tq/tqOffsetSnapshot.c index 2413a792c6..a4428aed43 100644 --- a/source/dnode/vnode/src/tq/tqOffsetSnapshot.c +++ b/source/dnode/vnode/src/tq/tqOffsetSnapshot.c @@ -56,7 +56,7 @@ int32_t tqOffsetSnapRead(STqOffsetReader* pReader, uint8_t** ppData) { TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ); if (pFile == NULL) { taosMemoryFree(fname); - return -1; + return 0; } int64_t sz = 0; From 8da92d818fceec500891e3c5124c426a316d55fa Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Sun, 29 Jan 2023 15:54:25 +0800 Subject: [PATCH 23/25] enh: flush database in testcase alter_replica_13.sim to trigger snapshot repl --- tests/script/tsim/db/alter_replica_13.sim | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/script/tsim/db/alter_replica_13.sim b/tests/script/tsim/db/alter_replica_13.sim index d75acb50ad..a9dc1741a1 100644 --- a/tests/script/tsim/db/alter_replica_13.sim +++ b/tests/script/tsim/db/alter_replica_13.sim @@ -79,6 +79,7 @@ sql insert into db.ctb6 values(now, 6, "6") sql insert into db.ctb7 values(now, 7, "7") sql insert into db.ctb8 values(now, 8, "8") sql insert into db.ctb9 values(now, 9, "9") +sql flush database db; print =============== step3: create dnodes sql create dnode $hostname port 7300 From 102ca2833ebcb2577add1afd3a859c2f8a371a9d Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sun, 29 Jan 2023 16:39:40 +0800 Subject: [PATCH 24/25] doc: fix the syntax of the show vnodes command --- docs/en/12-taos-sql/24-show.md | 2 +- docs/zh/12-taos-sql/24-show.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/12-taos-sql/24-show.md b/docs/en/12-taos-sql/24-show.md index 2db3e7cb31..f70d86570e 100644 --- a/docs/en/12-taos-sql/24-show.md +++ b/docs/en/12-taos-sql/24-show.md @@ -363,7 +363,7 @@ Shows information about all vgroups in the system or about the vgroups for a spe ## SHOW VNODES ```sql -SHOW VNODES [dnode_name]; +SHOW VNODES {dnode_id | dnode_endpoint}; ``` Shows information about all vnodes in the system or about the vnodes for a specified dnode. diff --git a/docs/zh/12-taos-sql/24-show.md b/docs/zh/12-taos-sql/24-show.md index 2b875199b5..0a326729f2 100644 --- a/docs/zh/12-taos-sql/24-show.md +++ b/docs/zh/12-taos-sql/24-show.md @@ -306,7 +306,7 @@ SHOW [db_name.]VGROUPS; ## SHOW VNODES ```sql -SHOW VNODES [dnode_name]; +SHOW VNODES {dnode_id | dnode_endpoint}; ``` 显示当前系统中所有 VNODE 或某个 DNODE 的 VNODE 的信息。 From c30cb1f5db1bf8475da196df27407d2ae12e98d7 Mon Sep 17 00:00:00 2001 From: sunpeng Date: Mon, 30 Jan 2023 09:58:52 +0800 Subject: [PATCH 25/25] build: add python demo to ci (#19641) * build: add python demo to ci * build: add fast write example to ci * build: add kafka demo to ci * fix kafka demo * fix python demo * fix python demo * fix python demo * fix python demo * fix python demo --- docs/examples/go/go.sum | 15 +++ docs/examples/python/conn_native_pandas.py | 7 +- docs/examples/python/conn_rest_pandas.py | 6 +- docs/examples/python/connect_rest_examples.py | 15 +-- .../connection_usage_native_reference.py | 6 +- docs/examples/python/fast_write_example.py | 119 ++++++++++++------ docs/examples/python/kafka_example.py | 75 +++++++++-- docs/examples/python/mockdatasource.py | 18 ++- docs/examples/python/sql_writer.py | 23 +++- docs/examples/python/tmq_example.py | 13 +- tests/docs-examples-test/python.sh | 43 ++++++- 11 files changed, 267 insertions(+), 73 deletions(-) create mode 100644 docs/examples/go/go.sum diff --git a/docs/examples/go/go.sum b/docs/examples/go/go.sum new file mode 100644 index 0000000000..13e13adaa1 --- /dev/null +++ b/docs/examples/go/go.sum @@ -0,0 +1,15 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/taosdata/driver-go/v3 v3.1.0/go.mod h1:H2vo/At+rOPY1aMzUV9P49SVX7NlXb3LAbKw+MCLrmU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/docs/examples/python/conn_native_pandas.py b/docs/examples/python/conn_native_pandas.py index 56942ef570..f3bab15efb 100644 --- a/docs/examples/python/conn_native_pandas.py +++ b/docs/examples/python/conn_native_pandas.py @@ -1,8 +1,11 @@ import pandas -from sqlalchemy import create_engine +from sqlalchemy import create_engine, text engine = create_engine("taos://root:taosdata@localhost:6030/power") -df = pandas.read_sql("SELECT * FROM meters", engine) +conn = engine.connect() +df = pandas.read_sql(text("SELECT * FROM power.meters"), conn) +conn.close() + # print index print(df.index) diff --git a/docs/examples/python/conn_rest_pandas.py b/docs/examples/python/conn_rest_pandas.py index 0164080cd5..1b207d6ff1 100644 --- a/docs/examples/python/conn_rest_pandas.py +++ b/docs/examples/python/conn_rest_pandas.py @@ -1,8 +1,10 @@ import pandas -from sqlalchemy import create_engine +from sqlalchemy import create_engine, text engine = create_engine("taosrest://root:taosdata@localhost:6041") -df: pandas.DataFrame = pandas.read_sql("SELECT * FROM power.meters", engine) +conn = engine.connect() +df: pandas.DataFrame = pandas.read_sql(text("SELECT * FROM power.meters"), conn) +conn.close() # print index print(df.index) diff --git a/docs/examples/python/connect_rest_examples.py b/docs/examples/python/connect_rest_examples.py index dba00b5a82..0f8625ae53 100644 --- a/docs/examples/python/connect_rest_examples.py +++ b/docs/examples/python/connect_rest_examples.py @@ -1,18 +1,19 @@ # ANCHOR: connect from taosrest import connect, TaosRestConnection, TaosRestCursor -conn: TaosRestConnection = connect(url="http://localhost:6041", - user="root", - password="taosdata", - timeout=30) +conn = connect(url="http://localhost:6041", + user="root", + password="taosdata", + timeout=30) # ANCHOR_END: connect # ANCHOR: basic # create STable -cursor: TaosRestCursor = conn.cursor() +cursor = conn.cursor() cursor.execute("DROP DATABASE IF EXISTS power") cursor.execute("CREATE DATABASE power") -cursor.execute("CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)") +cursor.execute( + "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)") # insert data cursor.execute("""INSERT INTO power.d1001 USING power.meters TAGS('California.SanFrancisco', 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000) @@ -28,7 +29,7 @@ print("queried row count:", cursor.rowcount) # get column names from cursor column_names = [meta[0] for meta in cursor.description] # get rows -data: list[tuple] = cursor.fetchall() +data = cursor.fetchall() print(column_names) for row in data: print(row) diff --git a/docs/examples/python/connection_usage_native_reference.py b/docs/examples/python/connection_usage_native_reference.py index a7179b4cf8..8b754ec722 100644 --- a/docs/examples/python/connection_usage_native_reference.py +++ b/docs/examples/python/connection_usage_native_reference.py @@ -8,7 +8,7 @@ conn.execute("CREATE DATABASE test") # change database. same as execute "USE db" conn.select_db("test") conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)") -affected_row: int = conn.execute("INSERT INTO t1 USING weather TAGS(1) VALUES (now, 23.5) (now+1m, 23.5) (now+2m, 24.4)") +affected_row = conn.execute("INSERT INTO t1 USING weather TAGS(1) VALUES (now, 23.5) (now+1m, 23.5) (now+2m, 24.4)") print("affected_row", affected_row) # output: # affected_row 3 @@ -16,10 +16,10 @@ print("affected_row", affected_row) # ANCHOR: query # Execute a sql and get its result set. It's useful for SELECT statement -result: taos.TaosResult = conn.query("SELECT * from weather") +result = conn.query("SELECT * from weather") # Get fields from result -fields: taos.field.TaosFields = result.fields +fields = result.fields for field in fields: print(field) # {name: ts, type: 9, bytes: 8} diff --git a/docs/examples/python/fast_write_example.py b/docs/examples/python/fast_write_example.py index c9d606388f..626e3310b1 100644 --- a/docs/examples/python/fast_write_example.py +++ b/docs/examples/python/fast_write_example.py @@ -1,15 +1,14 @@ # install dependencies: # recommend python >= 3.8 -# pip3 install faster-fifo # import logging import math +import multiprocessing import sys import time import os -from multiprocessing import Process -from faster_fifo import Queue +from multiprocessing import Process, Queue from mockdatasource import MockDataSource from queue import Empty from typing import List @@ -22,8 +21,7 @@ TABLE_COUNT = 1000 QUEUE_SIZE = 1000000 MAX_BATCH_SIZE = 3000 -read_processes = [] -write_processes = [] +_DONE_MESSAGE = '__DONE__' def get_connection(): @@ -44,41 +42,64 @@ def get_connection(): # ANCHOR: read -def run_read_task(task_id: int, task_queues: List[Queue]): +def run_read_task(task_id: int, task_queues: List[Queue], infinity): table_count_per_task = TABLE_COUNT // READ_TASK_COUNT - data_source = MockDataSource(f"tb{task_id}", table_count_per_task) + data_source = MockDataSource(f"tb{task_id}", table_count_per_task, infinity) try: for batch in data_source: + if isinstance(batch, tuple): + batch = [batch] for table_id, rows in batch: # hash data to different queue i = table_id % len(task_queues) # block putting forever when the queue is full - task_queues[i].put_many(rows, block=True, timeout=-1) + for row in rows: + task_queues[i].put(row) + if not infinity: + for queue in task_queues: + queue.put(_DONE_MESSAGE) except KeyboardInterrupt: pass + finally: + logging.info('read task over') # ANCHOR_END: read + # ANCHOR: write -def run_write_task(task_id: int, queue: Queue): +def run_write_task(task_id: int, queue: Queue, done_queue: Queue): from sql_writer import SQLWriter log = logging.getLogger(f"WriteTask-{task_id}") writer = SQLWriter(get_connection) lines = None try: while True: - try: - # get as many as possible - lines = queue.get_many(block=False, max_messages_to_get=MAX_BATCH_SIZE) + over = False + lines = [] + for _ in range(MAX_BATCH_SIZE): + try: + line = queue.get_nowait() + if line == _DONE_MESSAGE: + over = True + break + if line: + lines.append(line) + except Empty: + time.sleep(0.1) + if len(lines) > 0: writer.process_lines(lines) - except Empty: - time.sleep(0.01) + if over: + done_queue.put(_DONE_MESSAGE) + break except KeyboardInterrupt: pass except BaseException as e: log.debug(f"lines={lines}") raise e + finally: + writer.close() + log.debug('write task over') # ANCHOR_END: write @@ -103,47 +124,64 @@ def set_global_config(): # ANCHOR: monitor -def run_monitor_process(): +def run_monitor_process(done_queue: Queue): log = logging.getLogger("DataBaseMonitor") - conn = get_connection() - conn.execute("DROP DATABASE IF EXISTS test") - conn.execute("CREATE DATABASE test") - conn.execute("CREATE STABLE test.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) " - "TAGS (location BINARY(64), groupId INT)") + conn = None + try: + conn = get_connection() - def get_count(): - res = conn.query("SELECT count(*) FROM test.meters") - rows = res.fetch_all() - return rows[0][0] if rows else 0 + def get_count(): + res = conn.query("SELECT count(*) FROM test.meters") + rows = res.fetch_all() + return rows[0][0] if rows else 0 - last_count = 0 - while True: - time.sleep(10) - count = get_count() - log.info(f"count={count} speed={(count - last_count) / 10}") - last_count = count + last_count = 0 + while True: + try: + done = done_queue.get_nowait() + if done == _DONE_MESSAGE: + break + except Empty: + pass + time.sleep(10) + count = get_count() + log.info(f"count={count} speed={(count - last_count) / 10}") + last_count = count + finally: + conn.close() # ANCHOR_END: monitor # ANCHOR: main -def main(): +def main(infinity): set_global_config() logging.info(f"READ_TASK_COUNT={READ_TASK_COUNT}, WRITE_TASK_COUNT={WRITE_TASK_COUNT}, " f"TABLE_COUNT={TABLE_COUNT}, QUEUE_SIZE={QUEUE_SIZE}, MAX_BATCH_SIZE={MAX_BATCH_SIZE}") - monitor_process = Process(target=run_monitor_process) + conn = get_connection() + conn.execute("DROP DATABASE IF EXISTS test") + conn.execute("CREATE DATABASE IF NOT EXISTS test") + conn.execute("CREATE STABLE IF NOT EXISTS test.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) " + "TAGS (location BINARY(64), groupId INT)") + conn.close() + + done_queue = Queue() + monitor_process = Process(target=run_monitor_process, args=(done_queue,)) monitor_process.start() - time.sleep(3) # waiting for database ready. + logging.debug(f"monitor task started with pid {monitor_process.pid}") task_queues: List[Queue] = [] + write_processes = [] + read_processes = [] + # create task queues for i in range(WRITE_TASK_COUNT): - queue = Queue(max_size_bytes=QUEUE_SIZE) + queue = Queue() task_queues.append(queue) # create write processes for i in range(WRITE_TASK_COUNT): - p = Process(target=run_write_task, args=(i, task_queues[i])) + p = Process(target=run_write_task, args=(i, task_queues[i], done_queue)) p.start() logging.debug(f"WriteTask-{i} started with pid {p.pid}") write_processes.append(p) @@ -151,13 +189,19 @@ def main(): # create read processes for i in range(READ_TASK_COUNT): queues = assign_queues(i, task_queues) - p = Process(target=run_read_task, args=(i, queues)) + p = Process(target=run_read_task, args=(i, queues, infinity)) p.start() logging.debug(f"ReadTask-{i} started with pid {p.pid}") read_processes.append(p) try: monitor_process.join() + for p in read_processes: + p.join() + for p in write_processes: + p.join() + time.sleep(1) + return except KeyboardInterrupt: monitor_process.terminate() [p.terminate() for p in read_processes] @@ -176,5 +220,6 @@ def assign_queues(read_task_id, task_queues): if __name__ == '__main__': - main() + multiprocessing.set_start_method('spawn') + main(False) # ANCHOR_END: main diff --git a/docs/examples/python/kafka_example.py b/docs/examples/python/kafka_example.py index 735059eec0..43f9183f7e 100644 --- a/docs/examples/python/kafka_example.py +++ b/docs/examples/python/kafka_example.py @@ -26,7 +26,8 @@ class Consumer(object): 'bath_consume': True, 'batch_size': 1000, 'async_model': True, - 'workers': 10 + 'workers': 10, + 'testing': False } LOCATIONS = ['California.SanFrancisco', 'California.LosAngles', 'California.SanDiego', 'California.SanJose', @@ -46,11 +47,12 @@ class Consumer(object): def __init__(self, **configs): self.config: dict = self.DEFAULT_CONFIGS self.config.update(configs) - self.consumer = KafkaConsumer( - self.config.get('kafka_topic'), # topic - bootstrap_servers=self.config.get('kafka_brokers'), - group_id=self.config.get('kafka_group_id'), - ) + if not self.config.get('testing'): + self.consumer = KafkaConsumer( + self.config.get('kafka_topic'), # topic + bootstrap_servers=self.config.get('kafka_brokers'), + group_id=self.config.get('kafka_group_id'), + ) self.taos = taos.connect( host=self.config.get('taos_host'), user=self.config.get('taos_user'), @@ -60,7 +62,7 @@ class Consumer(object): ) if self.config.get('async_model'): self.pool = ThreadPoolExecutor(max_workers=self.config.get('workers')) - self.tasks: list[Future] = [] + self.tasks = [] # tags and table mapping # key: {location}_{groupId} value: self.tag_table_mapping = {} i = 0 @@ -115,14 +117,14 @@ class Consumer(object): if self.taos is not None: self.taos.close() - def _run(self, f: Callable[[ConsumerRecord], bool]): + def _run(self, f): for message in self.consumer: if self.config.get('async_model'): self.pool.submit(f(message)) else: f(message) - def _run_batch(self, f: Callable[[list[list[ConsumerRecord]]], None]): + def _run_batch(self, f): while True: messages = self.consumer.poll(timeout_ms=500, max_records=self.config.get('batch_size')) if messages: @@ -140,7 +142,7 @@ class Consumer(object): logging.info('## insert sql %s', sql) return self.taos.execute(sql=sql) == 1 - def _to_taos_batch(self, messages: list[list[ConsumerRecord]]): + def _to_taos_batch(self, messages): sql = self._build_sql_batch(messages=messages) if len(sql) == 0: # decode error, skip return @@ -162,7 +164,7 @@ class Consumer(object): table_name = self._get_table_name(location=location, group_id=group_id) return self.INSERT_PART_SQL.format(table_name, ts, current, voltage, phase) - def _build_sql_batch(self, messages: list[list[ConsumerRecord]]) -> str: + def _build_sql_batch(self, messages) -> str: sql_list = [] for partition_messages in messages: for message in partition_messages: @@ -186,7 +188,54 @@ def _get_location_and_group(key: str) -> (str, int): return fields[0], fields[1] +def test_to_taos(consumer: Consumer): + msg = { + 'location': 'California.SanFrancisco', + 'groupId': 1, + 'ts': '2022-12-06 15:13:38.643', + 'current': 3.41, + 'voltage': 105, + 'phase': 0.02027, + } + record = ConsumerRecord(checksum=None, headers=None, offset=1, key=None, value=json.dumps(msg), partition=1, + topic='test', serialized_key_size=None, serialized_header_size=None, + serialized_value_size=None, timestamp=time.time(), timestamp_type=None) + assert consumer._to_taos(message=record) + + +def test_to_taos_batch(consumer: Consumer): + records = [ + [ + ConsumerRecord(checksum=None, headers=None, offset=1, key=None, + value=json.dumps({'location': 'California.SanFrancisco', + 'groupId': 1, + 'ts': '2022-12-06 15:13:38.643', + 'current': 3.41, + 'voltage': 105, + 'phase': 0.02027, }), + partition=1, topic='test', serialized_key_size=None, serialized_header_size=None, + serialized_value_size=None, timestamp=time.time(), timestamp_type=None), + ConsumerRecord(checksum=None, headers=None, offset=1, key=None, + value=json.dumps({'location': 'California.LosAngles', + 'groupId': 2, + 'ts': '2022-12-06 15:13:39.643', + 'current': 3.41, + 'voltage': 102, + 'phase': 0.02027, }), + partition=1, topic='test', serialized_key_size=None, serialized_header_size=None, + serialized_value_size=None, timestamp=time.time(), timestamp_type=None), + ] + ] + + consumer._to_taos_batch(messages=records) + + if __name__ == '__main__': - consumer = Consumer(async_model=True) + consumer = Consumer(async_model=True, testing=True) + # init env consumer.init_env() - consumer.consume() \ No newline at end of file + # consumer.consume() + # test build sql + # test build sql batch + test_to_taos(consumer) + test_to_taos_batch(consumer) diff --git a/docs/examples/python/mockdatasource.py b/docs/examples/python/mockdatasource.py index 1c516a800e..9c702936ea 100644 --- a/docs/examples/python/mockdatasource.py +++ b/docs/examples/python/mockdatasource.py @@ -10,13 +10,14 @@ class MockDataSource: "9.4,118,0.141,California.SanFrancisco,4" ] - def __init__(self, tb_name_prefix, table_count): + def __init__(self, tb_name_prefix, table_count, infinity=True): self.table_name_prefix = tb_name_prefix + "_" self.table_count = table_count self.max_rows = 10000000 self.current_ts = round(time.time() * 1000) - self.max_rows * 100 # [(tableId, tableName, values),] self.data = self._init_data() + self.infinity = infinity def _init_data(self): lines = self.samples * (self.table_count // 5 + 1) @@ -28,14 +29,19 @@ class MockDataSource: def __iter__(self): self.row = 0 - return self + if not self.infinity: + return iter(self._iter_data()) + else: + return self def __next__(self): """ next 1000 rows for each table. return: {tableId:[row,...]} """ - # generate 1000 timestamps + return self._iter_data() + + def _iter_data(self): ts = [] for _ in range(1000): self.current_ts += 100 @@ -47,3 +53,9 @@ class MockDataSource: rows = [table_name + ',' + t + ',' + values for t in ts] result.append((table_id, rows)) return result + + +if __name__ == '__main__': + datasource = MockDataSource('t', 10, False) + for data in datasource: + print(data) diff --git a/docs/examples/python/sql_writer.py b/docs/examples/python/sql_writer.py index 758167376b..3456981a7b 100644 --- a/docs/examples/python/sql_writer.py +++ b/docs/examples/python/sql_writer.py @@ -10,6 +10,7 @@ class SQLWriter: self._tb_tags = {} self._conn = get_connection_func() self._max_sql_length = self.get_max_sql_length() + self._conn.execute("create database if not exists test") self._conn.execute("USE test") def get_max_sql_length(self): @@ -20,7 +21,7 @@ class SQLWriter: return int(r[1]) return 1024 * 1024 - def process_lines(self, lines: str): + def process_lines(self, lines: [str]): """ :param lines: [[tbName,ts,current,voltage,phase,location,groupId]] """ @@ -60,6 +61,7 @@ class SQLWriter: buf.append(q) sql_len += len(q) sql += " ".join(buf) + self.create_tables() self.execute_sql(sql) self._tb_values.clear() @@ -88,3 +90,22 @@ class SQLWriter: except BaseException as e: self.log.error("Execute SQL: %s", sql) raise e + + def close(self): + if self._conn: + self._conn.close() + + +if __name__ == '__main__': + def get_connection_func(): + conn = taos.connect() + return conn + + + writer = SQLWriter(get_connection_func=get_connection_func) + writer.execute_sql( + "create stable if not exists meters (ts timestamp, current float, voltage int, phase float) " + "tags (location binary(64), groupId int)") + writer.execute_sql( + "INSERT INTO d21001 USING meters TAGS ('California.SanFrancisco', 2) " + "VALUES ('2021-07-13 14:06:32.272', 10.2, 219, 0.32)") diff --git a/docs/examples/python/tmq_example.py b/docs/examples/python/tmq_example.py index fafa81e8b5..6f7fb87c89 100644 --- a/docs/examples/python/tmq_example.py +++ b/docs/examples/python/tmq_example.py @@ -19,8 +19,14 @@ def init_tmq_env(db, topic): conn.execute("insert into tb3 values (now, 3, 3.0, 'tmq test')") +def cleanup(db, topic): + conn = taos.connect() + conn.execute("drop topic if exists {}".format(topic)) + conn.execute("drop database if exists {}".format(db)) + + if __name__ == '__main__': - init_tmq_env("tmq_test", "tmq_test_topic") # init env + init_tmq_env("tmq_test", "tmq_test_topic") # init env consumer = Consumer( { "group.id": "tg2", @@ -33,9 +39,9 @@ if __name__ == '__main__': try: while True: - res = consumer.poll(100) + res = consumer.poll(1) if not res: - continue + break err = res.error() if err is not None: raise err @@ -46,3 +52,4 @@ if __name__ == '__main__': finally: consumer.unsubscribe() consumer.close() + cleanup("tmq_test", "tmq_test_topic") diff --git a/tests/docs-examples-test/python.sh b/tests/docs-examples-test/python.sh index 140d05395b..ccb391b752 100644 --- a/tests/docs-examples-test/python.sh +++ b/tests/docs-examples-test/python.sh @@ -23,7 +23,7 @@ python3 bind_param_example.py # 4 taos -s "drop database power" -python3 multi_bind_example.py +python3 multi_bind_example.py # 5 python3 query_example.py @@ -44,4 +44,43 @@ taos -s "drop database test" python3 json_protocol_example.py # 10 -# python3 subscribe_demo.py +pip install SQLAlchemy +pip install pandas +taosBenchmark -y -d power -t 10 -n 10 +python3 conn_native_pandas.py +python3 conn_rest_pandas.py +taos -s "drop database if exists power" + +# 11 +taos -s "create database if not exists test" +python3 connect_native_reference.py + +# 12 +python3 connect_rest_examples.py + +# 13 +python3 handle_exception.py + +# 14 +taosBenchmark -y -d power -t 2 -n 10 +python3 rest_client_example.py +taos -s "drop database if exists power" + +# 15 +python3 result_set_examples.py + +# 16 +python3 tmq_example.py + +# 17 +python3 sql_writer.py + +# 18 +python3 mockdatasource.py + +# 19 +python3 fast_write_example.py + +# 20 +pip3 install kafka-python +python3 kafka_example.py