diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in new file mode 100644 index 0000000000..af3b5af4a6 --- /dev/null +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -0,0 +1,13 @@ + +# zlib +ExternalProject_Add(taosadapter + GIT_REPOSITORY https://github.com/taosdata/taosadapter.git + GIT_TAG df8678f + SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" + BINARY_DIR "" + #BUILD_IN_SOURCE TRUE + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" +) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in new file mode 100644 index 0000000000..dfebcd2956 --- /dev/null +++ b/cmake/taostools_CMakeLists.txt.in @@ -0,0 +1,13 @@ + +# zlib +ExternalProject_Add(taos-tools + GIT_REPOSITORY https://github.com/taosdata/taos-tools.git + GIT_TAG 817cb6a + SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" + BINARY_DIR "" + #BUILD_IN_SOURCE TRUE + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" +) diff --git a/cmake/taosws_CMakeLists.txt.in b/cmake/taosws_CMakeLists.txt.in new file mode 100644 index 0000000000..4b7c264472 --- /dev/null +++ b/cmake/taosws_CMakeLists.txt.in @@ -0,0 +1,13 @@ + +# zlib +ExternalProject_Add(taosws-rs + GIT_REPOSITORY https://github.com/taosdata/taosws-rs.git + GIT_TAG 9de599d + SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosws-rs" + BINARY_DIR "" + #BUILD_IN_SOURCE TRUE + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" +) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index e80e7e4110..1887ba5365 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -9,6 +9,24 @@ endfunction(cat IN_FILE OUT_FILE) set(CONTRIB_TMP_FILE "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in") configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) +# taos-tools +if(${BUILD_TOOLS}) + cat("${TD_SUPPORT_DIR}/taostools_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) +endif() + +# taosws-rs +if(${WEBSOCKET}) + cat("${TD_SUPPORT_DIR}/taosws_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) +endif() + +# taosadapter +if(${BUILD_HTTP}) + MESSAGE("BUILD_HTTP is on") +else () + MESSAGE("BUILD_HTTP is off, use taosAdapter") + cat("${TD_SUPPORT_DIR}/taosadapter_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) +endif() + # pthread if(${BUILD_PTHREAD}) cat("${TD_SUPPORT_DIR}/pthread_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) diff --git a/docs/zh/12-taos-sql/02-database.md b/docs/zh/12-taos-sql/02-database.md index 38119dfb44..c28a833eb6 100644 --- a/docs/zh/12-taos-sql/02-database.md +++ b/docs/zh/12-taos-sql/02-database.md @@ -8,10 +8,10 @@ description: "创建、删除数据库，查看、修改数据库参数" ```sql CREATE DATABASE [IF NOT EXISTS] db_name [database_options] - + database_options: database_option ... - + database_option: { BUFFER value | CACHEMODEL {'none' | 'last_row' | 'last_value' | 'both'} @@ -39,41 +39,42 @@ database_option: { ``` ### 参数说明 -- buffer: 一个 VNODE 写入内存池大小，单位为MB，默认为96，最小为3，最大为16384。 -- CACHEMODEL：表示是否在内存中缓存子表的最近数据。默认为none。 - - none：表示不缓存。 - - last_row：表示缓存子表最近一行数据。这将显著改善 LAST_ROW 函数的性能表现。 - - last_value：表示缓存子表每一列的最近的非 NULL 值。这将显著改善无特殊影响（WHERE、ORDER BY、GROUP BY、INTERVAL）下的 LAST 函数的性能表现。 - - both：表示同时打开缓存最近行和列功能。 + +- buffer: 一个 VNODE 写入内存池大小，单位为 MB，默认为 96，最小为 3，最大为 16384。 +- CACHEMODEL：表示是否在内存中缓存子表的最近数据。默认为 none。 + - none：表示不缓存。 + - last_row：表示缓存子表最近一行数据。这将显著改善 LAST_ROW 函数的性能表现。 + - last_value：表示缓存子表每一列的最近的非 NULL 值。这将显著改善无特殊影响（WHERE、ORDER BY、GROUP BY、INTERVAL）下的 LAST 函数的性能表现。 + - both：表示同时打开缓存最近行和列功能。 - CACHESIZE：表示缓存子表最近数据的内存大小。默认为 1 ，范围是[1, 65536]，单位是 MB。 - COMP：表示数据库文件压缩标志位，缺省值为 2，取值范围为 [0, 2]。 - - 0：表示不压缩。 - - 1：表示一阶段压缩。 - - 2：表示两阶段压缩。 -- DURATION：数据文件存储数据的时间跨度。可以使用加单位的表示形式，如 DURATION 100h、DURATION 10d等，支持 m（分钟）、h（小时）和 d（天）三个单位。不加时间单位时默认单位为天，如 DURATION 50 表示 50 天。 -- FSYNC：当 WAL 参数设置为2时，落盘的周期。默认为3000，单位毫秒。最小为0，表示每次写入立即落盘；最大为180000，即三分钟。 -- MAXROWS：文件块中记录的最大条数，默认为4096条。 -- MINROWS：文件块中记录的最小条数，默认为100条。 -- KEEP：表示数据文件保存的天数，缺省值为 3650，取值范围 [1, 365000]，且必须大于或等于 DURATION 参数值。数据库会自动删除保存时间超过KEEP值的数据。KEEP 可以使用加单位的表示形式，如 KEEP 100h、KEEP 10d 等，支持m（分钟）、h（小时）和 d（天）三个单位。也可以不写单位，如 KEEP 50，此时默认单位为天。 -- PAGES：一个 VNODE 中元数据存储引擎的缓存页个数，默认为256，最小64。一个 VNODE 元数据存储占用 PAGESIZE * PAGES，默认情况下为1MB内存。 -- PAGESIZE：一个 VNODE 中元数据存储引擎的页大小，单位为KB，默认为4 KB。范围为1到16384，即1 KB到16 MB。 -- PRECISION：数据库的时间戳精度。ms表示毫秒，us表示微秒，ns表示纳秒，默认ms毫秒。 -- REPLICA：表示数据库副本数，取值为1或3，默认为1。在集群中使用，副本数必须小于或等于 DNODE 的数目。 -- RETENTIONS：表示数据的聚合周期和保存时长，如RETENTIONS 15s:7d,1m:21d,15m:50d表示数据原始采集周期为15秒，原始数据保存7天；按1分钟聚合的数据保存21天；按15分钟聚合的数据保存50天。目前支持且只支持三级存储周期。 -- STRICT：表示数据同步的一致性要求，默认为off。 - - on 表示强一致，即运行标准的 raft 协议，半数提交返回成功。 - - off表示弱一致，本地提交即返回成功。 -- WAL：WAL级别，默认为1。 - - 1：写WAL，但不执行fsync。 - - 2：写WAL，而且执行fsync。 -- VGROUPS：数据库中初始vgroup的数目。 + - 0：表示不压缩。 + - 1：表示一阶段压缩。 + - 2：表示两阶段压缩。 +- DURATION：数据文件存储数据的时间跨度。可以使用加单位的表示形式，如 DURATION 100h、DURATION 10d 等，支持 m（分钟）、h（小时）和 d（天）三个单位。不加时间单位时默认单位为天，如 DURATION 50 表示 50 天。 +- FSYNC：当 WAL 参数设置为 2 时，落盘的周期。默认为 3000，单位毫秒。最小为 0，表示每次写入立即落盘；最大为 180000，即三分钟。 +- MAXROWS：文件块中记录的最大条数，默认为 4096 条。 +- MINROWS：文件块中记录的最小条数，默认为 100 条。 +- KEEP：表示数据文件保存的天数，缺省值为 3650，取值范围 [1, 365000]，且必须大于或等于 DURATION 参数值。数据库会自动删除保存时间超过 KEEP 值的数据。KEEP 可以使用加单位的表示形式，如 KEEP 100h、KEEP 10d 等，支持 m（分钟）、h（小时）和 d（天）三个单位。也可以不写单位，如 KEEP 50，此时默认单位为天。 +- PAGES：一个 VNODE 中元数据存储引擎的缓存页个数，默认为 256，最小 64。一个 VNODE 元数据存储占用 PAGESIZE \* PAGES，默认情况下为 1MB 内存。 +- PAGESIZE：一个 VNODE 中元数据存储引擎的页大小，单位为 KB，默认为 4 KB。范围为 1 到 16384，即 1 KB 到 16 MB。 +- PRECISION：数据库的时间戳精度。ms 表示毫秒，us 表示微秒，ns 表示纳秒，默认 ms 毫秒。 +- REPLICA：表示数据库副本数，取值为 1 或 3，默认为 1。在集群中使用，副本数必须小于或等于 DNODE 的数目。 +- RETENTIONS：表示数据的聚合周期和保存时长，如 RETENTIONS 15s:7d,1m:21d,15m:50d 表示数据原始采集周期为 15 秒，原始数据保存 7 天；按 1 分钟聚合的数据保存 21 天；按 15 分钟聚合的数据保存 50 天。目前支持且只支持三级存储周期。 +- STRICT：表示数据同步的一致性要求，默认为 off。 + - on 表示强一致，即运行标准的 raft 协议，半数提交返回成功。 + - off 表示弱一致，本地提交即返回成功。 +- WAL：WAL 级别，默认为 1。 + - 1：写 WAL，但不执行 fsync。 + - 2：写 WAL，而且执行 fsync。 +- VGROUPS：数据库中初始 vgroup 的数目。 - SINGLE_STABLE：表示此数据库中是否只可以创建一个超级表，用于超级表列非常多的情况。 - - 0：表示可以创建多张超级表。 - - 1：表示只可以创建一张超级表。 -- WAL_RETENTION_PERIOD：wal文件的额外保留策略，用于数据订阅。wal的保存时长，单位为s。默认为0，即落盘后立即删除。-1表示不删除。 -- WAL_RETENTION_SIZE：wal文件的额外保留策略，用于数据订阅。wal的保存的最大上限，单位为KB。默认为0，即落盘后立即删除。-1表示不删除。 -- WAL_ROLL_PERIOD：wal文件切换时长，单位为s。当wal文件创建并写入后，经过该时间，会自动创建一个新的wal文件。默认为0，即仅在落盘时创建新文件。 -- WAL_SEGMENT_SIZE：wal单个文件大小，单位为KB。当前写入文件大小超过上限后会自动创建一个新的wal文件。默认为0，即仅在落盘时创建新文件。 + - 0：表示可以创建多张超级表。 + - 1：表示只可以创建一张超级表。 +- WAL_RETENTION_PERIOD：wal 文件的额外保留策略，用于数据订阅。wal 的保存时长，单位为 s。默认为 0，即落盘后立即删除。-1 表示不删除。 +- WAL_RETENTION_SIZE：wal 文件的额外保留策略，用于数据订阅。wal 的保存的最大上限，单位为 KB。默认为 0，即落盘后立即删除。-1 表示不删除。 +- WAL_ROLL_PERIOD：wal 文件切换时长，单位为 s。当 wal 文件创建并写入后，经过该时间，会自动创建一个新的 wal 文件。默认为 0，即仅在落盘时创建新文件。 +- WAL_SEGMENT_SIZE：wal 单个文件大小，单位为 KB。当前写入文件大小超过上限后会自动创建一个新的 wal 文件。默认为 0，即仅在落盘时创建新文件。 ### 创建数据库示例 @@ -104,10 +105,10 @@ DROP DATABASE [IF EXISTS] db_name ```sql ALTER DATABASE db_name [alter_database_options] - + alter_database_options: alter_database_option ... - + alter_database_option: { CACHEMODEL {'none' | 'last_row' | 'last_value' | 'both'} | CACHESIZE value @@ -118,7 +119,7 @@ alter_database_option: { ``` :::note -其它参数在3.0.0.0中暂不支持修改 +其它参数在 3.0.0.0 中暂不支持修改 ::: @@ -139,3 +140,17 @@ SHOW CREATE DATABASE db_name; 常用于数据库迁移。对一个已经存在的数据库，返回其创建语句；在另一个集群中执行该语句，就能得到一个设置完全相同的 Database。 ### 查看数据库参数 + +```sql +SHOW DATABASES \G; +``` + +会列出系统中所有数据库的配置参数，并且每行只显示一个参数。 + +## 删除过期数据 + +```sql +TRIM DATABASE db_name; +``` + +删除过期数据，并根据多级存储的配置归整数据。 diff --git a/docs/zh/12-taos-sql/03-table.md b/docs/zh/12-taos-sql/03-table.md index 9f3586ce6d..1e20f73541 100644 --- a/docs/zh/12-taos-sql/03-table.md +++ b/docs/zh/12-taos-sql/03-table.md @@ -139,6 +139,7 @@ ALTER TABLE tb_name RENAME COLUMN old_col_name new_col_name ## 修改子表 +```sql ALTER TABLE [db_name.]tb_name alter_table_clause alter_table_clause: { @@ -153,6 +154,7 @@ alter_table_option: { TTL value | COMMENT 'string_value' } +``` **使用说明** 1. 对子表的列和标签的修改，除了更改标签值以外，都要通过超级表才能进行。 @@ -192,5 +194,5 @@ SHOW CREATE TABLE tb_name; ### 获取表结构信息 ``` -DESCRIBE tb_name; +DESCRIBE [db_name.]tb_name; ``` \ No newline at end of file diff --git a/docs/zh/12-taos-sql/04-stable.md b/docs/zh/12-taos-sql/04-stable.md index 4d0f5e4765..59d9657694 100644 --- a/docs/zh/12-taos-sql/04-stable.md +++ b/docs/zh/12-taos-sql/04-stable.md @@ -46,7 +46,7 @@ SHOW CREATE STABLE stb_name; ### 获取超级表的结构信息 ``` -DESCRIBE stb_name; +DESCRIBE [db_name.]stb_name; ``` ## 删除超级表 diff --git a/docs/zh/12-taos-sql/05-insert.md b/docs/zh/12-taos-sql/05-insert.md index 04118303f3..c91e70c481 100644 --- a/docs/zh/12-taos-sql/05-insert.md +++ b/docs/zh/12-taos-sql/05-insert.md @@ -5,7 +5,7 @@ title: 数据写入 ## 写入语法 -``` +```sql INSERT INTO tb_name [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)] @@ -18,46 +18,64 @@ INSERT INTO ...]; ``` -## 插入一条或多条记录 +**关于时间戳** + +1. TDengine 要求插入的数据必须要有时间戳，插入数据的时间戳要注意以下几点： + +2. 时间戳不同的格式语法会有不同的精度影响。字符串格式的时间戳写法不受所在 DATABASE 的时间精度设置影响；而长整形格式的时间戳写法会受到所在 DATABASE 的时间精度设置影响。例如，时间戳"2021-07-13 16:16:48"的 UNIX 秒数为 1626164208。则其在毫秒精度下需要写作 1626164208000，在微秒精度设置下就需要写为 1626164208000000，纳秒精度设置下需要写为 1626164208000000000。 + +3. 一次插入多行数据时，不要把首列的时间戳的值都写 NOW。否则会导致语句中的多条记录使用相同的时间戳，于是就可能出现相互覆盖以致这些数据行无法全部被正确保存。其原因在于，NOW 函数在执行中会被解析为所在 SQL 语句的客户端执行时间，出现在同一语句中的多个 NOW 标记也就会被替换为完全相同的时间戳取值。 + 允许插入的最老记录的时间戳，是相对于当前服务器时间，减去配置的 KEEP 值（数据保留的天数）。允许插入的最新记录的时间戳，是相对于当前服务器时间，加上配置的 DURATION 值（数据文件存储数据的时间跨度，单位为天）。KEEP 和 DURATION 都是可以在创建数据库时指定的，缺省值分别是 3650 天和 10 天。 + +**语法说明** + +1. USING 子句是自动建表语法。如果用户在写数据时并不确定某个表是否存在，此时可以在写入数据时使用自动建表语法来创建不存在的表，若该表已存在则不会建立新表。自动建表时，要求必须以超级表为模板，并写明数据表的 TAGS 取值。可以只是指定部分 TAGS 列的取值，未被指定的 TAGS 列将置为 NULL。 + +2. 可以指定要插入值的列，对于为指定的列数据库将自动填充为 NULL。 + +3. VALUES 语法表示了要插入的一行或多行数据。 + +4. FILE 语法表示数据来自于 CSV 文件（英文逗号分隔、英文单引号括住每个值），CSV 文件无需表头。 + +5. 无论使用哪种语法，均可以在一条 INSERT 语句中同时向多个表插入数据。 + +6. INSERT 语句是完整解析后再执行的，对如下语句，不会再出现数据错误但建表成功的情况： + + ```sql + INSERT INTO d1001 USING meters TAGS('Beijing.Chaoyang', 2) VALUES('a'); + ``` + +7. 对于向多个子表插入数据的情况，依然会有部分数据写入失败，部分数据写入成功的情况。这是因为多个子表可能分布在不同的 VNODE 上，客户端将 INSERT 语句完整解析后，将数据发往各个涉及的 VNODE 上，每个 VNODE 独立进行写入操作。如果某个 VNODE 因为某些原因（比如网络问题或磁盘故障）导致写入失败，并不会影响其他 VNODE 节点的写入。 + +## 插入一条记录 指定已经创建好的数据子表的表名，并通过 VALUES 关键字提供一行或多行数据，即可向数据库写入这些数据。例如，执行如下语句可以写入一行记录： -``` +```sql INSERT INTO d1001 VALUES (NOW, 10.2, 219, 0.32); ``` +## 插入多条记录 + 或者，可以通过如下语句写入两行记录： -``` +```sql INSERT INTO d1001 VALUES ('2021-07-13 14:06:32.272', 10.2, 219, 0.32) (1626164208000, 10.15, 217, 0.33); ``` -:::note - -1. 在第二个例子中，两行记录的首列时间戳使用了不同格式的写法。其中字符串格式的时间戳写法不受所在 DATABASE 的时间精度设置影响；而长整形格式的时间戳写法会受到所在 DATABASE 的时间精度设置影响——例子中的时间戳在毫秒精度下可以写作 1626164208000，而如果是在微秒精度设置下就需要写为 1626164208000000，纳秒精度设置下需要写为 1626164208000000000。 -2. 在使用“插入多条记录”方式写入数据时，不能把第一列的时间戳取值都设为 NOW，否则会导致语句中的多条记录使用相同的时间戳，于是就可能出现相互覆盖以致这些数据行无法全部被正确保存。其原因在于，NOW 函数在执行中会被解析为所在 SQL 语句的实际执行时间，出现在同一语句中的多个 NOW 标记也就会被替换为完全相同的时间戳取值。 -3. 允许插入的最老记录的时间戳，是相对于当前服务器时间，减去配置的 keep 值（数据保留的天数）；允许插入的最新记录的时间戳，是相对于当前服务器时间，加上配置的 days 值（数据文件存储数据的时间跨度，单位为天）。keep 和 days 都是可以在创建数据库时指定的，缺省值分别是 3650 天和 10 天。 - -::: - -## 插入记录，数据对应到指定的列 +## 指定列插入 向数据子表中插入记录时，无论插入一行还是多行，都可以让数据对应到指定的列。对于 SQL 语句中没有出现的列，数据库将自动填充为 NULL。主键（时间戳）不能为 NULL。例如： -``` +```sql INSERT INTO d1001 (ts, current, phase) VALUES ('2021-07-13 14:06:33.196', 10.27, 0.31); ``` -:::info -如果不指定列，也即使用全列模式——那么在 VALUES 部分提供的数据，必须为数据表的每个列都显式地提供数据。全列模式写入速度会远快于指定列，因此建议尽可能采用全列写入方式，此时空列可以填入 NULL。 - -::: - ## 向多个表插入记录 可以在一条语句中，分别向多个表插入一条或多条记录，并且也可以在插入过程中指定列。例如： -``` +```sql INSERT INTO d1001 VALUES ('2021-07-13 14:06:34.630', 10.2, 219, 0.32) ('2021-07-13 14:06:35.779', 10.15, 217, 0.33) d1002 (ts, current, phase) VALUES ('2021-07-13 14:06:34.255', 10.27, 0.31）; ``` @@ -66,28 +84,24 @@ INSERT INTO d1001 VALUES ('2021-07-13 14:06:34.630', 10.2, 219, 0.32) ('2021-07- 如果用户在写数据时并不确定某个表是否存在，此时可以在写入数据时使用自动建表语法来创建不存在的表，若该表已存在则不会建立新表。自动建表时，要求必须以超级表为模板，并写明数据表的 TAGS 取值。例如： -``` +```sql INSERT INTO d21001 USING meters TAGS ('California.SanFrancisco', 2) VALUES ('2021-07-13 14:06:32.272', 10.2, 219, 0.32); ``` 也可以在自动建表时，只是指定部分 TAGS 列的取值，未被指定的 TAGS 列将置为 NULL。例如： -``` +```sql INSERT INTO d21001 USING meters (groupId) TAGS (2) VALUES ('2021-07-13 14:06:33.196', 10.15, 217, 0.33); ``` 自动建表语法也支持在一条语句中向多个表插入记录。例如： -``` +```sql INSERT INTO d21001 USING meters TAGS ('California.SanFrancisco', 2) VALUES ('2021-07-13 14:06:34.630', 10.2, 219, 0.32) ('2021-07-13 14:06:35.779', 10.15, 217, 0.33) d21002 USING meters (groupId) TAGS (2) VALUES ('2021-07-13 14:06:34.255', 10.15, 217, 0.33) d21003 USING meters (groupId) TAGS (2) (ts, current, phase) VALUES ('2021-07-13 14:06:34.255', 10.27, 0.31); ``` -:::info -在 2.0.20.5 版本之前，在使用自动建表语法并指定列时，子表的列名必须紧跟在子表名称后面，而不能如例子里那样放在 TAGS 和 VALUES 之间。从 2.0.20.5 版本开始，两种写法都可以，但不能在一条 SQL 语句中混用，否则会报语法错误。 -::: - ## 插入来自文件的数据记录 除了使用 VALUES 关键字插入一行或多行数据外，也可以把要写入的数据放在 CSV 文件中（英文逗号分隔、英文单引号括住每个值）供 SQL 指令读取。其中 CSV 文件无需表头。例如，如果 /tmp/csvfile.csv 文件的内容为： @@ -99,51 +113,19 @@ INSERT INTO d21001 USING meters TAGS ('California.SanFrancisco', 2) VALUES ('202 那么通过如下指令可以把这个文件中的数据写入子表中： -``` +```sql INSERT INTO d1001 FILE '/tmp/csvfile.csv'; ``` ## 插入来自文件的数据记录，并自动建表 -从 2.1.5.0 版本开始，支持在插入来自 CSV 文件的数据时，以超级表为模板来自动创建不存在的数据表。例如： - -``` +```sql INSERT INTO d21001 USING meters TAGS ('California.SanFrancisco', 2) FILE '/tmp/csvfile.csv'; ``` 也可以在一条语句中向多个表以自动建表的方式插入记录。例如： -``` +```sql INSERT INTO d21001 USING meters TAGS ('California.SanFrancisco', 2) FILE '/tmp/csvfile_21001.csv' d21002 USING meters (groupId) TAGS (2) FILE '/tmp/csvfile_21002.csv'; ``` - -## 历史记录写入 - -可使用 IMPORT 或者 INSERT 命令，IMPORT 的语法，功能与 INSERT 完全一样。 - -针对 insert 类型的 SQL 语句，我们采用的流式解析策略，在发现后面的错误之前，前面正确的部分 SQL 仍会执行。下面的 SQL 中，INSERT 语句是无效的，但是 d1001 仍会被创建。 - -``` -taos> CREATE TABLE meters(ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS(location BINARY(30), groupId INT); -Query OK, 0 row(s) affected (0.008245s) - -taos> SHOW STABLES; - name | created_time | columns | tags | tables | -============================================================================================ - meters | 2020-08-06 17:50:27.831 | 4 | 2 | 0 | -Query OK, 1 row(s) in set (0.001029s) - -taos> SHOW TABLES; -Query OK, 0 row(s) in set (0.000946s) - -taos> INSERT INTO d1001 USING meters TAGS('California.SanFrancisco', 2) VALUES('a'); - -DB error: invalid SQL: 'a' (invalid timestamp) (0.039494s) - -taos> SHOW TABLES; - table_name | created_time | columns | stable_name | -====================================================================================================== - d1001 | 2020-08-06 17:52:02.097 | 4 | meters | -Query OK, 1 row(s) in set (0.001091s) -``` diff --git a/docs/zh/12-taos-sql/06-select.md b/docs/zh/12-taos-sql/06-select.md index 92abc4344b..84fcda232d 100644 --- a/docs/zh/12-taos-sql/06-select.md +++ b/docs/zh/12-taos-sql/06-select.md @@ -5,121 +5,118 @@ title: 数据查询 ## 查询语法 -``` -SELECT select_expr [, select_expr ...] - FROM {tb_name_list} - [WHERE where_condition] - [SESSION(ts_col, tol_val)] - [STATE_WINDOW(col)] - [INTERVAL(interval_val [, interval_offset]) [SLIDING sliding_val]] - [FILL(fill_mod_and_val)] - [GROUP BY col_list] - [ORDER BY col_list { DESC | ASC }] +```sql +SELECT {DATABASE() | CLIENT_VERSION() | SERVER_VERSION() | SERVER_STATUS() | NOW() | TODAY() | TIMEZONE()} + +SELECT [DISTINCT] select_list + from_clause + [WHERE condition] + [PARTITION BY tag_list] + [window_clause] + [group_by_clause] + [order_by_clasue] [SLIMIT limit_val [SOFFSET offset_val]] [LIMIT limit_val [OFFSET offset_val]] - [>> export_file]; + [>> export_file] + +select_list: + select_expr [, select_expr] ... + +select_expr: { + * + | query_name.* + | [schema_name.] {table_name | view_name} .* + | t_alias.* + | expr [[AS] c_alias] +} + +from_clause: { + table_reference [, table_reference] ... + | join_clause [, join_clause] ... +} + +table_reference: + table_expr t_alias + +table_expr: { + table_name + | view_name + | ( subquery ) +} + +join_clause: + table_reference [INNER] JOIN table_reference ON condition + +window_clause: { + SESSION(ts_col, tol_val) + | STATE_WINDOW(col) + | INTERVAL(interval_val [, interval_offset]) [SLIDING (sliding_val)] [WATERMARK(watermark_val)] [FILL(fill_mod_and_val)] + +changes_option: { + DURATION duration_val + | ROWS rows_val +} + +group_by_clause: + GROUP BY expr [, expr] ... HAVING condition + +order_by_clasue: + ORDER BY order_expr [, order_expr] ... + +order_expr: + {expr | position | c_alias} [DESC | ASC] [NULLS FIRST | NULLS LAST] ``` -## 通配符 +## 列表 -通配符 \* 可以用于代指全部列。对于普通表，结果中只有普通列。 +查询语句可以指定部分或全部列作为返回结果。数据列和标签列都可以出现在列表中。 -``` -taos> SELECT * FROM d1001; - ts | current | voltage | phase | -====================================================================================== - 2018-10-03 14:38:05.000 | 10.30000 | 219 | 0.31000 | - 2018-10-03 14:38:15.000 | 12.60000 | 218 | 0.33000 | - 2018-10-03 14:38:16.800 | 12.30000 | 221 | 0.31000 | -Query OK, 3 row(s) in set (0.001165s) -``` +### 通配符 -在针对超级表，通配符包含 _标签列_ 。 +通配符 \* 可以用于代指全部列。对于普通表，结果中只有普通列。对于超级表和子表，还包含了 TAG 列。 -``` -taos> SELECT * FROM meters; - ts | current | voltage | phase | location | groupid | -===================================================================================================================================== - 2018-10-03 14:38:05.500 | 11.80000 | 221 | 0.28000 | California.LosAngeles | 2 | - 2018-10-03 14:38:16.600 | 13.40000 | 223 | 0.29000 | California.LosAngeles | 2 | - 2018-10-03 14:38:05.000 | 10.80000 | 223 | 0.29000 | California.LosAngeles | 3 | - 2018-10-03 14:38:06.500 | 11.50000 | 221 | 0.35000 | California.LosAngeles | 3 | - 2018-10-03 14:38:04.000 | 10.20000 | 220 | 0.23000 | California.SanFrancisco | 3 | - 2018-10-03 14:38:16.650 | 10.30000 | 218 | 0.25000 | California.SanFrancisco | 3 | - 2018-10-03 14:38:05.000 | 10.30000 | 219 | 0.31000 | California.SanFrancisco | 2 | - 2018-10-03 14:38:15.000 | 12.60000 | 218 | 0.33000 | California.SanFrancisco | 2 | - 2018-10-03 14:38:16.800 | 12.30000 | 221 | 0.31000 | California.SanFrancisco | 2 | -Query OK, 9 row(s) in set (0.002022s) +```sql +SELECT * FROM d1001; ``` 通配符支持表名前缀，以下两个 SQL 语句均为返回全部的列： -``` +```sql SELECT * FROM d1001; SELECT d1001.* FROM d1001; ``` -在 JOIN 查询中，带前缀的\*和不带前缀\*返回的结果有差别， \*返回全部表的所有列数据（不包含标签），带前缀的通配符，则只返回该表的列数据。 +在 JOIN 查询中，带表名前缀的\*和不带前缀\*返回的结果有差别， \*返回全部表的所有列数据（不包含标签），而带表名前缀的通配符，则只返回该表的列数据。 -``` -taos> SELECT * FROM d1001, d1003 WHERE d1001.ts=d1003.ts; - ts | current | voltage | phase | ts | current | voltage | phase | -================================================================================================================================== - 2018-10-03 14:38:05.000 | 10.30000| 219 | 0.31000 | 2018-10-03 14:38:05.000 | 10.80000| 223 | 0.29000 | -Query OK, 1 row(s) in set (0.017385s) +```sql +SELECT * FROM d1001, d1003 WHERE d1001.ts=d1003.ts; +SELECT d1001.* FROM d1001,d1003 WHERE d1001.ts = d1003.ts; ``` -``` -taos> SELECT d1001.* FROM d1001,d1003 WHERE d1001.ts = d1003.ts; - ts | current | voltage | phase | -====================================================================================== - 2018-10-03 14:38:05.000 | 10.30000 | 219 | 0.31000 | -Query OK, 1 row(s) in set (0.020443s) -``` +上面的查询语句中，前者返回 d1001 和 d1003 的全部列，而后者仅返回 d1001 的全部列。 在使用 SQL 函数来进行查询的过程中，部分 SQL 函数支持通配符操作。其中的区别在于： `count(*)`函数只返回一列。`first`、`last`、`last_row`函数则是返回全部列。 -``` -taos> SELECT COUNT(*) FROM d1001; - count(*) | -======================== - 3 | -Query OK, 1 row(s) in set (0.001035s) +### 标签列 + +在超级表和子表的查询中可以指定 _标签列_，且标签列的值会与普通列的数据一起返回。 + +```sql +ELECT location, groupid, current FROM d1001 LIMIT 2; ``` -``` -taos> SELECT FIRST(*) FROM d1001; - first(ts) | first(current) | first(voltage) | first(phase) | -========================================================================================= - 2018-10-03 14:38:05.000 | 10.30000 | 219 | 0.31000 | -Query OK, 1 row(s) in set (0.000849s) -``` +### 结果去重 -## 标签列 +`DISINTCT` 关键字可以对结果集中的一列或多列进行去重，去除的列既可以是标签列也可以是数据列。 -从 2.0.14 版本开始，支持在普通表的查询中指定 _标签列_，且标签列的值会与普通列的数据一起返回。 - -``` -taos> SELECT location, groupid, current FROM d1001 LIMIT 2; - location | groupid | current | -====================================================================== - California.SanFrancisco | 2 | 10.30000 | - California.SanFrancisco | 2 | 12.60000 | -Query OK, 2 row(s) in set (0.003112s) -``` - -注意：普通表的通配符 \* 中并不包含 _标签列_。 - -## 获取标签列或普通列的去重取值 - -从 2.0.15.0 版本开始，支持在超级表查询标签列时，指定 DISTINCT 关键字，这样将返回指定标签列的所有不重复取值。注意，在 2.1.6.0 版本之前，DISTINCT 只支持处理单个标签列，而从 2.1.6.0 版本开始，DISTINCT 可以对多个标签列进行处理，输出这些标签列取值不重复的组合。 +对标签列去重： ```sql SELECT DISTINCT tag_name [, tag_name ...] FROM stb_name; ``` -从 2.1.7.0 版本开始，DISTINCT 也支持对数据子表或普通表进行处理，也即支持获取单个普通列的不重复取值，或多个普通列取值的不重复组合。 +对数据列去重： ```sql SELECT DISTINCT col_name [, col_name ...] FROM tb_name; @@ -133,210 +130,162 @@ SELECT DISTINCT col_name [, col_name ...] FROM tb_name; ::: -## 结果集列名 +### 结果集列名 `SELECT`子句中，如果不指定返回结果集合的列名，结果集列名称默认使用`SELECT`子句中的表达式名称作为列名称。此外，用户可使用`AS`来重命名返回结果集合中列的名称。例如： -``` +```sql taos> SELECT ts, ts AS primary_key_ts FROM d1001; - ts | primary_key_ts | -==================================================== - 2018-10-03 14:38:05.000 | 2018-10-03 14:38:05.000 | - 2018-10-03 14:38:15.000 | 2018-10-03 14:38:15.000 | - 2018-10-03 14:38:16.800 | 2018-10-03 14:38:16.800 | -Query OK, 3 row(s) in set (0.001191s) ``` 但是针对`first(*)`、`last(*)`、`last_row(*)`不支持针对单列的重命名。 -## 隐式结果列 +### 隐式结果列 `Select_exprs`可以是表所属列的列名，也可以是基于列的函数表达式或计算式，数量的上限 256 个。当用户使用了`interval`或`group by tags`的子句以后，在最后返回结果中会强制返回时间戳列（第一列）和 group by 子句中的标签列。后续的版本中可以支持关闭 group by 子句中隐式列的输出，列输出完全由 select 子句控制。 -## 表（超级表）列表 +### 伪列 -FROM 关键字后面可以是若干个表（超级表）列表，也可以是子查询的结果。 -如果没有指定用户的当前数据库，可以在表名称之前使用数据库的名称来指定表所属的数据库。例如：`power.d1001` 方式来跨库使用表。 - -``` -SELECT * FROM power.d1001; ------------------------------- -USE power; -SELECT * FROM d1001; -``` - -## 特殊功能 - -部分特殊的查询功能可以不使用 FROM 子句执行。获取当前所在的数据库 database()： - -``` -taos> SELECT DATABASE(); - database() | -================================= - power | -Query OK, 1 row(s) in set (0.000079s) -``` - -如果登录的时候没有指定默认数据库，且没有使用`USE`命令切换数据，则返回 NULL。 - -``` -taos> SELECT DATABASE(); - database() | -================================= - NULL | -Query OK, 1 row(s) in set (0.000184s) -``` - -获取服务器和客户端版本号： - -``` -taos> SELECT CLIENT_VERSION(); - client_version() | -=================== - 2.0.0.0 | -Query OK, 1 row(s) in set (0.000070s) - -taos> SELECT SERVER_VERSION(); - server_version() | -=================== - 2.0.0.0 | -Query OK, 1 row(s) in set (0.000077s) -``` - -服务器状态检测语句。如果服务器正常，返回一个数字（例如 1）。如果服务器异常，返回 error code。该 SQL 语法能兼容连接池对于 TDengine 状态的检查及第三方工具对于数据库服务器状态的检查。并可以避免出现使用了错误的心跳检测 SQL 语句导致的连接池连接丢失的问题。 - -``` -taos> SELECT SERVER_STATUS(); - server_status() | -================== - 1 | -Query OK, 1 row(s) in set (0.000074s) - -taos> SELECT SERVER_STATUS() AS status; - status | -============== - 1 | -Query OK, 1 row(s) in set (0.000081s) -``` - -## \_block_dist 函数 - -**功能说明**: 用于获得指定的（超级）表的数据块分布信息 - -```txt title="语法" -SELECT _block_dist() FROM { tb_name | stb_name } -``` - -**返回结果类型**：字符串。 - -**适用数据类型**：不能输入任何参数。 - -**嵌套子查询支持**：不支持子查询或嵌套查询。 - -**返回结果**: - -- 返回 FROM 子句中输入的表或超级表的数据块分布情况。不支持查询条件。 -- 返回的结果是该表或超级表的数据块所包含的行数的数据分布直方图。 - -```txt title="返回结果" -summary: -5th=[392], 10th=[392], 20th=[392], 30th=[392], 40th=[792], 50th=[792] 60th=[792], 70th=[792], 80th=[792], 90th=[792], 95th=[792], 99th=[792] Min=[392(Rows)] Max=[800(Rows)] Avg=[666(Rows)] Stddev=[2.17] Rows=[2000], Blocks=[3], Size=[5.440(Kb)] Comp=[0.23] RowsInMem=[0] SeekHeaderTime=[1(us)] -``` - -**上述信息的说明如下**: - -- 查询的（超级）表所包含的存储在文件中的数据块（data block）中所包含的数据行的数量分布直方图信息：5%， 10%， 20%， 30%， 40%， 50%， 60%， 70%， 80%， 90%， 95%， 99% 的数值； -- 所有数据块中，包含行数最少的数据块所包含的行数量， 其中的 Min 指标 392 行。 -- 所有数据块中，包含行数最多的数据块所包含的行数量， 其中的 Max 指标 800 行。 -- 所有数据块行数的算数平均值 666 行（其中的 Avg 项）。 -- 所有数据块中行数分布的均方差为 2.17 ( stddev ）。 -- 数据块包含的行的总数为 2000 行（Rows）。 -- 数据块总数是 3 个数据块 （Blocks）。 -- 数据块占用磁盘空间大小 5.44 Kb （size）。 -- 压缩后的数据块的大小除以原始数据的所获得的压缩比例： 23%（Comp），及压缩后的数据规模是原始数据规模的 23%。 -- 内存中存在的数据行数是 0，表示内存中没有数据缓存。 -- 获取数据块信息的过程中读取头文件的时间开销 1 微秒（SeekHeaderTime）。 - -**支持版本**：指定计算算法的功能从 2.1.0.x 版本开始，2.1.0.0 之前的版本不支持指定使用算法的功能。 - -## TAOS SQL 中特殊关键词 - -- `TBNAME`： 在超级表查询中可视为一个特殊的标签，代表查询涉及的子表名 -- `_c0`: 表示表（超级表）的第一列 - -## 小技巧 +**TBNAME** +`TBNAME` 可以视为超级表中一个特殊的标签，代表子表的表名。 获取一个超级表所有的子表名及相关的标签信息： -``` +```mysql SELECT TBNAME, location FROM meters; ``` 统计超级表下辖子表数量： -``` -SELECT COUNT(TBNAME) FROM meters; +```mysql +SELECT COUNT(*) FROM (SELECT DISTINCT TBNAME FROM meters); ``` 以上两个查询均只支持在 WHERE 条件子句中添加针对标签（TAGS）的过滤条件。例如： -``` -taos> SELECT TBNAME, location FROM meters; - tbname | location | -================================================================== - d1004 | California.LosAngeles | - d1003 | California.LosAngeles | - d1002 | California.SanFrancisco | - d1001 | California.SanFrancisco | -Query OK, 4 row(s) in set (0.000881s) +**\_QSTART/\_QEND** -taos> SELECT COUNT(tbname) FROM meters WHERE groupId > 2; - count(tbname) | -======================== - 2 | -Query OK, 1 row(s) in set (0.001091s) +\_qstart 和\_qend 表示用户输入的查询时间范围，即 WHERE 子句中主键时间戳条件所限定的时间范围。如果 WHERE 子句中没有有效的主键时间戳条件，则时间范围为[-2^63, 2^63-1]。 + +\_qstart 和\_qend 不能用于 WHERE 子句中。 + +**\_WSTART/\_WEND/\_WDURATION** +\_wstart 伪列、\_wend 伪列和\_wduration 伪列 +\_wstart 表示窗口起始时间戳，\_wend 表示窗口结束时间戳，\_wduration 表示窗口持续时长。 + +这三个伪列只能用于时间窗口的窗口切分查询之中，且要在窗口切分子句之后出现。 + +**\_c0/\_ROWTS** + +TDengine 中，所有表的第一列都必须是时间戳类型，且为其主键，\_rowts 伪列和\_c0 伪列均代表了此列的值。相比实际的主键时间戳列，使用伪列更加灵活，语义也更加标准。例如，可以和 max\min 等函数一起使用。 + +```sql +select _rowts, max(current) from meters; ``` -- 可以使用 \* 返回所有列，或指定列名。可以对数字列进行四则运算，可以给输出的列取列名。 - - 暂不支持含列名的四则运算表达式用于条件过滤算子（例如，不支持 `where a*2>6;`，但可以写 `where a>6/2;`）。 - - 暂不支持含列名的四则运算表达式作为 SQL 函数的应用对象（例如，不支持 `select min(2*a) from t;`，但可以写 `select 2*min(a) from t;`）。 -- WHERE 语句可以使用各种逻辑判断来过滤数字值，或使用通配符来过滤字符串。 -- 输出结果缺省按首列时间戳升序排序，但可以指定按降序排序( \_c0 指首列时间戳)。使用 ORDER BY 对其他字段进行排序,排序结果顺序不确定。 -- 参数 LIMIT 控制输出条数，OFFSET 指定从第几条开始输出。LIMIT/OFFSET 对结果集的执行顺序在 ORDER BY 之后。且 `LIMIT 5 OFFSET 2` 可以简写为 `LIMIT 2, 5`。 - - 在有 GROUP BY 子句的情况下，LIMIT 参数控制的是每个分组中至多允许输出的条数。 -- 参数 SLIMIT 控制由 GROUP BY 指令划分的分组中，至多允许输出几个分组的数据。且 `SLIMIT 5 SOFFSET 2` 可以简写为 `SLIMIT 2, 5`。 -- 通过 “>>” 输出结果可以导出到指定文件。 +## 查询对象 -## 条件过滤操作 +FROM 关键字后面可以是若干个表（超级表）列表，也可以是子查询的结果。 +如果没有指定用户的当前数据库，可以在表名称之前使用数据库的名称来指定表所属的数据库。例如：`power.d1001` 方式来跨库使用表。 -| **Operation** | **Note** | **Applicable Data Types** | -| ------------- | ------------------------ | ----------------------------------------- | -| > | larger than | all types except bool | -| < | smaller than | all types except bool | -| >= | larger than or equal to | all types except bool | -| <= | smaller than or equal to | all types except bool | -| = | equal to | all types | -| <\> | not equal to | all types | -| is [not] null | is null or is not null | all types | -| between and | within a certain range | all types except bool | -| in | match any value in a set | all types except first column `timestamp` | -| like | match a wildcard string | **`binary`** **`nchar`** | -| match/nmatch | filter regex | **`binary`** **`nchar`** | +TDengine 支持基于时间戳主键的 INNER JOIN，规则如下： -**使用说明**: +1. 支持 FROM 表列表和显式的 JOIN 子句两种语法。 +2. 对于普通表和子表，ON 条件必须有且只有时间戳主键的等值条件。 +3. 对于超级表，ON 条件在时间戳主键的等值条件之外，还要求有可以一一对应的标签列等值条件，不支持 OR 条件。 +4. 参与 JOIN 计算的表只能是同一种类型，即只能都是超级表，或都是子表，或都是普通表。 +5. JOIN 两侧均支持子查询。 +6. 参与 JOIN 的表个数上限为 10 个。 +7. 不支持与 FILL 子句混合使用。 -- <\> 算子也可以写为 != ，请注意，这个算子不能用于数据表第一列的 timestamp 字段。 -- like 算子使用通配符字符串进行匹配检查。 - - 在通配符字符串中：'%'（百分号）匹配 0 到任意个字符；'\_'（下划线）匹配单个任意 ASCII 字符。 - - 如果希望匹配字符串中原本就带有的 \_（下划线）字符，那么可以在通配符字符串中写作 `\_`，也即加一个反斜线来进行转义。（从 2.2.0.0 版本开始支持） - - 通配符字符串最长不能超过 20 字节。（从 2.1.6.1 版本开始，通配符字符串的长度放宽到了 100 字节，并可以通过 taos.cfg 中的 maxWildCardsLength 参数来配置这一长度限制。但不建议使用太长的通配符字符串，将有可能严重影响 LIKE 操作的执行性能。） -- 同时进行多个字段的范围过滤，需要使用关键词 AND 来连接不同的查询条件，暂不支持 OR 连接的不同列之间的查询过滤条件。 - - 从 2.3.0.0 版本开始，已支持完整的同一列和/或不同列间的 AND/OR 运算。 -- 针对单一字段的过滤，如果是时间过滤条件，则一条语句中只支持设定一个；但针对其他的（普通）列或标签列，则可以使用 `OR` 关键字进行组合条件的查询过滤。例如： `((value > 20 AND value < 30) OR (value < 12))`。 - - 从 2.3.0.0 版本开始，允许使用多个时间过滤条件，但首列时间戳的过滤运算结果只能包含一个区间。 -- 从 2.0.17.0 版本开始，条件过滤开始支持 BETWEEN AND 语法，例如 `WHERE col2 BETWEEN 1.5 AND 3.25` 表示查询条件为“1.5 ≤ col2 ≤ 3.25”。 -- 从 2.1.4.0 版本开始，条件过滤开始支持 IN 算子，例如 `WHERE city IN ('California.SanFrancisco', 'California.SanDieo')`。说明：BOOL 类型写作 `{true, false}` 或 `{0, 1}` 均可，但不能写作 0、1 之外的整数；FLOAT 和 DOUBLE 类型会受到浮点数精度影响，集合内的值在精度范围内认为和数据行的值完全相等才能匹配成功；TIMESTAMP 类型支持非主键的列。 -- 从 2.3.0.0 版本开始，条件过滤开始支持正则表达式，关键字 match/nmatch，不区分大小写。 +## GROUP BY + +如果在语句中同时指定了 GROUP BY 子句，那么 SELECT 列表只能包含如下表达式： + +1. 常量 +2. 聚集函数 +3. 与 GROUP BY 后表达式相同的表达式。 +4. 包含前面表达式的表达式 + +GROUP BY 子句对每行数据按 GROUP BY 后的表达式的值进行分组，并为每个组返回一行汇总信息。 + +GROUP BY 子句中的表达式可以包含表或视图中的任何列，这些列不需要出现在 SELECT 列表中。 + +该子句对行进行分组，但不保证结果集的顺序。若要对分组进行排序，请使用 ORDER BY 子句 + + +## PARTITON BY + +PARTITION BY 子句是 TDengine 特色语法，按 part_list 对数据进行切分，在每个切分的分片中进行计算。 + +详见 [TDengine 特色查询](./distinguished) + +## ORDER BY + +ORDER BY 子句对结果集排序。如果没有指定 ORDER BY，无法保证同一语句多次查询的结果集返回顺序一致。 + +ORDER BY 后可以使用位置语法，位置标识为正整数，从 1 开始，表示使用 SELECT 列表的第几个表达式进行排序。 + +ASC 表示升序，DESC 表示降序。 + +NULLS 语法用来指定 NULL 值在排序中输出的位置。NULLS LAST 是升序的默认值，NULLS FIRST 是降序的默认值。 + +## LIMIT + +LIMIT 控制输出条数，OFFSET 指定从第几条之后开始输出。LIMIT/OFFSET 对结果集的执行顺序在 ORDER BY 之后。LIMIT 5 OFFSET 2 可以简写为 LIMIT 2, 5，都输出第 3 行到第 7 行数据。 + +在有 PARTITION BY 子句时，LIMIT 控制的是每个切分的分片中的输出，而不是总的结果集输出。 + +## SLIMIT + +SLIMIT 和 PARTITION BY 子句一起使用，用来控制输出的分片的数量。SLIMIT 5 SOFFSET 2 可以简写为 SLIMIT 2, 5，都表示输出第 3 个到第 7 个分片。 + +需要注意，如果有 ORDER BY 子句，则输出只有一个分片。 + +## 特殊功能 + +部分特殊的查询功能可以不使用 FROM 子句执行。 + +### 获取当前数据库 + +下面的命令可以获取当前所在的数据库 database()，如果登录的时候没有指定默认数据库，且没有使用`USE`命令切换数据，则返回 NULL。 + +```sql +SELECT DATABASE(); +``` + +### 获取服务器和客户端版本号 + +```sql +SELECT CLIENT_VERSION(); +SELECT SERVER_VERSION(); +``` + +### 获取服务器状态 + +服务器状态检测语句。如果服务器正常，返回一个数字（例如 1）。如果服务器异常，返回 error code。该 SQL 语法能兼容连接池对于 TDengine 状态的检查及第三方工具对于数据库服务器状态的检查。并可以避免出现使用了错误的心跳检测 SQL 语句导致的连接池连接丢失的问题。 + +```sql +SELECT SERVER_STATUS(); +``` + +### 获取当前时间 + +```sql +SELECT NOW(); +``` + +### 获取当前日期 + +```sql +SELECT TODAY(); +``` + +### 获取当前时区 + +```sql +SELECT TIMEZONE(); +``` ## 正则表达式过滤 @@ -358,7 +307,7 @@ WHERE (column|tbname) **match/MATCH/nmatch/NMATCH** _regex_ ## JOIN 子句 -从 2.2.0.0 版本开始，TDengine 对内连接（INNER JOIN）中的自然连接（Natural join）操作实现了完整的支持。也即支持“普通表与普通表之间”、“超级表与超级表之间”、“子查询与子查询之间”进行自然连接。自然连接与内连接的主要区别是，自然连接要求参与连接的字段在不同的表/超级表中必须是同名字段。也即，TDengine 在连接关系的表达中，要求必须使用同名数据列/标签列的相等关系。 +TDengine 支持“普通表与普通表之间”、“超级表与超级表之间”、“子查询与子查询之间” 进行自然连接。自然连接与内连接的主要区别是，自然连接要求参与连接的字段在不同的表/超级表中必须是同名字段。也即，TDengine 在连接关系的表达中，要求必须使用同名数据列/标签列的相等关系。 在普通表与普通表之间的 JOIN 操作中，只能使用主键时间戳之间的相等关系。例如： @@ -429,7 +378,7 @@ UNION ALL SELECT ... TDengine 支持 UNION ALL 操作符。也就是说，如果多个 SELECT 子句返回结果集的结构完全相同（列名、列类型、列数、顺序），那么可以通过 UNION ALL 把这些结果集合并到一起。目前只支持 UNION ALL 模式，也即在结果集的合并过程中是不去重的。在同一个 sql 语句中，UNION ALL 最多支持 100 个。 -### SQL 示例 +## SQL 示例 对于下面的例子，表 tb1 用以下语句创建： diff --git a/docs/zh/12-taos-sql/08-delete-data.mdx b/docs/zh/12-taos-sql/08-delete-data.mdx index eafe8cff26..15becd2593 100644 --- a/docs/zh/12-taos-sql/08-delete-data.mdx +++ b/docs/zh/12-taos-sql/08-delete-data.mdx @@ -5,8 +5,6 @@ title: "删除数据" --- 删除数据是 TDengine 提供的根据指定时间段删除指定表或超级表中数据记录的功能，方便用户清理由于设备故障等原因产生的异常数据。 -注意：本功能只在企业版 2.6.0.0 及以后的版本中提供，如需此功能请点击下面的链接访问[企业版产品](https://www.taosdata.com/products#enterprise-edition-link) - **语法：** @@ -17,21 +15,21 @@ DELETE FROM [ db_name. ] tb_name [WHERE condition]; **功能：** 删除指定表或超级表中的数据记录 **参数：** - -- `db_name` ： 可选参数，指定要删除表所在的数据库名，不填写则在当前数据库中 -- `tb_name` ： 必填参数，指定要删除数据的表名，可以是普通表、子表，也可以是超级表。 -- `condition`： 可选参数，指定删除数据的过滤条件，不指定过滤条件则为表中所有数据，请慎重使用。特别说明，这里的where 条件中只支持对第一列时间列的过滤，如果是超级表，支持对 tag 列过滤。 + +- `db_name` ： 可选参数，指定要删除表所在的数据库名，不填写则在当前数据库中 +- `tb_name` ： 必填参数，指定要删除数据的表名，可以是普通表、子表，也可以是超级表。 +- `condition`： 可选参数，指定删除数据的过滤条件，不指定过滤条件则为表中所有数据，请慎重使用。特别说明，这里的 where 条件中只支持对第一列时间列的过滤。 **特别说明：** - -数据删除后不可恢复，请慎重使用。为了确保删除的数据确实是自己要删除的，建议可以先使用 `select` 语句加 `where` 后的删除条件查看要删除的数据内容，确认无误后再执行 `delete` 命令。 + +数据删除后不可恢复，请慎重使用。为了确保删除的数据确实是自己要删除的，建议可以先使用 `select` 语句加 `where` 后的删除条件查看要删除的数据内容，确认无误后再执行 `delete` 命令。 **示例：** - -`meters` 是一个超级表，`groupid` 是 int 类型的 tag 列，现在要删除 `meters` 表中时间小于 2021-10-01 10:40:00.100 且 tag 列 `groupid` 值为 1 的所有数据，sql 如下： + +`meters` 是一个超级表，`groupid` 是 int 类型的 tag 列，现在要删除 `meters` 表中时间小于 2021-10-01 10:40:00.100 的所有数据，sql 如下： ```sql -delete from meters where ts < '2021-10-01 10:40:00.100' and groupid=1 ; +delete from meters where ts < '2021-10-01 10:40:00.100' ; ``` 执行后显示结果为： diff --git a/docs/zh/12-taos-sql/10-function.md b/docs/zh/12-taos-sql/10-function.md index 7674967f09..876aaa553e 100644 --- a/docs/zh/12-taos-sql/10-function.md +++ b/docs/zh/12-taos-sql/10-function.md @@ -13,7 +13,7 @@ toc_max_heading_level: 4 #### ABS ```sql - SELECT ABS(field_name) FROM { tb_name | stb_name } [WHERE clause] +SELECT ABS(field_name) FROM { tb_name | stb_name } [WHERE clause] ``` **功能说明**：获得指定列的绝对值 @@ -31,7 +31,7 @@ toc_max_heading_level: 4 #### ACOS ```sql - SELECT ACOS(field_name) FROM { tb_name | stb_name } [WHERE clause] +SELECT ACOS(field_name) FROM { tb_name | stb_name } [WHERE clause] ``` **功能说明**：获得指定列的反余弦结果 @@ -49,7 +49,7 @@ toc_max_heading_level: 4 #### ASIN ```sql - SELECT ASIN(field_name) FROM { tb_name | stb_name } [WHERE clause] +SELECT ASIN(field_name) FROM { tb_name | stb_name } [WHERE clause] ``` **功能说明**：获得指定列的反正弦结果 @@ -68,7 +68,7 @@ toc_max_heading_level: 4 #### ATAN ```sql - SELECT ATAN(field_name) FROM { tb_name | stb_name } [WHERE clause] +SELECT ATAN(field_name) FROM { tb_name | stb_name } [WHERE clause] ``` **功能说明**：获得指定列的反正切结果 @@ -86,7 +86,7 @@ toc_max_heading_level: 4 #### CEIL -``` +```sql SELECT CEIL(field_name) FROM { tb_name | stb_name } [WHERE clause]; ``` @@ -108,7 +108,7 @@ SELECT CEIL(field_name) FROM { tb_name | stb_name } [WHERE clause]; #### COS ```sql - SELECT COS(field_name) FROM { tb_name | stb_name } [WHERE clause] +SELECT COS(field_name) FROM { tb_name | stb_name } [WHERE clause] ``` **功能说明**：获得指定列的余弦结果 @@ -125,7 +125,7 @@ SELECT CEIL(field_name) FROM { tb_name | stb_name } [WHERE clause]; #### FLOOR -``` +```sql SELECT FLOOR(field_name) FROM { tb_name | stb_name } [WHERE clause]; ``` @@ -135,7 +135,7 @@ SELECT FLOOR(field_name) FROM { tb_name | stb_name } [WHERE clause]; #### LOG ```sql - SELECT LOG(field_name, base) FROM { tb_name | stb_name } [WHERE clause] +SELECT LOG(field_name, base) FROM { tb_name | stb_name } [WHERE clause] ``` **功能说明**：获得指定列对于底数 base 的对数 @@ -154,7 +154,7 @@ SELECT FLOOR(field_name) FROM { tb_name | stb_name } [WHERE clause]; #### POW ```sql - SELECT POW(field_name, power) FROM { tb_name | stb_name } [WHERE clause] +SELECT POW(field_name, power) FROM { tb_name | stb_name } [WHERE clause] ``` **功能说明**：获得指定列的指数为 power 的幂 @@ -172,7 +172,7 @@ SELECT FLOOR(field_name) FROM { tb_name | stb_name } [WHERE clause]; #### ROUND -``` +```sql SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; ``` @@ -183,7 +183,7 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; #### SIN ```sql - SELECT SIN(field_name) FROM { tb_name | stb_name } [WHERE clause] +SELECT SIN(field_name) FROM { tb_name | stb_name } [WHERE clause] ``` **功能说明**：获得指定列的正弦结果 @@ -201,7 +201,7 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; #### SQRT ```sql - SELECT SQRT(field_name) FROM { tb_name | stb_name } [WHERE clause] +SELECT SQRT(field_name) FROM { tb_name | stb_name } [WHERE clause] ``` **功能说明**：获得指定列的平方根 @@ -219,7 +219,7 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; #### TAN ```sql - SELECT TAN(field_name) FROM { tb_name | stb_name } [WHERE clause] +SELECT TAN(field_name) FROM { tb_name | stb_name } [WHERE clause] ``` **功能说明**：获得指定列的正切结果 @@ -240,8 +240,8 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; #### CHAR_LENGTH -``` - SELECT CHAR_LENGTH(str|column) FROM { tb_name | stb_name } [WHERE clause] +```sql +SELECT CHAR_LENGTH(str|column) FROM { tb_name | stb_name } [WHERE clause] ``` **功能说明**：以字符计数的字符串长度。 @@ -257,7 +257,7 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; #### CONCAT ```sql - SELECT CONCAT(str1|column1, str2|column2, ...) FROM { tb_name | stb_name } [WHERE clause] +SELECT CONCAT(str1|column1, str2|column2, ...) FROM { tb_name | stb_name } [WHERE clause] ``` **功能说明**：字符串连接函数。 @@ -273,8 +273,8 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; #### CONCAT_WS -``` - SELECT CONCAT_WS(separator, str1|column1, str2|column2, ...) FROM { tb_name | stb_name } [WHERE clause] +```sql +SELECT CONCAT_WS(separator, str1|column1, str2|column2, ...) FROM { tb_name | stb_name } [WHERE clause] ``` **功能说明**：带分隔符的字符串连接函数。 @@ -290,8 +290,8 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; #### LENGTH -``` - SELECT LENGTH(str|column) FROM { tb_name | stb_name } [WHERE clause] +```sql +SELECT LENGTH(str|column) FROM { tb_name | stb_name } [WHERE clause] ``` **功能说明**：以字节计数的字符串长度。 @@ -307,8 +307,8 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; #### LOWER -``` - SELECT LOWER(str|column) FROM { tb_name | stb_name } [WHERE clause] +```sql +SELECT LOWER(str|column) FROM { tb_name | stb_name } [WHERE clause] ``` **功能说明**：将字符串参数值转换为全小写字母。 @@ -324,8 +324,8 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; #### LTRIM -``` - SELECT LTRIM(str|column) FROM { tb_name | stb_name } [WHERE clause] +```sql +SELECT LTRIM(str|column) FROM { tb_name | stb_name } [WHERE clause] ``` **功能说明**：返回清除左边空格后的字符串。 @@ -341,8 +341,8 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; #### RTRIM -``` - SELECT LTRIM(str|column) FROM { tb_name | stb_name } [WHERE clause] +```sql +SELECT LTRIM(str|column) FROM { tb_name | stb_name } [WHERE clause] ``` **功能说明**：返回清除右边空格后的字符串。 @@ -358,8 +358,8 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; #### SUBSTR -``` - SELECT SUBSTR(str,pos[,len]) FROM { tb_name | stb_name } [WHERE clause] +```sql +SELECT SUBSTR(str,pos[,len]) FROM { tb_name | stb_name } [WHERE clause] ``` **功能说明**：从源字符串 str 中的指定位置 pos 开始取一个长度为 len 的子串并返回。 @@ -375,8 +375,8 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; #### UPPER -``` - SELECT UPPER(str|column) FROM { tb_name | stb_name } [WHERE clause] +```sql +SELECT UPPER(str|column) FROM { tb_name | stb_name } [WHERE clause] ``` **功能说明**：将字符串参数值转换为全大写字母。 @@ -397,7 +397,7 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; #### CAST ```sql - SELECT CAST(expression AS type_name) FROM { tb_name | stb_name } [WHERE clause] +SELECT CAST(expression AS type_name) FROM { tb_name | stb_name } [WHERE clause] ``` **功能说明**：数据类型转换函数，输入参数 expression 支持普通列、常量、标量函数及它们之间的四则运算，只适用于 select 子句中。 @@ -587,7 +587,7 @@ TDengine 支持针对数据的聚合查询。提供如下聚合函数。 ### AVG -``` +```sql SELECT AVG(field_name) FROM tb_name [WHERE clause]; ``` @@ -602,7 +602,7 @@ SELECT AVG(field_name) FROM tb_name [WHERE clause]; ### COUNT -``` +```sql SELECT COUNT([*|field_name]) FROM tb_name [WHERE clause]; ``` @@ -623,7 +623,7 @@ SELECT COUNT([*|field_name]) FROM tb_name [WHERE clause]; ### ELAPSED -```mysql +```sql SELECT ELAPSED(ts_primary_key [, time_unit]) FROM { tb_name | stb_name } [WHERE clause] [INTERVAL(interval [, offset]) [SLIDING sliding]]; ``` @@ -649,7 +649,7 @@ SELECT ELAPSED(ts_primary_key [, time_unit]) FROM { tb_name | stb_name } [WHERE ### LEASTSQUARES -``` +```sql SELECT LEASTSQUARES(field_name, start_val, step_val) FROM tb_name [WHERE clause]; ``` @@ -664,7 +664,7 @@ SELECT LEASTSQUARES(field_name, start_val, step_val) FROM tb_name [WHERE clause] ### MODE -``` +```sql SELECT MODE(field_name) FROM tb_name [WHERE clause]; ``` @@ -679,7 +679,7 @@ SELECT MODE(field_name) FROM tb_name [WHERE clause]; ### SPREAD -``` +```sql SELECT SPREAD(field_name) FROM { tb_name | stb_name } [WHERE clause]; ``` @@ -694,7 +694,7 @@ SELECT SPREAD(field_name) FROM { tb_name | stb_name } [WHERE clause]; ### STDDEV -``` +```sql SELECT STDDEV(field_name) FROM tb_name [WHERE clause]; ``` @@ -709,7 +709,7 @@ SELECT STDDEV(field_name) FROM tb_name [WHERE clause]; ### SUM -``` +```sql SELECT SUM(field_name) FROM tb_name [WHERE clause]; ``` @@ -724,7 +724,7 @@ SELECT SUM(field_name) FROM tb_name [WHERE clause]; ### HYPERLOGLOG -``` +```sql SELECT HYPERLOGLOG(field_name) FROM { tb_name | stb_name } [WHERE clause]; ``` @@ -741,7 +741,7 @@ SELECT HYPERLOGLOG(field_name) FROM { tb_name | stb_name } [WHERE clause]; ### HISTOGRAM -``` +```sql SELECT HISTOGRAM(field_name，bin_type, bin_description, normalized) FROM tb_name [WHERE clause]; ``` @@ -775,7 +775,7 @@ SELECT HISTOGRAM(field_name，bin_type, bin_description, normalized) FROM tb_nam ### APERCENTILE -``` +```sql SELECT APERCENTILE(field_name, P[, algo_type]) FROM { tb_name | stb_name } [WHERE clause] ``` @@ -790,7 +790,7 @@ FROM { tb_name | stb_name } [WHERE clause] ### BOTTOM -``` +```sql SELECT BOTTOM(field_name, K) FROM { tb_name | stb_name } [WHERE clause]; ``` @@ -810,7 +810,7 @@ SELECT BOTTOM(field_name, K) FROM { tb_name | stb_name } [WHERE clause]; ### FIRST -``` +```sql SELECT FIRST(field_name) FROM { tb_name | stb_name } [WHERE clause]; ``` @@ -830,7 +830,7 @@ SELECT FIRST(field_name) FROM { tb_name | stb_name } [WHERE clause]; ### INTERP -``` +```sql SELECT INTERP(field_name) FROM { tb_name | stb_name } [WHERE where_condition] [ RANGE(timestamp1,timestamp2) ] [EVERY(interval)] [FILL ({ VALUE | PREV | NULL | LINEAR | NEXT})]; ``` @@ -854,7 +854,7 @@ SELECT INTERP(field_name) FROM { tb_name | stb_name } [WHERE where_condition] [ ### LAST -``` +```sql SELECT LAST(field_name) FROM { tb_name | stb_name } [WHERE clause]; ``` @@ -875,7 +875,7 @@ SELECT LAST(field_name) FROM { tb_name | stb_name } [WHERE clause]; ### LAST_ROW -``` +```sql SELECT LAST_ROW(field_name) FROM { tb_name | stb_name }; ``` @@ -894,7 +894,7 @@ SELECT LAST_ROW(field_name) FROM { tb_name | stb_name }; ### MAX -``` +```sql SELECT MAX(field_name) FROM { tb_name | stb_name } [WHERE clause]; ``` @@ -909,7 +909,7 @@ SELECT MAX(field_name) FROM { tb_name | stb_name } [WHERE clause]; ### MIN -``` +```sql SELECT MIN(field_name) FROM {tb_name | stb_name} [WHERE clause]; ``` @@ -924,7 +924,7 @@ SELECT MIN(field_name) FROM {tb_name | stb_name} [WHERE clause]; ### PERCENTILE -``` +```sql SELECT PERCENTILE(field_name, P) FROM { tb_name } [WHERE clause]; ``` @@ -941,7 +941,7 @@ SELECT PERCENTILE(field_name, P) FROM { tb_name } [WHERE clause]; ### TAIL -``` +```sql SELECT TAIL(field_name, k, offset_val) FROM {tb_name | stb_name} [WHERE clause]; ``` @@ -958,7 +958,7 @@ SELECT TAIL(field_name, k, offset_val) FROM {tb_name | stb_name} [WHERE clause]; ### TOP -``` +```sql SELECT TOP(field_name, K) FROM { tb_name | stb_name } [WHERE clause]; ``` @@ -978,7 +978,7 @@ SELECT TOP(field_name, K) FROM { tb_name | stb_name } [WHERE clause]; ### UNIQUE -``` +```sql SELECT UNIQUE(field_name) FROM {tb_name | stb_name} [WHERE clause]; ``` @@ -998,7 +998,7 @@ SELECT UNIQUE(field_name) FROM {tb_name | stb_name} [WHERE clause]; ### CSUM ```sql - SELECT CSUM(field_name) FROM { tb_name | stb_name } [WHERE clause] +SELECT CSUM(field_name) FROM { tb_name | stb_name } [WHERE clause] ``` **功能说明**：累加和（Cumulative sum），输出行与输入行数相同。 @@ -1020,7 +1020,7 @@ SELECT UNIQUE(field_name) FROM {tb_name | stb_name} [WHERE clause]; ### DERIVATIVE -``` +```sql SELECT DERIVATIVE(field_name, time_interval, ignore_negative) FROM tb_name [WHERE clause]; ``` @@ -1037,9 +1037,9 @@ SELECT DERIVATIVE(field_name, time_interval, ignore_negative) FROM tb_name [WHER ### DIFF - ```sql - SELECT {DIFF(field_name, ignore_negative) | DIFF(field_name)} FROM tb_name [WHERE clause]; - ``` +```sql +SELECT {DIFF(field_name, ignore_negative) | DIFF(field_name)} FROM tb_name [WHERE clause]; +``` **功能说明**：统计表中某列的值与前一行对应值的差。 ignore_negative 取值为 0|1 , 可以不填，默认值为 0. 不忽略负值。ignore_negative 为 1 时表示忽略负数。 @@ -1054,7 +1054,7 @@ SELECT DERIVATIVE(field_name, time_interval, ignore_negative) FROM tb_name [WHER ### IRATE -``` +```sql SELECT IRATE(field_name) FROM tb_name WHERE clause; ``` @@ -1069,7 +1069,7 @@ SELECT IRATE(field_name) FROM tb_name WHERE clause; ### MAVG ```sql - SELECT MAVG(field_name, K) FROM { tb_name | stb_name } [WHERE clause] +SELECT MAVG(field_name, K) FROM { tb_name | stb_name } [WHERE clause] ``` **功能说明**： 计算连续 k 个值的移动平均数（moving average）。如果输入行数小于 k，则无结果输出。参数 k 的合法输入范围是 1≤ k ≤ 1000。 @@ -1091,7 +1091,7 @@ SELECT IRATE(field_name) FROM tb_name WHERE clause; ### SAMPLE ```sql - SELECT SAMPLE(field_name, K) FROM { tb_name | stb_name } [WHERE clause] +SELECT SAMPLE(field_name, K) FROM { tb_name | stb_name } [WHERE clause] ``` **功能说明**： 获取数据的 k 个采样值。参数 k 的合法输入范围是 1≤ k ≤ 1000。 @@ -1111,7 +1111,7 @@ SELECT IRATE(field_name) FROM tb_name WHERE clause; ### STATECOUNT -``` +```sql SELECT STATECOUNT(field_name, oper, val) FROM { tb_name | stb_name } [WHERE clause]; ``` @@ -1166,7 +1166,7 @@ SELECT stateDuration(field_name, oper, val, unit) FROM { tb_name | stb_name } [W ### TWA -``` +```sql SELECT TWA(field_name) FROM tb_name WHERE clause; ``` @@ -1185,7 +1185,7 @@ SELECT TWA(field_name) FROM tb_name WHERE clause; ### DATABASE -``` +```sql SELECT DATABASE(); ``` @@ -1194,7 +1194,7 @@ SELECT DATABASE(); ### CLIENT_VERSION -``` +```sql SELECT CLIENT_VERSION(); ``` @@ -1202,7 +1202,7 @@ SELECT CLIENT_VERSION(); ### SERVER_VERSION -``` +```sql SELECT SERVER_VERSION(); ``` @@ -1210,7 +1210,7 @@ SELECT SERVER_VERSION(); ### SERVER_STATUS -``` +```sql SELECT SERVER_VERSION(); ``` diff --git a/docs/zh/12-taos-sql/12-interval.md b/docs/zh/12-taos-sql/12-distinguished.md similarity index 91% rename from docs/zh/12-taos-sql/12-interval.md rename to docs/zh/12-taos-sql/12-distinguished.md index 8bf0c578e7..2dad49ece9 100644 --- a/docs/zh/12-taos-sql/12-interval.md +++ b/docs/zh/12-taos-sql/12-distinguished.md @@ -12,16 +12,16 @@ TDengine 提供的特色查询包括标签切分查询和窗口切分查询。 超级表查询中，当需要针对标签进行数据切分然后在切分出的数据空间内再进行一系列的计算时使用标签切分子句，标签切分的语句如下： ```sql -PARTITION BY tag_list +PARTITION BY part_list ``` -其中 `tag_list` 是标签列的列表，还可以包括 tbname 伪列。 +part_list 可以是任意的标量表达式，包括列、常量、标量函数和它们的组合。 -TDengine 按如下方式处理标签切分子句： +当 PARTITION BY 和标签一起使用时，TDengine 按如下方式处理标签切分子句： -标签切分子句位于 `WHERE` 子句之后，且不能和 `JOIN` 子句一起使用。 -标签切分子句将超级表数据按指定的标签组合进行切分，然后对每个切分的分片进行指定的计算。计算由之后的子句定义（窗口子句、`GROUP BY` 子句或`SELECT` 子句）。 -标签切分子句可以和窗口切分子句（或 `GROUP BY` 子句）一起使用，此时后面的子句作用在每个切分的分片上。例如，下面的示例将数据按标签 `location` 进行分组，并对每个组按 10 分钟进行降采样，取其最大值。 +- 标签切分子句位于 WHERE 子句之后，且不能和 JOIN 子句一起使用。 +- 标签切分子句将超级表数据按指定的标签组合进行切分，每个切分的分片进行指定的计算。计算由之后的子句定义（窗口子句、GROUP BY 子句或 SELECT 子句）。 +- 标签切分子句可以和窗口切分子句（或 GROUP BY 子句）一起使用，此时后面的子句作用在每个切分的分片上。例如，将数据按标签 location 进行分组，并对每个组按 10 分钟进行降采样，取其最大值。 ```sql select max(current) from meters partition by location interval(10m) diff --git a/docs/zh/12-taos-sql/13-tmq.md b/docs/zh/12-taos-sql/13-tmq.md new file mode 100644 index 0000000000..4d9c475a38 --- /dev/null +++ b/docs/zh/12-taos-sql/13-tmq.md @@ -0,0 +1,66 @@ +--- +sidebar_label: 消息队列 +title: 消息队列 +--- + +TDengine 3.0.0.0 开始对消息队列做了大幅的优化和增强以简化用户的解决方案。 + +## 创建订阅主题 + +```sql +CREATE TOPIC [IF NOT EXISTS] topic_name AS {subquery | DATABASE db_name | STABLE stb_name }; +``` + +订阅主题包括三种：列订阅、超级表订阅和数据库订阅。 + +**列订阅是**用 subquery 描述，支持过滤和标量函数和 UDF 标量函数，不支持 JOIN、GROUP BY、窗口切分子句、聚合函数和 UDF 聚合函数。列订阅规则如下： + +1. TOPIC 一旦创建则返回结果的字段确定 +2. 被订阅或用于计算的列不可被删除、修改 +3. 列可以新增，但新增的列不出现在订阅结果字段中 +4. 对于 select \*，则订阅展开为创建时所有的列（子表、普通表为数据列，超级表为数据列加标签列） + +**超级表订阅和数据库订阅**规则如下： + +1. 被订阅主体的 schema 变更不受限 +2. 返回消息中 schema 是块级别的，每块的 schema 可能不一样 +3. 列变更后写入的数据若未落盘，将以写入时的 schema 返回 +4. 列变更后写入的数据若未已落盘，将以落盘时的 schema 返回 + +## 删除订阅主题 + +```sql +DROP TOPIC [IF EXISTS] topic_name; +``` + +此时如果该订阅主题上存在 consumer，则此 consumer 会收到一个错误。 + +## 查看订阅主题 + +## SHOW TOPICS + +```sql +SHOW TOPICS; +``` + +显示当前数据库下的所有主题的信息。 + +## 创建消费组 + +消费组的创建只能通过 TDengine 客户端驱动或者连接器所提供的 API 创建。 + +## 删除消费组 + +```sql +DROP CONSUMER GROUP [IF EXISTS] cgroup_name ON topic_name; +``` + +删除主题 topic_name 上的消费组 cgroup_name。 + +## 查看消费组 + +```sql +SHOW CONSUMERS; +``` + +显示当前数据库下所有活跃的消费者的信息。 diff --git a/docs/zh/12-taos-sql/14-stream.md b/docs/zh/12-taos-sql/14-stream.md index e69de29bb2..7ff7da2bfb 100644 --- a/docs/zh/12-taos-sql/14-stream.md +++ b/docs/zh/12-taos-sql/14-stream.md @@ -0,0 +1,122 @@ +--- +sidebar_label: 流式计算 +title: 流式计算 +--- + +在时序数据的处理中，经常要对原始数据进行清洗、预处理，再使用时序数据库进行长久的储存。用户通常需要在时序数据库之外再搭建 Kafka、Flink、Spark 等流计算处理引擎，增加了用户的开发成本和维护成本。 + +使用 TDengine 3.0 的流式计算引擎能够最大限度的减少对这些额外中间件的依赖，真正将数据的写入、预处理、长期存储、复杂分析、实时计算、实时报警触发等功能融为一体，并且，所有这些任务只需要使用 SQL 完成，极大降低了用户的学习成本、使用成本。 + +## 创建流式计算 + +```sql +CREATE STREAM [IF NOT EXISTS] stream_name [stream_options] INTO stb_name AS subquery +stream_options: { + TRIGGER [AT_ONCE | WINDOW_CLOSE | MAX_DELAY time] + WATERMARK time +} + +``` + +其中 subquery 是 select 普通查询语法的子集: + +```sql +subquery: SELECT [DISTINCT] select_list + from_clause + [WHERE condition] + [PARTITION BY tag_list] + [window_clause] + [group_by_clause] +``` + +不支持 order_by，limit，slimit，fill 语句 + +例如，如下语句创建流式计算，同时自动创建名为 avg_vol 的超级表，此流计算以一分钟为时间窗口、30 秒为前向增量统计这些电表的平均电压，并将来自 meters 表的数据的计算结果写入 avg_vol 表，不同 partition 的数据会分别创建子表并写入不同子表。 + +```sql +CREATE STREAM avg_vol_s INTO avg_vol AS +SELECT _wstartts, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVAL(1m) SLIDING(30s); +``` + +## 删除流式计算 + +```sql +DROP STREAM [IF NOT EXISTS] stream_name +``` + +仅删除流式计算任务，由流式计算写入的数据不会被删除。 + +## 展示流式计算 + +```sql +SHOW STREAMS; +``` + +## 流式计算的触发模式 + +在创建流时，可以通过 TRIGGER 指令指定流式计算的触发模式。 + +对于非窗口计算，流式计算的触发是实时的；对于窗口计算，目前提供 3 种触发模式： + +1. AT_ONCE：写入立即触发 + +2. WINDOW_CLOSE：窗口关闭时触发（窗口关闭由事件时间决定，可配合 watermark 使用，详见《流式计算的乱序数据容忍策略》） + +3. MAX_DELAY time：若窗口关闭，则触发计算。若窗口未关闭，且未关闭时长超过 max delay 指定的时间，则触发计算。 + +由于窗口关闭是由事件时间决定的，如事件流中断、或持续延迟，则事件时间无法更新，可能导致无法得到最新的计算结果。 + +因此，流式计算提供了以事件时间结合处理时间计算的 MAX_DELAY 触发模式。 + +MAX_DELAY 模式在窗口关闭时会立即触发计算。此外，当数据写入后，计算触发的时间超过 max delay 指定的时间，则立即触发计算 + +## 流式计算的乱序数据容忍策略 + +在创建流时，可以在 stream_option 中指定 watermark。 + +流式计算通过 watermark 来度量对乱序数据的容忍程度，watermark 默认为 0。 + +T = 最新事件时间 - watermark + +每批到来的数据都会以上述公式更新窗口关闭时间，并将窗口结束时间 < T 的所有打开的窗口关闭，若触发模式为 WINDOW_CLOSE 或 MAX_DELAY，则推送窗口聚合结果。 + +流式计算的过期数据处理策略 +对于已关闭的窗口，再次落入该窗口中的数据被标记为过期数据，对于过期数据，流式计算提供两种处理方式： + +1. 直接丢弃：这是常见流式计算引擎提供的默认（甚至是唯一）计算模式 + +2. 重新计算：从 TSDB 中重新查找对应窗口的所有数据并重新计算得到最新结果 + +无论在哪种模式下，watermark 都应该被妥善设置，来得到正确结果（直接丢弃模式）或避免频繁触发重算带来的性能开销（重新计算模式）。 + +## 流式计算的数据填充策略 + +TODO + +## 流式计算与会话窗口（session window） + +```sql +window_clause: { + SESSION(ts_col, tol_val) + | STATE_WINDOW(col) + | INTERVAL(interval_val [, interval_offset]) [SLIDING (sliding_val)] [FILL(fill_mod_and_val)] +} +``` + +其中，SESSION 是会话窗口，tol_val 是时间间隔的最大范围。在 tol_val 时间间隔范围内的数据都属于同一个窗口，如果连续的两条数据的时间超过 tol_val，则自动开启下一个窗口。 + +## 流式计算的监控与流任务分布查询 + +TODO + +## 流式计算的内存控制与存算分离 + +TODO + +## 流式计算的暂停与恢复 + +```sql +STOP STREAM stream_name; + +RESUME STREAM stream_name; +``` diff --git a/docs/zh/12-taos-sql/19-limit.md b/docs/zh/12-taos-sql/19-limit.md index 95d1837ac6..ff552fc977 100644 --- a/docs/zh/12-taos-sql/19-limit.md +++ b/docs/zh/12-taos-sql/19-limit.md @@ -9,8 +9,8 @@ title: 命名与边界限制 2. 允许英文字符或下划线开头，不允许以数字开头 3. 不区分大小写 4. 转义后表（列）名规则： - 为了兼容支持更多形式的表（列）名，TDengine 引入新的转义符 "`"。可用让表名与关键词不冲突，同时不受限于上述表名称合法性约束检查。 - 转义后的表（列）名同样受到长度限制要求，且长度计算的时候不计算转义符。使用转义字符以后，不再对转义字符中的内容进行大小写统一。 + 为了兼容支持更多形式的表（列）名，TDengine 引入新的转义符 "`"。可用让表名与关键词不冲突，同时不受限于上述表名称合法性约束检查 + 转义后的表（列）名同样受到长度限制要求，且长度计算的时候不计算转义符。使用转义字符以后，不再对转义字符中的内容进行大小写统一 例如：\`aBc\` 和 \`abc\` 是不同的表（列）名，但是 abc 和 aBc 是相同的表（列）名。 需要注意的是转义字符中的内容必须是可打印字符。 @@ -23,28 +23,30 @@ title: 命名与边界限制 ## 一般限制 -- 数据库名最大长度为 32。 +- 数据库名最大长度为 32 - 表名最大长度为 192，不包括数据库名前缀和分隔符 -- 每行数据最大长度 48KB （注意：数据行内每个 BINARY/NCHAR 类型的列还会额外占用 2 个字节的存储位置）。 +- 每行数据最大长度 48KB （注意：数据行内每个 BINARY/NCHAR 类型的列还会额外占用 2 个字节的存储位置） - 列名最大长度为 64 - 最多允许 4096 列，最少需要 2 列，第一列必须是时间戳。 - 标签名最大长度为 64 -- 最多允许 128 个，至少要有 1 个标签，一个表中标签值的总长度不超过 16KB 。 -- SQL 语句最大长度 1048576 个字符，也可通过客户端配置参数 maxSQLLength 修改，取值范围 65480 ~ 1048576。 -- SELECT 语句的查询结果，最多允许返回 4096 列（语句中的函数调用可能也会占用一些列空间），超限时需要显式指定较少的返回数据列，以避免语句执行报错。 -- 库的数目，超级表的数目、表的数目，系统不做限制，仅受系统资源限制。 +- 最多允许 128 个，至少要有 1 个标签，一个表中标签值的总长度不超过 16KB +- SQL 语句最大长度 1048576 个字符，也可通过客户端配置参数 maxSQLLength 修改，取值范围 65480 ~ 1048576 +- SELECT 语句的查询结果，最多允许返回 4096 列（语句中的函数调用可能也会占用一些列空间），超限时需要显式指定较少的返回数据列，以避免语句执行报错 +- 库的数目，超级表的数目、表的数目，系统不做限制，仅受系统资源限制 - 数据库的副本数只能设置为 1 或 3 - 用户名的最大长度是 23 个字节 - 用户密码的最大长度是 15 个字节 -- 总数据行数取决于可用资源 +- 总数据行数取决于可用资源 - 单个数据库的虚拟结点数上限为 1024 ## 表(列)名合法性说明 ### TDengine 中的表（列）名命名规则如下： + 只能由字母、数字、下划线构成，数字不能在首位，长度不能超过 192 字节，不区分大小写。这里表名称不包括数据库名的前缀和分隔符。 ### 转义后表（列）名规则： + 为了兼容支持更多形式的表（列）名，TDengine 引入新的转义符 "`"，可以避免表名与关键词的冲突，同时不受限于上述表名合法性约束检查，转义符不计入表名的长度。 转义后的表（列）名同样受到长度限制要求，且长度计算的时候不计算转义符。使用转义字符以后，不再对转义字符中的内容进行大小写统一。 diff --git a/docs/zh/12-taos-sql/20-keywords.md b/docs/zh/12-taos-sql/20-keywords.md index d91c9be2ac..cac29d7863 100644 --- a/docs/zh/12-taos-sql/20-keywords.md +++ b/docs/zh/12-taos-sql/20-keywords.md @@ -1,5 +1,5 @@ --- -sidebar_label: 保留关键字 +sidebar_label: 保留关键字 title: TDengine 保留关键字 --- @@ -58,70 +58,70 @@ title: TDengine 保留关键字 ### D -- DATABASE -- DATABASES -- DAYS -- DBS -- DEFERRED +- DATABASE +- DATABASES +- DAYS +- DBS +- DEFERRED - DELETE - DELIMITERS -- DESC -- DESCRIBE -- DETACH -- DISTINCT -- DIVIDE -- DNODE -- DNODES -- DOT -- DOUBLE -- DROP +- DESC +- DESCRIBE +- DETACH +- DISTINCT +- DIVIDE +- DNODE +- DNODES +- DOT +- DOUBLE +- DROP ### E -- END -- EQ -- EXISTS -- EXPLAIN +- END +- EQ +- EXISTS +- EXPLAIN ### F -- FAIL -- FILE -- FILL -- FLOAT -- FOR -- FROM -- FSYNC +- FAIL +- FILE +- FILL +- FLOAT +- FOR +- FROM +- FSYNC ### G -- GE -- GLOB +- GE +- GLOB - GRANTS -- GROUP -- GT +- GROUP +- GT ### H -- HAVING +- HAVING ### I - ID - IF -- IGNORE +- IGNORE - IMMEDIA -- IMPORT -- IN +- IMPORT +- IN - INITIAL -- INSERT +- INSERT - INSTEAD -- INT +- INT - INTEGER - INTERVA -- INTO -- IS -- ISNULL +- INTO +- IS +- ISNULL ### J @@ -130,190 +130,147 @@ title: TDengine 保留关键字 ### K - KEEP -- KEY +- KEY - KILL ### L -- LE -- LIKE -- LIMIT +- LE +- LIKE +- LIMIT - LINEAR -- LOCAL -- LP +- LOCAL +- LP - LSHIFT -- LT +- LT ### M -- MATCH -- MAXROWS -- MINROWS -- MINUS -- MNODES -- MODIFY -- MODULES +- MATCH +- MAXROWS +- MINROWS +- MINUS +- MNODES +- MODIFY +- MODULES ### N -- NE -- NONE -- NOT +- NE +- NONE +- NOT - NOTNULL -- NOW +- NOW - NULL ### O -- OF +- OF - OFFSET -- OR -- ORDER +- OR +- ORDER ### P - PARTITION -- PASS -- PLUS -- PPS +- PASS +- PLUS +- PPS - PRECISION -- PREV +- PREV - PRIVILEGE ### Q -- QTIME +- QTIME - QUERIE -- QUERY +- QUERY - QUORUM ### R -- RAISE -- REM +- RAISE +- REM - REPLACE - REPLICA -- RESET +- RESET - RESTRIC -- ROW -- RP +- ROW +- RP - RSHIFT ### S -- SCORES -- SELECT -- SEMI +- SCORES +- SELECT +- SEMI - SESSION -- SET -- SHOW -- SLASH +- SET +- SHOW +- SLASH - SLIDING -- SLIMIT +- SLIMIT - SMALLIN - SOFFSET -- STable +- STable - STableS -- STAR -- STATE +- STAR +- STATE - STATEMEN - STATE_WI -- STORAGE -- STREAM -- STREAMS -- STRING -- SYNCDB +- STORAGE +- STREAM +- STREAMS +- STRING +- SYNCDB ### T -- TABLE -- TABLES -- TAG -- TAGS -- TBNAME -- TIMES -- TIMESTAMP -- TINYINT -- TOPIC -- TOPICS -- TRIGGER -- TSERIES +- TABLE +- TABLES +- TAG +- TAGS +- TBNAME +- TIMES +- TIMESTAMP +- TINYINT +- TOPIC +- TOPICS +- TRIGGER +- TSERIES ### U -- UMINUS -- UNION -- UNSIGNED -- UPDATE -- UPLUS -- USE -- USER -- USERS -- USING +- UMINUS +- UNION +- UNSIGNED +- UPDATE +- UPLUS +- USE +- USER +- USERS +- USING ### V -- VALUES -- VARIABLE +- VALUES +- VARIABLE - VARIABLES -- VGROUPS -- VIEW -- VNODES +- VGROUPS +- VIEW +- VNODES ### W - WAL - WHERE -### _ +### \_ -- _C0 -- _QSTART -- _QSTOP -- _QDURATION -- _WSTART -- _WSTOP -- _WDURATION - - -## 特殊说明 -### TBNAME -`TBNAME` 可以视为超级表中一个特殊的标签，代表子表的表名。 - -获取一个超级表所有的子表名及相关的标签信息： - -```mysql -SELECT TBNAME, location FROM meters; -``` - -统计超级表下辖子表数量： - -```mysql -SELECT COUNT(TBNAME) FROM meters; -``` - -以上两个查询均只支持在WHERE条件子句中添加针对标签（TAGS）的过滤条件。例如： -```mysql -taos> SELECT TBNAME, location FROM meters; - tbname | location | -================================================================== - d1004 | California.SanFrancisco | - d1003 | California.SanFrancisco | - d1002 | California.LosAngeles | - d1001 | California.LosAngeles | -Query OK, 4 row(s) in set (0.000881s) - -taos> SELECT COUNT(tbname) FROM meters WHERE groupId > 2; - count(tbname) | -======================== - 2 | -Query OK, 1 row(s) in set (0.001091s) -``` -### _QSTART/_QSTOP/_QDURATION -表示查询过滤窗口的起始，结束以及持续时间。 - -### _WSTART/_WSTOP/_WDURATION -窗口切分聚合查询（例如 interval/session window/state window）中表示每个切分窗口的起始，结束以及持续时间。 - -### _c0/_ROWTS -_c0 _ROWTS 等价，表示表或超级表的第一列 +- \_C0 +- \_QSTART +- \_QSTOP +- \_QDURATION +- \_WSTART +- \_WSTOP +- \_WDURATION diff --git a/docs/zh/12-taos-sql/21-node.md b/docs/zh/12-taos-sql/21-node.md new file mode 100644 index 0000000000..47aa2077a3 --- /dev/null +++ b/docs/zh/12-taos-sql/21-node.md @@ -0,0 +1,155 @@ +--- +sidebar_label: 集群管理 +title: 集群管理 +--- + +组成 TDengine 集群的物理实体是 dnode (data node 的缩写)，它是一个运行在操作系统之上的进程。在 dnode 中可以建立负责时序数据存储的 vnode (virtual node)，在多节点集群环境下当某个数据库的 replica 为 3 时，该数据库中的每个 vgroup 由 3 个 vnode 组成；当数据库的 replica 为 1 时，该数据库中的每个 vgroup 由 1 个 vnode 组成。如果要想配置某个数据库为多副本，则集群中的 dnode 数量至少为 3。在 dnode 还可以创建 mnode (management node)，单个集群中最多可以创建三个 mnode。在 TDengine 3.0.0.0 中为了支持存算分离，引入了一种新的逻辑节点 qnode (query node)，qnode 和 vnode 既可以共存在一个 dnode 中，也可以完全分离在不同的 dnode 上。 + +## 创建数据节点 + +```sql +CREATE DNODE {dnode_endpoint | dnode_host_name PORT port_val} +``` + +其中 `dnode_endpoint` 是形成 `hostname:port`的格式。也可以分开指定 hostname 和 port。 + +实际操作中推荐先创建 dnode，再启动相应的 dnode 进程，这样该 dnode 就可以立即根据其配置文件中的 firstEP 加入集群。每个 dnode 在加入成功后都会被分配一个 ID。 + +## 查看数据节点 + +```sql +SHOW DNODES; +``` + +可以列出集群中所有的数据节点，所列出的字段有 dnode 的 ID, endpoint, status。 + +## 删除数据节点 + +```sql +DROP DNODE {dnode_id | dnode_endpoint} +``` + +可以用 dnoe_id 或 endpoint 两种方式从集群中删除一个 dnode。注意删除 dnode 不等于停止相应的进程。实际中推荐先将一个 dnode 删除之后再停止其所对应的进程。 + +## 修改数据节点配置 + +```sql +ALTER DNODE dnode_id dnode_option + +ALTER ALL DNODES dnode_option + +dnode_option: { + 'resetLog' + | 'resetQueryCache' + | 'balance' value + | 'monitor' value + | 'debugFlag' value + | 'monDebugFlag' value + | 'vDebugFlag' value + | 'mDebugFlag' value + | 'cDebugFlag' value + | 'httpDebugFlag' value + | 'qDebugflag' value + | 'sdbDebugFlag' value + | 'uDebugFlag' value + | 'tsdbDebugFlag' value + | 'sDebugflag' value + | 'rpcDebugFlag' value + | 'dDebugFlag' value + | 'mqttDebugFlag' value + | 'wDebugFlag' value + | 'tmrDebugFlag' value + | 'cqDebugFlag' value +} +``` + +上面语法中的这些可修改配置项其配置方式与 dnode 配置文件中的配置方式相同，区别是修改是动态的立即生效，且不需要重启 dnode。 + +## 添加管理节点 + +```sql +CREATE MNODE ON DNODE dnode_id +``` + +系统启动默认在 firstEP 节点上创建一个 MNODE，用户可以使用此语句创建更多的 MNODE 来提高系统可用性。一个集群最多存在三个 MNODE，一个 DNODE 上只能创建一个 MNODE。 + +## 查看管理节点 + +```sql +SHOW MNODES; +``` + +列出集群中所有的管理节点，包括其 ID，所在 DNODE 以及状态。 + +## 删除管理节点 + +```sql +DROP MNODE ON DNODE dnode_id; +``` + +删除 dnode_id 所指定的 DNODE 上的 MNODE。 + +## 创建查询节点 + +```sql +CREATE QNODE ON DNODE dnode_id; +``` + +系统启动默认没有 QNODE，用户可以创建 QNODE 来实现计算和存储的分离。一个 DNODE 上只能创建一个 QNODE。一个 DNODE 的 `supportVnodes` 参数如果不为 0，同时又在其上创建上 QNODE，则在该 dnode 中既有负责存储管理的 vnode 又有负责查询计算的 qnode，如果还在该 dnode 上创建了 mnode，则一个 dnode 上最多三种逻辑节点都可以存在。但通过配置也可以使其彻底分离。将一个 dnode 的`supportVnodes`配置为 0，可以选择在其上创建 mnode 或者 qnode 中的一种，这样可以实现三种逻辑节点在物理上的彻底分离。 + +## 查看查询节点 + +```sql +SHOW QNODES; +``` + +列出集群中所有查询节点，包括 ID，及所在 DNODE。 + +## 删除查询节点 + +```sql +DROP QNODE ON DNODE dnode_id; +``` + +删除 ID 为 dnode_id 的 DNODE 上的 QNODE，但并不会影响该 dnode 的状态。 + +## 修改客户端配置 + +如果将客户端也看作广义的集群的一部分，可以通过如下命令动态修改客户端配置参数。 + +```sql +ALTER LOCAL local_option + +local_option: { + 'resetLog' + | 'rpcDebugFlag' value + | 'tmrDebugFlag' value + | 'cDebugFlag' value + | 'uDebugFlag' value + | 'debugFlag' value +} +``` + +上面语法中的参数与在配置文件中配置客户端的用法相同，但不需要重启客户端，修改后立即生效。 + +## 查看客户端配置 + +```sql +SHOW LOCAL VARIABLES; +``` + +## 合并 vgroup + +```sql +MERGE VGROUP vgroup_no1 vgroup_no2; +``` + +如果在系统实际运行一段时间后，因为不同时间线的数据特征不同导致在 vgroups 之间的数据和负载分布不均衡，可以通过合并或拆分 vgroups 的方式逐步实现负载均衡。 + +## 拆分 vgroup + +```sql +SPLIT VGROUP vgroup_no; +``` + +会创建一个新的 vgroup，并将指定 vgroup 中的数据按照一致性 HASH 迁移一部分到新的 vgroup 中。此过程中，原 vgroup 可以正常提供读写服务。 diff --git a/docs/zh/12-taos-sql/22-information.md b/docs/zh/12-taos-sql/22-information.md deleted file mode 100644 index 0695aa5172..0000000000 --- a/docs/zh/12-taos-sql/22-information.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -sidebar_label: Information内置数据库 -title: Information内置数据库 ---- - diff --git a/docs/zh/12-taos-sql/22-meta.md b/docs/zh/12-taos-sql/22-meta.md new file mode 100644 index 0000000000..8c262413f5 --- /dev/null +++ b/docs/zh/12-taos-sql/22-meta.md @@ -0,0 +1,179 @@ +--- +sidebar_label: 元数据库 +title: 元数据库 +--- + +TDengine 内置了一个名为 `INFORMATION_SCHEMA` 的数据库，提供对数据库元数据、数据库系统信息和状态的访问，例如数据库或表的名称，当前执行的 SQL 语句等。该数据库存储有关 TDengine 维护的所有其他数据库的信息。它包含多个只读表。实际上，这些表都是视图，而不是基表，因此没有与它们关联的文件。所以对这些表只能查询，不能进行 INSERT 等写入操作。`INFORMATION_SCHEMA` 数据库旨在以一种更一致的方式来提供对 TDengine 支持的各种 SHOW 语句（如 SHOW TABLES、SHOW DATABASES）所提供的信息的访问。与 SHOW 语句相比，使用 SELECT ... FROM INFORMATION_SCHEMA.tablename 具有以下优点： + +1. 可以使用 USE 语句将 INFORMATION_SCHEMA 设为默认数据库 +2. 可以使用 SELECT 语句熟悉的语法，只需要学习一些表名和列名 +3. 可以对查询结果进行筛选、排序等操作。事实上，可以使用任意 TDengine 支持的 SELECT 语句对 INFORMATION_SCHEMA 中的表进行查询 +4. TDengine 在后续演进中可以灵活的添加已有 INFORMATION_SCHEMA 中表的列，而不用担心对既有业务系统造成影响 +5. 与其他数据库系统更具互操作性。例如，Oracle 数据库用户熟悉查询 Oracle 数据字典中的表 + +Note: 由于 SHOW 语句已经被开发者熟悉和广泛使用，所以它们仍然被保留。 + +本章将详细介绍 `INFORMATION_SCHEMA` 这个内置元数据库中的表和表结构。 + +## DNODES + +提供 dnode 的相关信息。也可以使用 SHOW DNODES 来查询这些信息。 + +| # | **列名** | **数据类型** | **说明** | +| --- | :------------: | ------------ | --------------------- | +| 1 | vnodes | SMALLINT | dnode 中的 vnode 个数 | +| 2 | support_vnodes | SMALLINT | 支持的 vnode 个数 | +| 3 | status | BINARY(10) | 当前状态 | +| 4 | note | BINARY(256) | 离线原因等信息 | +| 5 | id | SMALLINT | dnode id | +| 6 | endpoint | BINARY(134) | dnode 的地址 | +| 7 | create | TIMESTAMP | 创建时间 | + +## MNODES + +提供 mnode 的相关信息。也可以使用 SHOW MNODES 来查询这些信息。 + +| # | **列名** | **数据类型** | **说明** | +| --- | :---------: | ------------ | ------------------ | +| 1 | id | SMALLINT | mnode id | +| 2 | endpoint | BINARY(134) | mnode 的地址 | +| 3 | role | BINARY(10) | 当前角色 | +| 4 | role_time | TIMESTAMP | 成为当前角色的时间 | +| 5 | create_time | TIMESTAMP | 创建时间 | + +## MODULES + +提供组件的相关信息。也可以使用 SHOW MODULES 来查询这些信息 + +| # | **列名** | **数据类型** | **说明** | +| --- | :------: | ------------ | ---------- | +| 1 | id | SMALLINT | module id | +| 2 | endpoint | BINARY(134) | 组件的地址 | +| 3 | module | BINARY(10) | 组件状态 | + +## QNODES + +当前系统中 QNODE 的信息。也可以使用 SHOW QNODES 来查询这些信息。 + +| # | **列名** | **数据类型** | **说明** | +| --- | :---------: | ------------ | ------------ | +| 1 | id | SMALLINT | module id | +| 2 | endpoint | BINARY(134) | qnode 的地址 | +| 3 | create_time | TIMESTAMP | 创建时间 | + +## USER_DATABASES + +提供用户创建的数据库对象的相关信息。也可以使用 SHOW DATABASES 来查询这些信息。 + +TODO + +| # | **列名** | **数据类型** | **说明** | +| --- | :---------: | ------------ | ------------------------------------------------ | +| 1 | name | BINARY(32) | 数据库名 | +| 2 | create_time | TIMESTAMP | 创建时间 | +| 3 | ntables | INT | 数据库中表的数量，包含子表和普通表但不包含超级表 | +| 4 | vgroups | INT | 数据库中有多少个 vgroup | +| 5 | replica | INT | 副本数 | +| 6 | quorum | INT | 写成功的确认数 | +| 7 | days | INT | 单文件存储数据的时间跨度 | +| 8 | keep | INT | 数据保留时长 | +| 9 | buffer | INT | 每个 vnode 写缓存的内存块大小，单位 MB | +| 10 | minrows | INT | 文件块中记录的最大条数 | +| 11 | maxrows | INT | 文件块中记录的最小条数 | +| 12 | wallevel | INT | WAL 级别 | +| 13 | fsync | INT | 数据落盘周期 | +| 14 | comp | INT | 数据压缩方式 | +| 15 | precision | BINARY(2) | 时间分辨率 | +| 16 | status | BINARY(10) | 数据库状态 | + +## USER_FUNCTIONS + +TODO + +## USER_INDEXES + +提供用户创建的索引的相关信息。也可以使用 SHOW INDEX 来查询这些信息。 + +| # | **列名** | **数据类型** | **说明** | +| --- | :--------------: | ------------ | ---------------------------------------------------------------------------------- | +| 1 | db_name | BINARY(32) | 包含此索引的表所在的数据库名 | +| 2 | table_name | BINARY(192) | 包含此索引的表的名称 | +| 3 | index_name | BINARY(192) | 索引名 | +| 4 | column_name | BINARY(64) | 建索引的列的列名 | +| 5 | index_type | BINARY(10) | 目前有 SMA 和 FULLTEXT | +| 6 | index_extensions | BINARY(256) | 索引的额外信息。对 SMA 类型的索引，是函数名的列表。对 FULLTEXT 类型的索引为 NULL。 | + +## USER_STABLES + +提供用户创建的超级表的相关信息。 + +| # | **列名** | **数据类型** | **说明** | +| --- | :-----------: | ------------ | ------------------------ | +| 1 | stable_name | BINARY(192) | 超级表表名 | +| 2 | db_name | BINARY(64) | 超级表所在的数据库的名称 | +| 3 | create_time | TIMESTAMP | 创建时间 | +| 4 | columns | INT | 列数目 | +| 5 | tags | INT | 标签数目 | +| 6 | last_update | TIMESTAMP | 最后更新时间 | +| 7 | table_comment | BINARY(1024) | 表注释 | +| 8 | watermark | BINARY(64) | 窗口的关闭时间 | +| 9 | max_delay | BINARY(64) | 推送计算结果的最大延迟 | +| 10 | rollup | BINARY(128) | rollup 聚合函数 | + +## USER_STREAMS + +提供用户创建的流计算的相关信息。 + +| # | **列名** | **数据类型** | **说明** | +| --- | :---------: | ------------ | --------------------------- | +| 1 | stream_name | BINARY(192) | 流计算名称 | +| 2 | user_name | BINARY(23) | 创建流计算的用户 | +| 3 | dest_table | BINARY(192) | 流计算写入的目标表 | +| 4 | create_time | TIMESTAMP | 创建时间 | +| 5 | sql | BLOB | 创建流计算时提供的 SQL 语句 | + +## USER_TABLES + +提供用户创建的普通表和子表的相关信息 + +| # | **列名** | **数据类型** | **说明** | +| --- | :-----------: | ------------ | ---------------- | +| 1 | table_name | BINARY(192) | 表名 | +| 2 | db_name | BINARY(64) | 数据库名 | +| 3 | create_time | TIMESTAMP | 创建时间 | +| 4 | columns | INT | 列数目 | +| 5 | stable_name | BINARY(192) | 所属的超级表表名 | +| 6 | uid | BIGINT | 表 id | +| 7 | vgroup_id | INT | vgroup id | +| 8 | ttl | INT | 表的生命周期 | +| 9 | table_comment | BINARY(1024) | 表注释 | +| 10 | type | BINARY(20) | 表类型 | + +## USER_USERS + +提供系统中创建的用户的相关信息。 + +| # | **列名** | **数据类型** | **说明** | +| --- | :---------: | ------------ | -------- | +| 1 | user_name | BINARY(23) | 用户名 | +| 2 | privilege | BINARY(256) | 权限 | +| 3 | create_time | TIMESTAMP | 创建时间 | + +## VGROUPS + +系统中所有 vgroups 的信息。 + +| # | **列名** | **数据类型** | **说明** | +| --- | :--------: | ------------ | ---------------------------- | +| 1 | vg_id | INT | vgroup id | +| 2 | db_name | BINARY(32) | 数据库名 | +| 3 | tables | INT | 此 vgroup 内有多少表 | +| 4 | status | BINARY(10) | 此 vgroup 的状态 | +| 5 | onlines | INT | 在线的成员数目 | +| 6 | v1_dnode | INT | 第一个成员所在的 dnode 的 id | +| 7 | v1_status | BINARY(10) | 第一个成员的状态 | +| 8 | v2_dnode | INT | 第二个成员所在的 dnode 的 id | +| 9 | v2_status | BINARY(10) | 第二个成员的状态 | +| 10 | v3_dnode | INT | 第三个成员所在的 dnode 的 id | +| 11 | v3_status | BINARY(10) | 第三个成员的状态 | +| 12 | compacting | INT | compact 状态 | diff --git a/docs/zh/12-taos-sql/23-show.md b/docs/zh/12-taos-sql/23-show.md new file mode 100644 index 0000000000..781f94324c --- /dev/null +++ b/docs/zh/12-taos-sql/23-show.md @@ -0,0 +1,270 @@ +--- +sidebar_label: SHOW 命令 +title: 使用 SHOW 命令查看系统元数据 +--- + +除了使用 `select` 语句查询 `INFORMATION_SCHEMA` 数据库中的表获得系统中的各种元数据、系统信息和状态之外，也可以用 `SHOW` 命令来实现同样的目的。 + +## SHOW ACCOUNTS + +```sql +SHOW ACCOUNTS; +``` + +显示当前系统中所有租户的信息。 + +注：企业版独有 + +## SHOW APPS + +```sql +SHOW APPS; +``` + +显示接入集群的应用（客户端）信息。 + +## SHOW BNODES + +```sql +SHOW BNODES; +``` + +显示当前系统中存在的 BNODE (backup node, 即备份节点）的信息。 + +## SHOW CLUSTER + +```sql +SHOW CLUSTER; +``` + +显示当前集群的信息 + +## SHOW CONNECTIONS + +```sql +SHOW CONNECTIONS; +``` + +显示当前系统中存在的连接的信息。 + +## SHOW CONSUMERS + +```sql +SHOW CONSUMERS; +``` + +显示当前数据库下所有活跃的消费者的信息。 + +## SHOW CREATE DATABASE + +```sql +SHOW CREATE DATABASE db_name; +``` + +显示 db_name 指定的数据库的创建语句。 + +## SHOW CREATE STABLE + +```sql +SHOW CREATE STABLE [db_name.]stb_name; +``` + +显示 tb_name 指定的超级表的创建语句 + +## SHOW CREATE TABLE + +```sql +SHOW CREATE TABLE [db_name.]tb_name +``` + +显示 tb_name 指定的表的创建语句。支持普通表、超级表和子表。 + +## SHOW DATABASES + +```sql +SHOW DATABASES; +``` + +显示用户定义的所有数据库。 + +## SHOW DNODES + +```sql +SHOW DNODES; +``` + +显示当前系统中 DNODE 的信息。 + +## SHOW FUNCTIONS + +```sql +SHOW FUNCTIONS; +``` + +显示用户定义的自定义函数。 + +## SHOW LICENSE + +```sql +SHOW LICENSE; +SHOW GRANTS; +``` + +显示企业版许可授权的信息。 + +注：企业版独有 + +## SHOW INDEXES + +```sql +SHOW INDEXES FROM tbl_name [FROM db_name]; +``` + +显示已创建的索引。 + +## SHOW LOCAL VARIABLES + +```sql +SHOW LOCAL VARIABLES; +``` + +显示当前客户端配置参数的运行值。 + +## SHOW MNODES + +```sql +SHOW MNODES; +``` + +显示当前系统中 MNODE 的信息。 + +## SHOW MODULES + +```sql +SHOW MODULES; +``` + +显示当前系统中所安装的组件的信息。 + +## SHOW QNODES + +```sql +SHOW QNODES; +``` + +显示当前系统中 QNODE （查询节点）的信息。 + +## SHOW SCORES + +```sql +SHOW SCORES; +``` + +显示系统被许可授权的容量的信息。 + +注：企业版独有 + +## SHOW SNODES + +```sql +SHOW SNODES; +``` + +显示当前系统中 SNODE （流计算节点）的信息。 + +## SHOW STABLES + +```sql +SHOW [db_name.]STABLES [LIKE 'pattern']; +``` + +显示当前数据库下的所有超级表的信息。可以使用 LIKE 对表名进行模糊匹配。 + +## SHOW STREAMS + +```sql +SHOW STREAMS; +``` + +显示当前系统内所有流计算的信息。 + +## SHOW SUBSCRIPTIONS + +```sql +SHOW SUBSCRIPTIONS; +``` + +显示当前数据库下的所有的订阅关系 + +## SHOW TABLES + +```sql +SHOW [db_name.]TABLES [LIKE 'pattern']; +``` + +显示当前数据库下的所有普通表和子表的信息。可以使用 LIKE 对表名进行模糊匹配。 + +## SHOW TABLE DISTRIBUTED + +```sql +SHOW TABLE DISTRIBUTED table_name; +``` + +显示表的数据分布信息。 + +## SHOW TAGS + +```sql +SHOW TAGS FROM child_table_name [FROM db_name]; +``` + +显示子表的标签信息。 + +## SHOW TOPICS + +```sql +SHOW TOPICS; +``` + +显示当前数据库下的所有主题的信息。 + +## SHOW TRANSACTIONS + +```sql +SHOW TRANSACTIONS; +``` + +显示当前系统中正在执行的事务的信息 + +## SHOW USERS + +```sql +SHOW USERS; +``` + +显示当前系统中所有用户的信息。包括用户自定义的用户和系统默认用户。 + +## SHOW VARIABLES + +```sql +SHOW VARIABLES; +SHOW DNODE dnode_id VARIABLES; +``` + +显示当前系统中各节点需要相同的配置参数的运行值，也可以指定 DNODE 来查看其的配置参数。 + +## SHOW VGROUPS + +```sql +SHOW [db_name.]VGROUPS; +``` + +显示当前系统中所有 VGROUP 或某个 db 的 VGROUPS 的信息。 + +## SHOW VNODES + +```sql +SHOW VNODES [dnode_name]; +``` + +显示当前系统中所有 VNODE 或某个 DNODE 的 VNODE 的信息。 diff --git a/docs/zh/12-taos-sql/25-grant.md b/docs/zh/12-taos-sql/25-grant.md new file mode 100644 index 0000000000..0c290350cc --- /dev/null +++ b/docs/zh/12-taos-sql/25-grant.md @@ -0,0 +1,94 @@ +--- +sidebar_label: 权限管理 +title: 权限管理 +--- + +本节讲述如何在 TDengine 中进行权限管理的相关操作。 + +## 创建用户 + +```sql +CREATE USER use_name PASS password; +``` + +创建用户。 + +use_name最长为23字节。 + +password最长为128字节，合法字符包括"a-zA-Z0-9!?$%^&*()_–+={[}]:;@~#|<,>.?/"，不可以出现单双引号、撇号、反斜杠和空格，且不可以为空。 + +## 删除用户 + +```sql +DROP USER user_name; +``` + +## 修改用户信息 + +```sql +ALTER USER user_name alter_user_clause + +alter_user_clause: { + PASS 'literal' + | ENABLE value + | SYSINFO value +} +``` + +- PASS：修改用户密码。 +- ENABLE：修改用户是否启用。1表示启用此用户，0表示禁用此用户。 +- SYSINFO：修改用户是否可查看系统信息。1表示可以查看系统信息，0表示不可以查看系统信息。 + + +## 授权 + +```sql +GRANT privileges ON priv_level TO user_name + +privileges : { + ALL + | priv_type [, priv_type] ... +} + +priv_type : { + READ + | WRITE +} + +priv_level : { + dbname.* + | *.* +} +``` + +对用户授权。 + +授权级别支持到DATABASE，权限有READ和WRITE两种。 + +TDengine 有超级用户和普通用户两类用户。超级用户缺省创建为root，拥有所有权限。使用超级用户创建出来的用户为普通用户。在未授权的情况下，普通用户可以创建DATABASE，并拥有自己创建的DATABASE的所有权限，包括删除数据库、修改数据库、查询时序数据和写入时序数据。超级用户可以给普通用户授予其他DATABASE的读写权限，使其可以在此DATABASE上读写数据，但不能对其进行删除和修改数据库的操作。 + +对于非DATABASE的对象，如USER、DNODE、UDF、QNODE等，普通用户只有读权限（一般为SHOW命令），不能创建和修改。 + +## 撤销授权 + +```sql +REVOKE privileges ON priv_level FROM user_name + +privileges : { + ALL + | priv_type [, priv_type] ... +} + +priv_type : { + READ + | WRITE +} + +priv_level : { + dbname.* + | *.* +} + +``` + +收回对用户的授权。 \ No newline at end of file diff --git a/docs/zh/12-taos-sql/26-udf.md b/docs/zh/12-taos-sql/26-udf.md new file mode 100644 index 0000000000..7b5acbfcad --- /dev/null +++ b/docs/zh/12-taos-sql/26-udf.md @@ -0,0 +1,28 @@ +--- +sidebar_label: 自定义函数 +title: 用户自定义函数 +--- + +除了 TDengine 的内置函数以外，用户还可以编写自己的函数逻辑并加入TDengine系统中。 + +## 创建函数 + +```sql +CREATE [AGGREGATE] FUNCTION func_name AS library_path OUTPUTTYPE type_name [BUFSIZE value] +``` + +语法说明： + +AGGREGATE：标识此函数是标量函数还是聚集函数。 +func_name：函数名，必须与函数实现中udfNormalFunc的实际名称一致。 +library_path：包含UDF函数实现的动态链接库的绝对路径，是在客户端侧主机上的绝对路径。 +OUTPUTTYPE：标识此函数的返回类型。 +BUFSIZE：中间结果的缓冲区大小，单位是字节。不设置则默认为0。最大不可超过512字节。 + +关于如何开发自定义函数，请参考 [UDF使用说明](../develop/udf)。 + +## 删除自定义函数 + +```sql +DROP FUNCTION func_name +``` \ No newline at end of file diff --git a/docs/zh/12-taos-sql/27-index.md b/docs/zh/12-taos-sql/27-index.md new file mode 100644 index 0000000000..2c0907723e --- /dev/null +++ b/docs/zh/12-taos-sql/27-index.md @@ -0,0 +1,47 @@ +--- +sidebar_label: 索引 +title: 使用索引 +--- + +TDengine 从 3.0.0.0 版本开始引入了索引功能，支持 SMA 索引和 FULLTEXT 索引。 + +## 创建索引 + +```sql +CREATE FULLTEXT INDEX index_name ON tb_name (col_name [, col_name] ...) + +CREATE SMA INDEX index_name ON tb_name index_option + +index_option: + FUNCTION(functions) INTERVAL(interval_val [, interval_offset]) [SLIDING(sliding_val)] [WATERMARK(watermark_val)] [MAX_DELAY(max_delay_val)] + +functions: + function [, function] ... +``` + +### SMA 索引 + +对指定列按 INTERVAL 子句定义的时间窗口创建进行预聚合计算，预聚合计算类型由 functions_string 指定。SMA 索引能提升指定时间段的聚合查询的性能。目前，限制一个超级表只能创建一个 SMA INDEX。 + +- 支持的函数包括 MAX、MIN 和 SUM。 +- WATERMARK: 最小单位毫秒，取值范围 [0ms, 900000ms]，默认值为 5 秒，只可用于超级表。 +- MAX_DELAY: 最小单位毫秒，取值范围 [1ms, 900000ms]，默认值为 interval 的值(但不能超过最大值)，只可用于超级表。注：不建议 MAX_DELAY 设置太小，否则会过于频繁的推送结果，影响存储和查询性能，如无特殊需求，取默认值即可。 + +### FULLTEXT 索引 + +对指定列建立文本索引，可以提升含有文本过滤的查询的性能。FULLTEXT 索引不支持 index_option 语法。现阶段只支持对 JSON 类型的标签列创建 FULLTEXT 索引。不支持多列联合索引，但可以为每个列分布创建 FULLTEXT 索引。 + +## 删除索引 + +```sql +DROP INDEX index_name; +``` + +## 查看索引 + +````sql +```sql +SHOW INDEXES FROM tbl_name [FROM db_name]; +```` + +显示在所指定的数据库或表上已创建的索引。 diff --git a/docs/zh/12-taos-sql/28-recovery.md b/docs/zh/12-taos-sql/28-recovery.md new file mode 100644 index 0000000000..72b220b8ff --- /dev/null +++ b/docs/zh/12-taos-sql/28-recovery.md @@ -0,0 +1,38 @@ +--- +sidebar_label: 异常恢复 +title: 异常恢复 +--- + +在一个复杂的应用场景中，连接和查询任务等有可能进入一种错误状态或者耗时过长迟迟无法结束，此时需要有能够终止这些连接或任务的方法。 + +## 终止连接 + +```sql +KILL CONNECTION conn_id; +``` + +conn_id 可以通过 `SHOW CONNECTIONS` 获取。 + +## 终止查询 + +```sql +SHOW QUERY query_id; +``` + +query_id 可以通过 `SHOW QUERIES` 获取。 + +## 终止事务 + +```sql +KILL TRANSACTION trans_id +``` + +trans_id 可以通过 `SHOW TRANSACTIONS` 获取。 + +## 重置客户端缓存 + +```sql +RESET QUERY CACHE; +``` + +如果在多客户端情况下出现元数据不同步的情况，可以用这条命令强制清空客户端缓存，随后客户端会从服务端拉取最新的元数据。 diff --git a/examples/c/tmq.c b/examples/c/tmq.c index 1d2b26624b..fd1f146618 100644 --- a/examples/c/tmq.c +++ b/examples/c/tmq.c @@ -29,7 +29,7 @@ static void msg_process(TAOS_RES* msg) { printf("vg: %d\n", tmq_get_vgroup_id(msg)); if (tmq_get_res_type(msg) == TMQ_RES_TABLE_META) { tmq_raw_data raw = {0}; - int32_t code = tmq_get_raw_meta(msg, &raw); + int32_t code = tmq_get_raw(msg, &raw); if (code == 0) { TAOS* pConn = taos_connect("192.168.1.86", "root", "taosdata", NULL, 0); if (pConn == NULL) { @@ -50,7 +50,7 @@ static void msg_process(TAOS_RES* msg) { } taos_free_result(pRes); - int32_t ret = taos_write_raw_meta(pConn, raw); + int32_t ret = tmq_write_raw(pConn, raw); printf("write raw data: %s\n", tmq_err2str(ret)); taos_close(pConn); } diff --git a/examples/c/tmq_taosx.c b/examples/c/tmq_taosx.c index 13f3b18e64..d0def44269 100644 --- a/examples/c/tmq_taosx.c +++ b/examples/c/tmq_taosx.c @@ -49,18 +49,25 @@ static void msg_process(TAOS_RES* msg) { printf("meta result: %s\n", result); } tmq_free_json_meta(result); - - - tmq_raw_data raw = {0}; - tmq_get_raw_meta(msg, &raw); - int32_t ret = taos_write_raw_meta(pConn, raw); - printf("write raw meta: %s\n", tmq_err2str(ret)); } - if(tmq_get_res_type(msg) == TMQ_RES_DATA){ - int32_t ret =taos_write_raw_data(pConn, msg); - printf("write raw data: %s\n", tmq_err2str(ret)); - } + tmq_raw_data raw = {0}; + tmq_get_raw(msg, &raw); + int32_t ret = tmq_write_raw(pConn, raw); + printf("write raw data: %s\n", tmq_err2str(ret)); + +// else{ +// while(1){ +// int numOfRows = 0; +// void *pData = NULL; +// taos_fetch_raw_block(msg, &numOfRows, &pData); +// if(numOfRows == 0) break; +// printf("write data: tbname:%s, numOfRows:%d\n", tmq_get_table_name(msg), numOfRows); +// int ret = taos_write_raw_block(pConn, numOfRows, pData, tmq_get_table_name(msg)); +// printf("write raw data: %s\n", tmq_err2str(ret)); +// } +// } + taos_close(pConn); } @@ -121,7 +128,7 @@ int32_t init_env() { } taos_free_result(pRes); - pRes = taos_query(pConn, "insert into ct0 values(now, 1, 2, 'a')"); + pRes = taos_query(pConn, "insert into ct0 values(1626006833600, 1, 2, 'a')"); if (taos_errno(pRes) != 0) { printf("failed to insert into ct0, reason:%s\n", taos_errstr(pRes)); return -1; @@ -142,7 +149,7 @@ int32_t init_env() { } taos_free_result(pRes); - pRes = taos_query(pConn, "insert into ct1 values(now, 3, 4, 'b')"); + pRes = taos_query(pConn, "insert into ct1 values(1626006833600, 3, 4, 'b')"); if (taos_errno(pRes) != 0) { printf("failed to insert into ct1, reason:%s\n", taos_errstr(pRes)); return -1; @@ -156,7 +163,7 @@ int32_t init_env() { } taos_free_result(pRes); - pRes = taos_query(pConn, "insert into ct3 values(now, 5, 6, 'c') ct1 values(now+1s, 2, 3, 'sds') (now+2s, 4, 5, 'ddd') ct0 values(now+1s, 4, 3, 'hwj') ct1 values(now+5s, 23, 32, 's21ds')"); + pRes = taos_query(pConn, "insert into ct3 values(1626006833600, 5, 6, 'c') ct1 values(1626006833601, 2, 3, 'sds') (1626006833602, 4, 5, 'ddd') ct0 values(1626006833602, 4, 3, 'hwj') ct1 values(now+5s, 23, 32, 's21ds')"); if (taos_errno(pRes) != 0) { printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); return -1; @@ -177,7 +184,14 @@ int32_t init_env() { } taos_free_result(pRes); - pRes = taos_query(pConn, "insert into ct3 values(now+7s, 53, 63, 'cffffffffffffffffffffffffffff', 8989898899999) (now+9s, 51, 62, 'c333', 940)"); + pRes = taos_query(pConn, "insert into ct3 values(1626006833605, 53, 63, 'cffffffffffffffffffffffffffff', 8989898899999) (1626006833609, 51, 62, 'c333', 940)"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into ct3 select * from ct1"); if (taos_errno(pRes) != 0) { printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); return -1; @@ -198,19 +212,26 @@ int32_t init_env() { } taos_free_result(pRes); -// pRes = taos_query(pConn, "drop table ct3 ct1"); -// if (taos_errno(pRes) != 0) { -// printf("failed to drop child table ct3, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "drop table st1"); -// if (taos_errno(pRes) != 0) { -// printf("failed to drop super table st1, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); + pRes = taos_query(pConn, "delete from abc1 .ct3 where ts < 1626006833606"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "drop table ct3 ct1"); + if (taos_errno(pRes) != 0) { + printf("failed to drop child table ct3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "drop table st1"); + if (taos_errno(pRes) != 0) { + printf("failed to drop super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); pRes = taos_query(pConn, "create table if not exists n1(ts timestamp, c1 int, c2 nchar(4))"); if (taos_errno(pRes) != 0) { @@ -261,12 +282,12 @@ int32_t init_env() { } taos_free_result(pRes); -// pRes = taos_query(pConn, "drop table n1"); -// if (taos_errno(pRes) != 0) { -// printf("failed to drop normal table n1, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); + pRes = taos_query(pConn, "drop table n1"); + if (taos_errno(pRes) != 0) { + printf("failed to drop normal table n1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); pRes = taos_query(pConn, "create table jt(ts timestamp, i int) tags(t json)"); if (taos_errno(pRes) != 0) { @@ -289,21 +310,21 @@ int32_t init_env() { } taos_free_result(pRes); -// pRes = taos_query(pConn, -// "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " -// "nchar(8), t4 bool)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "drop table st1"); -// if (taos_errno(pRes) != 0) { -// printf("failed to drop super table st1, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); + pRes = taos_query(pConn, + "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " + "nchar(8), t4 bool)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "drop table st1"); + if (taos_errno(pRes) != 0) { + printf("failed to drop super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); taos_close(pConn); return 0; diff --git a/include/client/taos.h b/include/client/taos.h index 5f147bb07c..6f3244ea82 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -260,17 +260,20 @@ enum tmq_res_t { }; typedef struct tmq_raw_data{ - void* raw_meta; - uint32_t raw_meta_len; - uint16_t raw_meta_type; + void* raw; + uint32_t raw_len; + uint16_t raw_type; } tmq_raw_data; typedef enum tmq_res_t tmq_res_t; DLL_EXPORT tmq_res_t tmq_get_res_type(TAOS_RES *res); -DLL_EXPORT int32_t tmq_get_raw_meta(TAOS_RES *res, tmq_raw_data *raw_meta); -DLL_EXPORT int32_t taos_write_raw_meta(TAOS *taos, tmq_raw_data raw_meta); -DLL_EXPORT int32_t taos_write_raw_data(TAOS *taos, TAOS_RES *res); +DLL_EXPORT int32_t tmq_get_raw(TAOS_RES *res, tmq_raw_data *raw); +DLL_EXPORT int32_t tmq_write_raw(TAOS *taos, tmq_raw_data raw); +DLL_EXPORT int taos_write_raw_block(TAOS *taos, int numOfRows, char *pData, const char* tbname); + + +DLL_EXPORT void tmq_free_raw(tmq_raw_data raw); DLL_EXPORT char *tmq_get_json_meta(TAOS_RES *res); // Returning null means error. Returned result need to be freed by tmq_free_json_meta DLL_EXPORT void tmq_free_json_meta(char* jsonMeta); DLL_EXPORT const char *tmq_get_topic_name(TAOS_RES *res); diff --git a/include/common/tcommon.h b/include/common/tcommon.h index e1aadd4486..c5404085bb 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -40,6 +40,7 @@ enum { || x == TDMT_VND_CREATE_TABLE \ || x == TDMT_VND_ALTER_TABLE \ || x == TDMT_VND_DROP_TABLE \ + || x == TDMT_VND_DELETE \ ) // clang-format on diff --git a/include/common/tmsg.h b/include/common/tmsg.h index eb12bb4073..dc83015a89 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -737,7 +737,7 @@ typedef struct { int32_t daysToKeep2; int32_t minRows; int32_t maxRows; - int32_t fsyncPeriod; + int32_t walFsyncPeriod; int8_t walLevel; int8_t precision; // time resolution int8_t compression; @@ -749,9 +749,9 @@ typedef struct { int32_t numOfRetensions; SArray* pRetensions; // SRetention int32_t walRetentionPeriod; - int32_t walRetentionSize; + int64_t walRetentionSize; int32_t walRollPeriod; - int32_t walSegmentSize; + int64_t walSegmentSize; } SCreateDbReq; int32_t tSerializeSCreateDbReq(void* buf, int32_t bufLen, SCreateDbReq* pReq); @@ -768,7 +768,7 @@ typedef struct { int32_t daysToKeep0; int32_t daysToKeep1; int32_t daysToKeep2; - int32_t fsyncPeriod; + int32_t walFsyncPeriod; int8_t walLevel; int8_t strict; int8_t cacheLast; @@ -859,7 +859,7 @@ typedef struct { int32_t daysToKeep2; int32_t minRows; int32_t maxRows; - int32_t fsyncPeriod; + int32_t walFsyncPeriod; int8_t walLevel; int8_t precision; int8_t compression; @@ -1137,7 +1137,7 @@ typedef struct { int32_t daysToKeep2; int32_t minRows; int32_t maxRows; - int32_t fsyncPeriod; + int32_t walFsyncPeriod; uint32_t hashBegin; uint32_t hashEnd; int8_t hashMethod; @@ -1192,7 +1192,7 @@ typedef struct { int32_t daysToKeep0; int32_t daysToKeep1; int32_t daysToKeep2; - int32_t fsyncPeriod; + int32_t walFsyncPeriod; int8_t walLevel; int8_t strict; int8_t cacheLast; @@ -3044,6 +3044,7 @@ typedef struct SDeleteRes { int64_t skey; int64_t ekey; int64_t affectedRows; + char tableFName[TSDB_TABLE_FNAME_LEN]; } SDeleteRes; int32_t tEncodeDeleteRes(SEncoder* pCoder, const SDeleteRes* pRes); diff --git a/include/libs/executor/dataSinkMgt.h b/include/libs/executor/dataSinkMgt.h index 8d5a8abcb4..90b804b382 100644 --- a/include/libs/executor/dataSinkMgt.h +++ b/include/libs/executor/dataSinkMgt.h @@ -38,6 +38,7 @@ typedef struct SDeleterRes { int64_t skey; int64_t ekey; int64_t affectedRows; + char tableFName[TSDB_TABLE_FNAME_LEN]; } SDeleterRes; typedef struct SDeleterParam { diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index ba16acf7b0..1147e875f2 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -503,6 +503,7 @@ typedef struct SDataDeleterNode { uint64_t tableId; int8_t tableType; // table type char tableFName[TSDB_TABLE_FNAME_LEN]; + char tsColName[TSDB_COL_NAME_LEN]; STimeWindow deleteTimeRange; SNode* pAffectedRows; } SDataDeleterNode; diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index cc040594b1..b62f822313 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -251,8 +251,8 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t (_code) == TSDB_CODE_APP_NOT_READY || (_code) == TSDB_CODE_RPC_BROKEN_LINK) #define NEED_CLIENT_RM_TBLMETA_REQ(_type) \ - ((_type) == TDMT_VND_CREATE_TABLE || (_type) == TDMT_VND_CREATE_STB || (_type) == TDMT_VND_DROP_TABLE || \ - (_type) == TDMT_VND_DROP_STB) + ((_type) == TDMT_VND_CREATE_TABLE || (_type) == TDMT_MND_CREATE_STB || (_type) == TDMT_VND_DROP_TABLE || \ + (_type) == TDMT_MND_DROP_STB) #define NEED_SCHEDULER_REDIRECT_ERROR(_code) \ ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_NODE_NOT_DEPLOYED || \ diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index eb83da1803..6199732cc8 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -171,8 +171,8 @@ typedef struct { } STaskDispatcherFixedEp; typedef struct { - // int8_t hashMethod; char stbFullName[TSDB_TABLE_FNAME_LEN]; + int32_t waitingRspCnt; SUseDbRsp dbInfo; } STaskDispatcherShuffle; diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 4f4d000134..bba03718ba 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -192,6 +192,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_NO_USER_FROM_CONN TAOS_DEF_ERROR_CODE(0, 0x0354) #define TSDB_CODE_MND_TOO_MANY_USERS TAOS_DEF_ERROR_CODE(0, 0x0355) #define TSDB_CODE_MND_INVALID_ALTER_OPER TAOS_DEF_ERROR_CODE(0, 0x0356) +#define TSDB_CODE_MND_AUTH_FAILURE TAOS_DEF_ERROR_CODE(0, 0x0357) // mnode-stable-part1 #define TSDB_CODE_MND_STB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0360) diff --git a/include/util/tlog.h b/include/util/tlog.h index 76d04a5997..68b004cda7 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -63,6 +63,7 @@ extern int32_t metaDebugFlag; extern int32_t udfDebugFlag; extern int32_t smaDebugFlag; extern int32_t idxDebugFlag; +extern int32_t tdbDebugFlag; int32_t taosInitLog(const char *logName, int32_t maxFiles); void taosCloseLog(); diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 74d5d4270b..df9072fe1a 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -1206,6 +1206,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { SDecoder decoder; tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); tDecodeSMqDataRsp(&decoder, &pRspWrapper->dataRsp); + tDecoderClear(&decoder); memcpy(&pRspWrapper->dataRsp, pMsg->pData, sizeof(SMqRspHead)); } else { ASSERT(rspType == TMQ_MSG_TYPE__POLL_META_RSP); @@ -1859,6 +1860,10 @@ tmq_res_t tmq_get_res_type(TAOS_RES* res) { if (TD_RES_TMQ(res)) { return TMQ_RES_DATA; } else if (TD_RES_TMQ_META(res)) { + SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; + if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_DELETE) { + return TMQ_RES_DATA; + } return TMQ_RES_TABLE_META; } else { return TMQ_RES_INVALID; @@ -1913,17 +1918,6 @@ const char* tmq_get_table_name(TAOS_RES* res) { return NULL; } -int32_t tmq_get_raw_meta(TAOS_RES* res, tmq_raw_data *raw) { - if (TD_RES_TMQ_META(res) && raw) { - SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; - raw->raw_meta = pMetaRspObj->metaRsp.metaRsp; - raw->raw_meta_len = pMetaRspObj->metaRsp.metaRspLen; - raw->raw_meta_type = pMetaRspObj->metaRsp.resMsgType; - return TSDB_CODE_SUCCESS; - } - return TSDB_CODE_INVALID_PARA; -} - static char* buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id, int8_t t) { char* string = NULL; @@ -2436,30 +2430,6 @@ _exit: return string; } -char* tmq_get_json_meta(TAOS_RES* res) { - if (!TD_RES_TMQ_META(res)) { - return NULL; - } - - SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; - if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_CREATE_STB) { - return processCreateStb(&pMetaRspObj->metaRsp); - } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_ALTER_STB) { - return processAlterStb(&pMetaRspObj->metaRsp); - } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_DROP_STB) { - return processDropSTable(&pMetaRspObj->metaRsp); - } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_CREATE_TABLE) { - return processCreateTable(&pMetaRspObj->metaRsp); - } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_ALTER_TABLE) { - return processAlterTable(&pMetaRspObj->metaRsp); - } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_DROP_TABLE) { - return processDropTable(&pMetaRspObj->metaRsp); - } - return NULL; -} - -void tmq_free_json_meta(char* jsonMeta) { taosMemoryFreeClear(jsonMeta); } - static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) { SVCreateStbReq req = {0}; SDecoder coder; @@ -2531,6 +2501,13 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) { pQuery.stableQuery = true; launchQueryImpl(pRequest, &pQuery, true, NULL); + + if(pRequest->code == TSDB_CODE_SUCCESS){ + SCatalog* pCatalog = NULL; + catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog); + catalogRemoveTableMeta(pCatalog, &tableName); + } + code = pRequest->code; taosMemoryFree(pCmdMsg.pMsg); @@ -2572,7 +2549,7 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { pReq.suid = req.suid; STscObj* pTscObj = pRequest->pTscObj; - SName tableName; + SName tableName = {0}; tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName), pReq.name); SCmdMsgInfo pCmdMsg = {0}; @@ -2593,6 +2570,13 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { pQuery.stableQuery = true; launchQueryImpl(pRequest, &pQuery, true, NULL); + + if(pRequest->code == TSDB_CODE_SUCCESS){ + SCatalog* pCatalog = NULL; + catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog); + catalogRemoveTableMeta(pCatalog, &tableName); + } + code = pRequest->code; taosMemoryFree(pCmdMsg.pMsg); @@ -2659,17 +2643,20 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { .requestId = pRequest->requestId, .requestObjRefId = pRequest->self, .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; + + pRequest->tableList = taosArrayInit(req.nReqs, sizeof(SName)); // loop to create table for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { pCreateReq = req.pReqs + iReq; SVgroupInfo pInfo = {0}; - SName pName; + SName pName = {0}; toName(pTscObj->acctId, pRequest->pDb, pCreateReq->name, &pName); code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &pInfo); if (code != TSDB_CODE_SUCCESS) { goto end; } + taosArrayPush(pRequest->tableList, &pName); SVgroupCreateTableBatch* pTableBatch = taosHashGet(pVgroupHashmap, &pInfo.vgId, sizeof(pInfo.vgId)); if (pTableBatch == NULL) { @@ -2703,8 +2690,11 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { goto end; } - launchQueryImpl(pRequest, pQuery, false, NULL); - pQuery = NULL; // no need to free in the end + launchQueryImpl(pRequest, pQuery, true, NULL); + if (pRequest->code == TSDB_CODE_SUCCESS){ + removeMeta(pTscObj, pRequest->tableList); + } + code = pRequest->code; end: @@ -2772,19 +2762,21 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { .requestId = pRequest->requestId, .requestObjRefId = pRequest->self, .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; + pRequest->tableList = taosArrayInit(req.nReqs, sizeof(SName)); // loop to create table for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { pDropReq = req.pReqs + iReq; pDropReq->igNotExists = true; SVgroupInfo pInfo = {0}; - SName pName; + SName pName = {0}; toName(pTscObj->acctId, pRequest->pDb, pDropReq->name, &pName); code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &pInfo); if (code != TSDB_CODE_SUCCESS) { goto end; } + taosArrayPush(pRequest->tableList, &pName); SVgroupDropTableBatch* pTableBatch = taosHashGet(pVgroupHashmap, &pInfo.vgId, sizeof(pInfo.vgId)); if (pTableBatch == NULL) { SVgroupDropTableBatch tBatch = {0}; @@ -2815,8 +2807,10 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { goto end; } - launchQueryImpl(pRequest, pQuery, false, NULL); - pQuery = NULL; // no need to free in the end + launchQueryImpl(pRequest, pQuery, true, NULL); + if (pRequest->code == TSDB_CODE_SUCCESS){ + removeMeta(pTscObj, pRequest->tableList); + } code = pRequest->code; end: @@ -2827,6 +2821,70 @@ end: return code; } +// delete from db.tabl where .. -> delete from tabl where .. +// delete from db .tabl where .. -> delete from tabl where .. +static void getTbName(char *sql){ + char *ch = sql; + + bool inBackQuote = false; + int8_t dotIndex = 0; + while(*ch != '\0'){ + if(!inBackQuote && *ch == '`'){ + inBackQuote = true; + ch++; + continue; + } + + if(inBackQuote && *ch == '`'){ + inBackQuote = false; + ch++; + + continue; + } + + if(!inBackQuote && *ch == '.'){ + dotIndex ++; + if(dotIndex == 2){ + memmove(sql, ch + 1, strlen(ch + 1) + 1); + break; + } + } + ch++; + } +} + +static int32_t taosDeleteData(TAOS* taos, void* meta, int32_t metaLen) { + SDeleteRes req = {0}; + SDecoder coder = {0}; + int32_t code = TSDB_CODE_SUCCESS; + + // decode and process req + void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); + int32_t len = metaLen - sizeof(SMsgHead); + tDecoderInit(&coder, data, len); + if (tDecodeDeleteRes(&coder, &req) < 0) { + code = TSDB_CODE_INVALID_PARA; + goto end; + } + + getTbName(req.tableFName); + char sql[256] = {0}; + sprintf(sql, "delete from `%s` where `%s` >= %" PRId64" and `%s` <= %" PRId64, req.tableFName, "ts", req.skey, "ts", req.ekey); + printf("delete sql:%s\n", sql); + + TAOS_RES* res = taos_query(taos, sql); + SRequestObj *pRequest = (SRequestObj *)res; + code = pRequest->code; + if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) { + code = TSDB_CODE_SUCCESS; + } + taos_free_result(res); + +end: + tDecoderClear(&coder); + return code; +} + static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) { SVAlterTbReq req = {0}; SDecoder coder = {0}; @@ -2914,15 +2972,21 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) { goto end; } - launchQueryImpl(pRequest, pQuery, false, NULL); - pQuery = NULL; // no need to free in the end + launchQueryImpl(pRequest, pQuery, true, NULL); + pVgData = NULL; pArray = NULL; code = pRequest->code; if (code == TSDB_CODE_VND_TABLE_NOT_EXIST) { - code = 0; + code = TSDB_CODE_SUCCESS; } + if(pRequest->code == TSDB_CODE_SUCCESS){ + SExecResult* pRes = &pRequest->body.resInfo.execRes; + if(pRes->res != NULL){ + code = handleAlterTbExecRes(pRes->res, pCatalog); + } + } end: taosArrayDestroy(pArray); if (pVgData) taosMemoryFreeClear(pVgData->pData); @@ -2933,27 +2997,6 @@ end: return code; } -int32_t taos_write_raw_meta(TAOS *taos, tmq_raw_data raw_meta){ - if (!taos) { - return TSDB_CODE_INVALID_PARA; - } - - if(raw_meta.raw_meta_type == TDMT_VND_CREATE_STB) { - return taosCreateStb(taos, raw_meta.raw_meta, raw_meta.raw_meta_len); - }else if(raw_meta.raw_meta_type == TDMT_VND_ALTER_STB){ - return taosCreateStb(taos, raw_meta.raw_meta, raw_meta.raw_meta_len); - }else if(raw_meta.raw_meta_type == TDMT_VND_DROP_STB){ - return taosDropStb(taos, raw_meta.raw_meta, raw_meta.raw_meta_len); - }else if(raw_meta.raw_meta_type == TDMT_VND_CREATE_TABLE){ - return taosCreateTable(taos, raw_meta.raw_meta, raw_meta.raw_meta_len); - }else if(raw_meta.raw_meta_type == TDMT_VND_ALTER_TABLE){ - return taosAlterTable(taos, raw_meta.raw_meta, raw_meta.raw_meta_len); - }else if(raw_meta.raw_meta_type == TDMT_VND_DROP_TABLE){ - return taosDropTable(taos, raw_meta.raw_meta, raw_meta.raw_meta_len); - } - return TSDB_CODE_INVALID_PARA; -} - typedef struct{ SVgroupInfo vg; void *data; @@ -2964,15 +3007,196 @@ static void destroyVgHash(void* data) { taosMemoryFreeClear(vgData->data); } -int32_t taos_write_raw_data(TAOS *taos, TAOS_RES *msg){ - if (!TD_RES_TMQ(msg)) { - uError("WriteRaw:msg is not tmq : %d", *(int8_t*)msg); - return TSDB_CODE_TMQ_INVALID_MSG; +int taos_write_raw_block(TAOS *taos, int rows, char *pData, const char* tbname){ + int32_t code = TSDB_CODE_SUCCESS; + STableMeta* pTableMeta = NULL; + SQuery *pQuery = NULL; + + SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT); + if(!pRequest){ + uError("WriteRaw:createRequest error request is null"); + code = terrno; + goto end; } + if (!pRequest->pDb) { + uError("WriteRaw:not use db"); + code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; + goto end; + } + + SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}}; + strcpy(pName.dbname, pRequest->pDb); + strcpy(pName.tname, tbname); + + struct SCatalog *pCatalog = NULL; + code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); + if(code != TSDB_CODE_SUCCESS){ + uError("WriteRaw: get gatlog error"); + goto end; + } + + SRequestConnInfo conn = {0}; + conn.pTrans = pRequest->pTscObj->pAppInfo->pTransporter; + conn.requestId = pRequest->requestId; + conn.requestObjRefId = pRequest->self; + conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); + + SVgroupInfo vgData = {0}; + code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &vgData); + if (code != TSDB_CODE_SUCCESS) { + uError("WriteRaw:catalogGetTableHashVgroup failed. table name: %s", tbname); + goto end; + } + + code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta); + if (code != TSDB_CODE_SUCCESS) { + uError("WriteRaw:catalogGetTableMeta failed. table name: %s", tbname); + goto end; + } + uint64_t suid = (TSDB_NORMAL_TABLE == pTableMeta->tableType ? 0 : pTableMeta->suid); + uint64_t uid = pTableMeta->uid; + int32_t numOfCols = pTableMeta->tableInfo.numOfColumns; + + uint16_t fLen = 0; + int32_t rowSize = 0; + int16_t nVar = 0; + for (int i = 0; i < numOfCols; i++) { + SSchema *schema = pTableMeta->schema + i; + fLen += TYPE_BYTES[schema->type]; + rowSize += schema->bytes; + if(IS_VAR_DATA_TYPE(schema->type)){ + nVar ++; + } + } + + int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) + + (int32_t)TD_BITMAP_BYTES(numOfCols - 1); + int32_t schemaLen = 0; + int32_t submitLen = sizeof(SSubmitBlk) + schemaLen + rows * extendedRowSize; + + int32_t totalLen = sizeof(SSubmitReq) + submitLen; + SSubmitReq* subReq = taosMemoryCalloc(1, totalLen); + SSubmitBlk* blk = POINTER_SHIFT(subReq, sizeof(SSubmitReq)); + void* blkSchema = POINTER_SHIFT(blk, sizeof(SSubmitBlk)); + STSRow* rowData = POINTER_SHIFT(blkSchema, schemaLen); + + SRowBuilder rb = {0}; + tdSRowInit(&rb, pTableMeta->sversion); + tdSRowSetTpInfo(&rb, numOfCols, fLen); + int32_t dataLen = 0; + + char* pStart = pData + sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int16_t) + sizeof(int32_t)); + int32_t* colLength = (int32_t*)pStart; + pStart += sizeof(int32_t) * numOfCols; + + SResultColumn *pCol = taosMemoryCalloc(numOfCols, sizeof(SResultColumn)); + + for (int32_t i = 0; i < numOfCols; ++i) { + if (IS_VAR_DATA_TYPE(pTableMeta->schema[i].type)) { + pCol[i].offset = (int32_t*)pStart; + pStart += rows * sizeof(int32_t); + } else { + pCol[i].nullbitmap = pStart; + pStart += BitmapLen(rows); + } + + pCol[i].pData = pStart; + pStart += colLength[i]; + } + + for (int32_t j = 0; j < rows; j++) { + tdSRowResetBuf(&rb, rowData); + int32_t offset = 0; + for (int32_t k = 0; k < numOfCols; k++) { + const SSchema* pColumn = &pTableMeta->schema[k]; + + if (IS_VAR_DATA_TYPE(pColumn->type)) { + if (pCol[k].offset[j] != -1) { + char* data = pCol[k].pData + pCol[k].offset[j]; + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k); + } else { + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); + } + } else { + if (!colDataIsNull_f(pCol[k].nullbitmap, j)) { + char* data = pCol[k].pData + pColumn->bytes * j; + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k); + } else { + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); + } + } + + offset += TYPE_BYTES[pColumn->type]; + } + int32_t rowLen = TD_ROW_LEN(rowData); + rowData = POINTER_SHIFT(rowData, rowLen); + dataLen += rowLen; + } + + taosMemoryFree(pCol); + + blk->uid = htobe64(uid); + blk->suid = htobe64(suid); + blk->padding = htonl(blk->padding); + blk->sversion = htonl(pTableMeta->sversion); + blk->schemaLen = htonl(schemaLen); + blk->numOfRows = htons(rows); + blk->dataLen = htonl(dataLen); + subReq->length = sizeof(SSubmitReq) + sizeof(SSubmitBlk) + schemaLen + dataLen; + subReq->numOfBlocks = 1; + + pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); + if (NULL == pQuery) { + uError("create SQuery error"); + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; + pQuery->haveResultSet = false; + pQuery->msgType = TDMT_VND_SUBMIT; + pQuery->pRoot = (SNode *)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT); + if (NULL == pQuery->pRoot) { + uError("create pQuery->pRoot error"); + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + SVnodeModifOpStmt *nodeStmt = (SVnodeModifOpStmt *)(pQuery->pRoot); + nodeStmt->payloadType = PAYLOAD_TYPE_KV; + nodeStmt->pDataBlocks = taosArrayInit(1, POINTER_BYTES); + + SVgDataBlocks *dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); + if (NULL == dst) { + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto end; + } + dst->vg = vgData; + dst->numOfTables = subReq->numOfBlocks; + dst->size = subReq->length; + dst->pData = (char*)subReq; + subReq->header.vgId = htonl(dst->vg.vgId); + subReq->version = htonl(1); + subReq->header.contLen = htonl(subReq->length); + subReq->length = htonl(subReq->length); + subReq->numOfBlocks = htonl(subReq->numOfBlocks); + subReq = NULL; // no need free + taosArrayPush(nodeStmt->pDataBlocks, &dst); + + launchQueryImpl(pRequest, pQuery, true, NULL); + code = pRequest->code; + +end: + taosMemoryFreeClear(pTableMeta); + qDestroyQuery(pQuery); + return code; +} + +static int32_t tmqWriteRaw(TAOS *taos, void* data, int32_t dataLen){ int32_t code = TSDB_CODE_SUCCESS; SHashObj *pVgHash = NULL; SQuery *pQuery = NULL; + SMqRspObj rspObj = {0}; + SDecoder decoder = {0}; terrno = TSDB_CODE_SUCCESS; SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT); @@ -2981,6 +3205,17 @@ int32_t taos_write_raw_data(TAOS *taos, TAOS_RES *msg){ return terrno; } + rspObj.resIter = -1; + rspObj.resType = RES_TYPE__TMQ; + + tDecoderInit(&decoder, data, dataLen); + code = tDecodeSMqDataRsp(&decoder, &rspObj.rsp); + if (code != 0){ + uError("WriteRaw:decode smqDataRsp error"); + code = TSDB_CODE_INVALID_MSG; + goto end; + } + if (!pRequest->pDb) { uError("WriteRaw:not use db"); code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; @@ -3001,18 +3236,18 @@ int32_t taos_write_raw_data(TAOS *taos, TAOS_RES *msg){ conn.requestId = pRequest->requestId; conn.requestObjRefId = pRequest->self; conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); - SMqRspObj *rspObj = ((SMqRspObj*)msg); - printf("raw data block num:%d\n", rspObj->rsp.blockNum); - while (++rspObj->resIter < rspObj->rsp.blockNum) { - SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(rspObj->rsp.blockData, rspObj->resIter); - if (!rspObj->rsp.withSchema) { - uError("WriteRaw:no schema, iter:%d", rspObj->resIter); + + printf("raw data block num:%d\n", rspObj.rsp.blockNum); + while (++rspObj.resIter < rspObj.rsp.blockNum) { + SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(rspObj.rsp.blockData, rspObj.resIter); + if (!rspObj.rsp.withSchema) { + uError("WriteRaw:no schema, iter:%d", rspObj.resIter); goto end; } - SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj->rsp.blockSchema, rspObj->resIter); - setResSchemaInfo(&rspObj->resInfo, pSW->pSchema, pSW->nCols); + SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.blockSchema, rspObj.resIter); + setResSchemaInfo(&rspObj.resInfo, pSW->pSchema, pSW->nCols); - code = setQueryResultFromRsp(&rspObj->resInfo, pRetrieve, false, false); + code = setQueryResultFromRsp(&rspObj.resInfo, pRetrieve, false, false); if(code != TSDB_CODE_SUCCESS){ uError("WriteRaw: setQueryResultFromRsp error"); goto end; @@ -3030,13 +3265,13 @@ int32_t taos_write_raw_data(TAOS *taos, TAOS_RES *msg){ } } - int32_t rows = rspObj->resInfo.numOfRows; + int32_t rows = rspObj.resInfo.numOfRows; int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) + (int32_t)TD_BITMAP_BYTES(pSW->nCols - 1); int32_t schemaLen = 0; int32_t submitLen = sizeof(SSubmitBlk) + schemaLen + rows * extendedRowSize; - const char* tbName = tmq_get_table_name(msg); + const char* tbName = (const char*)taosArrayGetP(rspObj.rsp.blockTbName, rspObj.resIter); if(!tbName){ uError("WriteRaw: tbname is null"); code = TSDB_CODE_TMQ_INVALID_MSG; @@ -3108,13 +3343,13 @@ int32_t taos_write_raw_data(TAOS *taos, TAOS_RES *msg){ for (int32_t j = 0; j < rows; j++) { tdSRowResetBuf(&rb, rowData); - doSetOneRowPtr(&rspObj->resInfo); - rspObj->resInfo.current += 1; + doSetOneRowPtr(&rspObj.resInfo); + rspObj.resInfo.current += 1; int32_t offset = 0; for (int32_t k = 0; k < pSW->nCols; k++) { const SSchema* pColumn = &pSW->pSchema[k]; - char *data = rspObj->resInfo.row[k]; + char *data = rspObj.resInfo.row[k]; if (!data) { tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); } else { @@ -3186,13 +3421,105 @@ int32_t taos_write_raw_data(TAOS *taos, TAOS_RES *msg){ launchQueryImpl(pRequest, pQuery, true, NULL); code = pRequest->code; + end: + tDecoderClear(&decoder); + taos_free_result(&rspObj); qDestroyQuery(pQuery); destroyRequest(pRequest); taosHashCleanup(pVgHash); return code; } +char* tmq_get_json_meta(TAOS_RES* res) { + if (!TD_RES_TMQ_META(res)) { + return NULL; + } + + SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; + if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_CREATE_STB) { + return processCreateStb(&pMetaRspObj->metaRsp); + } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_ALTER_STB) { + return processAlterStb(&pMetaRspObj->metaRsp); + } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_DROP_STB) { + return processDropSTable(&pMetaRspObj->metaRsp); + } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_CREATE_TABLE) { + return processCreateTable(&pMetaRspObj->metaRsp); + } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_ALTER_TABLE) { + return processAlterTable(&pMetaRspObj->metaRsp); + } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_DROP_TABLE) { + return processDropTable(&pMetaRspObj->metaRsp); + } + return NULL; +} + +void tmq_free_json_meta(char* jsonMeta) { taosMemoryFreeClear(jsonMeta); } + +int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data *raw) { + if (!raw || !res){ + return TSDB_CODE_INVALID_PARA; + } + if (TD_RES_TMQ_META(res)) { + SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; + raw->raw = pMetaRspObj->metaRsp.metaRsp; + raw->raw_len = pMetaRspObj->metaRsp.metaRspLen; + raw->raw_type = pMetaRspObj->metaRsp.resMsgType; + } else if(TD_RES_TMQ(res)){ + SMqRspObj *rspObj = ((SMqRspObj*)res); + + int32_t len = 0; + int32_t code = 0; + tEncodeSize(tEncodeSMqDataRsp, &rspObj->rsp, len, code); + if (code < 0) { + return -1; + } + + void *buf = taosMemoryCalloc(1, len); + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, len); + tEncodeSMqDataRsp(&encoder, &rspObj->rsp); + tEncoderClear(&encoder); + + raw->raw = buf; + raw->raw_len = len; + raw->raw_type = RES_TYPE__TMQ; + } else { + return TSDB_CODE_TMQ_INVALID_MSG; + } + return TSDB_CODE_SUCCESS; +} + +void tmq_free_raw(tmq_raw_data raw) { + if (raw.raw_type == RES_TYPE__TMQ){ + taosMemoryFree(raw.raw); + } +} + +int32_t tmq_write_raw(TAOS *taos, tmq_raw_data raw){ + if (!taos) { + return TSDB_CODE_INVALID_PARA; + } + + if(raw.raw_type == TDMT_VND_CREATE_STB) { + return taosCreateStb(taos, raw.raw, raw.raw_len); + }else if(raw.raw_type == TDMT_VND_ALTER_STB){ + return taosCreateStb(taos, raw.raw, raw.raw_len); + }else if(raw.raw_type == TDMT_VND_DROP_STB){ + return taosDropStb(taos, raw.raw, raw.raw_len); + }else if(raw.raw_type == TDMT_VND_CREATE_TABLE){ + return taosCreateTable(taos, raw.raw, raw.raw_len); + }else if(raw.raw_type == TDMT_VND_ALTER_TABLE){ + return taosAlterTable(taos, raw.raw, raw.raw_len); + }else if(raw.raw_type == TDMT_VND_DROP_TABLE) { + return taosDropTable(taos, raw.raw, raw.raw_len); + }else if(raw.raw_type == TDMT_VND_DELETE){ + return taosDeleteData(taos, raw.raw, raw.raw_len); + }else if(raw.raw_type == RES_TYPE__TMQ){ + return tmqWriteRaw(taos, raw.raw, raw.raw_len); + } + return TSDB_CODE_INVALID_PARA; +} + void tmq_commit_async(tmq_t* tmq, const TAOS_RES* msg, tmq_commit_cb* cb, void* param) { // tmqCommitInner2(tmq, msg, 0, 1, cb, param); diff --git a/source/common/src/systable.c b/source/common/src/systable.c index bae00bbd57..589a4a9c81 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -84,17 +84,19 @@ static const SSysDbTableSchema userDBSchema[] = { {.name = "pages", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "minrows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "maxrows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "wal", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, - {.name = "fsync", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "comp", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, - {.name = "cacheModel", .bytes = TSDB_CACHE_MODEL_STR_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "precision", .bytes = 2 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "single_stable", .bytes = 1, .type = TSDB_DATA_TYPE_BOOL}, {.name = "status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - // {.name = "schemaless", .bytes = 1, .type = TSDB_DATA_TYPE_BOOL}, {.name = "retention", .bytes = 60 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - - // {.name = "update", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, // disable update + {.name = "single_stable", .bytes = 1, .type = TSDB_DATA_TYPE_BOOL}, + {.name = "cache_model", .bytes = TSDB_CACHE_MODEL_STR_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "cache_size", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "wal_level", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, + {.name = "wal_fsync_period", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "wal_retention_period", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "wal_retention_size", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, + {.name = "wal_roll_period", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "wal_seg_size", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, }; static const SSysDbTableSchema userFuncSchema[] = { diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 7a20969a63..cb1f3ca91c 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -316,6 +316,7 @@ static int32_t taosAddServerLogCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "udfDebugFlag", udfDebugFlag, 0, 255, 0) != 0) return -1; if (cfgAddInt32(pCfg, "smaDebugFlag", smaDebugFlag, 0, 255, 0) != 0) return -1; if (cfgAddInt32(pCfg, "idxDebugFlag", idxDebugFlag, 0, 255, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "tdbDebugFlag", tdbDebugFlag, 0, 255, 0) != 0) return -1; return 0; } @@ -506,6 +507,7 @@ static void taosSetServerLogCfg(SConfig *pCfg) { udfDebugFlag = cfgGetItem(pCfg, "udfDebugFlag")->i32; smaDebugFlag = cfgGetItem(pCfg, "smaDebugFlag")->i32; idxDebugFlag = cfgGetItem(pCfg, "idxDebugFlag")->i32; + tdbDebugFlag = cfgGetItem(pCfg, "tdbDebugFlag")->i32; } static int32_t taosSetClientCfg(SConfig *pCfg) { @@ -950,6 +952,8 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) { uError("failed to create tempDir:%s since %s", tsTempDir, terrstr()); return -1; } + } else if (strcasecmp("tdbDebugFlag", name) == 0) { + tdbDebugFlag = cfgGetItem(pCfg, "tdbDebugFlag")->i32; } else if (strcasecmp("telemetryReporting", name) == 0) { tsEnableTelem = cfgGetItem(pCfg, "telemetryReporting")->bval; } else if (strcasecmp("telemetryInterval", name) == 0) { @@ -1151,14 +1155,14 @@ void taosCfgDynamicOptions(const char *option, const char *value) { } const char *options[] = { - "dDebugFlag", "vDebugFlag", "mDebugFlag", "wDebugFlag", "sDebugFlag", "tsdbDebugFlag", - "tqDebugFlag", "fsDebugFlag", "udfDebugFlag", "smaDebugFlag", "idxDebugFlag", "tmrDebugFlag", - "uDebugFlag", "smaDebugFlag", "rpcDebugFlag", "qDebugFlag", + "dDebugFlag", "vDebugFlag", "mDebugFlag", "wDebugFlag", "sDebugFlag", "tsdbDebugFlag", + "tqDebugFlag", "fsDebugFlag", "udfDebugFlag", "smaDebugFlag", "idxDebugFlag", "tdbDebugFlag", + "tmrDebugFlag", "uDebugFlag", "smaDebugFlag", "rpcDebugFlag", "qDebugFlag", }; int32_t *optionVars[] = { - &dDebugFlag, &vDebugFlag, &mDebugFlag, &wDebugFlag, &sDebugFlag, &tsdbDebugFlag, - &tqDebugFlag, &fsDebugFlag, &udfDebugFlag, &smaDebugFlag, &idxDebugFlag, &tmrDebugFlag, - &uDebugFlag, &smaDebugFlag, &rpcDebugFlag, &qDebugFlag, + &dDebugFlag, &vDebugFlag, &mDebugFlag, &wDebugFlag, &sDebugFlag, &tsdbDebugFlag, + &tqDebugFlag, &fsDebugFlag, &udfDebugFlag, &smaDebugFlag, &idxDebugFlag, &tdbDebugFlag, + &tmrDebugFlag, &uDebugFlag, &smaDebugFlag, &rpcDebugFlag, &qDebugFlag, }; int32_t optionSize = tListLen(options); @@ -1204,5 +1208,6 @@ void taosSetAllDebugFlag(int32_t flag) { taosSetDebugFlag(&udfDebugFlag, "udfDebugFlag", flag); taosSetDebugFlag(&smaDebugFlag, "smaDebugFlag", flag); taosSetDebugFlag(&idxDebugFlag, "idxDebugFlag", flag); + taosSetDebugFlag(&tdbDebugFlag, "tdbDebugFlag", flag); uInfo("all debug flag are set to %d", flag); } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 6bdb6a4e65..22b7dd827d 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2010,7 +2010,7 @@ int32_t tSerializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) { if (tEncodeI32(&encoder, pReq->daysToKeep2) < 0) return -1; if (tEncodeI32(&encoder, pReq->minRows) < 0) return -1; if (tEncodeI32(&encoder, pReq->maxRows) < 0) return -1; - if (tEncodeI32(&encoder, pReq->fsyncPeriod) < 0) return -1; + if (tEncodeI32(&encoder, pReq->walFsyncPeriod) < 0) return -1; if (tEncodeI8(&encoder, pReq->walLevel) < 0) return -1; if (tEncodeI8(&encoder, pReq->precision) < 0) return -1; if (tEncodeI8(&encoder, pReq->compression) < 0) return -1; @@ -2019,9 +2019,9 @@ int32_t tSerializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) { if (tEncodeI8(&encoder, pReq->cacheLast) < 0) return -1; if (tEncodeI8(&encoder, pReq->schemaless) < 0) return -1; if (tEncodeI32(&encoder, pReq->walRetentionPeriod) < 0) return -1; - if (tEncodeI32(&encoder, pReq->walRetentionSize) < 0) return -1; + if (tEncodeI64(&encoder, pReq->walRetentionSize) < 0) return -1; if (tEncodeI32(&encoder, pReq->walRollPeriod) < 0) return -1; - if (tEncodeI32(&encoder, pReq->walSegmentSize) < 0) return -1; + if (tEncodeI64(&encoder, pReq->walSegmentSize) < 0) return -1; if (tEncodeI8(&encoder, pReq->ignoreExist) < 0) return -1; if (tEncodeI32(&encoder, pReq->numOfRetensions) < 0) return -1; for (int32_t i = 0; i < pReq->numOfRetensions; ++i) { @@ -2056,7 +2056,7 @@ int32_t tDeserializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) if (tDecodeI32(&decoder, &pReq->daysToKeep2) < 0) return -1; if (tDecodeI32(&decoder, &pReq->minRows) < 0) return -1; if (tDecodeI32(&decoder, &pReq->maxRows) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->fsyncPeriod) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->walFsyncPeriod) < 0) return -1; if (tDecodeI8(&decoder, &pReq->walLevel) < 0) return -1; if (tDecodeI8(&decoder, &pReq->precision) < 0) return -1; if (tDecodeI8(&decoder, &pReq->compression) < 0) return -1; @@ -2065,9 +2065,9 @@ int32_t tDeserializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) if (tDecodeI8(&decoder, &pReq->cacheLast) < 0) return -1; if (tDecodeI8(&decoder, &pReq->schemaless) < 0) return -1; if (tDecodeI32(&decoder, &pReq->walRetentionPeriod) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->walRetentionSize) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->walRetentionSize) < 0) return -1; if (tDecodeI32(&decoder, &pReq->walRollPeriod) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->walSegmentSize) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->walSegmentSize) < 0) return -1; if (tDecodeI8(&decoder, &pReq->ignoreExist) < 0) return -1; if (tDecodeI32(&decoder, &pReq->numOfRetensions) < 0) return -1; pReq->pRetensions = taosArrayInit(pReq->numOfRetensions, sizeof(SRetention)); @@ -2113,7 +2113,7 @@ int32_t tSerializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) { if (tEncodeI32(&encoder, pReq->daysToKeep0) < 0) return -1; if (tEncodeI32(&encoder, pReq->daysToKeep1) < 0) return -1; if (tEncodeI32(&encoder, pReq->daysToKeep2) < 0) return -1; - if (tEncodeI32(&encoder, pReq->fsyncPeriod) < 0) return -1; + if (tEncodeI32(&encoder, pReq->walFsyncPeriod) < 0) return -1; if (tEncodeI8(&encoder, pReq->walLevel) < 0) return -1; if (tEncodeI8(&encoder, pReq->strict) < 0) return -1; if (tEncodeI8(&encoder, pReq->cacheLast) < 0) return -1; @@ -2139,7 +2139,7 @@ int32_t tDeserializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) { if (tDecodeI32(&decoder, &pReq->daysToKeep0) < 0) return -1; if (tDecodeI32(&decoder, &pReq->daysToKeep1) < 0) return -1; if (tDecodeI32(&decoder, &pReq->daysToKeep2) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->fsyncPeriod) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->walFsyncPeriod) < 0) return -1; if (tDecodeI8(&decoder, &pReq->walLevel) < 0) return -1; if (tDecodeI8(&decoder, &pReq->strict) < 0) return -1; if (tDecodeI8(&decoder, &pReq->cacheLast) < 0) return -1; @@ -2748,7 +2748,7 @@ int32_t tSerializeSDbCfgRsp(void *buf, int32_t bufLen, const SDbCfgRsp *pRsp) { if (tEncodeI32(&encoder, pRsp->daysToKeep2) < 0) return -1; if (tEncodeI32(&encoder, pRsp->minRows) < 0) return -1; if (tEncodeI32(&encoder, pRsp->maxRows) < 0) return -1; - if (tEncodeI32(&encoder, pRsp->fsyncPeriod) < 0) return -1; + if (tEncodeI32(&encoder, pRsp->walFsyncPeriod) < 0) return -1; if (tEncodeI8(&encoder, pRsp->walLevel) < 0) return -1; if (tEncodeI8(&encoder, pRsp->precision) < 0) return -1; if (tEncodeI8(&encoder, pRsp->compression) < 0) return -1; @@ -2787,7 +2787,7 @@ int32_t tDeserializeSDbCfgRsp(void *buf, int32_t bufLen, SDbCfgRsp *pRsp) { if (tDecodeI32(&decoder, &pRsp->daysToKeep2) < 0) return -1; if (tDecodeI32(&decoder, &pRsp->minRows) < 0) return -1; if (tDecodeI32(&decoder, &pRsp->maxRows) < 0) return -1; - if (tDecodeI32(&decoder, &pRsp->fsyncPeriod) < 0) return -1; + if (tDecodeI32(&decoder, &pRsp->walFsyncPeriod) < 0) return -1; if (tDecodeI8(&decoder, &pRsp->walLevel) < 0) return -1; if (tDecodeI8(&decoder, &pRsp->precision) < 0) return -1; if (tDecodeI8(&decoder, &pRsp->compression) < 0) return -1; @@ -3720,7 +3720,7 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR if (tEncodeI32(&encoder, pReq->daysToKeep2) < 0) return -1; if (tEncodeI32(&encoder, pReq->minRows) < 0) return -1; if (tEncodeI32(&encoder, pReq->maxRows) < 0) return -1; - if (tEncodeI32(&encoder, pReq->fsyncPeriod) < 0) return -1; + if (tEncodeI32(&encoder, pReq->walFsyncPeriod) < 0) return -1; if (tEncodeU32(&encoder, pReq->hashBegin) < 0) return -1; if (tEncodeU32(&encoder, pReq->hashEnd) < 0) return -1; if (tEncodeI8(&encoder, pReq->hashMethod) < 0) return -1; @@ -3782,7 +3782,7 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq * if (tDecodeI32(&decoder, &pReq->daysToKeep2) < 0) return -1; if (tDecodeI32(&decoder, &pReq->minRows) < 0) return -1; if (tDecodeI32(&decoder, &pReq->maxRows) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->fsyncPeriod) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->walFsyncPeriod) < 0) return -1; if (tDecodeU32(&decoder, &pReq->hashBegin) < 0) return -1; if (tDecodeU32(&decoder, &pReq->hashEnd) < 0) return -1; if (tDecodeI8(&decoder, &pReq->hashMethod) < 0) return -1; @@ -3910,7 +3910,7 @@ int32_t tSerializeSAlterVnodeReq(void *buf, int32_t bufLen, SAlterVnodeReq *pReq if (tEncodeI32(&encoder, pReq->daysToKeep0) < 0) return -1; if (tEncodeI32(&encoder, pReq->daysToKeep1) < 0) return -1; if (tEncodeI32(&encoder, pReq->daysToKeep2) < 0) return -1; - if (tEncodeI32(&encoder, pReq->fsyncPeriod) < 0) return -1; + if (tEncodeI32(&encoder, pReq->walFsyncPeriod) < 0) return -1; if (tEncodeI8(&encoder, pReq->walLevel) < 0) return -1; if (tEncodeI8(&encoder, pReq->strict) < 0) return -1; if (tEncodeI8(&encoder, pReq->cacheLast) < 0) return -1; @@ -3941,7 +3941,7 @@ int32_t tDeserializeSAlterVnodeReq(void *buf, int32_t bufLen, SAlterVnodeReq *pR if (tDecodeI32(&decoder, &pReq->daysToKeep0) < 0) return -1; if (tDecodeI32(&decoder, &pReq->daysToKeep1) < 0) return -1; if (tDecodeI32(&decoder, &pReq->daysToKeep2) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->fsyncPeriod) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->walFsyncPeriod) < 0) return -1; if (tDecodeI8(&decoder, &pReq->walLevel) < 0) return -1; if (tDecodeI8(&decoder, &pReq->strict) < 0) return -1; if (tDecodeI8(&decoder, &pReq->cacheLast) < 0) return -1; @@ -5681,6 +5681,7 @@ int32_t tEncodeDeleteRes(SEncoder *pCoder, const SDeleteRes *pRes) { if (tEncodeI64(pCoder, pRes->ekey) < 0) return -1; if (tEncodeI64v(pCoder, pRes->affectedRows) < 0) return -1; + if (tEncodeCStr(pCoder, pRes->tableFName) < 0) return -1; return 0; } @@ -5692,12 +5693,13 @@ int32_t tDecodeDeleteRes(SDecoder *pCoder, SDeleteRes *pRes) { if (tDecodeI32v(pCoder, &nUid) < 0) return -1; for (int32_t iUid = 0; iUid < nUid; iUid++) { if (tDecodeU64(pCoder, &uid) < 0) return -1; - taosArrayPush(pRes->uidList, &uid); + if (pRes->uidList) taosArrayPush(pRes->uidList, &uid); } if (tDecodeI64(pCoder, &pRes->skey) < 0) return -1; if (tDecodeI64(pCoder, &pRes->ekey) < 0) return -1; if (tDecodeI64v(pCoder, &pRes->affectedRows) < 0) return -1; + if (tDecodeCStrTo(pCoder, pRes->tableFName) < 0) return -1; return 0; } int32_t tEncodeSMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) { diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index cb061e6d1c..0c80e69384 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -160,7 +160,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { } pCfg->walCfg.vgId = pCreate->vgId; - pCfg->walCfg.fsyncPeriod = pCreate->fsyncPeriod; + pCfg->walCfg.fsyncPeriod = pCreate->walFsyncPeriod; pCfg->walCfg.retentionPeriod = pCreate->walRetentionPeriod; pCfg->walCfg.rollPeriod = pCreate->walRollPeriod; pCfg->walCfg.retentionSize = pCreate->walRetentionSize; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 0ff9b4102d..06c64dcea6 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -294,7 +294,7 @@ typedef struct { int32_t daysToKeep2; int32_t minRows; int32_t maxRows; - int32_t fsyncPeriod; + int32_t walFsyncPeriod; int8_t walLevel; int8_t precision; int8_t compression; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 86787fcd01..9684753ace 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -102,7 +102,7 @@ static SSdbRaw *mndDbActionEncode(SDbObj *pDb) { SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysToKeep2, _OVER) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.minRows, _OVER) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.maxRows, _OVER) - SDB_SET_INT32(pRaw, dataPos, pDb->cfg.fsyncPeriod, _OVER) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.walFsyncPeriod, _OVER) SDB_SET_INT8(pRaw, dataPos, pDb->cfg.walLevel, _OVER) SDB_SET_INT8(pRaw, dataPos, pDb->cfg.precision, _OVER) SDB_SET_INT8(pRaw, dataPos, pDb->cfg.compression, _OVER) @@ -179,7 +179,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.daysToKeep2, _OVER) SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.minRows, _OVER) SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.maxRows, _OVER) - SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.fsyncPeriod, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.walFsyncPeriod, _OVER) SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.walLevel, _OVER) SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.precision, _OVER) SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.compression, _OVER) @@ -249,7 +249,7 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) { pOld->cfg.daysToKeep0 = pNew->cfg.daysToKeep0; pOld->cfg.daysToKeep1 = pNew->cfg.daysToKeep1; pOld->cfg.daysToKeep2 = pNew->cfg.daysToKeep2; - pOld->cfg.fsyncPeriod = pNew->cfg.fsyncPeriod; + pOld->cfg.walFsyncPeriod = pNew->cfg.walFsyncPeriod; pOld->cfg.walLevel = pNew->cfg.walLevel; pOld->cfg.strict = pNew->cfg.strict; pOld->cfg.cacheLast = pNew->cfg.cacheLast; @@ -312,7 +312,7 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { if (pCfg->minRows < TSDB_MIN_MINROWS_FBLOCK || pCfg->minRows > TSDB_MAX_MINROWS_FBLOCK) return -1; if (pCfg->maxRows < TSDB_MIN_MAXROWS_FBLOCK || pCfg->maxRows > TSDB_MAX_MAXROWS_FBLOCK) return -1; if (pCfg->minRows > pCfg->maxRows) return -1; - if (pCfg->fsyncPeriod < TSDB_MIN_FSYNC_PERIOD || pCfg->fsyncPeriod > TSDB_MAX_FSYNC_PERIOD) return -1; + if (pCfg->walFsyncPeriod < TSDB_MIN_FSYNC_PERIOD || pCfg->walFsyncPeriod > TSDB_MAX_FSYNC_PERIOD) return -1; if (pCfg->walLevel < TSDB_MIN_WAL_LEVEL || pCfg->walLevel > TSDB_MAX_WAL_LEVEL) return -1; if (pCfg->precision < TSDB_MIN_PRECISION && pCfg->precision > TSDB_MAX_PRECISION) return -1; if (pCfg->compression < TSDB_MIN_COMP_LEVEL || pCfg->compression > TSDB_MAX_COMP_LEVEL) return -1; @@ -347,7 +347,7 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->daysToKeep2 < 0) pCfg->daysToKeep2 = pCfg->daysToKeep1; if (pCfg->minRows < 0) pCfg->minRows = TSDB_DEFAULT_MINROWS_FBLOCK; if (pCfg->maxRows < 0) pCfg->maxRows = TSDB_DEFAULT_MAXROWS_FBLOCK; - if (pCfg->fsyncPeriod < 0) pCfg->fsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD; + if (pCfg->walFsyncPeriod < 0) pCfg->walFsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD; if (pCfg->walLevel < 0) pCfg->walLevel = TSDB_DEFAULT_WAL_LEVEL; if (pCfg->precision < 0) pCfg->precision = TSDB_DEFAULT_PRECISION; if (pCfg->compression < 0) pCfg->compression = TSDB_DEFAULT_COMP_LEVEL; @@ -466,7 +466,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate, .daysToKeep2 = pCreate->daysToKeep2, .minRows = pCreate->minRows, .maxRows = pCreate->maxRows, - .fsyncPeriod = pCreate->fsyncPeriod, + .walFsyncPeriod = pCreate->walFsyncPeriod, .walLevel = pCreate->walLevel, .precision = pCreate->precision, .compression = pCreate->compression, @@ -642,8 +642,8 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) { terrno = 0; } - if (pAlter->fsyncPeriod >= 0 && pAlter->fsyncPeriod != pDb->cfg.fsyncPeriod) { - pDb->cfg.fsyncPeriod = pAlter->fsyncPeriod; + if (pAlter->walFsyncPeriod >= 0 && pAlter->walFsyncPeriod != pDb->cfg.walFsyncPeriod) { + pDb->cfg.walFsyncPeriod = pAlter->walFsyncPeriod; terrno = 0; } @@ -833,7 +833,7 @@ static int32_t mndProcessGetDbCfgReq(SRpcMsg *pReq) { cfgRsp.daysToKeep2 = pDb->cfg.daysToKeep2; cfgRsp.minRows = pDb->cfg.minRows; cfgRsp.maxRows = pDb->cfg.maxRows; - cfgRsp.fsyncPeriod = pDb->cfg.fsyncPeriod; + cfgRsp.walFsyncPeriod = pDb->cfg.walFsyncPeriod; cfgRsp.walLevel = pDb->cfg.walLevel; cfgRsp.precision = pDb->cfg.precision; cfgRsp.compression = pDb->cfg.compression; @@ -1521,7 +1521,7 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in colDataAppend(pColInfo, rows, buf, false); } else if (i == 3) { colDataAppend(pColInfo, rows, (const char *)&numOfTables, false); - } else if (i == 20) { + } else if (i == 15) { colDataAppend(pColInfo, rows, statusVstr, false); } else { colDataAppendNULL(pColInfo, rows); @@ -1582,21 +1582,9 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.maxRows, false); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.walLevel, false); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.fsyncPeriod, false); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.compression, false); - const char *cacheModelStr = getCacheModelStr(pDb->cfg.cacheLast); - char cacheModelVstr[24] = {0}; - STR_WITH_SIZE_TO_VARSTR(cacheModelVstr, cacheModelStr, strlen(cacheModelStr)); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)cacheModelVstr, false); - const char *precStr = NULL; switch (pDb->cfg.precision) { case TSDB_TIME_PRECISION_MILLI: @@ -1617,20 +1605,47 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, rows, (const char *)precVstr, false); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.numOfStables, false); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, rows, (const char *)statusVstr, false); char *rentensionVstr = buildRetension(pDb->cfg.pRetensions); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); if (rentensionVstr == NULL) { colDataAppendNULL(pColInfo, rows); } else { colDataAppend(pColInfo, rows, (const char *)rentensionVstr, false); taosMemoryFree(rentensionVstr); } + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.numOfStables, false); + + const char *cacheModelStr = getCacheModelStr(pDb->cfg.cacheLast); + char cacheModelVstr[24] = {0}; + STR_WITH_SIZE_TO_VARSTR(cacheModelVstr, cacheModelStr, strlen(cacheModelStr)); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, rows, (const char *)cacheModelVstr, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.cacheLastSize, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.walLevel, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.walFsyncPeriod, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.walRetentionPeriod, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.walRollPeriod, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.walRetentionSize, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.walSegmentSize, false); } taosMemoryFree(buf); diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 7141a62be5..d84455ac94 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -788,9 +788,9 @@ _OVER: static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; const char *options[] = { - "debugFlag", "dDebugFlag", "vDebugFlag", "mDebugFlag", "wDebugFlag", "sDebugFlag", - "tsdbDebugFlag", "tqDebugFlag", "fsDebugFlag", "udfDebugFlag", "smaDebugFlag", "idxDebugFlag", - "tmrDebugFlag", "uDebugFlag", "smaDebugFlag", "rpcDebugFlag", "qDebugFlag", + "debugFlag", "dDebugFlag", "vDebugFlag", "mDebugFlag", "wDebugFlag", "sDebugFlag", + "tsdbDebugFlag", "tqDebugFlag", "fsDebugFlag", "udfDebugFlag", "smaDebugFlag", "idxDebugFlag", + "tdbDebugFlag", "tmrDebugFlag", "uDebugFlag", "smaDebugFlag", "rpcDebugFlag", "qDebugFlag", }; int32_t optionSize = tListLen(options); @@ -813,7 +813,6 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { SEpSet epSet = mndGetDnodeEpset(pDnode); mndReleaseDnode(pMnode, pDnode); - SDCfgDnodeReq dcfgReq = {0}; if (strcasecmp(cfgReq.config, "resetlog") == 0) { strcpy(dcfgReq.config, "resetlog"); @@ -839,7 +838,7 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { if (strncasecmp(cfgReq.config, optName, optLen) != 0) continue; const char *value = cfgReq.value; - int32_t flag = atoi(value); + int32_t flag = atoi(value); if (flag <= 0) { flag = atoi(cfgReq.config + optLen + 1); } diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index f6ecd4493d..4b57f45631 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -240,7 +240,7 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { if (strncmp(connReq.passwd, pUser->pass, TSDB_PASSWORD_LEN - 1) != 0) { mGError("user:%s, failed to login from %s since invalid pass, input:%s", pReq->info.conn.user, ip, connReq.passwd); - code = TSDB_CODE_RPC_AUTH_FAILURE; + code = TSDB_CODE_MND_AUTH_FAILURE; goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index a46938590e..9882b0a9ae 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -114,18 +114,26 @@ int32_t mndAddSinkToTask(SMnode* pMnode, SStreamObj* pStream, SStreamTask* pTask int32_t mndAddDispatcherToInnerTask(SMnode* pMnode, SStreamObj* pStream, SStreamTask* pTask) { pTask->sinkType = TASK_SINK__NONE; + + bool isShuffle = false; + if (pStream->fixedSinkVgId == 0) { - pTask->dispatchType = TASK_DISPATCH__SHUFFLE; - pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH; SDbObj* pDb = mndAcquireDb(pMnode, pStream->targetDb); ASSERT(pDb); - - if (mndExtractDbInfo(pMnode, pDb, &pTask->shuffleDispatcher.dbInfo, NULL) < 0) { - ASSERT(0); - return -1; + if (pDb->cfg.numOfVgroups > 1) { + isShuffle = true; + pTask->dispatchType = TASK_DISPATCH__SHUFFLE; + pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH; + if (mndExtractDbInfo(pMnode, pDb, &pTask->shuffleDispatcher.dbInfo, NULL) < 0) { + ASSERT(0); + return -1; + } } - sdbRelease(pMnode->pSdb, pDb); + sdbRelease(pMnode->pSdb, pDb); + } + + if (isShuffle) { memcpy(pTask->shuffleDispatcher.stbFullName, pStream->targetSTbName, TSDB_TABLE_FNAME_LEN); SArray* pVgs = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; int32_t sz = taosArrayGetSize(pVgs); diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 4625b2ab01..0567ec4e14 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -214,7 +214,7 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg createReq.daysToKeep2 = pDb->cfg.daysToKeep2; createReq.minRows = pDb->cfg.minRows; createReq.maxRows = pDb->cfg.maxRows; - createReq.fsyncPeriod = pDb->cfg.fsyncPeriod; + createReq.walFsyncPeriod = pDb->cfg.walFsyncPeriod; createReq.walLevel = pDb->cfg.walLevel; createReq.precision = pDb->cfg.precision; createReq.compression = pDb->cfg.compression; @@ -286,7 +286,7 @@ void *mndBuildAlterVnodeReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup, int32_ alterReq.daysToKeep0 = pDb->cfg.daysToKeep0; alterReq.daysToKeep1 = pDb->cfg.daysToKeep1; alterReq.daysToKeep2 = pDb->cfg.daysToKeep2; - alterReq.fsyncPeriod = pDb->cfg.fsyncPeriod; + alterReq.walFsyncPeriod = pDb->cfg.walFsyncPeriod; alterReq.walLevel = pDb->cfg.walLevel; alterReq.strict = pDb->cfg.strict; alterReq.cacheLast = pDb->cfg.cacheLast; diff --git a/source/dnode/mnode/impl/test/db/db.cpp b/source/dnode/mnode/impl/test/db/db.cpp index 0fb8e9d530..06916b375c 100644 --- a/source/dnode/mnode/impl/test/db/db.cpp +++ b/source/dnode/mnode/impl/test/db/db.cpp @@ -44,7 +44,7 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) { createReq.daysToKeep2 = 3650; createReq.minRows = 100; createReq.maxRows = 4096; - createReq.fsyncPeriod = 3000; + createReq.walFsyncPeriod = 3000; createReq.walLevel = 1; createReq.precision = 0; createReq.compression = 2; @@ -81,7 +81,7 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) { alterdbReq.daysToKeep0 = -1; alterdbReq.daysToKeep1 = -1; alterdbReq.daysToKeep2 = -1; - alterdbReq.fsyncPeriod = 4000; + alterdbReq.walFsyncPeriod = 4000; alterdbReq.walLevel = 2; alterdbReq.strict = 1; alterdbReq.cacheLast = 1; @@ -140,7 +140,7 @@ TEST_F(MndTestDb, 03_Create_Use_Restart_Use_Db) { createReq.daysToKeep2 = 3650; createReq.minRows = 100; createReq.maxRows = 4096; - createReq.fsyncPeriod = 3000; + createReq.walFsyncPeriod = 3000; createReq.walLevel = 1; createReq.precision = 0; createReq.compression = 2; diff --git a/source/dnode/mnode/impl/test/sma/sma.cpp b/source/dnode/mnode/impl/test/sma/sma.cpp index ce6954279f..06bb735f54 100644 --- a/source/dnode/mnode/impl/test/sma/sma.cpp +++ b/source/dnode/mnode/impl/test/sma/sma.cpp @@ -49,7 +49,7 @@ void* MndTestSma::BuildCreateDbReq(const char* dbname, int32_t* pContLen) { createReq.daysToKeep2 = 3650 * 1440; createReq.minRows = 100; createReq.maxRows = 4096; - createReq.fsyncPeriod = 3000; + createReq.walFsyncPeriod = 3000; createReq.walLevel = 1; createReq.precision = 0; createReq.compression = 2; diff --git a/source/dnode/mnode/impl/test/stb/stb.cpp b/source/dnode/mnode/impl/test/stb/stb.cpp index dfdd8f3a49..d275231c70 100644 --- a/source/dnode/mnode/impl/test/stb/stb.cpp +++ b/source/dnode/mnode/impl/test/stb/stb.cpp @@ -50,7 +50,7 @@ void* MndTestStb::BuildCreateDbReq(const char* dbname, int32_t* pContLen) { createReq.daysToKeep2 = 3650; createReq.minRows = 100; createReq.maxRows = 4096; - createReq.fsyncPeriod = 3000; + createReq.walFsyncPeriod = 3000; createReq.walLevel = 1; createReq.precision = 0; createReq.compression = 2; diff --git a/source/dnode/mnode/impl/test/topic/topic.cpp b/source/dnode/mnode/impl/test/topic/topic.cpp index 353cedf636..84b3a21a8f 100644 --- a/source/dnode/mnode/impl/test/topic/topic.cpp +++ b/source/dnode/mnode/impl/test/topic/topic.cpp @@ -42,7 +42,7 @@ void* MndTestTopic::BuildCreateDbReq(const char* dbname, int32_t* pContLen) { createReq.daysToKeep2 = 3650 * 1440; createReq.minRows = 100; createReq.maxRows = 4096; - createReq.fsyncPeriod = 3000; + createReq.walFsyncPeriod = 3000; createReq.walLevel = 1; createReq.precision = 0; createReq.compression = 2; diff --git a/source/dnode/mnode/impl/test/user/user.cpp b/source/dnode/mnode/impl/test/user/user.cpp index d8b6964114..dd2bda6b19 100644 --- a/source/dnode/mnode/impl/test/user/user.cpp +++ b/source/dnode/mnode/impl/test/user/user.cpp @@ -309,7 +309,7 @@ TEST_F(MndTestUser, 03_Alter_User) { createReq.daysToKeep2 = 3650 * 1440; createReq.minRows = 100; createReq.maxRows = 4096; - createReq.fsyncPeriod = 3000; + createReq.walFsyncPeriod = 3000; createReq.walLevel = 1; createReq.precision = 0; createReq.compression = 2; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 2adfc92ab1..01f2f659ff 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -146,8 +146,8 @@ int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, con } } - int32_t len; - int32_t code; + int32_t len = 0; + int32_t code = 0; tEncodeSize(tEncodeSMqDataRsp, pRsp, len, code); if (code < 0) { return -1; @@ -164,9 +164,10 @@ int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, con void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); - SEncoder encoder; + SEncoder encoder = {0}; tEncoderInit(&encoder, abuf, len); tEncodeSMqDataRsp(&encoder, pRsp); + tEncoderClear(&encoder); SRpcMsg rsp = { .info = pMsg->info, @@ -176,8 +177,8 @@ int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, con }; tmsgSendRsp(&rsp); - char buf1[80]; - char buf2[80]; + char buf1[80] = {0}; + char buf2[80] = {0}; tFormatOffset(buf1, 80, &pRsp->reqOffset); tFormatOffset(buf2, 80, &pRsp->rspOffset); tqDebug("vgId:%d from consumer:%" PRId64 ", (epoch %d) send rsp, block num: %d, reqOffset:%s, rspOffset:%s", diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 9d06fbffdd..b5bf92ee75 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -106,7 +106,9 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { .meta = pVnode->pMeta, .config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb}; code = qWorkerProcessDeleteMsg(&handle, pVnode->pQuery, pMsg, &res); - if (code) goto _err; + if (code) { + goto _err; + } // malloc and encode tEncodeSize(tEncodeDeleteRes, &res, size, ret); @@ -993,6 +995,11 @@ static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq SDecoder *pCoder = &(SDecoder){0}; SDeleteRes *pRes = &(SDeleteRes){0}; + pRsp->msgType = TDMT_VND_DELETE_RSP; + pRsp->pCont = NULL; + pRsp->contLen = 0; + pRsp->code = TSDB_CODE_SUCCESS; + pRes->uidList = taosArrayInit(0, sizeof(tb_uid_t)); if (pRes->uidList == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -1010,6 +1017,15 @@ static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq tDecoderClear(pCoder); taosArrayDestroy(pRes->uidList); + + SVDeleteRsp rsp = {.affectedRows = pRes->affectedRows}; + int32_t ret = 0; + tEncodeSize(tEncodeSVDeleteRsp, &rsp, pRsp->contLen, ret); + pRsp->pCont = rpcMallocCont(pRsp->contLen); + SEncoder ec = {0}; + tEncoderInit(&ec, pRsp->pCont, pRsp->contLen); + tEncodeSVDeleteRsp(&ec, &rsp); + tEncoderClear(&ec); return code; _err: diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 2b760efba0..4323fa0aff 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -501,49 +501,54 @@ static void vnodeSyncReconfig(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SReCon } static void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { - SVnode *pVnode = pFsm->data; - vTrace("vgId:%d, commit-cb is excuted, fsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s, msgtype:%d %s", - syncGetVgId(pVnode->sync), pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, - syncUtilState2String(cbMeta.state), pMsg->msgType, TMSG_INFO(pMsg->msgType)); + if (cbMeta.isWeak == 0) { + SVnode *pVnode = pFsm->data; + vTrace("vgId:%d, commit-cb is excuted, fsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s, msgtype:%d %s", + syncGetVgId(pVnode->sync), pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, + syncUtilState2String(cbMeta.state), pMsg->msgType, TMSG_INFO(pMsg->msgType)); - if (cbMeta.code == 0 && cbMeta.isWeak == 0) { - SRpcMsg rpcMsg = {.msgType = pMsg->msgType, .contLen = pMsg->contLen}; - rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); - memcpy(rpcMsg.pCont, pMsg->pCont, pMsg->contLen); - syncGetAndDelRespRpc(pVnode->sync, cbMeta.seqNum, &rpcMsg.info); - rpcMsg.info.conn.applyIndex = cbMeta.index; - rpcMsg.info.conn.applyTerm = cbMeta.term; - tmsgPutToQueue(&pVnode->msgCb, APPLY_QUEUE, &rpcMsg); - } else { - SRpcMsg rsp = {.code = cbMeta.code, .info = pMsg->info}; - vError("vgId:%d, sync commit error, msgtype:%d,%s, error:0x%X, errmsg:%s", syncGetVgId(pVnode->sync), pMsg->msgType, - TMSG_INFO(pMsg->msgType), cbMeta.code, tstrerror(cbMeta.code)); - if (rsp.info.handle != NULL) { - tmsgSendRsp(&rsp); + if (cbMeta.code == 0) { + SRpcMsg rpcMsg = {.msgType = pMsg->msgType, .contLen = pMsg->contLen}; + rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); + memcpy(rpcMsg.pCont, pMsg->pCont, pMsg->contLen); + syncGetAndDelRespRpc(pVnode->sync, cbMeta.seqNum, &rpcMsg.info); + rpcMsg.info.conn.applyIndex = cbMeta.index; + rpcMsg.info.conn.applyTerm = cbMeta.term; + tmsgPutToQueue(&pVnode->msgCb, APPLY_QUEUE, &rpcMsg); + } else { + SRpcMsg rsp = {.code = cbMeta.code, .info = pMsg->info}; + vError("vgId:%d, sync commit error, msgtype:%d,%s, error:0x%X, errmsg:%s", syncGetVgId(pVnode->sync), + pMsg->msgType, TMSG_INFO(pMsg->msgType), cbMeta.code, tstrerror(cbMeta.code)); + if (rsp.info.handle != NULL) { + tmsgSendRsp(&rsp); + } } } } static void vnodeSyncPreCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { - SVnode *pVnode = pFsm->data; - vTrace("vgId:%d, pre-commit-cb is excuted, fsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s, msgtype:%d %s", - syncGetVgId(pVnode->sync), pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, - syncUtilState2String(cbMeta.state), pMsg->msgType, TMSG_INFO(pMsg->msgType)); + if (cbMeta.isWeak == 1) { + SVnode *pVnode = pFsm->data; + vTrace("vgId:%d, pre-commit-cb is excuted, fsm:%p, index:%" PRId64 + ", isWeak:%d, code:%d, state:%d %s, msgtype:%d %s", + syncGetVgId(pVnode->sync), pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, + syncUtilState2String(cbMeta.state), pMsg->msgType, TMSG_INFO(pMsg->msgType)); - if (cbMeta.code == 0 && cbMeta.isWeak == 1) { - SRpcMsg rpcMsg = {.msgType = pMsg->msgType, .contLen = pMsg->contLen}; - rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); - memcpy(rpcMsg.pCont, pMsg->pCont, pMsg->contLen); - syncGetAndDelRespRpc(pVnode->sync, cbMeta.seqNum, &rpcMsg.info); - rpcMsg.info.conn.applyIndex = cbMeta.index; - rpcMsg.info.conn.applyTerm = cbMeta.term; - tmsgPutToQueue(&pVnode->msgCb, APPLY_QUEUE, &rpcMsg); - } else { - SRpcMsg rsp = {.code = cbMeta.code, .info = pMsg->info}; - vError("vgId:%d, sync pre-commit error, msgtype:%d,%s, error:0x%X, errmsg:%s", syncGetVgId(pVnode->sync), - pMsg->msgType, TMSG_INFO(pMsg->msgType), cbMeta.code, tstrerror(cbMeta.code)); - if (rsp.info.handle != NULL) { - tmsgSendRsp(&rsp); + if (cbMeta.code == 0) { + SRpcMsg rpcMsg = {.msgType = pMsg->msgType, .contLen = pMsg->contLen}; + rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); + memcpy(rpcMsg.pCont, pMsg->pCont, pMsg->contLen); + syncGetAndDelRespRpc(pVnode->sync, cbMeta.seqNum, &rpcMsg.info); + rpcMsg.info.conn.applyIndex = cbMeta.index; + rpcMsg.info.conn.applyTerm = cbMeta.term; + tmsgPutToQueue(&pVnode->msgCb, APPLY_QUEUE, &rpcMsg); + } else { + SRpcMsg rsp = {.code = cbMeta.code, .info = pMsg->info}; + vError("vgId:%d, sync pre-commit error, msgtype:%d,%s, error:0x%X, errmsg:%s", syncGetVgId(pVnode->sync), + pMsg->msgType, TMSG_INFO(pMsg->msgType), cbMeta.code, tstrerror(cbMeta.code)); + if (rsp.info.handle != NULL) { + tmsgSendRsp(&rsp); + } } } } diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 51b721f818..0be85333dc 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -103,7 +103,7 @@ void sendCreateDbMsg(void *shandle, SEpSet *pEpSet) { createReq.daysToKeep2 = 3650; createReq.minRows = 100; createReq.maxRows = 4096; - createReq.fsyncPeriod = 3000; + createReq.walFsyncPeriod = 3000; createReq.walLevel = 1; createReq.precision = 0; createReq.compression = 2; diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 3cb2553f2e..7c95e71823 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -225,7 +225,7 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbFName, S "CREATE DATABASE `%s` BUFFER %d CACHEMODEL %d COMP %d DURATION %dm " "FSYNC %d MAXROWS %d MINROWS %d KEEP %dm,%dm,%dm PAGES %d PAGESIZE %d PRECISION '%s' REPLICA %d " "STRICT %d WAL %d VGROUPS %d SINGLE_STABLE %d", - dbFName, pCfg->buffer, pCfg->cacheLast, pCfg->compression, pCfg->daysPerFile, pCfg->fsyncPeriod, + dbFName, pCfg->buffer, pCfg->cacheLast, pCfg->compression, pCfg->daysPerFile, pCfg->walFsyncPeriod, pCfg->maxRows, pCfg->minRows, pCfg->daysToKeep0, pCfg->daysToKeep1, pCfg->daysToKeep2, pCfg->pages, pCfg->pageSize, prec, pCfg->replications, pCfg->strict, pCfg->walLevel, pCfg->numOfVgroups, 1 == pCfg->numOfStables); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 6c0f4b2c12..d5486d62b1 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -468,7 +468,6 @@ typedef struct SStreamScanInfo { SSDataBlock* pUpdateDataRes; // status for tmq // SSchemaWrapper schema; - STqOffset offset; SNodeList* pGroupTags; SNode* pTagCond; SNode* pTagIndexCond; @@ -1021,6 +1020,7 @@ void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsCol int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* groupKey); SSDataBlock* createSpecialDataBlock(EStreamType type); +void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput); #ifdef __cplusplus } diff --git a/source/libs/executor/src/dataDeleter.c b/source/libs/executor/src/dataDeleter.c index a0d4d64bff..391aef529f 100644 --- a/source/libs/executor/src/dataDeleter.c +++ b/source/libs/executor/src/dataDeleter.c @@ -90,6 +90,7 @@ static void toDataCacheEntry(SDataDeleterHandle* pHandle, const SInputData* pInp pRes->uidList = pHandle->pParam->pUidList; pRes->skey = pHandle->pDeleter->deleteTimeRange.skey; pRes->ekey = pHandle->pDeleter->deleteTimeRange.ekey; + strcpy(pRes->tableFName, pHandle->pDeleter->tableFName); pRes->affectedRows = *(int64_t*)pColRes->pData; pBuf->useSize += pEntry->dataLen; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index f249321a76..8b1cbb5ae8 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -14,12 +14,12 @@ */ #include "executor.h" -#include "tref.h" #include "executorimpl.h" #include "planner.h" #include "tdatablock.h" -#include "vnode.h" +#include "tref.h" #include "tudf.h" +#include "vnode.h" static TdThreadOnce initPoolOnce = PTHREAD_ONCE_INIT; int32_t exchangeObjRefPool = -1; @@ -95,16 +95,6 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu } } -#if 0 -int32_t qStreamScanSnapshot(qTaskInfo_t tinfo) { - if (tinfo == NULL) { - return TSDB_CODE_QRY_APP_ERROR; - } - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - return doSetStreamBlock(pTaskInfo->pRoot, NULL, 0, STREAM_INPUT__TABLE_SCAN, 0, NULL); -} -#endif - int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input, int32_t type, bool assignUid) { return qSetMultiStreamInput(tinfo, input, 1, type, assignUid); } @@ -258,8 +248,8 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo } // todo refactor STableList - size_t bufLen = (pScanInfo->pGroupTags != NULL)? getTableTagsBufLen(pScanInfo->pGroupTags):0; - char* keyBuf = NULL; + size_t bufLen = (pScanInfo->pGroupTags != NULL) ? getTableTagsBufLen(pScanInfo->pGroupTags) : 0; + char* keyBuf = NULL; if (bufLen > 0) { keyBuf = taosMemoryMalloc(bufLen); if (keyBuf == NULL) { @@ -267,13 +257,13 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo } } - for(int32_t i = 0; i < taosArrayGetSize(qa); ++i) { + for (int32_t i = 0; i < taosArrayGetSize(qa); ++i) { uint64_t* uid = taosArrayGet(qa, i); STableKeyInfo keyInfo = {.uid = *uid, .groupId = 0}; if (bufLen > 0) { code = getGroupIdFromTagsVal(pScanInfo->readHandle.meta, keyInfo.uid, pScanInfo->pGroupTags, keyBuf, - &keyInfo.groupId); + &keyInfo.groupId); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -347,9 +337,12 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, } code = dsCreateDataSinker(pSubplan->pDataSink, handle, pSinkParam); + if(code != TSDB_CODE_SUCCESS){ + taosMemoryFreeClear(pSinkParam); + } } - _error: +_error: // if failed to add ref for all tables in this query, abort current query return code; } @@ -573,11 +566,6 @@ const SSchemaWrapper* qExtractSchemaFromStreamScanner(void* scanner) { return pInfo->tqReader->pSchemaWrapper; } -const STqOffset* qExtractStatusFromStreamScanner(void* scanner) { - SStreamScanInfo* pInfo = scanner; - return &pInfo->offset; -} - void* qStreamExtractMetaMsg(qTaskInfo_t tinfo) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE); @@ -600,12 +588,17 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, const STqOffsetVal* pOffset) { while (1) { uint8_t type = pOperator->operatorType; pOperator->status = OP_OPENED; - if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { - SStreamScanInfo* pInfo = pOperator->info; - if (pOffset->type == TMQ_OFFSET__LOG) { - STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; - tsdbReaderClose(pTSInfo->dataReader); - pTSInfo->dataReader = NULL; + // TODO add more check + if (type != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { + ASSERT(pOperator->numOfDownstream == 1); + pOperator = pOperator->pDownstream[0]; + } + + SStreamScanInfo* pInfo = pOperator->info; + if (pOffset->type == TMQ_OFFSET__LOG) { + STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; + tsdbReaderClose(pTSInfo->dataReader); + pTSInfo->dataReader = NULL; #if 0 if (tOffsetEqual(pOffset, &pTaskInfo->streamInfo.lastStatus) && pInfo->tqReader->pWalReader->curVersion != pOffset->version) { @@ -614,102 +607,74 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, const STqOffsetVal* pOffset) { ASSERT(0); } #endif - if (tqSeekVer(pInfo->tqReader, pOffset->version + 1) < 0) { + if (tqSeekVer(pInfo->tqReader, pOffset->version + 1) < 0) { + return -1; + } + ASSERT(pInfo->tqReader->pWalReader->curVersion == pOffset->version + 1); + } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { + /*pInfo->blockType = STREAM_INPUT__TABLE_SCAN;*/ + int64_t uid = pOffset->uid; + int64_t ts = pOffset->ts; + + if (uid == 0) { + if (taosArrayGetSize(pTaskInfo->tableqinfoList.pTableList) != 0) { + STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, 0); + uid = pTableInfo->uid; + ts = INT64_MIN; + } else { return -1; } - ASSERT(pInfo->tqReader->pWalReader->curVersion == pOffset->version + 1); - } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { - /*pInfo->blockType = STREAM_INPUT__TABLE_SCAN;*/ - int64_t uid = pOffset->uid; - int64_t ts = pOffset->ts; + } - if (uid == 0) { - if (taosArrayGetSize(pTaskInfo->tableqinfoList.pTableList) != 0) { - STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, 0); - uid = pTableInfo->uid; - ts = INT64_MIN; - } else { - return -1; - } - } - - /*if (pTaskInfo->streamInfo.lastStatus.type != TMQ_OFFSET__SNAPSHOT_DATA ||*/ - /*pTaskInfo->streamInfo.lastStatus.uid != uid || pTaskInfo->streamInfo.lastStatus.ts != ts) {*/ - STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; - int32_t tableSz = taosArrayGetSize(pTaskInfo->tableqinfoList.pTableList); + /*if (pTaskInfo->streamInfo.lastStatus.type != TMQ_OFFSET__SNAPSHOT_DATA ||*/ + /*pTaskInfo->streamInfo.lastStatus.uid != uid || pTaskInfo->streamInfo.lastStatus.ts != ts) {*/ + STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; + int32_t tableSz = taosArrayGetSize(pTaskInfo->tableqinfoList.pTableList); #ifndef NDEBUG - qDebug("switch to next table %ld (cursor %d), %ld rows returned", uid, pTableScanInfo->currentTable, - pInfo->pTableScanOp->resultInfo.totalRows); - pInfo->pTableScanOp->resultInfo.totalRows = 0; + qDebug("switch to next table %ld (cursor %d), %ld rows returned", uid, pTableScanInfo->currentTable, + pInfo->pTableScanOp->resultInfo.totalRows); + pInfo->pTableScanOp->resultInfo.totalRows = 0; #endif - bool found = false; - for (int32_t i = 0; i < tableSz; i++) { - STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, i); - if (pTableInfo->uid == uid) { - found = true; - pTableScanInfo->currentTable = i; - break; - } + bool found = false; + for (int32_t i = 0; i < tableSz; i++) { + STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, i); + if (pTableInfo->uid == uid) { + found = true; + pTableScanInfo->currentTable = i; + break; } - - // TODO after dropping table, table may be not found - ASSERT(found); - - if (pTableScanInfo->dataReader == NULL) { - if (tsdbReaderOpen(pTableScanInfo->readHandle.vnode, &pTableScanInfo->cond, - pTaskInfo->tableqinfoList.pTableList, &pTableScanInfo->dataReader, NULL) < 0 || - pTableScanInfo->dataReader == NULL) { - ASSERT(0); - } - } - - tsdbSetTableId(pTableScanInfo->dataReader, uid); - int64_t oldSkey = pTableScanInfo->cond.twindows.skey; - pTableScanInfo->cond.twindows.skey = ts + 1; - tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond); - pTableScanInfo->cond.twindows.skey = oldSkey; - pTableScanInfo->scanTimes = 0; - - qDebug("tsdb reader offset seek to uid %ld ts %ld, table cur set to %d , all table num %d", uid, ts, - pTableScanInfo->currentTable, tableSz); - /*}*/ - - } else { - ASSERT(0); } - return 0; + + // TODO after dropping table, table may be not found + ASSERT(found); + + if (pTableScanInfo->dataReader == NULL) { + if (tsdbReaderOpen(pTableScanInfo->readHandle.vnode, &pTableScanInfo->cond, + pTaskInfo->tableqinfoList.pTableList, &pTableScanInfo->dataReader, NULL) < 0 || + pTableScanInfo->dataReader == NULL) { + ASSERT(0); + } + } + + tsdbSetTableId(pTableScanInfo->dataReader, uid); + int64_t oldSkey = pTableScanInfo->cond.twindows.skey; + pTableScanInfo->cond.twindows.skey = ts + 1; + tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond); + pTableScanInfo->cond.twindows.skey = oldSkey; + pTableScanInfo->scanTimes = 0; + + qDebug("tsdb reader offset seek to uid %ld ts %ld, table cur set to %d , all table num %d", uid, ts, + pTableScanInfo->currentTable, tableSz); + /*}*/ + } else { - ASSERT(pOperator->numOfDownstream == 1); - pOperator = pOperator->pDownstream[0]; + ASSERT(0); } + return 0; } } return 0; } - - -#if 0 -int32_t qStreamPrepareTsdbScan(qTaskInfo_t tinfo, uint64_t uid, int64_t ts) { - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - - if (uid == 0) { - if (taosArrayGetSize(pTaskInfo->tableqinfoList.pTableList) != 0) { - STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, 0); - uid = pTableInfo->uid; - ts = INT64_MIN; - } - } - - return doPrepareScan(pTaskInfo->pRoot, uid, ts); -} - -int32_t qGetStreamScanStatus(qTaskInfo_t tinfo, uint64_t* uid, int64_t* ts) { - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - - return doGetScanStatus(pTaskInfo->pRoot, uid, ts); -} -#endif - diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 3f2f528a2d..962f37a8f7 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -3436,7 +3436,7 @@ void initBasicInfo(SOptrBasicInfo* pInfo, SSDataBlock* pBlock) { initResultRowInfo(&pInfo->resultRowInfo); } -static void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) { +void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) { if (pCtx == NULL) { return NULL; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 463ac0e69c..2015ae8f08 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1396,13 +1396,11 @@ SOperatorInfo* createRawScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNo static void destroyStreamScanOperatorInfo(void* param, int32_t numOfOutput) { SStreamScanInfo* pStreamScan = (SStreamScanInfo*)param; -#if 1 if (pStreamScan->pTableScanOp && pStreamScan->pTableScanOp->info) { STableScanInfo* pTableScanInfo = pStreamScan->pTableScanOp->info; destroyTableScanOperatorInfo(pTableScanInfo, numOfOutput); taosMemoryFreeClear(pStreamScan->pTableScanOp); } -#endif if (pStreamScan->tqReader) { tqCloseReader(pStreamScan->tqReader); } @@ -2855,7 +2853,8 @@ int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) { return TSDB_CODE_SUCCESS; } -SSDataBlock* getSortedTableMergeScanBlockData(SSortHandle* pHandle, SSDataBlock* pResBlock, int32_t capacity, SOperatorInfo* pOperator) { +SSDataBlock* getSortedTableMergeScanBlockData(SSortHandle* pHandle, SSDataBlock* pResBlock, int32_t capacity, + SOperatorInfo* pOperator) { STableMergeScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -2874,7 +2873,6 @@ SSDataBlock* getSortedTableMergeScanBlockData(SSortHandle* pHandle, SSDataBlock* } } - qDebug("%s get sorted row blocks, rows:%d", GET_TASKID(pTaskInfo), pResBlock->info.rows); return (pResBlock->info.rows > 0) ? pResBlock : NULL; } @@ -2905,7 +2903,8 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) { } SSDataBlock* pBlock = NULL; while (pInfo->tableStartIndex < tableListSize) { - pBlock = getSortedTableMergeScanBlockData(pInfo->pSortHandle, pInfo->pResBlock, pOperator->resultInfo.capacity, pOperator); + pBlock = getSortedTableMergeScanBlockData(pInfo->pSortHandle, pInfo->pResBlock, pOperator->resultInfo.capacity, + pOperator); if (pBlock != NULL) { pBlock->info.groupId = pInfo->groupId; pOperator->resultInfo.totalRows += pBlock->info.rows; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index c2cf19167a..9a82b194a9 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -773,9 +773,9 @@ int32_t binarySearch(void* keyList, int num, TSKEY key, int order, __get_value_f } int32_t comparePullWinKey(void* pKey, void* data, int32_t index) { - SArray* res = (SArray*)data; + SArray* res = (SArray*)data; SPullWindowInfo* pos = taosArrayGet(res, index); - SPullWindowInfo* pData = (SPullWindowInfo*) pKey; + SPullWindowInfo* pData = (SPullWindowInfo*)pKey; if (pData->window.skey == pos->window.skey) { if (pData->groupId > pos->groupId) { return 1; @@ -810,7 +810,7 @@ static int32_t savePullWindow(SPullWindowInfo* pPullInfo, SArray* pPullWins) { int32_t compareResKey(void* pKey, void* data, int32_t index) { SArray* res = (SArray*)data; SResKeyPos* pos = taosArrayGetP(res, index); - SWinRes* pData = (SWinRes*) pKey; + SWinRes* pData = (SWinRes*)pKey; if (pData->ts == *(int64_t*)pos->key) { if (pData->groupId > pos->groupId) { return 1; @@ -880,7 +880,7 @@ int64_t getWinReskey(void* data, int32_t index) { int32_t compareWinRes(void* pKey, void* data, int32_t index) { SArray* res = (SArray*)data; SWinRes* pos = taosArrayGetP(res, index); - SResKeyPos* pData = (SResKeyPos*) pKey; + SResKeyPos* pData = (SResKeyPos*)pKey; if (*(int64_t*)pData->key == pos->ts) { if (pData->groupId > pos->groupId) { return 1; @@ -1417,15 +1417,15 @@ static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval* SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); pGpDatas = (uint64_t*)pGpCol->pData; } - int32_t step = 0; - int32_t startPos = 0; + int32_t step = 0; + int32_t startPos = 0; for (int32_t i = 0; i < pBlock->info.rows; i++) { SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTsCols[i], pInterval, TSDB_ORDER_ASC); while (win.ekey <= endTsCols[i]) { uint64_t winGpId = pGpDatas ? pGpDatas[startPos] : pBlock->info.groupId; - bool res = doClearWindow(pAggSup, pSup1, (char*)&win.skey, sizeof(TSKEY), winGpId, numOfOutput); + bool res = doClearWindow(pAggSup, pSup1, (char*)&win.skey, sizeof(TSKEY), winGpId, numOfOutput); if (pUpWins && res) { SWinRes winRes = {.ts = win.skey, .groupId = winGpId}; taosArrayPush(pUpWins, &winRes); @@ -1596,7 +1596,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { } if (pBlock->info.type == STREAM_NORMAL) { - //set input version + // set input version pTaskInfo->version = pBlock->info.version; } @@ -1644,7 +1644,7 @@ static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput) { } static void freeItem(void* param) { - SGroupKeys *pKey = (SGroupKeys*) param; + SGroupKeys* pKey = (SGroupKeys*)param; taosMemoryFree(pKey->pData); } @@ -2347,10 +2347,10 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode initResultSizeInfo(&pOperator->resultInfo, 4096); pInfo->pFillColInfo = createFillColInfo(pExprInfo, numOfExprs, (SNodeListNode*)pInterpPhyNode->pFillValues); - pInfo->pRes = createResDataBlock(pPhyNode->pOutputDataBlockDesc); - pInfo->win = pInterpPhyNode->timeRange; + pInfo->pRes = createResDataBlock(pPhyNode->pOutputDataBlockDesc); + pInfo->win = pInterpPhyNode->timeRange; pInfo->interval.interval = pInterpPhyNode->interval; - pInfo->current = pInfo->win.skey; + pInfo->current = pInfo->win.skey; pOperator->name = "TimeSliceOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC; @@ -2970,8 +2970,8 @@ SSDataBlock* createSpecialDataBlock(EStreamType type) { pBlock->info.groupId = 0; pBlock->info.rows = 0; pBlock->info.type = type; - pBlock->info.rowSize = sizeof(TSKEY) + sizeof(TSKEY) + sizeof(uint64_t) + - sizeof(uint64_t) + sizeof(TSKEY) + sizeof(TSKEY); + pBlock->info.rowSize = + sizeof(TSKEY) + sizeof(TSKEY) + sizeof(uint64_t) + sizeof(uint64_t) + sizeof(TSKEY) + sizeof(TSKEY); pBlock->pDataBlock = taosArrayInit(6, sizeof(SColumnInfoData)); SColumnInfoData infoData = {0}; @@ -3147,6 +3147,8 @@ void destroyStreamSessionAggOperatorInfo(void* param, int32_t numOfOutput) { blockDataDestroy(pInfo->pDelRes); blockDataDestroy(pInfo->pWinBlock); blockDataDestroy(pInfo->pUpdateRes); + destroySqlFunctionCtx(pInfo->pDummyCtx, 0); + taosHashCleanup(pInfo->pStDeleted); taosMemoryFreeClear(param); } @@ -4323,10 +4325,10 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl } else { return; } - + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; blockDataEnsureCapacity(pAggSup->pScanBlock, pSDataBlock->info.rows); - SColumnInfoData* pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId); + SColumnInfoData* pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId); for (int32_t i = 0; i < pSDataBlock->info.rows; i += winRows) { if (pInfo->ignoreExpiredData && isOverdue(tsCols[i], &pInfo->twAggSup)) { i++; @@ -4341,7 +4343,7 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl &allEqual, pInfo->pSeDeleted); if (!allEqual) { appendOneRow(pAggSup->pScanBlock, &pCurWin->winInfo.win.skey, &pCurWin->winInfo.win.ekey, - &pSDataBlock->info.groupId); + &pSDataBlock->info.groupId); taosHashRemove(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition)); deleteWindow(pAggSup->pCurWins, winIndex); continue; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 4a578e9b86..1424a522c4 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2975,7 +2975,7 @@ static int32_t buildCreateDbReq(STranslateContext* pCxt, SCreateDatabaseStmt* pS pReq->daysToKeep2 = pStmt->pOptions->keep[2]; pReq->minRows = pStmt->pOptions->minRowsPerBlock; pReq->maxRows = pStmt->pOptions->maxRowsPerBlock; - pReq->fsyncPeriod = pStmt->pOptions->fsyncPeriod; + pReq->walFsyncPeriod = pStmt->pOptions->fsyncPeriod; pReq->walLevel = pStmt->pOptions->walLevel; pReq->precision = pStmt->pOptions->precision; pReq->compression = pStmt->pOptions->compressionLevel; @@ -3334,7 +3334,7 @@ static void buildAlterDbReq(STranslateContext* pCxt, SAlterDatabaseStmt* pStmt, pReq->daysToKeep0 = pStmt->pOptions->keep[0]; pReq->daysToKeep1 = pStmt->pOptions->keep[1]; pReq->daysToKeep2 = pStmt->pOptions->keep[2]; - pReq->fsyncPeriod = pStmt->pOptions->fsyncPeriod; + pReq->walFsyncPeriod = pStmt->pOptions->fsyncPeriod; pReq->walLevel = pStmt->pOptions->walLevel; pReq->strict = pStmt->pOptions->strict; pReq->cacheLast = pStmt->pOptions->cacheModel; diff --git a/source/libs/parser/test/parAlterToBalanceTest.cpp b/source/libs/parser/test/parAlterToBalanceTest.cpp index 39052738f6..17a457625b 100644 --- a/source/libs/parser/test/parAlterToBalanceTest.cpp +++ b/source/libs/parser/test/parAlterToBalanceTest.cpp @@ -106,7 +106,7 @@ TEST_F(ParserInitialATest, alterDatabase) { expect.daysToKeep0 = -1; expect.daysToKeep1 = -1; expect.daysToKeep2 = -1; - expect.fsyncPeriod = -1; + expect.walFsyncPeriod = -1; expect.walLevel = -1; expect.strict = -1; expect.cacheLast = -1; @@ -123,7 +123,7 @@ TEST_F(ParserInitialATest, alterDatabase) { expect.daysToKeep1 = (-1 == daysToKeep1 ? expect.daysToKeep0 : daysToKeep1); expect.daysToKeep2 = (-1 == daysToKeep1 ? expect.daysToKeep1 : daysToKeep2); }; - auto setAlterDbFsync = [&](int32_t fsync) { expect.fsyncPeriod = fsync; }; + auto setAlterDbFsync = [&](int32_t fsync) { expect.walFsyncPeriod = fsync; }; auto setAlterDbWal = [&](int8_t wal) { expect.walLevel = wal; }; auto setAlterDbStrict = [&](int8_t strict) { expect.strict = strict; }; auto setAlterDbCacheModel = [&](int8_t cacheModel) { expect.cacheLast = cacheModel; }; @@ -141,7 +141,7 @@ TEST_F(ParserInitialATest, alterDatabase) { ASSERT_EQ(req.daysToKeep0, expect.daysToKeep0); ASSERT_EQ(req.daysToKeep1, expect.daysToKeep1); ASSERT_EQ(req.daysToKeep2, expect.daysToKeep2); - ASSERT_EQ(req.fsyncPeriod, expect.fsyncPeriod); + ASSERT_EQ(req.walFsyncPeriod, expect.walFsyncPeriod); ASSERT_EQ(req.walLevel, expect.walLevel); ASSERT_EQ(req.strict, expect.strict); ASSERT_EQ(req.cacheLast, expect.cacheLast); diff --git a/source/libs/parser/test/parInitialCTest.cpp b/source/libs/parser/test/parInitialCTest.cpp index e62beb3a35..c7dc55b458 100644 --- a/source/libs/parser/test/parInitialCTest.cpp +++ b/source/libs/parser/test/parInitialCTest.cpp @@ -102,7 +102,7 @@ TEST_F(ParserInitialCTest, createDatabase) { expect.cacheLastSize = TSDB_DEFAULT_CACHE_SIZE; expect.compression = TSDB_DEFAULT_COMP_LEVEL; expect.daysPerFile = TSDB_DEFAULT_DAYS_PER_FILE; - expect.fsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD; + expect.walFsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD; expect.maxRows = TSDB_DEFAULT_MAXROWS_FBLOCK; expect.minRows = TSDB_DEFAULT_MINROWS_FBLOCK; expect.daysToKeep0 = TSDB_DEFAULT_KEEP; @@ -124,7 +124,7 @@ TEST_F(ParserInitialCTest, createDatabase) { auto setDbCachelastSize = [&](int8_t cachelastSize) { expect.cacheLastSize = cachelastSize; }; auto setDbCompressionFunc = [&](int8_t compressionLevel) { expect.compression = compressionLevel; }; auto setDbDaysFunc = [&](int32_t daysPerFile) { expect.daysPerFile = daysPerFile; }; - auto setDbFsyncFunc = [&](int32_t fsyncPeriod) { expect.fsyncPeriod = fsyncPeriod; }; + auto setDbFsyncFunc = [&](int32_t fsyncPeriod) { expect.walFsyncPeriod = fsyncPeriod; }; auto setDbMaxRowsFunc = [&](int32_t maxRowsPerBlock) { expect.maxRows = maxRowsPerBlock; }; auto setDbMinRowsFunc = [&](int32_t minRowsPerBlock) { expect.minRows = minRowsPerBlock; }; auto setDbKeepFunc = [&](int32_t keep0, int32_t keep1 = 0, int32_t keep2 = 0) { @@ -175,7 +175,7 @@ TEST_F(ParserInitialCTest, createDatabase) { ASSERT_EQ(req.daysToKeep2, expect.daysToKeep2); ASSERT_EQ(req.minRows, expect.minRows); ASSERT_EQ(req.maxRows, expect.maxRows); - ASSERT_EQ(req.fsyncPeriod, expect.fsyncPeriod); + ASSERT_EQ(req.walFsyncPeriod, expect.walFsyncPeriod); ASSERT_EQ(req.walLevel, expect.walLevel); ASSERT_EQ(req.precision, expect.precision); ASSERT_EQ(req.compression, expect.compression); diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index e09887e651..df57d0fef1 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -283,7 +283,7 @@ int32_t qwGetDeleteResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SDeleteRes *pRes pRes->skey = pDelRes->skey; pRes->ekey = pDelRes->ekey; pRes->affectedRows = pDelRes->affectedRows; - + strcpy(pRes->tableFName, pDelRes->tableFName); taosMemoryFree(output.pData); return TSDB_CODE_SUCCESS; diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index b794cb91f5..67050241e3 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -230,6 +230,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa SVDeleteRsp rsp = {0}; tDecoderInit(&coder, msg, msgSize); tDecodeSVDeleteRsp(&coder, &rsp); + tDecoderClear(&coder); atomic_add_fetch_32(&pJob->resNumOfRows, rsp.affectedRows); SCH_TASK_DLOG("delete succeed, affectedRows:%" PRId64, rsp.affectedRows); diff --git a/source/libs/stream/inc/streamInc.h b/source/libs/stream/inc/streamInc.h index 093242c610..06bd6539fd 100644 --- a/source/libs/stream/inc/streamInc.h +++ b/source/libs/stream/inc/streamInc.h @@ -38,14 +38,14 @@ int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum); int32_t streamDispatch(SStreamTask* pTask, SMsgCb* pMsgCb); int32_t streamDispatchReqToData(const SStreamDispatchReq* pReq, SStreamDataBlock* pData); int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock* pData); -int32_t streamBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* data, SRpcMsg* pMsg, SEpSet** ppEpSet); +int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* data); int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock); int32_t tEncodeStreamRetrieveReq(SEncoder* pEncoder, const SStreamRetrieveReq* pReq); SStreamQueueItem* streamAppendQueueItem(SStreamQueueItem* dst, SStreamQueueItem* elem); -void streamFreeQitem(SStreamQueueItem* data); +void streamFreeQitem(SStreamQueueItem* data); #ifdef __cplusplus } diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 99a06575a9..8c9e8ca2db 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -219,6 +219,12 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp) { qDebug("task %d receive dispatch rsp", pTask->taskId); + if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) { + int32_t leftRsp = atomic_sub_fetch_32(&pTask->shuffleDispatcher.waitingRspCnt, 1); + qDebug("task %d is shuffle, left waiting rsp %d", pTask->taskId, leftRsp); + if (leftRsp > 0) return 0; + } + int8_t old = atomic_exchange_8(&pTask->outputStatus, pRsp->inputStatus); ASSERT(old == TASK_OUTPUT_STATUS__WAIT); if (pRsp->inputStatus == TASK_INPUT_STATUS__BLOCKED) { diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 5d4adb2896..2e33632f12 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -198,6 +198,158 @@ static int32_t streamAddBlockToDispatchMsg(const SSDataBlock* pBlock, SStreamDis return 0; } +int32_t streamDispatchOneReq(SStreamTask* pTask, const SStreamDispatchReq* pReq, int32_t vgId, SEpSet* pEpSet) { + void* buf = NULL; + int32_t code = -1; + SRpcMsg msg = {0}; + + // serialize + int32_t tlen; + tEncodeSize(tEncodeStreamDispatchReq, pReq, tlen, code); + if (code < 0) goto FAIL; + code = -1; + buf = rpcMallocCont(sizeof(SMsgHead) + tlen); + if (buf == NULL) { + goto FAIL; + } + + ((SMsgHead*)buf)->vgId = htonl(vgId); + void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + + SEncoder encoder; + tEncoderInit(&encoder, abuf, tlen); + if ((code = tEncodeStreamDispatchReq(&encoder, pReq)) < 0) { + goto FAIL; + } + tEncoderClear(&encoder); + + msg.contLen = tlen + sizeof(SMsgHead); + msg.pCont = buf; + msg.msgType = pTask->dispatchMsgType; + + tmsgSendReq(pEpSet, &msg); + + code = 0; +FAIL: + if (code < 0 && buf) rpcFreeCont(buf); + return 0; +} + +int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pData) { + int32_t code = -1; + int32_t blockNum = taosArrayGetSize(pData->blocks); + ASSERT(blockNum != 0); + + if (pTask->dispatchType == TASK_DISPATCH__FIXED) { + SStreamDispatchReq req = { + .streamId = pTask->streamId, + .dataSrcVgId = pData->srcVgId, + .upstreamTaskId = pTask->taskId, + .upstreamChildId = pTask->selfChildId, + .upstreamNodeId = pTask->nodeId, + .blockNum = blockNum, + }; + + req.data = taosArrayInit(blockNum, sizeof(void*)); + req.dataLen = taosArrayInit(blockNum, sizeof(int32_t)); + if (req.data == NULL || req.dataLen == NULL) { + goto FAIL_FIXED_DISPATCH; + } + + for (int32_t i = 0; i < blockNum; i++) { + SSDataBlock* pDataBlock = taosArrayGet(pData->blocks, i); + if (streamAddBlockToDispatchMsg(pDataBlock, &req) < 0) { + goto FAIL_FIXED_DISPATCH; + } + } + int32_t vgId = pTask->fixedEpDispatcher.nodeId; + SEpSet* pEpSet = &pTask->fixedEpDispatcher.epSet; + int32_t downstreamTaskId = pTask->fixedEpDispatcher.taskId; + + req.taskId = downstreamTaskId; + + qDebug("dispatch from task %d (child id %d) to down stream task %d in vnode %d", pTask->taskId, pTask->selfChildId, + downstreamTaskId, vgId); + + if (streamDispatchOneReq(pTask, &req, vgId, pEpSet) < 0) { + goto FAIL_FIXED_DISPATCH; + } + code = 0; + FAIL_FIXED_DISPATCH: + taosArrayDestroy(req.data); + taosArrayDestroy(req.dataLen); + return code; + + } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) { + SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; + ASSERT(pTask->shuffleDispatcher.waitingRspCnt == 0); + int32_t vgSz = taosArrayGetSize(vgInfo); + SStreamDispatchReq* pReqs = taosMemoryCalloc(vgSz, sizeof(SStreamDispatchReq)); + if (pReqs == NULL) { + return -1; + } + + for (int32_t i = 0; i < vgSz; i++) { + pReqs[i].streamId = pTask->streamId; + pReqs[i].dataSrcVgId = pData->srcVgId; + pReqs[i].upstreamTaskId = pTask->taskId; + pReqs[i].upstreamChildId = pTask->selfChildId; + pReqs[i].upstreamNodeId = pTask->nodeId; + pReqs[i].blockNum = 0; + pReqs[i].data = taosArrayInit(0, sizeof(void*)); + pReqs[i].dataLen = taosArrayInit(0, sizeof(int32_t)); + if (pReqs[i].data == NULL || pReqs[i].dataLen == NULL) { + goto FAIL_SHUFFLE_DISPATCH; + } + } + for (int32_t i = 0; i < blockNum; i++) { + SSDataBlock* pDataBlock = taosArrayGet(pData->blocks, i); + char* ctbName = buildCtbNameByGroupId(pTask->shuffleDispatcher.stbFullName, pDataBlock->info.groupId); + + // TODO: get hash function by hashMethod + uint32_t hashValue = MurmurHash3_32(ctbName, strlen(ctbName)); + + // TODO: optimize search + int32_t j; + for (j = 0; j < vgSz; j++) { + SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, j); + ASSERT(pVgInfo->vgId > 0); + if (hashValue >= pVgInfo->hashBegin && hashValue <= pVgInfo->hashEnd) { + if (streamAddBlockToDispatchMsg(pDataBlock, &pReqs[j]) < 0) { + goto FAIL_SHUFFLE_DISPATCH; + } + pReqs[j].taskId = pVgInfo->taskId; + pReqs[j].blockNum++; + break; + } + } + } + for (int32_t i = 0; i < vgSz; i++) { + if (pReqs[i].blockNum > 0) { + // send + SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i); + if (streamDispatchOneReq(pTask, &pReqs[i], pVgInfo->vgId, &pVgInfo->epSet) < 0) { + goto FAIL_SHUFFLE_DISPATCH; + } + pTask->shuffleDispatcher.waitingRspCnt++; + } + } + code = 0; + FAIL_SHUFFLE_DISPATCH: + if (pReqs) { + for (int32_t i = 0; i < vgSz; i++) { + taosArrayDestroy(pReqs[i].data); + taosArrayDestroy(pReqs[i].dataLen); + } + taosMemoryFree(pReqs); + } + return code; + } else { + ASSERT(0); + } + return 0; +} + int32_t streamBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* data, SRpcMsg* pMsg, SEpSet** ppEpSet) { void* buf = NULL; int32_t code = -1; @@ -262,29 +414,7 @@ int32_t streamBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* data, qDebug("dispatch from task %d (child id %d) to down stream task %d in vnode %d", pTask->taskId, pTask->selfChildId, downstreamTaskId, vgId); - // serialize - int32_t tlen; - tEncodeSize(tEncodeStreamDispatchReq, &req, tlen, code); - if (code < 0) goto FAIL; - code = -1; - buf = rpcMallocCont(sizeof(SMsgHead) + tlen); - if (buf == NULL) { - goto FAIL; - } - - ((SMsgHead*)buf)->vgId = htonl(vgId); - void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); - - SEncoder encoder; - tEncoderInit(&encoder, abuf, tlen); - if ((code = tEncodeStreamDispatchReq(&encoder, &req)) < 0) { - goto FAIL; - } - tEncoderClear(&encoder); - - pMsg->contLen = tlen + sizeof(SMsgHead); - pMsg->pCont = buf; - pMsg->msgType = pTask->dispatchMsgType; + streamDispatchOneReq(pTask, &req, vgId, *ppEpSet); code = 0; FAIL: @@ -314,6 +444,18 @@ int32_t streamDispatch(SStreamTask* pTask, SMsgCb* pMsgCb) { qDebug("stream continue dispatching: task %d", pTask->taskId); + int32_t code = 0; + if (streamDispatchAllBlocks(pTask, pBlock) < 0) { + ASSERT(0); + code = -1; + // TODO set status fail + goto FREE; + } + /*atomic_store_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL);*/ +FREE: + taosArrayDestroyEx(pBlock->blocks, (FDelete)blockDataFreeRes); + taosFreeQitem(pBlock); +#if 0 SRpcMsg dispatchMsg = {0}; SEpSet* pEpSet = NULL; if (streamBuildDispatchMsg(pTask, pBlock, &dispatchMsg, &pEpSet) < 0) { @@ -325,5 +467,6 @@ int32_t streamDispatch(SStreamTask* pTask, SMsgCb* pMsgCb) { taosFreeQitem(pBlock); tmsgSendReq(pEpSet, &dispatchMsg); - return 0; +#endif + return code; } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 935d89b99b..ceca201506 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -2501,19 +2501,15 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg, SyncI // if mulit replica, start replicate right now if (ths->replicaNum > 1) { syncNodeReplicate(ths); - } - // pre commit - syncNodePreCommit(ths, pEntry, 0); + // pre commit + syncNodePreCommit(ths, pEntry, 0); + } // if only myself, maybe commit right now if (ths->replicaNum == 1) { syncMaybeAdvanceCommitIndex(ths); } - - } else { - // pre commit - syncNodePreCommit(ths, pEntry, 0); } if (pRetIndex != NULL) { diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index f256c96037..a81d6db80f 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -7,8 +7,7 @@ * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * + * FITNESS FOR A PARTICULAR PURPOSE. * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ @@ -211,6 +210,7 @@ typedef struct SConnBuffer { char* buf; int len; int cap; + int left; int total; } SConnBuffer; @@ -282,6 +282,8 @@ int transClearBuffer(SConnBuffer* buf); int transDestroyBuffer(SConnBuffer* buf); int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf); bool transReadComplete(SConnBuffer* connBuf); +int transResetBuffer(SConnBuffer* connBuf); +int transDumpFromBuffer(SConnBuffer* connBuf, char** buf); int transSetConnOption(uv_tcp_t* stream); diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index c747e69339..62277a7569 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -17,6 +17,7 @@ #ifdef USE_UV #include #endif +// clang-format off #include "zlib.h" #include "thttp.h" #include "taoserror.h" @@ -174,7 +175,7 @@ int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32 #else int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32_t contLen, EHttpCompFlag flag) { - int32_t code = -1; + int32_t code = -1; TdSocketPtr pSocket = NULL; uint32_t ip = taosGetIpv4FromFqdn(server); @@ -231,4 +232,5 @@ SEND_OVER: return code; } -#endif \ No newline at end of file +// clang-format on +#endif diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 70d56dca13..54ffcabc8d 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -323,7 +323,8 @@ void cliHandleResp(SCliConn* conn) { SCliThrd* pThrd = conn->hostThrd; STrans* pTransInst = pThrd->pTransInst; - STransMsgHead* pHead = (STransMsgHead*)(conn->readBuf.buf); + STransMsgHead* pHead = NULL; + transDumpFromBuffer(&conn->readBuf, (char**)&pHead); pHead->code = htonl(pHead->code); pHead->msgLen = htonl(pHead->msgLen); @@ -366,7 +367,6 @@ void cliHandleResp(SCliConn* conn) { } } // buf's mem alread translated to transMsg.pCont - transClearBuffer(&conn->readBuf); if (!CONN_NO_PERSIST_BY_APP(conn)) { transMsg.info.handle = (void*)conn->refId; tDebug("%s conn %p ref by app", CONN_GET_INST_LABEL(conn), conn); @@ -636,6 +636,8 @@ static SCliConn* cliCreateConn(SCliThrd* pThrd) { transReqQueueInit(&conn->wreqQueue); transQueueInit(&conn->cliMsgs, NULL); + + transInitBuffer(&conn->readBuf); QUEUE_INIT(&conn->q); conn->hostThrd = pThrd; conn->status = ConnNormal; @@ -651,8 +653,9 @@ static void cliDestroyConn(SCliConn* conn, bool clear) { QUEUE_REMOVE(&conn->q); QUEUE_INIT(&conn->q); transRemoveExHandle(transGetRefMgt(), conn->refId); - conn->refId = -1; + transDestroyBuffer(&conn->readBuf); + conn->refId = -1; if (conn->task != NULL) transDQCancel(((SCliThrd*)conn->hostThrd)->timeoutQueue, conn->task); if (clear) { @@ -678,7 +681,6 @@ static void cliDestroy(uv_handle_t* handle) { tTrace("%s conn %p destroy successfully", CONN_GET_INST_LABEL(conn), conn); transReqQueueClear(&conn->wreqQueue); - transDestroyBuffer(&conn->readBuf); taosMemoryFree(conn); } static bool cliHandleNoResp(SCliConn* conn) { diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 155cdd1062..fb59aafb33 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -16,6 +16,8 @@ #include "transComm.h" +#define BUFFER_CAP 4096 + static TdThreadOnce transModuleInit = PTHREAD_ONCE_INIT; static int32_t refMgt; @@ -111,12 +113,56 @@ int transGetSockDebugInfo(struct sockaddr* sockname, char* dst) { return r; } int transInitBuffer(SConnBuffer* buf) { - transClearBuffer(buf); + buf->cap = BUFFER_CAP; + buf->buf = taosMemoryCalloc(1, BUFFER_CAP); + buf->left = -1; + buf->len = 0; + buf->total = 0; return 0; } +int transDestroyBuffer(SConnBuffer* buf) { + taosMemoryFree(buf->buf); + return 0; +} + int transClearBuffer(SConnBuffer* buf) { - memset(buf, 0, sizeof(*buf)); - buf->total = -1; + SConnBuffer* p = buf; + if (p->cap > BUFFER_CAP) { + p->cap = BUFFER_CAP; + p->buf = taosMemoryRealloc(p->buf, BUFFER_CAP); + } + p->left = -1; + p->len = 0; + p->total = 0; + return 0; +} + +int transDumpFromBuffer(SConnBuffer* connBuf, char** buf) { + SConnBuffer* p = connBuf; + if (p->left != 0) { + return -1; + } + int total = connBuf->total; + *buf = taosMemoryCalloc(1, total); + memcpy(*buf, p->buf, total); + + transResetBuffer(connBuf); + return total; +} + +int transResetBuffer(SConnBuffer* connBuf) { + SConnBuffer* p = connBuf; + if (p->total <= p->len) { + int left = p->len - p->total; + memmove(p->buf, p->buf + p->total, left); + p->left = -1; + p->total = 0; + p->len = left; + } else { + p->left = -1; + p->total = 0; + p->len = 0; + } return 0; } int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf) { @@ -126,54 +172,39 @@ int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf) { * |<------STransMsgHead------->|<-------------------userdata--------------->|<-----auth data----->|<----user * info--->| */ - static const int CAPACITY = sizeof(STransMsgHead); - SConnBuffer* p = connBuf; - if (p->cap == 0) { - p->buf = (char*)taosMemoryCalloc(CAPACITY, sizeof(char)); - tTrace("internal malloc mem:%p, size:%d", p->buf, CAPACITY); - p->len = 0; - p->cap = CAPACITY; - p->total = -1; - uvBuf->base = p->buf; - uvBuf->len = CAPACITY; - } else if (p->total == -1 && p->len < CAPACITY) { - uvBuf->base = p->buf + p->len; - uvBuf->len = CAPACITY - p->len; - } else { - p->cap = p->total; - p->buf = taosMemoryRealloc(p->buf, p->cap); - tTrace("internal realloc mem:%p, size:%d", p->buf, p->cap); - - uvBuf->base = p->buf + p->len; + uvBuf->base = p->buf + p->len; + if (p->left == -1) { uvBuf->len = p->cap - p->len; + } else { + if (p->left < p->cap - p->len) { + uvBuf->len = p->left; + } else { + p->buf = taosMemoryRealloc(p->buf, p->left + p->len); + uvBuf->base = p->buf + p->len; + uvBuf->len = p->left; + } } return 0; } // check whether already read complete bool transReadComplete(SConnBuffer* connBuf) { - if (connBuf->total == -1 && connBuf->len >= sizeof(STransMsgHead)) { - STransMsgHead head; - memcpy((char*)&head, connBuf->buf, sizeof(head)); - int32_t msgLen = (int32_t)htonl(head.msgLen); - connBuf->total = msgLen; + SConnBuffer* p = connBuf; + if (p->len >= sizeof(STransMsgHead)) { + if (p->left == -1) { + STransMsgHead head; + memcpy((char*)&head, connBuf->buf, sizeof(head)); + int32_t msgLen = (int32_t)htonl(head.msgLen); + p->total = msgLen; + } + if (p->total >= p->len) { + p->left = p->total - p->len; + } else { + p->left = 0; + } } - if (connBuf->len == connBuf->cap && connBuf->total == connBuf->cap) { - return true; - } - return false; -} -int transPackMsg(STransMsgHead* msgHead, bool sercured, bool auth) { return 0; } - -int transUnpackMsg(STransMsgHead* msgHead) { return 0; } -int transDestroyBuffer(SConnBuffer* buf) { - if (buf->cap > 0) { - taosMemoryFreeClear(buf->buf); - } - transClearBuffer(buf); - - return 0; + return p->left == 0 ? true : false; } int transSetConnOption(uv_tcp_t* stream) { diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index fe7ab47fee..e360926b40 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -212,9 +212,10 @@ static void uvHandleActivityTimeout(uv_timer_t* handle) { } static void uvHandleReq(SSvrConn* pConn) { - SConnBuffer* pBuf = &pConn->readBuf; - char* msg = pBuf->buf; - uint32_t msgLen = pBuf->len; + STransMsgHead* msg = NULL; + int msgLen = 0; + + msgLen = transDumpFromBuffer(&pConn->readBuf, (char**)&msg); STransMsgHead* pHead = (STransMsgHead*)msg; pHead->code = htonl(pHead->code); @@ -761,6 +762,7 @@ static SSvrConn* createConn(void* hThrd) { memset(&pConn->regArg, 0, sizeof(pConn->regArg)); pConn->broken = false; pConn->status = ConnNormal; + transInitBuffer(&pConn->readBuf); SExHandle* exh = taosMemoryMalloc(sizeof(SExHandle)); exh->handle = pConn; diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index 047354c4aa..c939c8c436 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -83,7 +83,16 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { // set config memcpy(&pWal->cfg, pCfg, sizeof(SWalCfg)); + pWal->fsyncSeq = pCfg->fsyncPeriod / 1000; + if (pWal->cfg.retentionSize > 0) { + pWal->cfg.retentionSize *= 1024; + } + + if (pWal->cfg.segSize > 0) { + pWal->cfg.segSize *= 1024; + } + if (pWal->fsyncSeq <= 0) pWal->fsyncSeq = 1; tstrncpy(pWal->path, path, sizeof(pWal->path)); diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index d869e6e2ce..491e5b0e08 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -401,6 +401,7 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy pWal->writeHead.head.version = index; pWal->writeHead.head.bodyLen = bodyLen; pWal->writeHead.head.msgType = msgType; + pWal->writeHead.head.ingestTs = taosGetTimestampMs(); // sync info for sync module pWal->writeHead.head.syncMeta = syncMeta; @@ -457,14 +458,14 @@ int64_t walAppendLog(SWal *pWal, tmsg_t msgType, SWalSyncInfo syncMeta, const vo return -1; } - if (pWal->pIdxFile == NULL || pWal->pIdxFile == NULL || pWal->writeCur < 0) { + if (pWal->pLogFile == NULL || pWal->pIdxFile == NULL || pWal->writeCur < 0) { if (walInitWriteFile(pWal) < 0) { taosThreadMutexUnlock(&pWal->mutex); return -1; } } - ASSERT(pWal->pIdxFile != NULL && pWal->pLogFile != NULL && pWal->writeCur >= 0); + ASSERT(pWal->pLogFile != NULL && pWal->pIdxFile != NULL && pWal->writeCur >= 0); if (walWriteImpl(pWal, index, msgType, syncMeta, body, bodyLen) < 0) { taosThreadMutexUnlock(&pWal->mutex); diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 8450e8baea..fa94bc6a13 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -374,9 +374,10 @@ int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) { size_t size = 0; int32_t done = 0; int32_t code = -1; + float coreCount = 0; TdFilePtr pFile = taosOpenFile("/proc/cpuinfo", TD_FILE_READ | TD_FILE_STREAM); - if (pFile == NULL) return false; + if (pFile == NULL) return code; while (done != 3 && (size = taosGetLineFile(pFile, &line)) != -1) { line[size - 1] = '\0'; @@ -390,11 +391,26 @@ int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) { *numOfCores = atof(v); done |= 2; } + if (strncmp(line, "processor", 9) == 0) coreCount += 1; } if (line != NULL) taosMemoryFree(line); taosCloseFile(&pFile); + if (code != 0) { + TdFilePtr pFile1 = taosOpenFile("/proc/device-tree/model", TD_FILE_READ | TD_FILE_STREAM); + if (pFile1 == NULL) return code; + taosGetsFile(pFile1, maxLen, cpuModel); + taosCloseFile(&pFile1); + code = 0; + done |= 1; + } + + if ((done & 2) == 0) { + *numOfCores = coreCount; + done |= 2; + } + return code; #endif } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index ad6eff3c12..3b95adc970 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -195,8 +195,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_PASS_FORMAT, "Invalid password form TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_USER_FROM_CONN, "Can not get user from conn") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_USERS, "Too many users") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ALTER_OPER, "Invalid alter operation") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_AUTH_FAILURE, "Authentication failure") -// mnode-stable-part1 +//mnode-stable-part1 TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_ALREADY_EXIST, "STable already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_NOT_EXIST, "STable not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_TAGS, "Too many tags") diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 0c19e4a2fe..4ec3a14cc8 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -165,13 +165,13 @@ # TD-17661 ./test.sh -f tsim/parser/where.sim # ---- query -./test.sh -f tsim/query/interval.sim -./test.sh -f tsim/query/interval-offset.sim -./test.sh -f tsim/query/scalarFunction.sim ./test.sh -f tsim/query/charScalarFunction.sim -./test.sh -f tsim/query/explain.sim -./test.sh -f tsim/query/session.sim +# ./test.sh -f tsim/query/explain.sim +./test.sh -f tsim/query/interval-offset.sim +./test.sh -f tsim/query/interval.sim +./test.sh -f tsim/query/scalarFunction.sim ./test.sh -f tsim/query/scalarNull.sim +./test.sh -f tsim/query/session.sim ./test.sh -f tsim/query/udf.sim # ---- qnode diff --git a/tests/script/tsim/db/alter_option.sim b/tests/script/tsim/db/alter_option.sim index 1648eb56fa..0590355ccf 100644 --- a/tests/script/tsim/db/alter_option.sim +++ b/tests/script/tsim/db/alter_option.sim @@ -93,19 +93,34 @@ endi if $data12_db != 8000 then # maxrows return -1 endi -if $data13_db != 2 then # wal +if $data13_db != 0 then # comp return -1 endi -if $data14_db != 1000 then # fsync +if $data14_db != ns then # precision return -1 endi -if $data15_db != 0 then # comp +if $data18_db != both then # cache_model return -1 endi -if $data16_db != both then # cachelast +if $data19_db != 1 then # cash_size return -1 endi -if $data17_db != ns then # precision +if $data20_db != 2 then # wal level + return -1 +endi +if $data21_db != 1000 then # wal fsyncperiod + return -1 +endi +if $data22_db != 0 then # + return -1 +endi +if $data23_db != 0 then # + return -1 +endi +if $data24_db != 0 then # + return -1 +endi +if $data25_db != 0 then # return -1 endi @@ -284,14 +299,14 @@ sql_error alter database db maxrows 10 # little than minrows print ============== step wal sql alter database db wal 1 sql show databases -print wal $data13_db -if $data13_db != 1 then +print wal $data20_db +if $data20_db != 1 then return -1 endi sql alter database db wal 2 sql show databases -print wal $data13_db -if $data13_db != 2 then +print wal $data20_db +if $data20_db != 2 then return -1 endi @@ -303,20 +318,20 @@ sql_error alter database db wal -1 print ============== modify fsync sql alter database db fsync 2000 sql show databases -print fsync $data14_db -if $data14_db != 2000 then +print fsync $data21_db +if $data21_db != 2000 then return -1 endi sql alter database db fsync 500 sql show databases -print fsync $data14_db -if $data14_db != 500 then +print fsync $data21_db +if $data21_db != 500 then return -1 endi sql alter database db fsync 0 sql show databases -print fsync $data14_db -if $data14_db != 0 then +print fsync $data21_db +if $data21_db != 0 then return -1 endi sql_error alter database db fsync 180001 @@ -335,32 +350,32 @@ sql_error alter database db comp -1 print ============== modify cachelast [0, 1, 2, 3] sql alter database db cachemodel 'last_value' sql show databases -print cachelast $data16_db -if $data16_db != last_value then +print cachelast $data18_db +if $data18_db != last_value then return -1 endi sql alter database db cachemodel 'last_row' sql show databases -print cachelast $data16_db -if $data16_db != last_row then +print cachelast $data18_db +if $data18_db != last_row then return -1 endi sql alter database db cachemodel 'none' sql show databases -print cachelast $data16_db -if $data16_db != none then +print cachelast $data18_db +if $data18_db != none then return -1 endi sql alter database db cachemodel 'last_value' sql show databases -print cachelast $data16_db -if $data16_db != last_value then +print cachelast $data18_db +if $data18_db != last_value then return -1 endi sql alter database db cachemodel 'both' sql show databases -print cachelast $data16_db -if $data16_db != both then +print cachelast $data18_db +if $data18_db != both then return -1 endi diff --git a/tests/script/tsim/db/create_all_options.sim b/tests/script/tsim/db/create_all_options.sim index 9de44ca335..259044cb98 100644 --- a/tests/script/tsim/db/create_all_options.sim +++ b/tests/script/tsim/db/create_all_options.sim @@ -113,21 +113,22 @@ endi if $data12_db != 4096 then # maxrows return -1 endi -if $data13_db != 1 then # wal +if $data13_db != 2 then # comp return -1 endi -if $data14_db != 3000 then # fsync +if $data14_db != ms then # precision return -1 endi -if $data15_db != 2 then # comp +if $data18_db != none then # cachelast return -1 endi -if $data16_db != none then # cachelast +if $data20_db != 1 then # wal return -1 endi -if $data17_db != ms then # precision +if $data21_db != 3000 then # fsync return -1 endi + sql drop database db #print ====> BLOCKS value [3~1000, default: 6] @@ -171,7 +172,7 @@ print ====> CACHEMODEL value [0, 1, 2, 3, default: 0] sql create database db CACHEMODEL 'last_row' sql show databases print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db -if $data16_db != last_row then +if $data18_db != last_row then return -1 endi sql drop database db @@ -179,7 +180,7 @@ sql drop database db sql create database db CACHEMODEL 'last_value' sql show databases print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db -if $data16_db != last_value then +if $data18_db != last_value then return -1 endi sql drop database db @@ -187,7 +188,7 @@ sql drop database db sql create database db CACHEMODEL 'both' sql show databases print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db -if $data16_db != both then +if $data18_db != both then return -1 endi sql drop database db @@ -198,7 +199,7 @@ print ====> COMP [0 | 1 | 2, default: 2] sql create database db COMP 1 sql show databases print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db -if $data15_db != 1 then +if $data13_db != 1 then return -1 endi sql drop database db @@ -206,7 +207,7 @@ sql drop database db sql create database db COMP 0 sql show databases print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db -if $data15_db != 0 then +if $data13_db != 0 then return -1 endi sql drop database db @@ -257,7 +258,7 @@ print ====> FSYNC value [0 ~ 180000 ms, default: 3000] sql create database db FSYNC 0 sql show databases print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db -if $data14_db != 0 then +if $data21_db != 0 then return -1 endi sql drop database db @@ -265,7 +266,7 @@ sql drop database db sql create database db FSYNC 180000 sql show databases print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db -if $data14_db != 180000 then +if $data21_db != 180000 then return -1 endi sql drop database db @@ -308,7 +309,7 @@ print ====> PRECISION ['ms' | 'us' | 'ns', default: ms] sql create database db PRECISION 'us' sql show databases print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db -if $data17_db != us then +if $data14_db != us then return -1 endi sql drop database db @@ -316,7 +317,7 @@ sql drop database db sql create database db PRECISION 'ns' sql show databases print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db -if $data17_db != ns then +if $data14_db != ns then return -1 endi sql drop database db @@ -389,7 +390,7 @@ print ====> WAL value [1 | 2, default: 1] sql create database db WAL 2 sql show databases print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db -if $data13_db != 2 then +if $data20_db != 2 then return -1 endi sql drop database db @@ -397,7 +398,7 @@ sql drop database db sql create database db WAL 1 sql show databases print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db -if $data13_db != 1 then +if $data20_db != 1 then return -1 endi sql drop database db diff --git a/tests/script/tsim/query/charScalarFunction.sim b/tests/script/tsim/query/charScalarFunction.sim index f1575d7293..49d3499738 100644 --- a/tests/script/tsim/query/charScalarFunction.sim +++ b/tests/script/tsim/query/charScalarFunction.sim @@ -1,26 +1,6 @@ -#### length, char_length, lower, upper, ltrim, rtrim, concat, concat_ws, substr. - system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start - -$loop_cnt = 0 -check_dnode_ready: - $loop_cnt = $loop_cnt + 1 - sleep 200 - if $loop_cnt == 10 then - print ====> dnode not ready! - return -1 - endi -sql show dnodes -print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 -if $data00 != 1 then - return -1 -endi -if $data04 != ready then - goto check_dnode_ready -endi - sql connect $vgroups = 4 diff --git a/tests/script/tsim/query/complex_group.sim b/tests/script/tsim/query/complex_group.sim index 6c9a7c5a7a..a0cb727253 100644 --- a/tests/script/tsim/query/complex_group.sim +++ b/tests/script/tsim/query/complex_group.sim @@ -1,25 +1,6 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start - -$loop_cnt = 0 -check_dnode_ready: - $loop_cnt = $loop_cnt + 1 - sleep 200 - if $loop_cnt == 10 then - print ====> dnode not ready! - return -1 - endi - -sql show dnodes -print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 -if $data00 != 1 then - return -1 -endi -if $data04 != ready then - goto check_dnode_ready -endi - sql connect print =============== create database diff --git a/tests/script/tsim/query/complex_having.sim b/tests/script/tsim/query/complex_having.sim index 6a4aa6ea28..29a600dab5 100644 --- a/tests/script/tsim/query/complex_having.sim +++ b/tests/script/tsim/query/complex_having.sim @@ -1,25 +1,6 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start - -$loop_cnt = 0 -check_dnode_ready: - $loop_cnt = $loop_cnt + 1 - sleep 200 - if $loop_cnt == 10 then - print ====> dnode not ready! - return -1 - endi - -sql show dnodes -print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 -if $data00 != 1 then - return -1 -endi -if $data04 != ready then - goto check_dnode_ready -endi - sql connect print =============== create database diff --git a/tests/script/tsim/query/complex_limit.sim b/tests/script/tsim/query/complex_limit.sim index 4942fec4ee..edcf5734aa 100644 --- a/tests/script/tsim/query/complex_limit.sim +++ b/tests/script/tsim/query/complex_limit.sim @@ -1,25 +1,6 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start - -$loop_cnt = 0 -check_dnode_ready: - $loop_cnt = $loop_cnt + 1 - sleep 200 - if $loop_cnt == 10 then - print ====> dnode not ready! - return -1 - endi - -sql show dnodes -print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 -if $data00 != 1 then - return -1 -endi -if $data04 != ready then - goto check_dnode_ready -endi - sql connect print =============== create database diff --git a/tests/script/tsim/query/complex_select.sim b/tests/script/tsim/query/complex_select.sim index 1f41783383..ed6c40f616 100644 --- a/tests/script/tsim/query/complex_select.sim +++ b/tests/script/tsim/query/complex_select.sim @@ -1,25 +1,6 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start - -$loop_cnt = 0 -check_dnode_ready: - $loop_cnt = $loop_cnt + 1 - sleep 200 - if $loop_cnt == 10 then - print ====> dnode not ready! - return -1 - endi - -sql show dnodes -print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 -if $data00 != 1 then - return -1 -endi -if $data04 != ready then - goto check_dnode_ready -endi - sql connect print =============== create database diff --git a/tests/script/tsim/query/complex_where.sim b/tests/script/tsim/query/complex_where.sim index 8e22a12fcf..c634efabfe 100644 --- a/tests/script/tsim/query/complex_where.sim +++ b/tests/script/tsim/query/complex_where.sim @@ -1,25 +1,6 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start - -$loop_cnt = 0 -check_dnode_ready: - $loop_cnt = $loop_cnt + 1 - sleep 200 - if $loop_cnt == 10 then - print ====> dnode not ready! - return -1 - endi - -sql show dnodes -print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 -if $data00 != 1 then - return -1 -endi -if $data04 != ready then - goto check_dnode_ready -endi - sql connect print =============== create database diff --git a/tests/script/tsim/query/crash_sql.sim b/tests/script/tsim/query/crash_sql.sim index b2b9239232..88ff812d68 100644 --- a/tests/script/tsim/query/crash_sql.sim +++ b/tests/script/tsim/query/crash_sql.sim @@ -1,25 +1,6 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start - -$loop_cnt = 0 -check_dnode_ready: - $loop_cnt = $loop_cnt + 1 - sleep 200 - if $loop_cnt == 10 then - print ====> dnode not ready! - return -1 - endi - -sql show dnodes -print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 -if $data00 != 1 then - return -1 -endi -if $data04 != ready then - goto check_dnode_ready -endi - sql connect print =============== create database diff --git a/tests/script/tsim/query/diff.sim b/tests/script/tsim/query/diff.sim index 7bfeeeba7f..8604859dc5 100644 --- a/tests/script/tsim/query/diff.sim +++ b/tests/script/tsim/query/diff.sim @@ -1,26 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start - -$loop_cnt = 0 -check_dnode_ready: - $loop_cnt = $loop_cnt + 1 - sleep 200 - if $loop_cnt == 10 then - print ====> dnode not ready! - return -1 - endi -sql show dnodes -print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 -if $data00 != 1 then - return -1 -endi -if $data04 != ready then - goto check_dnode_ready -endi - sql connect $dbPrefix = db diff --git a/tests/script/tsim/query/explain.sim b/tests/script/tsim/query/explain.sim index c853022281..44f5eb74e7 100644 --- a/tests/script/tsim/query/explain.sim +++ b/tests/script/tsim/query/explain.sim @@ -1,7 +1,5 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 - -print ========= start dnode1 as leader system sh/exec.sh -n dnode1 -s start sql connect diff --git a/tests/script/tsim/query/interval-offset.sim b/tests/script/tsim/query/interval-offset.sim index 1399be7b53..b7d367ad90 100644 --- a/tests/script/tsim/query/interval-offset.sim +++ b/tests/script/tsim/query/interval-offset.sim @@ -1,7 +1,6 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start -sleep 500 sql connect print =============== create database diff --git a/tests/script/tsim/query/interval.sim b/tests/script/tsim/query/interval.sim index 280a66de00..bd607848d0 100644 --- a/tests/script/tsim/query/interval.sim +++ b/tests/script/tsim/query/interval.sim @@ -1,9 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c wal -v 1 system sh/exec.sh -n dnode1 -s start -sleep 500 sql connect $dbPrefix = m_in_db diff --git a/tests/script/tsim/query/read.sim b/tests/script/tsim/query/read.sim index c6bec2586d..87c1d93f17 100644 --- a/tests/script/tsim/query/read.sim +++ b/tests/script/tsim/query/read.sim @@ -1,12 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c wallevel -v 2 -system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 - -print ========= start dnode1 as leader system sh/exec.sh -n dnode1 -s start -sleep 2000 sql connect sql create database abc1 vgroups 2; diff --git a/tests/script/tsim/query/scalarFunction.sim b/tests/script/tsim/query/scalarFunction.sim index 2946a89ff6..27aa1a7e10 100644 --- a/tests/script/tsim/query/scalarFunction.sim +++ b/tests/script/tsim/query/scalarFunction.sim @@ -1,26 +1,6 @@ -#### abs, log, pow, sqrt, sin, cos, tan, asin, acos, atan, ceil, floor, round - system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start - -$loop_cnt = 0 -check_dnode_ready: - $loop_cnt = $loop_cnt + 1 - sleep 200 - if $loop_cnt == 10 then - print ====> dnode not ready! - return -1 - endi -sql show dnodes -print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 -if $data00 != 1 then - return -1 -endi -if $data04 != ready then - goto check_dnode_ready -endi - sql connect $vgroups = 4 diff --git a/tests/script/tsim/query/scalarNull.sim b/tests/script/tsim/query/scalarNull.sim index 1b437747ce..77aae17afc 100644 --- a/tests/script/tsim/query/scalarNull.sim +++ b/tests/script/tsim/query/scalarNull.sim @@ -1,12 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c wallevel -v 2 -system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 - -print ========= start dnode1 as leader system sh/exec.sh -n dnode1 -s start -sleep 2000 sql connect print ======== step1 diff --git a/tests/script/tsim/query/session.sim b/tests/script/tsim/query/session.sim index 29559fdee5..3f219f7be9 100644 --- a/tests/script/tsim/query/session.sim +++ b/tests/script/tsim/query/session.sim @@ -1,26 +1,6 @@ -#### session windows - system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start - -$loop_cnt = 0 -check_dnode_ready: - $loop_cnt = $loop_cnt + 1 - sleep 200 - if $loop_cnt == 10 then - print ====> dnode not ready! - return -1 - endi -sql show dnodes -print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 -if $data00 != 1 then - return -1 -endi -if $data04 != ready then - goto check_dnode_ready -endi - sql connect $vgroups = 4 diff --git a/tests/script/tsim/query/stddev.sim b/tests/script/tsim/query/stddev.sim index 15041623bc..291ee32e74 100644 --- a/tests/script/tsim/query/stddev.sim +++ b/tests/script/tsim/query/stddev.sim @@ -1,25 +1,6 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start - -$loop_cnt = 0 -check_dnode_ready: - $loop_cnt = $loop_cnt + 1 - sleep 200 - if $loop_cnt == 10 then - print ====> dnode not ready! - return -1 - endi - -sql show dnodes -print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 -if $data00 != 1 then - return -1 -endi -if $data04 != ready then - goto check_dnode_ready -endi - sql connect print =============== create database diff --git a/tests/script/tsim/query/time_process.sim b/tests/script/tsim/query/time_process.sim index 0b77c41bb2..e0ca724ef1 100644 --- a/tests/script/tsim/query/time_process.sim +++ b/tests/script/tsim/query/time_process.sim @@ -1,25 +1,6 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start - -$loop_cnt = 0 -check_dnode_ready: - $loop_cnt = $loop_cnt + 1 - sleep 200 - if $loop_cnt == 10 then - print ====> dnode not ready! - return -1 - endi - -sql show dnodes -print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 -if $data00 != 1 then - return -1 -endi -if $data04 != ready then - goto check_dnode_ready -endi - sql connect print =============== create database diff --git a/tests/script/tsim/sync/3Replica1VgElect.sim b/tests/script/tsim/sync/3Replica1VgElect.sim index a451b1cba2..1cadf7c81d 100644 --- a/tests/script/tsim/sync/3Replica1VgElect.sim +++ b/tests/script/tsim/sync/3Replica1VgElect.sim @@ -65,7 +65,7 @@ print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $d if $rows != 3 then return -1 endi -if $data[2][19] != ready then +if $data[2][15] != ready then goto check_db_ready endi @@ -406,7 +406,7 @@ print $data(db1)[0] $data(db1)[1] $data(db1)[2] $data(db1)[3] $data(db1)[4] $dat if $rows != 4 then return -1 endi -if $data(db1)[19] != ready then +if $data(db1)[15] != ready then goto check_db_ready1 endi diff --git a/tests/script/tsim/sync/3Replica5VgElect.sim b/tests/script/tsim/sync/3Replica5VgElect.sim index aec5666e6c..2a9f8c30a2 100644 --- a/tests/script/tsim/sync/3Replica5VgElect.sim +++ b/tests/script/tsim/sync/3Replica5VgElect.sim @@ -65,7 +65,7 @@ print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $d if $rows != 3 then return -1 endi -if $data[2][19] != ready then +if $data[2][15] != ready then goto check_db_ready endi @@ -590,7 +590,7 @@ print $data(db1)[0] $data(db1)[1] $data(db1)[2] $data(db1)[3] $data(db1)[4] $dat if $rows != 4 then return -1 endi -if $data(db1)[19] != ready then +if $data(db1)[15] != ready then goto check_db_ready1 endi diff --git a/tests/script/tsim/sync/3Replica5VgElect3mnode.sim b/tests/script/tsim/sync/3Replica5VgElect3mnode.sim index 8a69d5ca07..22ff28a485 100644 --- a/tests/script/tsim/sync/3Replica5VgElect3mnode.sim +++ b/tests/script/tsim/sync/3Replica5VgElect3mnode.sim @@ -63,7 +63,7 @@ print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $d if $rows != 3 then return -1 endi -if $data[2][19] != ready then +if $data[2][15] != ready then goto check_db_ready endi @@ -698,7 +698,7 @@ print $data(db1)[0] $data(db1)[1] $data(db1)[2] $data(db1)[3] $data(db1)[4] $dat if $rows != 4 then return -1 endi -if $data(db1)[19] != ready then +if $data(db1)[15] != ready then goto check_db_ready1 endi diff --git a/tests/script/tsim/sync/3Replica5VgElect3mnodedrop.sim b/tests/script/tsim/sync/3Replica5VgElect3mnodedrop.sim index 960f579f55..5d906d8857 100644 --- a/tests/script/tsim/sync/3Replica5VgElect3mnodedrop.sim +++ b/tests/script/tsim/sync/3Replica5VgElect3mnodedrop.sim @@ -65,7 +65,7 @@ print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $d if $rows != 3 then return -1 endi -if $data[2][19] != ready then +if $data[2][15] != ready then goto check_db_ready endi diff --git a/tests/script/tsim/sync/electTest.sim b/tests/script/tsim/sync/electTest.sim index 750ab9d63a..e421517961 100644 --- a/tests/script/tsim/sync/electTest.sim +++ b/tests/script/tsim/sync/electTest.sim @@ -65,7 +65,7 @@ print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $d if $rows != 3 then return -1 endi -if $data[2][19] != ready then +if $data[2][15] != ready then goto check_db_ready endi diff --git a/tests/script/tsim/sync/oneReplica1VgElect.sim b/tests/script/tsim/sync/oneReplica1VgElect.sim index 829cf029b4..423f70646e 100644 --- a/tests/script/tsim/sync/oneReplica1VgElect.sim +++ b/tests/script/tsim/sync/oneReplica1VgElect.sim @@ -66,7 +66,7 @@ print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $dat if $rows != 3 then return -1 endi -if $data(db1)[19] != ready then +if $data(db1)[15] != ready then goto check_db_ready endi diff --git a/tests/script/tsim/sync/oneReplica1VgElectWithInsert.sim b/tests/script/tsim/sync/oneReplica1VgElectWithInsert.sim index aff458c02a..d2b5565308 100644 --- a/tests/script/tsim/sync/oneReplica1VgElectWithInsert.sim +++ b/tests/script/tsim/sync/oneReplica1VgElectWithInsert.sim @@ -66,7 +66,7 @@ print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $dat if $rows != 3 then return -1 endi -if $data(db)[19] != ready then +if $data(db)[15] != ready then goto check_db_ready endi diff --git a/tests/script/tsim/sync/oneReplica5VgElect.sim b/tests/script/tsim/sync/oneReplica5VgElect.sim index 2cf9b7f1f5..765ba35ebd 100644 --- a/tests/script/tsim/sync/oneReplica5VgElect.sim +++ b/tests/script/tsim/sync/oneReplica5VgElect.sim @@ -65,7 +65,7 @@ print $data(db1)[0] $data(db)[1] $data(db)[2] $data(db)[3] $data(db)[4] $data(db if $rows != 3 then return -1 endi -if $data(db1)[19] != ready then +if $data(db1)[15] != ready then goto check_db_ready endi diff --git a/tests/script/tsim/sync/threeReplica1VgElect.sim b/tests/script/tsim/sync/threeReplica1VgElect.sim index bf2cb05570..7a4fa1c2a6 100644 --- a/tests/script/tsim/sync/threeReplica1VgElect.sim +++ b/tests/script/tsim/sync/threeReplica1VgElect.sim @@ -66,7 +66,7 @@ print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $dat if $rows != 3 then return -1 endi -if $data(db)[19] != ready then +if $data(db)[15] != ready then goto check_db_ready endi diff --git a/tests/script/tsim/sync/threeReplica1VgElectWihtInsert.sim b/tests/script/tsim/sync/threeReplica1VgElectWihtInsert.sim index a6a0dda8a7..967b3f1406 100644 --- a/tests/script/tsim/sync/threeReplica1VgElectWihtInsert.sim +++ b/tests/script/tsim/sync/threeReplica1VgElectWihtInsert.sim @@ -66,7 +66,7 @@ print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $dat if $rows != 3 then return -1 endi -if $data(db)[19] != ready then +if $data(db)[15] != ready then goto check_db_ready endi diff --git a/tests/script/tsim/sync/vnode-insert.sim b/tests/script/tsim/sync/vnode-insert.sim index 221e3dd2b2..18ed40a881 100644 --- a/tests/script/tsim/sync/vnode-insert.sim +++ b/tests/script/tsim/sync/vnode-insert.sim @@ -65,7 +65,7 @@ print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $d if $rows != 3 then return -1 endi -if $data[2][19] != ready then +if $data[2][15] != ready then goto check_db_ready endi diff --git a/tests/script/tsim/sync/vnodeLeaderTransfer.sim b/tests/script/tsim/sync/vnodeLeaderTransfer.sim index cbeb7063ff..4fa08a8fbb 100644 --- a/tests/script/tsim/sync/vnodeLeaderTransfer.sim +++ b/tests/script/tsim/sync/vnodeLeaderTransfer.sim @@ -65,7 +65,7 @@ print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $d if $rows != 3 then return -1 endi -if $data[2][19] != ready then +if $data[2][15] != ready then goto check_db_ready endi diff --git a/tests/script/tsim/sync/vnodesnapshot-test.sim b/tests/script/tsim/sync/vnodesnapshot-test.sim index 2f0eccf02e..c3d8a243d3 100644 --- a/tests/script/tsim/sync/vnodesnapshot-test.sim +++ b/tests/script/tsim/sync/vnodesnapshot-test.sim @@ -65,7 +65,7 @@ print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $d if $rows != 3 then return -1 endi -if $data[2][19] != ready then +if $data[2][15] != ready then goto check_db_ready endi diff --git a/tests/script/tsim/sync/vnodesnapshot.sim b/tests/script/tsim/sync/vnodesnapshot.sim index b9452d2147..bec13d7e79 100644 --- a/tests/script/tsim/sync/vnodesnapshot.sim +++ b/tests/script/tsim/sync/vnodesnapshot.sim @@ -65,7 +65,7 @@ print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $d if $rows != 3 then return -1 endi -if $data[2][19] != ready then +if $data[2][15] != ready then goto check_db_ready endi diff --git a/tests/script/tsim/tmq/basic1Of2Cons.sim b/tests/script/tsim/tmq/basic1Of2Cons.sim index 9d4b0e75da..11b645c4d1 100644 --- a/tests/script/tsim/tmq/basic1Of2Cons.sim +++ b/tests/script/tsim/tmq/basic1Of2Cons.sim @@ -115,7 +115,7 @@ $expectmsgcnt = $totalMsgOfStb sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from stb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start @@ -217,7 +217,7 @@ $totalMsgOfCtb = $rowsPerCtb $expectmsgcnt = $totalMsgOfCtb sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ctb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -319,7 +319,7 @@ $totalMsgOfNtb = $rowsPerCtb $expectmsgcnt = $totalMsgOfNtb sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ntb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start diff --git a/tests/script/tsim/tmq/basic2Of2Cons.sim b/tests/script/tsim/tmq/basic2Of2Cons.sim index 0494ddb5b8..87559305ba 100644 --- a/tests/script/tsim/tmq/basic2Of2Cons.sim +++ b/tests/script/tsim/tmq/basic2Of2Cons.sim @@ -85,7 +85,7 @@ $totalMsgOfStb = $totalMsgOfStb * $topicNum $expectmsgcnt = $totalMsgOfStb sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from stb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start @@ -172,7 +172,7 @@ $totalMsgOfCtb = $rowsPerCtb * $topicNum $expectmsgcnt = $totalMsgOfCtb sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ctb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -259,7 +259,7 @@ $totalMsgOfNtb = $rowsPerCtb * $topicNum $expectmsgcnt = $totalMsgOfNtb sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ntb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start diff --git a/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim b/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim index d796d00f03..d9c6b195d2 100644 --- a/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim +++ b/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim @@ -89,7 +89,7 @@ $topicList = $topicList . , $topicList = $topicList . topic_stb_function $topicList = $topicList . ' $consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now +1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from stb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start @@ -197,7 +197,7 @@ $topicList = $topicList . , $topicList = $topicList . topic_ctb_all $topicList = $topicList . ' $consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now +1s, $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ctb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start @@ -289,7 +289,7 @@ $topicList = $topicList . , $topicList = $topicList . topic_ntb_all $topicList = $topicList . ' $consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ntb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start diff --git a/tests/script/tsim/tmq/basic3Of2Cons.sim b/tests/script/tsim/tmq/basic3Of2Cons.sim index afaf824acb..4921c86c45 100644 --- a/tests/script/tsim/tmq/basic3Of2Cons.sim +++ b/tests/script/tsim/tmq/basic3Of2Cons.sim @@ -113,7 +113,7 @@ $totalMsgOfStb = $ctbNum * $rowsPerCtb $expectmsgcnt = $totalMsgOfStb sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from stb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start @@ -228,7 +228,7 @@ $totalMsgOfCtb = $rowsPerCtb $expectmsgcnt = $totalMsgOfCtb sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ctb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -330,7 +330,7 @@ $totalMsgOfNtb = $rowsPerCtb $expectmsgcnt = $totalMsgOfNtb sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ntb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start diff --git a/tests/script/tsim/tmq/basic4Of2Cons.sim b/tests/script/tsim/tmq/basic4Of2Cons.sim index 510aaf0e1a..f1755f732b 100644 --- a/tests/script/tsim/tmq/basic4Of2Cons.sim +++ b/tests/script/tsim/tmq/basic4Of2Cons.sim @@ -82,7 +82,7 @@ $totalMsgOfStb = $totalMsgOfStb * $topicNum $expectmsgcnt = $totalMsgOfStb sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from stb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start @@ -181,7 +181,7 @@ $totalMsgOfCtb = $rowsPerCtb * $topicNum $expectmsgcnt = $totalMsgOfCtb sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ctb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -269,7 +269,7 @@ $totalMsgOfNtb = $rowsPerCtb * $topicNum $expectmsgcnt = $totalMsgOfNtb sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ntb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start diff --git a/tests/script/tsim/tmq/prepareBasicEnv-1vgrp.sim b/tests/script/tsim/tmq/prepareBasicEnv-1vgrp.sim index 43b93b7372..4630e295a9 100644 --- a/tests/script/tsim/tmq/prepareBasicEnv-1vgrp.sim +++ b/tests/script/tsim/tmq/prepareBasicEnv-1vgrp.sim @@ -39,7 +39,7 @@ sql show databases print ==> rows: $rows print ==> $data(db)[0] $data(db)[1] $data(db)[2] $data(db)[3] $data(db)[4] $data(db)[5] $data(db)[6] $data(db)[7] $data(db)[8] $data(db)[9] $data(db)[10] $data(db)[11] $data(db)[12] print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $data(db)[18] $data(db)[19] $data(db)[20] -if $data(db)[19] != ready then +if $data(db)[15] != ready then sleep 100 $loop_cnt = $loop_cnt + 1 goto check_db_ready diff --git a/tests/script/tsim/tmq/prepareBasicEnv-4vgrp.sim b/tests/script/tsim/tmq/prepareBasicEnv-4vgrp.sim index 5e81137ffa..bce2292f97 100644 --- a/tests/script/tsim/tmq/prepareBasicEnv-4vgrp.sim +++ b/tests/script/tsim/tmq/prepareBasicEnv-4vgrp.sim @@ -39,7 +39,7 @@ sql show databases print ==> rows: $rows print ==> $data(db)[0] $data(db)[1] $data(db)[2] $data(db)[3] $data(db)[4] $data(db)[5] $data(db)[6] $data(db)[7] $data(db)[8] $data(db)[9] $data(db)[10] $data(db)[11] $data(db)[12] print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $data(db)[18] $data(db)[19] $data(db)[20] -if $data(db)[19] != ready then +if $data(db)[15] != ready then sleep 100 $loop_cnt = $loop_cnt + 1 goto check_db_ready diff --git a/tests/script/tsim/tmq/topic.sim b/tests/script/tsim/tmq/topic.sim index 9add349a18..33a5b861b6 100644 --- a/tests/script/tsim/tmq/topic.sim +++ b/tests/script/tsim/tmq/topic.sim @@ -31,7 +31,7 @@ sql show databases print ==> rows: $rows print ==> $data(db)[0] $data(db)[1] $data(db)[2] $data(db)[3] $data(db)[4] $data(db)[5] $data(db)[6] $data(db)[7] $data(db)[8] $data(db)[9] $data(db)[10] $data(db)[11] $data(db)[12] print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $data(db)[18] $data(db)[19] $data(db)[20] -if $data(db)[19] != ready then +if $data(db)[15] != ready then sleep 100 $loop_cnt = $loop_cnt + 1 goto check_db_ready diff --git a/tests/script/tsim/trans/create_db.sim b/tests/script/tsim/trans/create_db.sim index d4d158ede0..4a20f73e6c 100644 --- a/tests/script/tsim/trans/create_db.sim +++ b/tests/script/tsim/trans/create_db.sim @@ -128,8 +128,8 @@ if $system_content != Windows_NT then if $rows != 4 then return -1 endi - print d2 ==> $data(d2)[19] - if $data(d2)[19] != creating then + print d2 ==> $data(d2)[15] + if $data(d2)[15] != creating then return -1 endi diff --git a/tests/script/tsim/valgrind/basic1.sim b/tests/script/tsim/valgrind/basic1.sim index 49f0b20702..a259dd9219 100644 --- a/tests/script/tsim/valgrind/basic1.sim +++ b/tests/script/tsim/valgrind/basic1.sim @@ -45,7 +45,17 @@ while $i < $tbNum endw print =============== step3: tb -sql select count(1) from tb1 +sql explain analyze select ts from stb where -2; +sql explain analyze select ts from tb1; +sql explain analyze select ts from stb order by ts; +sql explain analyze select * from information_schema.user_stables; +sql explain analyze select count(*),sum(tbcol) from tb1; +sql explain analyze select count(*),sum(tbcol) from stb; +sql explain analyze select count(*),sum(tbcol) from stb group by tbcol; +sql explain analyze select * from information_schema.user_stables; +sql explain analyze verbose true select * from information_schema.user_stables where db_name='db2'; +sql explain analyze verbose true select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), count(tbcol) from stb where ts <= 1601481840000 and ts >= 1601481800000 partition by tgcol interval(1m) fill(value, 0) +sql explain select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), count(tbcol) from stb where ts <= 1601481840000 and ts >= 1601481800000 partition by tgcol interval(1m) fill(value, 0) _OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/valgrind/checkError6.sim b/tests/script/tsim/valgrind/checkError6.sim index 27fd291f64..804961e7b0 100644 --- a/tests/script/tsim/valgrind/checkError6.sim +++ b/tests/script/tsim/valgrind/checkError6.sim @@ -29,7 +29,7 @@ $rowNum = 10 print =============== step2: prepare data sql create database db vgroups 2 sql use db -sql create table if not exists stb (ts timestamp, tbcol int, tbcol2 float, tbcol3 double) tags (tgcol int unsigned) +sql create table if not exists stb (ts timestamp, tbcol int, tbcol2 float, tbcol3 double, tbcol4 binary(30), tbcol5 binary(30)) tags (tgcol int unsigned) $i = 0 while $i < $tbNum @@ -39,13 +39,13 @@ while $i < $tbNum while $x < $rowNum $cc = $x * 60000 $ms = 1601481600000 + $cc - sql insert into $tb values ($ms , $x , $x , $x ) + sql insert into $tb values ($ms , $x , $x , $x , "abcd1234=-+*" , "123456 0" ) $x = $x + 1 endw $cc = $x * 60000 $ms = 1601481600000 + $cc - sql insert into $tb values ($ms , NULL , NULL , NULL ) + sql insert into $tb values ($ms , NULL , NULL , NULL , NULL , NULL ) $i = $i + 1 endw @@ -60,13 +60,21 @@ sql select top(tbcol, 2) from tb1 where ts <= 1601481840000 sql select percentile(tbcol, 2) from tb1 where ts <= 1601481840000 sql select leastsquares(tbcol, 1, 1) as b from tb1 where ts <= 1601481840000 sql show table distributed tb1 +sql select count(1) from tb1 sql select count(tbcol) as b from tb1 where ts <= 1601481840000 interval(1m) sql select diff(tbcol) from tb1 where ts <= 1601481840000 sql select diff(tbcol) from tb1 where tbcol > 5 and tbcol < 20 sql select first(tbcol), last(tbcol) as b from tb1 where ts <= 1601481840000 interval(1m) sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), sum(tbcol), stddev(tbcol) from tb1 where ts <= 1601481840000 partition by tgcol interval(1m) -#sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), count(tbcol) from tb1 where ts <= 1601481840000 and ts >= 1601481800000 partition by tgcol interval(1m) fill(value, 0) +sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), count(tbcol) from tb1 where ts <= 1601481840000 and ts >= 1601481800000 partition by tgcol interval(1m) fill(value, 0) sql select last_row(*) from tb1 where tbcol > 5 and tbcol < 20 +sql select _wstart, _wend, _wduration, _qstart, _qend, count(*) from tb1 interval(10s, 2s) sliding(10s) +sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), count(tbcol) from tb1 where ts <= 1601481840000 and ts >= 1601481800000 partition by tgcol interval(1m) fill(value, 0) +sql select log(tbcol), abs(tbcol), pow(tbcol, 2), sqrt(tbcol), sin(tbcol), cos(tbcol), tan(tbcol), asin(tbcol), acos(tbcol), atan(tbcol), ceil(tbcol), floor(tbcol), round(tbcol), atan(tbcol) from tb1 +sql select length("abcd1234"), char_length("abcd1234=-+*") from tb1 +sql select tbcol4, length(tbcol4), lower(tbcol4), upper(tbcol4), ltrim(tbcol4), rtrim(tbcol4), concat(tbcol4, tbcol5), concat_ws('_', tbcol4, tbcol5), substr(tbcol4, 1, 4) from tb1 +sql select * from tb1 where tbcol not in (1,2,3,null); +sql select * from tb1 where tbcol + 3 <> null; print =============== step4: stb sql select avg(tbcol) as c from stb @@ -78,14 +86,24 @@ sql select avg(tbcol) as b from stb where ts <= 1601481840000 interval(1m) sql select avg(tbcol) as c from stb group by tgcol sql select avg(tbcol) as b from stb where ts <= 1601481840000 partition by tgcol interval(1m) sql show table distributed stb +sql select count(1) from stb sql select count(tbcol) as b from stb where ts <= 1601481840000 partition by tgcol interval(1m) sql select diff(tbcol) from stb where ts <= 1601481840000 sql select first(tbcol), last(tbcol) as c from stb group by tgcol sql select first(tbcol), last(tbcol) as b from stb where ts <= 1601481840000 and tbcol2 is null partition by tgcol interval(1m) sql select first(tbcol), last(tbcol) as b from stb where ts <= 1601481840000 partition by tgcol interval(1m) sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), sum(tbcol), stddev(tbcol) from stb where ts <= 1601481840000 partition by tgcol interval(1m) -#sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), count(tbcol) from stb where ts <= 1601481840000 and ts >= 1601481800000 partition by tgcol interval(1m) fill(value, 0) +sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), count(tbcol) from stb where ts <= 1601481840000 and ts >= 1601481800000 partition by tgcol interval(1m) fill(value, 0) sql select last_row(tbcol), stddev(tbcol) from stb where tbcol > 5 and tbcol < 20 group by tgcol +sql select _wstart, _wend, _wduration, _qstart, _qend, count(*) from stb interval(10s, 2s) sliding(10s) +sql select log(tbcol), abs(tbcol), pow(tbcol, 2), sqrt(tbcol), sin(tbcol), cos(tbcol), tan(tbcol), asin(tbcol), acos(tbcol), atan(tbcol), ceil(tbcol), floor(tbcol), round(tbcol), atan(tbcol) from stb +sql select length("abcd1234"), char_length("abcd1234=-+*") from stb +sql select tbcol4, length(tbcol4), lower(tbcol4), upper(tbcol4), ltrim(tbcol4), rtrim(tbcol4), concat(tbcol4, tbcol5), concat_ws('_', tbcol4, tbcol5), substr(tbcol4, 1, 4) from stb +sql select * from stb where tbcol not in (1,2,3,null); +sql select * from stb where tbcol + 3 <> null; + +print =============== step5: explain + _OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/system-test/0-others/cachemodel.py b/tests/system-test/0-others/cachemodel.py index 09538e6678..102a34612d 100644 --- a/tests/system-test/0-others/cachemodel.py +++ b/tests/system-test/0-others/cachemodel.py @@ -90,7 +90,7 @@ class TDTestCase: for db_info in databases_infos: dbname = db_info[0] # print(dbname) - cache_last_value = db_info[16] + cache_last_value = db_info[18] # print(cache_last_value) if dbname in ["information_schema" , "performance_schema"]: continue diff --git a/tests/system-test/0-others/fsync.py b/tests/system-test/0-others/fsync.py index 964550cdbc..eedf3629ed 100644 --- a/tests/system-test/0-others/fsync.py +++ b/tests/system-test/0-others/fsync.py @@ -43,9 +43,9 @@ class TDTestCase: fsync_index = 0 tdSql.query("show databases") for i in range(tdSql.queryCols): - if tdSql.cursor.description[i][0] == "wal": + if tdSql.cursor.description[i][0] == "wal_level": wal_index = i - if tdSql.cursor.description[i][0] == "fsync": + if tdSql.cursor.description[i][0] == "wal_fsync_period": fsync_index = i tdSql.execute("drop database if exists db1") diff --git a/tests/system-test/2-query/cast.py b/tests/system-test/2-query/cast.py index bdac2b6175..4045b6ad88 100644 --- a/tests/system-test/2-query/cast.py +++ b/tests/system-test/2-query/cast.py @@ -566,8 +566,7 @@ class TDTestCase: if data_ct4_c10[i] is None: tdSql.checkData( i, 0, None ) else: - # time2str = str(int((data_ct4_c10[i]-datetime.datetime.fromtimestamp(0)).total_seconds()*1000)) - time2str = str(int((datetime.datetime.timestamp(data_ct4_c10[i])-datetime.datetime.timestamp(datetime.datetime.fromtimestamp(0)))*1000)) + time2str = str(int((data_ct4_c10[i]-datetime.datetime.fromtimestamp(0)).total_seconds()*1000)) tdSql.checkData( i, 0, time2str ) tdSql.query(f"select cast(c10 as nchar(32)) as b from {self.dbname}.t1") for i in range(len(data_t1_c10)): @@ -576,8 +575,7 @@ class TDTestCase: elif i == 10: continue else: - # time2str = str(int((data_t1_c10[i]-datetime.datetime.fromtimestamp(0)).total_seconds()*1000)) - time2str = str(int((datetime.datetime.timestamp(data_t1_c10[i])-datetime.datetime.timestamp(datetime.datetime.fromtimestamp(0)))*1000)) + time2str = str(int((data_t1_c10[i]-datetime.datetime.fromtimestamp(0)).total_seconds()*1000)) tdSql.checkData( i, 0, time2str ) tdLog.printNoPrefix("==========step38: cast timestamp to binary, expect no changes ") @@ -586,8 +584,7 @@ class TDTestCase: if data_ct4_c10[i] is None: tdSql.checkData( i, 0, None ) else: - # time2str = str(int((data_ct4_c10[i]-datetime.datetime.fromtimestamp(0)).total_seconds()*1000)) - time2str = str(int((datetime.datetime.timestamp(data_ct4_c10[i])-datetime.datetime.timestamp(datetime.datetime.fromtimestamp(0)))*1000)) + time2str = str(int((data_ct4_c10[i]-datetime.datetime.fromtimestamp(0)).total_seconds()*1000)) tdSql.checkData( i, 0, time2str ) tdSql.query(f"select cast(c10 as binary(32)) as b from {self.dbname}.t1") for i in range(len(data_t1_c10)): @@ -596,8 +593,7 @@ class TDTestCase: elif i == 10: continue else: - # time2str = str(int((data_t1_c10[i]-datetime.datetime.fromtimestamp(0)).total_seconds()*1000)) - time2str = str(int((datetime.datetime.timestamp(data_t1_c10[i])-datetime.datetime.timestamp(datetime.datetime.fromtimestamp(0)))*1000)) + time2str = str(int((data_t1_c10[i]-datetime.datetime.fromtimestamp(0)).total_seconds()*1000)) tdSql.checkData( i, 0, time2str ) tdLog.printNoPrefix("==========step39: cast constant operation to bigint, expect change to int ") diff --git a/tests/system-test/2-query/csum.py b/tests/system-test/2-query/csum.py index bdb8c095e6..260528be04 100644 --- a/tests/system-test/2-query/csum.py +++ b/tests/system-test/2-query/csum.py @@ -30,7 +30,7 @@ class TDTestCase: tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) - def csum_query_form(self, col="c1", alias="", table_expr="t1", condition=""): + def csum_query_form(self, col="c1", alias="", table_expr="db.t1", condition=""): ''' csum function: @@ -44,7 +44,7 @@ class TDTestCase: return f"select csum({col}) {alias} from {table_expr} {condition}" - def checkcsum(self,col="c1", alias="", table_expr="t1", condition="" ): + def checkcsum(self,col="c1", alias="", table_expr="db.t1", condition="" ): line = sys._getframe().f_back.f_lineno pre_sql = self.csum_query_form( col=col, table_expr=table_expr, condition=condition @@ -59,11 +59,11 @@ class TDTestCase: tdSql.checkRows(0) return - if "order by tbname" in condition: - tdSql.error(self.csum_query_form( - col=col, alias=alias, table_expr=table_expr, condition=condition - )) - return + # if "order by tbname" in condition: + # tdSql.error(self.csum_query_form( + # col=col, alias=alias, table_expr=table_expr, condition=condition + # )) + # return if "group" in condition: @@ -123,7 +123,8 @@ class TDTestCase: return else: - tdSql.query(f"select {col} from {table_expr} {re.sub('limit [0-9]*|offset [0-9]*','',condition)}") + + tdSql.query(f"select {col} from {table_expr} {re.sub('limit [0-9]*|offset [0-9]*','',condition)} " ) offset_val = condition.split("offset")[1].split(" ")[1] if "offset" in condition else 0 pre_result = np.array(tdSql.queryResult)[np.array(tdSql.queryResult) != None] if (platform.system().lower() == 'windows' and pre_result.dtype == 'int32'): @@ -161,12 +162,12 @@ class TDTestCase: self.checkcsum(**case6) # case7~8: nested query - # case7 = {"table_expr": "(select c1 from stb1)"} - # self.checkcsum(**case7) - # case8 = {"table_expr": "(select csum(c1) c1 from stb1 group by tbname)"} - # self.checkcsum(**case8) + case7 = {"table_expr": "(select c1 from db.stb1 order by tbname ,ts )"} + self.checkcsum(**case7) + case8 = {"table_expr": "(select csum(c1) c1 from db.t1 partition by tbname)"} + self.checkcsum(**case8) - # case9~10: mix with tbname/ts/tag/col + # case9~10: mix with tbname/ts/tag/col not support , must partition by alias ,such as select tbname ,csum(c1) partition by tbname # case9 = {"alias": ", tbname"} # self.checkcsum(**case9) # case10 = {"alias": ", _c0"} @@ -196,37 +197,37 @@ class TDTestCase: } self.checkcsum(**case17) # case18~19: with group by - # case18 = { - # "table_expr": "t1", - # "condition": "group by c6" - # } - # self.checkcsum(**case18) - # case19 = { - # "table_expr": "stb1", - # "condition": "partition by tbname" # partition by tbname - # } - # self.checkcsum(**case19) + case18 = { + "table_expr": "db.t1", + "condition": "where c6 <0 partition by c6 order by c6" + } + self.checkcsum(**case18) + case19 = { + "table_expr": "db.t1", + "condition": " " # partition by tbname + } + self.checkcsum(**case19) - # # case20~21: with order by - # case20 = {"condition": "order by ts"} - # self.checkcsum(**case20) + # case20~21: with order by + case20 = {"condition": "partition by tbname order by tbname "} + # self.checkcsum(**case20) # order by without order by tbname ,because check will random failed - # # case22: with union - # case22 = { - # "condition": "union all select csum(c1) from t2" - # } - # self.checkcsum(**case22) + # case22: with union + case22 = { + "condition": "union all select csum(c1) from db.t2" + } + # self.checkcsum(**case22) union all without check result becasue ,it will random return table_records # case23: with limit/slimit case23 = { "condition": "limit 1" } self.checkcsum(**case23) - # case24 = { - # "table_expr": "stb1", - # "condition": "group by tbname slimit 1 soffset 1" - # } - # self.checkcsum(**case24) + case24 = { + "table_expr": "db.t1", + "condition": "partition by tbname " + } + self.checkcsum(**case24) pass @@ -291,17 +292,17 @@ class TDTestCase: } tdSql.error(self.csum_query_form(**interval_sql)) # interval group_normal_col = { - "table_expr": "t1", + "table_expr": "db.t1", "condition": "group by c6" } tdSql.error(self.csum_query_form(**group_normal_col)) # group by normal col slimit_soffset_sql = { - "table_expr": "stb1", + "table_expr": "db.stb1", "condition": "group by tbname slimit 1 soffset 1" } # tdSql.error(self.csum_query_form(**slimit_soffset_sql)) order_by_tbname_sql = { - "table_expr": "stb1", + "table_expr": "db.stb1", "condition": "group by tbname order by tbname" } tdSql.error(self.csum_query_form(**order_by_tbname_sql)) @@ -346,8 +347,8 @@ class TDTestCase: "create stable db.stb2 (ts timestamp, c1 int) tags(st2 int)" ) for i in range(tbnum): - tdSql.execute(f"create table t{i} using stb1 tags({i})") - tdSql.execute(f"create table tt{i} using stb2 tags({i})") + tdSql.execute(f"create table db.t{i} using db.stb1 tags({i})") + tdSql.execute(f"create table db.tt{i} using db.stb2 tags({i})") pass @@ -364,25 +365,25 @@ class TDTestCase: tdLog.printNoPrefix("######## insert only NULL test:") for i in range(tbnum): - tdSql.execute(f"insert into t{i}(ts) values ({nowtime - 5})") - tdSql.execute(f"insert into t{i}(ts) values ({nowtime + 5})") + tdSql.execute(f"insert into db.t{i}(ts) values ({nowtime - 5})") + tdSql.execute(f"insert into db.t{i}(ts) values ({nowtime + 5})") self.csum_current_query() self.csum_error_query() tdLog.printNoPrefix("######## insert data in the range near the max(bigint/double):") self.csum_test_table(tbnum) - tdSql.execute(f"insert into t1(ts, c1,c2,c5,c7) values " + tdSql.execute(f"insert into db.t1(ts, c1,c2,c5,c7) values " f"({nowtime - (per_table_rows + 1) * 10}, {2**31-1}, {3.4*10**38}, {1.7*10**308}, {2**63-1})") - tdSql.execute(f"insert into t1(ts, c1,c2,c5,c7) values " + tdSql.execute(f"insert into db.t1(ts, c1,c2,c5,c7) values " f"({nowtime - (per_table_rows + 2) * 10}, {2**31-1}, {3.4*10**38}, {1.7*10**308}, {2**63-1})") self.csum_current_query() self.csum_error_query() tdLog.printNoPrefix("######## insert data in the range near the min(bigint/double):") self.csum_test_table(tbnum) - tdSql.execute(f"insert into t1(ts, c1,c2,c5,c7) values " + tdSql.execute(f"insert into db.t1(ts, c1,c2,c5,c7) values " f"({nowtime - (per_table_rows + 1) * 10}, {1-2**31}, {-3.4*10**38}, {-1.7*10**308}, {1-2**63})") - tdSql.execute(f"insert into t1(ts, c1,c2,c5,c7) values " + tdSql.execute(f"insert into db.t1(ts, c1,c2,c5,c7) values " f"({nowtime - (per_table_rows + 2) * 10}, {1-2**31}, {-3.4*10**38}, {-1.7*10**308}, {512-2**63})") self.csum_current_query() self.csum_error_query() @@ -396,9 +397,9 @@ class TDTestCase: tdLog.printNoPrefix("######## insert data mix with NULL test:") for i in range(tbnum): - tdSql.execute(f"insert into t{i}(ts) values ({nowtime})") - tdSql.execute(f"insert into t{i}(ts) values ({nowtime-(per_table_rows+3)*10})") - tdSql.execute(f"insert into t{i}(ts) values ({nowtime+(per_table_rows+3)*10})") + tdSql.execute(f"insert into db.t{i}(ts) values ({nowtime})") + tdSql.execute(f"insert into db.t{i}(ts) values ({nowtime-(per_table_rows+3)*10})") + tdSql.execute(f"insert into db.t{i}(ts) values ({nowtime+(per_table_rows+3)*10})") self.csum_current_query() self.csum_error_query() @@ -411,65 +412,65 @@ class TDTestCase: tdDnodes.start(index) self.csum_current_query() self.csum_error_query() - tdSql.query("select csum(1) from t1 ") + tdSql.query("select csum(1) from db.t1 ") tdSql.checkRows(7) tdSql.checkData(0,0,1) tdSql.checkData(1,0,2) tdSql.checkData(2,0,3) tdSql.checkData(3,0,4) - tdSql.query("select csum(abs(c1))+2 from t1 ") + tdSql.query("select csum(abs(c1))+2 from db.t1 ") tdSql.checkRows(4) def csum_support_stable(self): - tdSql.query(" select csum(1) from stb1 ") + tdSql.query(" select csum(1) from db.stb1 ") tdSql.checkRows(70) - tdSql.query("select csum(c1) from stb1 partition by tbname ") + tdSql.query("select csum(c1) from db.stb1 partition by tbname ") tdSql.checkRows(40) - tdSql.query("select csum(st1) from stb1 partition by tbname") + tdSql.query("select csum(st1) from db.stb1 partition by tbname") tdSql.checkRows(70) - tdSql.query("select csum(st1+c1) from stb1 partition by tbname") + tdSql.query("select csum(st1+c1) from db.stb1 partition by tbname") tdSql.checkRows(40) - tdSql.query("select csum(st1+c1) from stb1 partition by tbname") + tdSql.query("select csum(st1+c1) from db.stb1 partition by tbname") tdSql.checkRows(40) - tdSql.query("select csum(st1+c1) from stb1 partition by tbname") + tdSql.query("select csum(st1+c1) from db.stb1 partition by tbname") tdSql.checkRows(40) # # bug need fix - tdSql.query("select csum(st1+c1) from stb1 partition by tbname slimit 1 ") + tdSql.query("select csum(st1+c1) from db.stb1 partition by tbname slimit 1 ") tdSql.checkRows(4) - # tdSql.error("select csum(st1+c1) from stb1 partition by tbname limit 1 ") + # tdSql.error("select csum(st1+c1) from db.stb1 partition by tbname limit 1 ") # bug need fix - tdSql.query("select csum(st1+c1) from stb1 partition by tbname") + tdSql.query("select csum(st1+c1) from db.stb1 partition by tbname") tdSql.checkRows(40) # bug need fix - tdSql.query("select tbname , csum(c1) from stb1 partition by tbname") + tdSql.query("select tbname , csum(c1) from db.stb1 partition by tbname") tdSql.checkRows(40) - tdSql.query("select tbname , csum(st1) from stb1 partition by tbname") + tdSql.query("select tbname , csum(st1) from db.stb1 partition by tbname") tdSql.checkRows(70) - tdSql.query("select tbname , csum(st1) from stb1 partition by tbname slimit 1") + tdSql.query("select tbname , csum(st1) from db.stb1 partition by tbname slimit 1") tdSql.checkRows(7) # partition by tags - tdSql.query("select st1 , csum(c1) from stb1 partition by st1") + tdSql.query("select st1 , csum(c1) from db.stb1 partition by st1") tdSql.checkRows(40) - tdSql.query("select csum(c1) from stb1 partition by st1") + tdSql.query("select csum(c1) from db.stb1 partition by st1") tdSql.checkRows(40) - tdSql.query("select st1 , csum(c1) from stb1 partition by st1 slimit 1") + tdSql.query("select st1 , csum(c1) from db.stb1 partition by st1 slimit 1") tdSql.checkRows(4) - tdSql.query("select csum(c1) from stb1 partition by st1 slimit 1") + tdSql.query("select csum(c1) from db.stb1 partition by st1 slimit 1") tdSql.checkRows(4) # partition by col - # tdSql.query("select c1 , csum(c1) from stb1 partition by c1") + # tdSql.query("select c1 , csum(c1) from db.stb1 partition by c1") # tdSql.checkRows(41) - # tdSql.query("select csum(c1) from stb1 partition by c1") + # tdSql.query("select csum(c1) from db.stb1 partition by c1") # tdSql.checkRows(41) - # tdSql.query("select c1 , csum(c1) from stb1 partition by st1 slimit 1") + # tdSql.query("select c1 , csum(c1) from db.stb1 partition by st1 slimit 1") # tdSql.checkRows(4) - # tdSql.query("select csum(c1) from stb1 partition by st1 slimit 1") + # tdSql.query("select csum(c1) from db.stb1 partition by st1 slimit 1") # tdSql.checkRows(4) diff --git a/tests/system-test/6-cluster/clusterCommonCheck.py b/tests/system-test/6-cluster/clusterCommonCheck.py index 992d77b03b..7ed1322b53 100644 --- a/tests/system-test/6-cluster/clusterCommonCheck.py +++ b/tests/system-test/6-cluster/clusterCommonCheck.py @@ -82,7 +82,7 @@ class ClusterComCheck: for i in range(alldbNumbers): tdSql.query("show databases;") if "%s_%d"%(dbNameIndex,j) == tdSql.queryResult[i][0] : - if tdSql.queryResult[i][19] == "ready": + if tdSql.queryResult[i][15] == "ready": query_status+=1 tdLog.debug("check %s_%d that status is ready "%(dbNameIndex,j)) else: diff --git a/tests/system-test/7-tmq/basic5.py b/tests/system-test/7-tmq/basic5.py index 4ed3be967e..70813eba96 100644 --- a/tests/system-test/7-tmq/basic5.py +++ b/tests/system-test/7-tmq/basic5.py @@ -265,7 +265,7 @@ class TDTestCase: else: continue - if tdSql.getData(index,19) == 'ready': + if tdSql.getData(index,15) == 'ready': print("******************** index: %d"%index) break @@ -395,7 +395,7 @@ class TDTestCase: while 1: tdSql.query("show databases") if tdSql.getRows() == 5: - print ('==================================================') + print ('==================================================dbname: %s'%parameterDict['dbName']) print (tdSql.getData(0,0), tdSql.getData(1,0),tdSql.getData(2,0),tdSql.getData(3,0),tdSql.getData(4,0)) index = 0 if tdSql.getData(0,0) == parameterDict['dbName']: @@ -409,9 +409,9 @@ class TDTestCase: elif tdSql.getData(4,0) == parameterDict['dbName']: index = 4 else: - continue + continue - if tdSql.getData(index,19) == 'ready': + if tdSql.getData(index,15) == 'ready': print("******************** index: %d"%index) break diff --git a/tests/system-test/7-tmq/dropDbR3ConflictTransaction.py b/tests/system-test/7-tmq/dropDbR3ConflictTransaction.py new file mode 100644 index 0000000000..4dac872fde --- /dev/null +++ b/tests/system-test/7-tmq/dropDbR3ConflictTransaction.py @@ -0,0 +1,193 @@ +from distutils.log import error +import taos +import sys +import time +import socket +import os +import threading +import subprocess +import platform + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +sys.path.append("./7-tmq") +from tmqCommon import * + + + +class TDTestCase: + def __init__(self): + self.snapshot = 0 + self.replica = 3 + self.vgroups = 3 + self.ctbNum = 2 + self.rowsPerTbl = 2 + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + #tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def checkFileContent(self, consumerId, queryString): + buildPath = tdCom.getBuildPath() + cfgPath = tdCom.getClientCfgPath() + dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId) + cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile) + tdLog.info(cmdStr) + os.system(cmdStr) + + consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId) + tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile)) + + consumeFile = open(consumeRowsFile, mode='r') + queryFile = open(dstFile, mode='r') + + # skip first line for it is schema + queryFile.readline() + + while True: + dst = queryFile.readline() + src = consumeFile.readline() + + if dst: + if dst != src: + tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId) + else: + break + return + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 2, + 'rowsPerTbl': 1000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 3, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=self.replica) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + # tmqCom.insert_data_with_autoCreateTbl(tsql=tdSql,dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix="ctbx", + # ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + # startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + # tmqCom.asyncInsertDataByInterlace(paraDict) + tmqCom.create_ntable(tdSql, dbname=paraDict["dbName"], tbname_prefix="ntb", tbname_index_start_num = 1, column_elm_list=paraDict["colSchema"], colPrefix='c', tblNum=1) + tmqCom.insert_rows_into_ntbl(tdSql, dbname=paraDict["dbName"], tbname_prefix="ntb", tbname_index_start_num = 1, column_ele_list=paraDict["colSchema"], startTs=paraDict["startTs"], tblNum=1, rows=2) # tdLog.info("restart taosd to ensure that the data falls into the disk") + tdSql.query("drop database %s"%paraDict["dbName"]) + return + + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: ") + + # create and start thread + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 100, + 'rowsPerTbl': 1000, + 'batchNum': 100, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 3, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 1} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tdLog.info("create topics from stb1") + topicFromStb1 = 'topic_stb1' + queryString = "select ts, c1, c2 from %s.%s where t4 == 'beijing' or t4 == 'changsha' "%(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicFromStb1, queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + topicList = topicFromStb1 + ifcheckdata = 0 + ifManualCommit = 0 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:false,\ + auto.commit.interval.ms:6000,\ + auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + pollDelay = 100 + showMsg = 1 + showRow = 1 + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + + tdLog.info("start to check consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + tdSql.query(queryString) + totalRowsInserted = tdSql.getRows() + + tdLog.info("act consume rows: %d, act insert rows: %d, expect consume rows: %d, "%(totalConsumeRows, totalRowsInserted, expectrowcnt)) + + if totalConsumeRows != expectrowcnt: + tdLog.exit("tmq consume rows error!") + + # tmqCom.checkFileContent(consumerId, queryString) + + tmqCom.waitSubscriptionExit(tdSql, topicFromStb1) + tdSql.query("drop topic %s"%topicFromStb1) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def run(self): + self.prepareTestEnv() + # self.tmqCase1() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index f074bd8850..8c19a88ac6 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -27,7 +27,7 @@ python3 ./test.py -f 1-insert/alter_stable.py python3 ./test.py -f 1-insert/alter_table.py python3 ./test.py -f 1-insert/insertWithMoreVgroup.py python3 ./test.py -f 1-insert/table_comment.py -python3 ./test.py -f 1-insert/time_range_wise.py +#python3 ./test.py -f 1-insert/time_range_wise.py python3 ./test.py -f 1-insert/block_wise.py python3 ./test.py -f 1-insert/create_retentions.py python3 ./test.py -f 1-insert/table_param_ttl.py @@ -59,8 +59,8 @@ python3 ./test.py -f 2-query/ceil.py python3 ./test.py -f 2-query/ceil.py -R python3 ./test.py -f 2-query/char_length.py python3 ./test.py -f 2-query/char_length.py -R -python3 ./test.py -f 2-query/check_tsdb.py -python3 ./test.py -f 2-query/check_tsdb.py -R +#python3 ./test.py -f 2-query/check_tsdb.py +#python3 ./test.py -f 2-query/check_tsdb.py -R python3 ./test.py -f 1-insert/update_data.py @@ -173,7 +173,7 @@ python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 6 -M 3 -C 5 # python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 # python3 test.py -f 6-cluster/5dnode3mnodeStopConnect.py -N 5 -M 3 - +python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3 python3 ./test.py -f 7-tmq/basic5.py python3 ./test.py -f 7-tmq/subscribeDb.py python3 ./test.py -f 7-tmq/subscribeDb0.py @@ -209,8 +209,8 @@ python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py -python3 ./test.py -f 7-tmq/tmqAutoCreateTbl.py -python3 ./test.py -f 7-tmq/tmqDnodeRestart.py +#python3 ./test.py -f 7-tmq/tmqAutoCreateTbl.py +#python3 ./test.py -f 7-tmq/tmqDnodeRestart.py python3 ./test.py -f 7-tmq/tmqUpdate-1ctb.py python3 ./test.py -f 7-tmq/tmqUpdateWithConsume.py python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot0.py @@ -250,7 +250,7 @@ python3 ./test.py -f 2-query/concat.py -Q 2 python3 ./test.py -f 2-query/concat2.py -Q 2 python3 ./test.py -f 2-query/concat_ws.py -Q 2 python3 ./test.py -f 2-query/concat_ws2.py -Q 2 -python3 ./test.py -f 2-query/check_tsdb.py -Q 2 +#python3 ./test.py -f 2-query/check_tsdb.py -Q 2 python3 ./test.py -f 2-query/spread.py -Q 2 python3 ./test.py -f 2-query/hyperloglog.py -Q 2 python3 ./test.py -f 2-query/explain.py -Q 2 @@ -340,7 +340,7 @@ python3 ./test.py -f 2-query/concat.py -Q 3 python3 ./test.py -f 2-query/concat2.py -Q 3 python3 ./test.py -f 2-query/concat_ws.py -Q 3 python3 ./test.py -f 2-query/concat_ws2.py -Q 3 -python3 ./test.py -f 2-query/check_tsdb.py -Q 3 +#python3 ./test.py -f 2-query/check_tsdb.py -Q 3 python3 ./test.py -f 2-query/spread.py -Q 3 python3 ./test.py -f 2-query/hyperloglog.py -Q 3 python3 ./test.py -f 2-query/explain.py -Q 3 diff --git a/tests/test/c/sdbDump.c b/tests/test/c/sdbDump.c index 4d0f582dc6..aef5056031 100644 --- a/tests/test/c/sdbDump.c +++ b/tests/test/c/sdbDump.c @@ -79,8 +79,6 @@ void dumpDb(SSdb *pSdb, SJson *json) { tjsonAddIntegerToObject(item, "daysToKeep2", pObj->cfg.daysToKeep2); tjsonAddIntegerToObject(item, "minRows", pObj->cfg.minRows); tjsonAddIntegerToObject(item, "maxRows", pObj->cfg.maxRows); - tjsonAddIntegerToObject(item, "fsyncPeriod", pObj->cfg.fsyncPeriod); - tjsonAddIntegerToObject(item, "walLevel", pObj->cfg.walLevel); tjsonAddIntegerToObject(item, "precision", pObj->cfg.precision); tjsonAddIntegerToObject(item, "compression", pObj->cfg.compression); tjsonAddIntegerToObject(item, "replications", pObj->cfg.replications); @@ -89,7 +87,13 @@ void dumpDb(SSdb *pSdb, SJson *json) { tjsonAddIntegerToObject(item, "hashMethod", pObj->cfg.hashMethod); tjsonAddIntegerToObject(item, "numOfRetensions", pObj->cfg.numOfRetensions); tjsonAddIntegerToObject(item, "schemaless", pObj->cfg.schemaless); - + tjsonAddIntegerToObject(item, "walLevel", pObj->cfg.walLevel); + tjsonAddIntegerToObject(item, "walFsyncPeriod", pObj->cfg.walFsyncPeriod); + tjsonAddIntegerToObject(item, "walRetentionPeriod", pObj->cfg.walRetentionPeriod); + tjsonAddIntegerToObject(item, "walRetentionSize", pObj->cfg.walRetentionSize); + tjsonAddIntegerToObject(item, "walRollPeriod", pObj->cfg.walRollPeriod); + tjsonAddIntegerToObject(item, "walSegmentSize", pObj->cfg.walSegmentSize); + sdbRelease(pSdb, pObj); } } diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index 6a18263d50..d39ade7e91 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -630,7 +630,7 @@ static int32_t meta_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIn { tmq_raw_data raw = {0}; - int32_t code = tmq_get_raw_meta(msg, &raw); + int32_t code = tmq_get_raw(msg, &raw); if(code == TSDB_CODE_SUCCESS){ int retCode = queryDB(pInfo->taos, "use metadb"); @@ -641,7 +641,7 @@ static int32_t meta_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIn } taosFprintfFile(g_fp, "raw:%p\n", &raw); - taos_write_raw_meta(pInfo->taos, raw); + tmq_write_raw(pInfo->taos, raw); } char* result = tmq_get_json_meta(msg);