diff --git a/docs/en/07-develop/02-model/index.mdx b/docs/en/07-develop/02-model/index.mdx index e0378cc77c..b647c845d0 100644 --- a/docs/en/07-develop/02-model/index.mdx +++ b/docs/en/07-develop/02-model/index.mdx @@ -9,15 +9,15 @@ The data model employed by TDengine is similar to that of a relational database. The [characteristics of time-series data](https://www.taosdata.com/blog/2019/07/09/86.html) from different data collection points may be different. Characteristics include collection frequency, retention policy and others which determine how you create and configure the database. For e.g. days to keep, number of replicas, data block size, whether data updates are allowed and other configurable parameters would be determined by the characteristics of your data and your business requirements. For TDengine to operate with the best performance, we strongly recommend that you create and configure different databases for data with different characteristics. This allows you, for example, to set up different storage and retention policies. When creating a database, there are a lot of parameters that can be configured such as, the days to keep data, the number of replicas, the number of memory blocks, time precision, the minimum and maximum number of rows in each data block, whether compression is enabled, the time range of the data in single data file and so on. Below is an example of the SQL statement to create a database. ```sql -CREATE DATABASE power KEEP 365 DAYS 10 BLOCKS 6 UPDATE 1; +CREATE DATABASE power KEEP 365 DURATION 10 BUFFER 16 VGROUPS 100 WAL 1; ``` In the above SQL statement: - a database named "power" will be created - the data in it will be kept for 365 days, which means that data older than 365 days will be deleted automatically - a new data file will be created every 10 days -- the number of memory blocks is 6 -- data is allowed to be updated +- the size of memory cache for writing is 16 MB +- data will be firstly written to WAL without FSYNC For more details please refer to [Database](/taos-sql/database). @@ -30,7 +30,6 @@ USE power; :::note - Any table or STable must belong to a database. To create a table or STable, the database it belongs to must be ready. -- JOIN operations can't be performed on tables from two different databases. - Timestamp needs to be specified when inserting rows or querying historical rows. ::: @@ -52,7 +51,7 @@ Similar to creating a regular table, when creating a STable, the name and schema For each kind of data collection point, a corresponding STable must be created. There may be many STables in an application. For electrical power system, we need to create a STable respectively for meters, transformers, busbars, switches. There may be multiple kinds of data collection points on a single device, for example there may be one data collection point for electrical data like current and voltage and another data collection point for environmental data like temperature, humidity and wind direction. Multiple STables are required for these kinds of devices. -At most 4096 (or 1024 prior to version 2.1.7.0) columns are allowed in a STable. If there are more than 4096 of metrics to be collected for a data collection point, multiple STables are required. There can be multiple databases in a system, while one or more STables can exist in a database. +At most 4096 columns are allowed in a STable. If there are more than 4096 of metrics to be collected for a data collection point, multiple STables are required. There can be multiple databases in a system, while one or more STables can exist in a database. ## Create Table @@ -66,12 +65,11 @@ In the above SQL statement, "d1001" is the table name, "meters" is the STable na In the TDengine system, it's recommended to create a table for a data collection point via STable. A table created via STable is called subtable in some parts of the TDengine documentation. All SQL commands applied on regular tables can be applied on subtables. -:::warning -It's not recommended to create a table in a database while using a STable from another database as template. - :::tip It's suggested to use the globally unique ID of a data collection point as the table name. For example the device serial number could be used as a unique ID. If a unique ID doesn't exist, multiple IDs that are not globally unique can be combined to form a globally unique ID. It's not recommended to use a globally unique ID as tag value. +::: + ## Create Table Automatically In some circumstances, it's unknown whether the table already exists when inserting rows. The table can be created automatically using the SQL statement below, and nothing will happen if the table already exists. diff --git a/docs/en/07-develop/03-insert-data/01-sql-writing.mdx b/docs/en/07-develop/03-insert-data/01-sql-writing.mdx index d8c4453f40..da0e87abbc 100644 --- a/docs/en/07-develop/03-insert-data/01-sql-writing.mdx +++ b/docs/en/07-develop/03-insert-data/01-sql-writing.mdx @@ -42,7 +42,7 @@ INSERT INTO d1001 VALUES (1538548684000, 10.2, 220, 0.23) (1538548696650, 10.3, ### Insert into Multiple Tables -Data can be inserted into multiple tables in the same SQL statement. The example below inserts 2 rows into table "d1001" and 1 row into table "d1002". +Data can be inserted into multiple tables in single SQL statement. The example below inserts 2 rows into table "d1001" and 1 row into table "d1002". ```sql INSERT INTO d1001 VALUES (1538548685000, 10.3, 219, 0.31) (1538548695000, 12.6, 218, 0.33) d1002 VALUES (1538548696800, 12.3, 221, 0.31); @@ -52,15 +52,15 @@ For more details about `INSERT` please refer to [INSERT](/taos-sql/insert). :::info -- Inserting in batches can improve performance. Normally, the higher the batch size, the better the performance. Please note that a single row can't exceed 48K bytes and each SQL statement can't exceed 1MB. -- Inserting with multiple threads can also improve performance. However, depending on the system resources on the application side and the server side, when the number of inserting threads grows beyond a specific point the performance may drop instead of improving. The proper number of threads needs to be tested in a specific environment to find the best number. +- Inserting in batches can improve performance. Normally, the higher the batch size, the better the performance. Please note that a single row can't exceed 48 KB bytes and each SQL statement can't exceed 1 MB. +- Inserting with multiple threads can also improve performance. However, depending on the system resources on the application side and the server side, when the number of inserting threads grows beyond a specific point the performance may drop instead of improving. The proper number of threads needs to be tested in a specific environment to find the best number. The proper number of threads may be impacted by the system resources on the server side, the system resources on the client side, the table schemas, etc. ::: :::warning -- If the timestamp for the row to be inserted already exists in the table, the behavior depends on the value of parameter `UPDATE`. If it's set to 0 (the default value), the row will be discarded. If it's set to 1, the new values will override the old values for the same row. -- The timestamp to be inserted must be newer than the timestamp of subtracting current time by the parameter `KEEP`. If `KEEP` is set to 3650 days, then the data older than 3650 days ago can't be inserted. The timestamp to be inserted can't be newer than the timestamp of current time plus parameter `DAYS`. If `DAYS` is set to 2, the data newer than 2 days later can't be inserted. +- If the timestamp for the row to be inserted already exists in the table, the old data will be overritten by the new values for the columns for which new values are provided, columns for which no new values are provided are not impacted. +- The timestamp to be inserted must be newer than the timestamp of subtracting current time by the parameter `KEEP`. If `KEEP` is set to 3650 days, then the data older than 3650 days ago can't be inserted. The timestamp to be inserted can't be newer than the timestamp of current time plus parameter `DURATION`. If `DAYS` is set to 2, the data newer than 2 days later can't be inserted. ::: @@ -101,7 +101,7 @@ For more details about `INSERT` please refer to [INSERT](/taos-sql/insert). ### Insert with Parameter Binding -TDengine also provides API support for parameter binding. Similar to MySQL, only `?` can be used in these APIs to represent the parameters to bind. From version 2.1.1.0 and 2.1.2.0, parameter binding support for inserting data has improved significantly to improve the insert performance by avoiding the cost of parsing SQL statements. +TDengine also provides API support for parameter binding. Similar to MySQL, only `?` can be used in these APIs to represent the parameters to bind. Parameter binding support for inserting data has improved significantly to improve the insert performance by avoiding the cost of parsing SQL statements. Parameter binding is available only with native connection. diff --git a/docs/en/12-taos-sql/02-database.md b/docs/en/12-taos-sql/02-database.md index 80581b2f1b..c2961d6241 100644 --- a/docs/en/12-taos-sql/02-database.md +++ b/docs/en/12-taos-sql/02-database.md @@ -32,7 +32,6 @@ CREATE DATABASE [IF NOT EXISTS] db_name [KEEP keep] [DAYS days] [UPDATE 1]; - cacheLast: [Description](/reference/config/#cachelast) - replica: [Description](/reference/config/#replica) - quorum: [Description](/reference/config/#quorum) - - maxVgroupsPerDb: [Description](/reference/config/#maxvgroupsperdb) - comp: [Description](/reference/config/#comp) - precision: [Description](/reference/config/#precision) 6. Please note that all of the parameters mentioned in this section are configured in configuration file `taos.cfg` on the TDengine server. If not specified in the `create database` statement, the values from taos.cfg are used by default. To override default parameters, they must be specified in the `create database` statement. diff --git a/docs/zh/07-develop/02-model/index.mdx b/docs/zh/07-develop/02-model/index.mdx index 7e2762b6e7..be545e8813 100644 --- a/docs/zh/07-develop/02-model/index.mdx +++ b/docs/zh/07-develop/02-model/index.mdx @@ -8,13 +8,13 @@ TDengine 采用类关系型数据模型，需要建库、建表。因此对于 ## 创建库 -不同类型的数据采集点往往具有不同的数据特征，包括数据采集频率的高低，数据保留时间的长短，副本的数目，数据块的大小，是否允许更新数据等等。为了在各种场景下 TDengine 都能最大效率的工作，TDengine 建议将不同数据特征的表创建在不同的库里，因为每个库可以配置不同的存储策略。创建一个库时，除 SQL 标准的选项外，还可以指定保留时长、副本数、内存块个数、时间精度、文件块里最大最小记录条数、是否压缩、一个数据文件覆盖的天数等多种参数。比如： +不同类型的数据采集点往往具有不同的数据特征，包括数据采集频率的高低，数据保留时间的长短，副本的数目，数据块的大小，是否允许更新数据等等。为了在各种场景下 TDengine 都能最大效率的工作，TDengine 建议将不同数据特征的表创建在不同的库里，因为每个库可以配置不同的存储策略。创建一个库时，除 SQL 标准的选项外，还可以指定保留时长、副本数、缓存大小、时间精度、文件块里最大最小记录条数、是否压缩、一个数据文件覆盖的天数等多种参数。比如： ```sql -CREATE DATABASE power KEEP 365 DAYS 10 BLOCKS 6 UPDATE 1; +CREATE DATABASE power KEEP 365 DURATION 10 BUFFER 16 VGROUPS 100 WAL 1; ``` -上述语句将创建一个名为 power 的库，这个库的数据将保留 365 天（超过 365 天将被自动删除），每 10 天一个数据文件，内存块数为 6，允许更新数据。详细的语法及参数请见 [数据库管理](/taos-sql/database) 章节。 +上述语句将创建一个名为 power 的库，这个库的数据将保留 365 天（超过 365 天将被自动删除），每 10 天一个数据文件，每个 VNODE 的写入内存池的大小为 16 MB，数据库的 VGROUPS 数量，对该数据库入会写 WAL 但不执行 FSYNC。详细的语法及参数请见 [数据库管理](/taos-sql/database) 章节。 创建库之后，需要使用 SQL 命令 `USE` 将当前库切换过来，例如： @@ -27,7 +27,6 @@ USE power; :::note - 任何一张表或超级表必须属于某个库，在创建表之前，必须先创建库。 -- 处于两个不同库的表是不能进行 JOIN 操作的。 - 创建并插入记录、查询历史记录的时候，均需要指定时间戳。 ::: @@ -40,15 +39,11 @@ USE power; CREATE STABLE meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupId int); ``` -:::note -这一指令中的 STABLE 关键字，在 2.0.15 之前的版本中需写作 TABLE 。 -::: - 与创建普通表一样，创建超级表时，需要提供表名（示例中为 meters），表结构 Schema，即数据列的定义。第一列必须为时间戳（示例中为 ts)，其他列为采集的物理量（示例中为 current, voltage, phase)，数据类型可以为整型、浮点型、字符串等。除此之外，还需要提供标签的 schema (示例中为 location, groupId)，标签的数据类型可以为整型、浮点型、字符串等。采集点的静态属性往往可以作为标签，比如采集点的地理位置、设备型号、设备组 ID、管理员 ID 等等。标签的 schema 可以事后增加、删除、修改。具体定义以及细节请见 [TAOS SQL 的超级表管理](/taos-sql/stable) 章节。 每一种类型的数据采集点需要建立一个超级表，因此一个物联网系统，往往会有多个超级表。对于电网，我们就需要对智能电表、变压器、母线、开关等都建立一个超级表。在物联网中，一个设备就可能有多个数据采集点（比如一台风力发电的风机，有的采集点采集电流、电压等电参数，有的采集点采集温度、湿度、风向等环境参数），这个时候，对这一类型的设备，需要建立多张超级表。 -一张超级表最多容许 4096 列 （在 2.1.7.0 版本之前，列数限制为 1024 列），如果一个采集点采集的物理量个数超过 4096，需要建多张超级表来处理。一个系统可以有多个 DB，一个 DB 里可以有一到多个超级表。 +一张超级表最多容许 4096 列，如果一个采集点采集的物理量个数超过 4096，需要建多张超级表来处理。一个系统可以有多个 DB，一个 DB 里可以有一到多个超级表。 ## 创建表 @@ -60,11 +55,6 @@ CREATE TABLE d1001 USING meters TAGS ("California.SanFrancisco", 2); 其中 d1001 是表名，meters 是超级表的表名，后面紧跟标签 Location 的具体标签值 "California.SanFrancisco"，标签 groupId 的具体标签值 2。虽然在创建表时，需要指定标签值，但可以事后修改。详细细则请见 [TAOS SQL 的表管理](/taos-sql/table) 章节。 -:::warning -目前 TDengine 没有从技术层面限制使用一个 database (db1) 的超级表作为模板建立另一个 database (db2) 的子表，后续会禁止这种用法，不建议使用这种方法建表。 - -::: - TDengine 建议将数据采集点的全局唯一 ID 作为表名(比如设备序列号）。但对于有的场景，并没有唯一的 ID，可以将多个 ID 组合成一个唯一的 ID。不建议将具有唯一性的 ID 作为标签值。 ### 自动建表 diff --git a/docs/zh/07-develop/03-insert-data/01-sql-writing.mdx b/docs/zh/07-develop/03-insert-data/01-sql-writing.mdx index 99a92573c8..214cbdaa96 100644 --- a/docs/zh/07-develop/03-insert-data/01-sql-writing.mdx +++ b/docs/zh/07-develop/03-insert-data/01-sql-writing.mdx @@ -52,15 +52,15 @@ INSERT INTO d1001 VALUES (1538548685000, 10.3, 219, 0.31) (1538548695000, 12.6, :::info -- 要提高写入效率，需要批量写入。一批写入的记录条数越多，插入效率就越高。但一条记录不能超过 48K，一条 SQL 语句总长度不能超过 1M 。 -- TDengine 支持多线程同时写入，要进一步提高写入速度，一个客户端需要打开 20 个以上的线程同时写。但线程数达到一定数量后，无法再提高，甚至还会下降，因为线程频繁切换，带来额外开销。 +- 要提高写入效率，需要批量写入。一般来说一批写入的记录条数越多，插入效率就越高。但一条记录不能超过 48K，一条 SQL 语句总长度不能超过 1M 。 +- TDengine 支持多线程同时写入，要进一步提高写入速度，一个客户端需要打开多个同时写。但线程数达到一定数量后，无法再提高，甚至还会下降，因为线程频繁切换，会带来额外开销，合适的线程数量与服务端的处理能力，服务端的具体配置，数据库的参数，数据定义的 Schema，写入数据的 Batch Size 等很多因素相关。一般来说，服务端和客户端处理能力越强，所能支持的并发写入的线程可以越多；数据库配置时的 vgroups 越多（但仍然要在服务端的处理能力以内）则所能支持的并发写入越多；数据定义的 Schema 越简单，所能支持的并发写入越多。 ::: :::warning -- 对同一张表，如果新插入记录的时间戳已经存在，默认情形下（UPDATE=0）新记录将被直接抛弃，也就是说，在一张表里，时间戳必须是唯一的。如果应用自动生成记录，很有可能生成的时间戳是一样的，这样，成功插入的记录条数会小于应用插入的记录条数。如果在创建数据库时使用了 UPDATE 1 选项，插入相同时间戳的新记录将覆盖原有记录。 -- 写入的数据的时间戳必须大于当前时间减去配置参数 keep 的时间。如果 keep 配置为 3650 天，那么无法写入比 3650 天还早的数据。写入数据的时间戳也不能大于当前时间加配置参数 days。如果 days 为 2，那么无法写入比当前时间还晚 2 天的数据。 +- 对同一张表，如果新插入记录的时间戳已经存在，则指定了新值的列会用新值覆盖旧值，而没有指定新值的列则不受影响。 +- 写入的数据的时间戳必须大于当前时间减去配置参数 keep 的时间。如果 keep 配置为 3650 天，那么无法写入比 3650 天还早的数据。写入数据的时间戳也不能大于当前时间加配置参数 duration。如果 duration 为 2，那么无法写入比当前时间还晚 2 天的数据。 ::: @@ -104,7 +104,7 @@ INSERT INTO d1001 VALUES (1538548685000, 10.3, 219, 0.31) (1538548695000, 12.6, ### 参数绑定写入 -TDengine 也提供了支持参数绑定的 Prepare API，与 MySQL 类似，这些 API 目前也仅支持用问号 `?` 来代表待绑定的参数。从 2.1.1.0 和 2.1.2.0 版本开始，TDengine 大幅改进了参数绑定接口对数据写入（INSERT）场景的支持。这样在通过参数绑定接口写入数据时，就避免了 SQL 语法解析的资源消耗，从而在绝大多数情况下显著提升写入性能。 +TDengine 也提供了支持参数绑定的 Prepare API，与 MySQL 类似，这些 API 目前也仅支持用问号 `?` 来代表待绑定的参数。在通过参数绑定接口写入数据时，就避免了 SQL 语法解析的资源消耗，从而在绝大多数情况下显著提升写入性能。 需要注意的是，只有使用原生连接的连接器，才能使用参数绑定功能。 diff --git a/docs/zh/12-taos-sql/02-database.md b/docs/zh/12-taos-sql/02-database.md index 566fec3241..e3a0aa7c87 100644 --- a/docs/zh/12-taos-sql/02-database.md +++ b/docs/zh/12-taos-sql/02-database.md @@ -32,7 +32,6 @@ CREATE DATABASE [IF NOT EXISTS] db_name [KEEP keep] [DAYS days] [UPDATE 1]; - cacheLast: [详细说明](/reference/config/#cachelast) - replica: [详细说明](/reference/config/#replica) - quorum: [详细说明](/reference/config/#quorum) - - maxVgroupsPerDb: [详细说明](/reference/config/#maxvgroupsperdb) - comp: [详细说明](/reference/config/#comp) - precision: [详细说明](/reference/config/#precision) 6. 请注意上面列出的所有参数都可以配置在配置文件 `taosd.cfg` 中作为创建数据库时使用的默认配置， `create database` 的参数中明确指定的会覆盖配置文件中的设置。 diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 41674b7a70..944eaa28bc 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -67,7 +67,6 @@ extern int32_t tsNumOfVnodeQueryThreads; extern int32_t tsNumOfVnodeFetchThreads; extern int32_t tsNumOfVnodeWriteThreads; extern int32_t tsNumOfVnodeSyncThreads; -extern int32_t tsNumOfVnodeMergeThreads; extern int32_t tsNumOfQnodeQueryThreads; extern int32_t tsNumOfQnodeFetchThreads; extern int32_t tsNumOfSnodeSharedThreads; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index e09de1e23e..0ab65e7978 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -723,7 +723,7 @@ typedef struct { int32_t buffer; // MB int32_t pageSize; int32_t pages; - int32_t lastRowMem; + int32_t cacheLastSize; int32_t daysPerFile; int32_t daysToKeep0; int32_t daysToKeep1; @@ -736,7 +736,7 @@ typedef struct { int8_t compression; int8_t replications; int8_t strict; - int8_t cacheLastRow; + int8_t cacheLast; int8_t schemaless; int8_t ignoreExist; int32_t numOfRetensions; @@ -752,7 +752,7 @@ typedef struct { int32_t buffer; int32_t pageSize; int32_t pages; - int32_t lastRowMem; + int32_t cacheLastSize; int32_t daysPerFile; int32_t daysToKeep0; int32_t daysToKeep1; @@ -760,7 +760,7 @@ typedef struct { int32_t fsyncPeriod; int8_t walLevel; int8_t strict; - int8_t cacheLastRow; + int8_t cacheLast; int8_t replications; } SAlterDbReq; @@ -822,6 +822,20 @@ typedef struct { int32_t tSerializeSTrimDbReq(void* buf, int32_t bufLen, STrimDbReq* pReq); int32_t tDeserializeSTrimDbReq(void* buf, int32_t bufLen, STrimDbReq* pReq); +typedef struct { + int32_t timestamp; +} SVTrimDbReq; + +int32_t tSerializeSVTrimDbReq(void* buf, int32_t bufLen, SVTrimDbReq* pReq); +int32_t tDeserializeSVTrimDbReq(void* buf, int32_t bufLen, SVTrimDbReq* pReq); + +typedef struct { + int32_t timestamp; +} SVDropTtlTableReq; + +int32_t tSerializeSVDropTtlTableReq(void* buf, int32_t bufLen, SVDropTtlTableReq* pReq); +int32_t tDeserializeSVDropTtlTableReq(void* buf, int32_t bufLen, SVDropTtlTableReq* pReq); + typedef struct { int32_t numOfVgroups; int32_t numOfStables; @@ -840,7 +854,7 @@ typedef struct { int8_t compression; int8_t replications; int8_t strict; - int8_t cacheLastRow; + int8_t cacheLast; int32_t numOfRetensions; SArray* pRetensions; int8_t schemaless; @@ -1105,7 +1119,7 @@ typedef struct { int32_t buffer; int32_t pageSize; int32_t pages; - int32_t lastRowMem; + int32_t cacheLastSize; int32_t daysPerFile; int32_t daysToKeep0; int32_t daysToKeep1; @@ -1120,7 +1134,7 @@ typedef struct { int8_t precision; int8_t compression; int8_t strict; - int8_t cacheLastRow; + int8_t cacheLast; int8_t isTsma; int8_t standby; int8_t replica; @@ -1158,7 +1172,7 @@ typedef struct { int32_t buffer; int32_t pageSize; int32_t pages; - int32_t lastRowMem; + int32_t cacheLastSize; int32_t daysPerFile; int32_t daysToKeep0; int32_t daysToKeep1; @@ -1166,7 +1180,7 @@ typedef struct { int32_t fsyncPeriod; int8_t walLevel; int8_t strict; - int8_t cacheLastRow; + int8_t cacheLast; int8_t selfIndex; int8_t replica; SReplica replicas[TSDB_MAX_REPLICA]; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 1dbfbfb2b9..488bc6346e 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -199,9 +199,10 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_ALTER_REPLICA, "alter-replica", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIRM, "alter-confirm", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_HASHRANGE, "alter-hashrange", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_COMPACT, "compact", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_DROP_TTL_TABLE, "drop-ttl-stb", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_COMMIT, "commit vnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_COMPACT, "vnode-compact", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_DROP_TTL_TABLE, "vnode-drop-ttl-stb", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_TRIM, "vnode-trim", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_COMMIT, "vnode-commit", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_MAX_MSG, "vnd-max", NULL, NULL) TD_NEW_MSG_SEG(TDMT_SCH_MSG) diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index 70056783ea..3bed77d682 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -52,6 +52,7 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption); * @param pMnode The mnode object to close. */ void mndClose(SMnode *pMnode); +void mndPreClose(SMnode *pMnode); /** * @brief Start mnode diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 0a8e55bb4f..7e2d09dd63 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -45,7 +45,6 @@ extern "C" { #define WAL_MAGIC 0xFAFBFCFDULL typedef enum { - TAOS_WAL_NOLOG = 0, TAOS_WAL_WRITE = 1, TAOS_WAL_FSYNC = 2, } EWalType; @@ -74,7 +73,7 @@ typedef struct { int8_t isWeek; uint64_t seqNum; uint64_t term; -} SSyncLogMeta; +} SWalSyncInfo; typedef struct { int8_t protoVer; @@ -84,7 +83,7 @@ typedef struct { int64_t ingestTs; // not implemented // sync meta - SSyncLogMeta syncMeta; + SWalSyncInfo syncMeta; char body[]; } SWalCont; @@ -149,11 +148,22 @@ SWal *walOpen(const char *path, SWalCfg *pCfg); int32_t walAlter(SWal *, SWalCfg *pCfg); void walClose(SWal *); -// write -int32_t walWriteWithSyncInfo(SWal *, int64_t index, tmsg_t msgType, SSyncLogMeta syncMeta, const void *body, - int32_t bodyLen); +// write interfaces + +// By assigning index by the caller, wal gurantees linearizability int32_t walWrite(SWal *, int64_t index, tmsg_t msgType, const void *body, int32_t bodyLen); -void walFsync(SWal *, bool force); +int32_t walWriteWithSyncInfo(SWal *, int64_t index, tmsg_t msgType, SWalSyncInfo syncMeta, const void *body, + int32_t bodyLen); + +// This interface assign version automatically and return to caller. +// When using this interface with concurrent writes, +// wal will write all logs atomically, +// but not sure which one will be actually write first, +// and then the unique index of successful writen is returned. +// -1 will be returned for failed writes +int64_t walAppendLog(SWal *, tmsg_t msgType, SWalSyncInfo syncMeta, const void *body, int32_t bodyLen); + +void walFsync(SWal *, bool force); // apis for lifecycle management int32_t walCommit(SWal *, int64_t ver); diff --git a/include/util/tRealloc.h b/include/util/tRealloc.h index 8d40f6cc5d..f3593d5818 100644 --- a/include/util/tRealloc.h +++ b/include/util/tRealloc.h @@ -25,10 +25,11 @@ extern "C" { static FORCE_INLINE int32_t tRealloc(uint8_t **ppBuf, int64_t size) { int32_t code = 0; int64_t bsize = 0; - uint8_t *pBuf; + uint8_t *pBuf = NULL; if (*ppBuf) { - bsize = *(int64_t *)((*ppBuf) - sizeof(int64_t)); + pBuf = (*ppBuf) - sizeof(int64_t); + bsize = *(int64_t *)pBuf; } if (bsize >= size) goto _exit; @@ -38,7 +39,7 @@ static FORCE_INLINE int32_t tRealloc(uint8_t **ppBuf, int64_t size) { bsize *= 2; } - pBuf = (uint8_t *)taosMemoryRealloc(*ppBuf ? (*ppBuf) - sizeof(int64_t) : *ppBuf, bsize + sizeof(int64_t)); + pBuf = (uint8_t *)taosMemoryRealloc(pBuf, bsize + sizeof(int64_t)); if (pBuf == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; diff --git a/include/util/tdef.h b/include/util/tdef.h index 84bc30b9e7..55194d8647 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -331,12 +331,12 @@ typedef enum ELogicConditionType { #define TSDB_DB_STRICT_OFF 0 #define TSDB_DB_STRICT_ON 1 #define TSDB_DEFAULT_DB_STRICT 0 -#define TSDB_MIN_DB_CACHE_LAST_ROW 0 -#define TSDB_MAX_DB_CACHE_LAST_ROW 3 -#define TSDB_DEFAULT_CACHE_LAST_ROW 0 -#define TSDB_MIN_DB_LAST_ROW_MEM 1 // MB -#define TSDB_MAX_DB_LAST_ROW_MEM 65536 -#define TSDB_DEFAULT_LAST_ROW_MEM 1 +#define TSDB_MIN_DB_CACHE_LAST 0 +#define TSDB_MAX_DB_CACHE_LAST 3 +#define TSDB_DEFAULT_CACHE_LAST 0 +#define TSDB_MIN_DB_CACHE_LAST_SIZE 1 // MB +#define TSDB_MAX_DB_CACHE_LAST_SIZE 65536 +#define TSDB_DEFAULT_CACHE_LAST_SIZE 1 #define TSDB_DB_STREAM_MODE_OFF 0 #define TSDB_DB_STREAM_MODE_ON 1 #define TSDB_DEFAULT_DB_STREAM_MODE 0 diff --git a/include/util/thash.h b/include/util/thash.h index fc8785a8fb..781c22a56a 100644 --- a/include/util/thash.h +++ b/include/util/thash.h @@ -188,7 +188,7 @@ void *taosHashGetKey(void *data, size_t* keyLen); void *taosHashAcquire(SHashObj *pHashObj, const void *key, size_t keyLen); /** - * release the prevous acquired obj + * release the previous acquired obj * * @param pHashObj * @param data diff --git a/include/util/tlockfree.h b/include/util/tlockfree.h index 54a90d7b71..8db6be8860 100644 --- a/include/util/tlockfree.h +++ b/include/util/tlockfree.h @@ -69,10 +69,9 @@ typedef void (*_ref_fn_t)(const void *pObj); #define T_REF_VAL_GET(x) (x)->_ref.val // single writer multiple reader lock -typedef volatile int64_t SRWLatch; +typedef volatile int32_t SRWLatch; void taosInitRWLatch(SRWLatch *pLatch); -void taosInitReentrantRWLatch(SRWLatch *pLatch); void taosWLockLatch(SRWLatch *pLatch); void taosWUnLockLatch(SRWLatch *pLatch); void taosRLockLatch(SRWLatch *pLatch); diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 77817e5cf6..f409834091 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -328,7 +328,7 @@ function install_header() { ${csudo}ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h ${csudo}ln -s ${install_main_dir}/include/taosudf.h ${inc_link_dir}/taosudf.h - ${csudo}ln -s ${install_main_dir}/include/taosws.h ${inc_link_dir}/taosws.h || : + [ -f ${install_main_dir}/include/taosws.h ] && ${csudo}ln -s ${install_main_dir}/include/taosws.h ${inc_link_dir}/taosws.h || : } function add_newHostname_to_hosts() { diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index 0f1080521e..ca3a9a19be 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -161,13 +161,11 @@ if [[ $productName == "TDengine" ]]; then mkdir -p ${install_dir}/connector if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then if [ "$osType" != "Darwin" ]; then - cp ${build_dir}/lib/*.jar ${install_dir}/connector || : - fi - if find ${connector_dir}/go -mindepth 1 -maxdepth 1 | read; then - cp -r ${connector_dir}/go ${install_dir}/connector - else - echo "WARNING: go connector not found, please check if want to use it!" + [ -f ${build_dir}/lib/*.jar ] && cp ${build_dir}/lib/*.jar ${install_dir}/connector || : fi + git clone --depth 1 https://github.com/taosdata/driver-go ${install_dir}/connector/go + rm -rf ${install_dir}/connector/go/.git ||: + git clone --depth 1 https://github.com/taosdata/taos-connector-python ${install_dir}/connector/python rm -rf ${install_dir}/connector/python/.git ||: # cp -r ${connector_dir}/python ${install_dir}/connector diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 7edab2141b..c37397005c 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -290,19 +290,17 @@ fi # Copy driver mkdir -p ${install_dir}/driver && cp ${lib_files} ${install_dir}/driver && echo "${versionComp}" >${install_dir}/driver/vercomp.txt -cp ${wslib_files} ${install_dir}/driver || : +[ -f ${wslib_files} ] && cp ${wslib_files} ${install_dir}/driver || : # Copy connector if [ "$verMode" == "cluster" ]; then connector_dir="${code_dir}/connector" mkdir -p ${install_dir}/connector if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then - cp ${build_dir}/lib/*.jar ${install_dir}/connector || : - if find ${connector_dir}/go -mindepth 1 -maxdepth 1 | read; then - cp -r ${connector_dir}/go ${install_dir}/connector - else - echo "WARNING: go connector not found, please check if want to use it!" - fi + [ -f ${build_dir}/lib/*.jar ] && cp ${build_dir}/lib/*.jar ${install_dir}/connector || : + git clone --depth 1 https://github.com/taosdata/driver-go ${install_dir}/connector/go + rm -rf ${install_dir}/connector/go/.git ||: + git clone --depth 1 https://github.com/taosdata/taos-connector-python ${install_dir}/connector/python rm -rf ${install_dir}/connector/python/.git ||: @@ -314,6 +312,7 @@ if [ "$verMode" == "cluster" ]; then git clone --depth 1 https://github.com/taosdata/libtaos-rs ${install_dir}/connector/rust rm -rf ${install_dir}/connector/rust/.git ||: + # cp -r ${connector_dir}/python ${install_dir}/connector # cp -r ${connector_dir}/nodejs ${install_dir}/connector fi diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh index ec836f2eac..1bd5ed3ea4 100755 --- a/packaging/tools/remove.sh +++ b/packaging/tools/remove.sh @@ -114,6 +114,7 @@ function clean_header() { ${csudo}rm -f ${inc_link_dir}/taos.h || : ${csudo}rm -f ${inc_link_dir}/taosdef.h || : ${csudo}rm -f ${inc_link_dir}/taoserror.h || : + ${csudo}rm -f ${inc_link_dir}/taosudf.h || : ${csudo}rm -f ${inc_link_dir}/taosws.h || : } diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index ba92ed238b..6805e4d501 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -189,12 +189,15 @@ void destroyTscObj(void *pObj) { SClientHbKey connKey = {.tscRid = pTscObj->id, .connType = pTscObj->connType}; hbDeregisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey); - int64_t connNum = atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); + destroyAllRequests(pTscObj->pRequests); + taosHashCleanup(pTscObj->pRequests); + schedulerStopQueryHb(pTscObj->pAppInfo->pTransporter); tscDebug("connObj 0x%" PRIx64 " p:%p destroyed, remain inst totalConn:%" PRId64, pTscObj->id, pTscObj, pTscObj->pAppInfo->numOfConns); + int64_t connNum = atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); if (0 == connNum) { destroyAppInst(pTscObj->pAppInfo); } diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index e2d75d39e3..2a9d113108 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -671,8 +671,7 @@ static void *hbThreadFunc(void *param) { } #endif while (1) { - int8_t threadStop = atomic_val_compare_exchange_8(&clientHbMgr.threadStop, 1, 2); - if (1 == threadStop) { + if (1 == clientHbMgr.threadStop) { break; } @@ -760,9 +759,7 @@ static void hbStopThread() { return; } - while (2 != atomic_load_8(&clientHbMgr.threadStop)) { - taosUsleep(10); - } + taosThreadJoin(clientHbMgr.thread, NULL); tscDebug("hb thread stopped"); } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 9f53b6dcd3..25ba63fd34 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -810,11 +810,16 @@ int32_t handleQueryExecRsp(SRequestObj* pRequest) { void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) { SRequestObj* pRequest = (SRequestObj*)param; pRequest->code = code; - memcpy(&pRequest->body.resInfo.execRes, pResult, sizeof(*pResult)); + + if (pResult) { + memcpy(&pRequest->body.resInfo.execRes, pResult, sizeof(*pResult)); + } if (TDMT_VND_SUBMIT == pRequest->type || TDMT_VND_DELETE == pRequest->type || TDMT_VND_CREATE_TABLE == pRequest->type) { - pRequest->body.resInfo.numOfRows = pResult->numOfRows; + if (pResult) { + pRequest->body.resInfo.numOfRows = pResult->numOfRows; + } schedulerFreeJob(&pRequest->body.queryJob, 0); } @@ -1476,12 +1481,16 @@ void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertU tsem_wait(&pParam->sem); } - if (pRequest->code == TSDB_CODE_SUCCESS && pResultInfo->numOfRows > 0 && setupOneRowPtr) { - doSetOneRowPtr(pResultInfo); - pResultInfo->current += 1; - } + if (pResultInfo->numOfRows == 0 || pRequest->code != TSDB_CODE_SUCCESS) { + return NULL; + } else { + if (setupOneRowPtr) { + doSetOneRowPtr(pResultInfo); + pResultInfo->current += 1; + } - return pResultInfo->row; + return pResultInfo->row; + } } static int32_t doPrepareResPtr(SReqResultInfo* pResInfo) { diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index d653b8720f..92ed830dee 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -357,6 +357,7 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { pStmt->bInfo.inExecCache = true; if (pStmt->sql.autoCreateTbl) { + tscDebug("reuse stmt block for tb %s in execBlock", pStmt->bInfo.tbFName); return TSDB_CODE_SUCCESS; } } @@ -365,9 +366,11 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { if (pStmt->bInfo.inExecCache) { ASSERT(taosHashGetSize(pStmt->exec.pBlockHash) == 1); pStmt->bInfo.needParse = false; + tscDebug("reuse stmt block for tb %s in execBlock", pStmt->bInfo.tbFName); return TSDB_CODE_SUCCESS; } + tscDebug("no stmt block cache for tb %s", pStmt->bInfo.tbFName); return TSDB_CODE_SUCCESS; } @@ -391,6 +394,8 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } + tscDebug("reuse stmt block for tb %s in sqlBlock, suid:0x%" PRIx64 , pStmt->bInfo.tbFName, pStmt->bInfo.tbSuid); + return TSDB_CODE_SUCCESS; } @@ -406,6 +411,8 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { if (TSDB_CODE_PAR_TABLE_NOT_EXIST == code) { STMT_ERR_RET(stmtCleanBindInfo(pStmt)); + tscDebug("tb %s not exist", pStmt->bInfo.tbFName); + return TSDB_CODE_SUCCESS; } @@ -420,6 +427,8 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { if (uid == pStmt->bInfo.tbUid) { pStmt->bInfo.needParse = false; + tscDebug("tb %s is current table", pStmt->bInfo.tbFName); + return TSDB_CODE_SUCCESS; } @@ -440,6 +449,8 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { pStmt->bInfo.boundTags = pCache->boundTags; pStmt->bInfo.tagsCached = true; + tscDebug("tb %s in execBlock list, set to current", pStmt->bInfo.tbFName); + return TSDB_CODE_SUCCESS; } @@ -461,6 +472,8 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } + tscDebug("tb %s in sqlBlock list, set to current", pStmt->bInfo.tbFName); + return TSDB_CODE_SUCCESS; } @@ -510,6 +523,8 @@ TAOS_STMT* stmtInit(STscObj* taos) { int stmtPrepare(TAOS_STMT* stmt, const char* sql, unsigned long length) { STscStmt* pStmt = (STscStmt*)stmt; + tscDebug("stmt start to prepare"); + if (pStmt->sql.status >= STMT_PREPARE) { STMT_ERR_RET(stmtResetStmt(pStmt)); } @@ -529,6 +544,8 @@ int stmtPrepare(TAOS_STMT* stmt, const char* sql, unsigned long length) { int stmtSetTbName(TAOS_STMT* stmt, const char* tbName) { STscStmt* pStmt = (STscStmt*)stmt; + tscDebug("stmt start to set tbName: %s", tbName); + STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_SETTBNAME)); int32_t insert = 0; @@ -559,6 +576,8 @@ int stmtSetTbName(TAOS_STMT* stmt, const char* tbName) { int stmtSetTbTags(TAOS_STMT* stmt, TAOS_MULTI_BIND* tags) { STscStmt* pStmt = (STscStmt*)stmt; + tscDebug("stmt start to set tbTags"); + STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_SETTAGS)); if (pStmt->bInfo.inExecCache) { @@ -572,6 +591,7 @@ int stmtSetTbTags(TAOS_STMT* stmt, TAOS_MULTI_BIND* tags) { STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } + tscDebug("start to bind stmt tag values"); STMT_ERR_RET(qBindStmtTagsValue(*pDataBlock, pStmt->bInfo.boundTags, pStmt->bInfo.tbSuid, pStmt->bInfo.sname.tname, tags, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen)); @@ -617,6 +637,8 @@ int stmtFetchColFields(STscStmt* pStmt, int32_t* fieldNum, TAOS_FIELD_E** fields int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) { STscStmt* pStmt = (STscStmt*)stmt; + tscDebug("start to bind stmt data, colIdx: %d", colIdx); + STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_BIND)); if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && @@ -707,6 +729,8 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) { int stmtAddBatch(TAOS_STMT* stmt) { STscStmt* pStmt = (STscStmt*)stmt; + tscDebug("stmt start to add batch"); + STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_ADD_BATCH)); STMT_ERR_RET(stmtCacheBlock(pStmt)); @@ -715,6 +739,8 @@ int stmtAddBatch(TAOS_STMT* stmt) { } int stmtUpdateTableUid(STscStmt* pStmt, SSubmitRsp* pRsp) { + tscDebug("stmt start to update tbUid, blockNum: %d", pRsp->nBlocks); + if (pRsp->nBlocks <= 0) { tscError("invalid submit resp block number %d", pRsp->nBlocks); STMT_ERR_RET(TSDB_CODE_TSC_APP_ERROR); @@ -727,11 +753,6 @@ int stmtUpdateTableUid(STscStmt* pStmt, SSubmitRsp* pRsp) { char* key = taosHashGetKey(pIter, &keyLen); STableMeta* pMeta = qGetTableMetaInDataBlock(pBlock); - if (pMeta->uid != pStmt->bInfo.tbUid) { - tscError("table uid %" PRIx64 " mis-match with current table uid %" PRIx64, pMeta->uid, pStmt->bInfo.tbUid); - STMT_ERR_RET(TSDB_CODE_TSC_APP_ERROR); - } - if (pMeta->uid) { pIter = taosHashIterate(pStmt->exec.pBlockHash, pIter); continue; @@ -775,6 +796,8 @@ int stmtExec(TAOS_STMT* stmt) { SSubmitRsp* pRsp = NULL; bool autoCreateTbl = pStmt->exec.autoCreateTbl; + tscDebug("stmt start to exec"); + STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_EXECUTE)); if (STMT_TYPE_QUERY == pStmt->sql.type) { diff --git a/source/common/src/systable.c b/source/common/src/systable.c index ba8a8e1220..08997bcaf8 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -123,6 +123,9 @@ static const SSysDbTableSchema userStbsSchema[] = { {.name = "tags", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "last_update", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, {.name = "table_comment", .bytes = TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "watermark", .bytes = 64 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "max_delay", .bytes = 64 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "rollup", .bytes = 128 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, }; static const SSysDbTableSchema streamSchema[] = { @@ -146,8 +149,8 @@ static const SSysDbTableSchema userTblsSchema[] = { {.name = "uid", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "ttl", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "table_comment", .bytes = TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "type", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "table_comment", .bytes = TSDB_TB_COMMENT_LEN - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "type", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, }; static const SSysDbTableSchema userTblDistSchema[] = { diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 1fa37d5ef0..38f46b9b11 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1747,7 +1747,7 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) for (int32_t k = 0; k < colNum; k++) { SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); - if (colDataIsNull(pColInfoData, rows, j, NULL) || !var) { + if (colDataIsNull(pColInfoData, rows, j, NULL) || !pColInfoData->pData) { len += snprintf(dumpBuf + len, size - len, " %15s |", "NULL"); if (len >= size -1) return dumpBuf; continue; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 496c7beb47..f19d17d034 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -58,9 +58,8 @@ int32_t tsNumOfVnodeQueryThreads = 2; int32_t tsNumOfVnodeFetchThreads = 4; int32_t tsNumOfVnodeWriteThreads = 2; int32_t tsNumOfVnodeSyncThreads = 2; -int32_t tsNumOfVnodeMergeThreads = 2; int32_t tsNumOfQnodeQueryThreads = 2; -int32_t tsNumOfQnodeFetchThreads = 1; +int32_t tsNumOfQnodeFetchThreads = 4; int32_t tsNumOfSnodeSharedThreads = 2; int32_t tsNumOfSnodeUniqueThreads = 2; @@ -106,11 +105,6 @@ int32_t tsCompressMsgSize = -1; */ int32_t tsCompressColData = -1; -/* - * denote if 3.0 query pattern compatible for 2.0 - */ -int32_t tsCompatibleModel = 1; - // count/hyperloglog function always return values in case of all NULL data or Empty data set. int32_t tsCountAlwaysReturnValue = 1; @@ -414,7 +408,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { tsNumOfMnodeReadThreads = TRANGE(tsNumOfMnodeReadThreads, 1, 4); if (cfgAddInt32(pCfg, "numOfMnodeReadThreads", tsNumOfMnodeReadThreads, 1, 1024, 0) != 0) return -1; - tsNumOfVnodeQueryThreads = tsNumOfCores / 2; + tsNumOfVnodeQueryThreads = tsNumOfCores / 4; tsNumOfVnodeQueryThreads = TMAX(tsNumOfVnodeQueryThreads, 2); if (cfgAddInt32(pCfg, "numOfVnodeQueryThreads", tsNumOfVnodeQueryThreads, 1, 1024, 0) != 0) return -1; @@ -425,19 +419,16 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { tsNumOfVnodeWriteThreads = TMAX(tsNumOfVnodeWriteThreads, 1); if (cfgAddInt32(pCfg, "numOfVnodeWriteThreads", tsNumOfVnodeWriteThreads, 1, 1024, 0) != 0) return -1; - tsNumOfVnodeSyncThreads = tsNumOfCores / 2; + tsNumOfVnodeSyncThreads = tsNumOfCores; tsNumOfVnodeSyncThreads = TMAX(tsNumOfVnodeSyncThreads, 1); if (cfgAddInt32(pCfg, "numOfVnodeSyncThreads", tsNumOfVnodeSyncThreads, 1, 1024, 0) != 0) return -1; - tsNumOfVnodeMergeThreads = tsNumOfCores / 8; - tsNumOfVnodeMergeThreads = TRANGE(tsNumOfVnodeMergeThreads, 1, 1); - if (cfgAddInt32(pCfg, "numOfVnodeMergeThreads", tsNumOfVnodeMergeThreads, 1, 1024, 0) != 0) return -1; - tsNumOfQnodeQueryThreads = tsNumOfCores / 2; tsNumOfQnodeQueryThreads = TMAX(tsNumOfQnodeQueryThreads, 1); if (cfgAddInt32(pCfg, "numOfQnodeQueryThreads", tsNumOfQnodeQueryThreads, 1, 1024, 0) != 0) return -1; - tsNumOfQnodeFetchThreads = TRANGE(tsNumOfQnodeFetchThreads, 1, 1); + tsNumOfQnodeFetchThreads = tsNumOfCores / 2; + tsNumOfQnodeFetchThreads = TMAX(tsNumOfQnodeFetchThreads, 4); if (cfgAddInt32(pCfg, "numOfQnodeFetchThreads", tsNumOfQnodeFetchThreads, 1, 1024, 0) != 0) return -1; tsNumOfSnodeSharedThreads = tsNumOfCores / 4; @@ -598,7 +589,6 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsNumOfVnodeFetchThreads = cfgGetItem(pCfg, "numOfVnodeFetchThreads")->i32; tsNumOfVnodeWriteThreads = cfgGetItem(pCfg, "numOfVnodeWriteThreads")->i32; tsNumOfVnodeSyncThreads = cfgGetItem(pCfg, "numOfVnodeSyncThreads")->i32; - tsNumOfVnodeMergeThreads = cfgGetItem(pCfg, "numOfVnodeMergeThreads")->i32; tsNumOfQnodeQueryThreads = cfgGetItem(pCfg, "numOfQnodeQueryThreads")->i32; tsNumOfQnodeFetchThreads = cfgGetItem(pCfg, "numOfQnodeFetchThreads")->i32; tsNumOfSnodeSharedThreads = cfgGetItem(pCfg, "numOfSnodeSharedThreads")->i32; @@ -840,8 +830,6 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) { tsNumOfVnodeWriteThreads = cfgGetItem(pCfg, "numOfVnodeWriteThreads")->i32; } else if (strcasecmp("numOfVnodeSyncThreads", name) == 0) { tsNumOfVnodeSyncThreads = cfgGetItem(pCfg, "numOfVnodeSyncThreads")->i32; - } else if (strcasecmp("numOfVnodeMergeThreads", name) == 0) { - tsNumOfVnodeMergeThreads = cfgGetItem(pCfg, "numOfVnodeMergeThreads")->i32; } else if (strcasecmp("numOfQnodeQueryThreads", name) == 0) { tsNumOfQnodeQueryThreads = cfgGetItem(pCfg, "numOfQnodeQueryThreads")->i32; } else if (strcasecmp("numOfQnodeFetchThreads", name) == 0) { diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 9d8f49bdd5..adfa5b0f1e 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2001,7 +2001,7 @@ int32_t tSerializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) { if (tEncodeI32(&encoder, pReq->buffer) < 0) return -1; if (tEncodeI32(&encoder, pReq->pageSize) < 0) return -1; if (tEncodeI32(&encoder, pReq->pages) < 0) return -1; - if (tEncodeI32(&encoder, pReq->lastRowMem) < 0) return -1; + if (tEncodeI32(&encoder, pReq->cacheLastSize) < 0) return -1; if (tEncodeI32(&encoder, pReq->daysPerFile) < 0) return -1; if (tEncodeI32(&encoder, pReq->daysToKeep0) < 0) return -1; if (tEncodeI32(&encoder, pReq->daysToKeep1) < 0) return -1; @@ -2014,7 +2014,7 @@ int32_t tSerializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) { if (tEncodeI8(&encoder, pReq->compression) < 0) return -1; if (tEncodeI8(&encoder, pReq->replications) < 0) return -1; if (tEncodeI8(&encoder, pReq->strict) < 0) return -1; - if (tEncodeI8(&encoder, pReq->cacheLastRow) < 0) return -1; + if (tEncodeI8(&encoder, pReq->cacheLast) < 0) return -1; if (tEncodeI8(&encoder, pReq->schemaless) < 0) return -1; if (tEncodeI8(&encoder, pReq->ignoreExist) < 0) return -1; if (tEncodeI32(&encoder, pReq->numOfRetensions) < 0) return -1; @@ -2043,7 +2043,7 @@ int32_t tDeserializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) if (tDecodeI32(&decoder, &pReq->buffer) < 0) return -1; if (tDecodeI32(&decoder, &pReq->pageSize) < 0) return -1; if (tDecodeI32(&decoder, &pReq->pages) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->lastRowMem) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->cacheLastSize) < 0) return -1; if (tDecodeI32(&decoder, &pReq->daysPerFile) < 0) return -1; if (tDecodeI32(&decoder, &pReq->daysToKeep0) < 0) return -1; if (tDecodeI32(&decoder, &pReq->daysToKeep1) < 0) return -1; @@ -2056,7 +2056,7 @@ int32_t tDeserializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) if (tDecodeI8(&decoder, &pReq->compression) < 0) return -1; if (tDecodeI8(&decoder, &pReq->replications) < 0) return -1; if (tDecodeI8(&decoder, &pReq->strict) < 0) return -1; - if (tDecodeI8(&decoder, &pReq->cacheLastRow) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->cacheLast) < 0) return -1; if (tDecodeI8(&decoder, &pReq->schemaless) < 0) return -1; if (tDecodeI8(&decoder, &pReq->ignoreExist) < 0) return -1; if (tDecodeI32(&decoder, &pReq->numOfRetensions) < 0) return -1; @@ -2098,7 +2098,7 @@ int32_t tSerializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) { if (tEncodeI32(&encoder, pReq->buffer) < 0) return -1; if (tEncodeI32(&encoder, pReq->pageSize) < 0) return -1; if (tEncodeI32(&encoder, pReq->pages) < 0) return -1; - if (tEncodeI32(&encoder, pReq->lastRowMem) < 0) return -1; + if (tEncodeI32(&encoder, pReq->cacheLastSize) < 0) return -1; if (tEncodeI32(&encoder, pReq->daysPerFile) < 0) return -1; if (tEncodeI32(&encoder, pReq->daysToKeep0) < 0) return -1; if (tEncodeI32(&encoder, pReq->daysToKeep1) < 0) return -1; @@ -2106,7 +2106,7 @@ int32_t tSerializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) { if (tEncodeI32(&encoder, pReq->fsyncPeriod) < 0) return -1; if (tEncodeI8(&encoder, pReq->walLevel) < 0) return -1; if (tEncodeI8(&encoder, pReq->strict) < 0) return -1; - if (tEncodeI8(&encoder, pReq->cacheLastRow) < 0) return -1; + if (tEncodeI8(&encoder, pReq->cacheLast) < 0) return -1; if (tEncodeI8(&encoder, pReq->replications) < 0) return -1; tEndEncode(&encoder); @@ -2124,7 +2124,7 @@ int32_t tDeserializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) { if (tDecodeI32(&decoder, &pReq->buffer) < 0) return -1; if (tDecodeI32(&decoder, &pReq->pageSize) < 0) return -1; if (tDecodeI32(&decoder, &pReq->pages) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->lastRowMem) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->cacheLastSize) < 0) return -1; if (tDecodeI32(&decoder, &pReq->daysPerFile) < 0) return -1; if (tDecodeI32(&decoder, &pReq->daysToKeep0) < 0) return -1; if (tDecodeI32(&decoder, &pReq->daysToKeep1) < 0) return -1; @@ -2132,7 +2132,7 @@ int32_t tDeserializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) { if (tDecodeI32(&decoder, &pReq->fsyncPeriod) < 0) return -1; if (tDecodeI8(&decoder, &pReq->walLevel) < 0) return -1; if (tDecodeI8(&decoder, &pReq->strict) < 0) return -1; - if (tDecodeI8(&decoder, &pReq->cacheLastRow) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->cacheLast) < 0) return -1; if (tDecodeI8(&decoder, &pReq->replications) < 0) return -1; tEndDecode(&decoder); @@ -2672,6 +2672,56 @@ int32_t tDeserializeSTrimDbReq(void *buf, int32_t bufLen, STrimDbReq *pReq) { return 0; } +int32_t tSerializeSVTrimDbReq(void *buf, int32_t bufLen, SVTrimDbReq *pReq) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI32(&encoder, pReq->timestamp) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSVTrimDbReq(void *buf, int32_t bufLen, SVTrimDbReq *pReq) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->timestamp) < 0) return -1; + tEndDecode(&decoder); + + tDecoderClear(&decoder); + return 0; +} + +int32_t tSerializeSVDropTtlTableReq(void *buf, int32_t bufLen, SVDropTtlTableReq *pReq) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI32(&encoder, pReq->timestamp) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSVDropTtlTableReq(void *buf, int32_t bufLen, SVDropTtlTableReq *pReq) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->timestamp) < 0) return -1; + tEndDecode(&decoder); + + tDecoderClear(&decoder); + return 0; +} + int32_t tSerializeSDbCfgRsp(void *buf, int32_t bufLen, const SDbCfgRsp *pRsp) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -2694,7 +2744,7 @@ int32_t tSerializeSDbCfgRsp(void *buf, int32_t bufLen, const SDbCfgRsp *pRsp) { if (tEncodeI8(&encoder, pRsp->compression) < 0) return -1; if (tEncodeI8(&encoder, pRsp->replications) < 0) return -1; if (tEncodeI8(&encoder, pRsp->strict) < 0) return -1; - if (tEncodeI8(&encoder, pRsp->cacheLastRow) < 0) return -1; + if (tEncodeI8(&encoder, pRsp->cacheLast) < 0) return -1; if (tEncodeI32(&encoder, pRsp->numOfRetensions) < 0) return -1; for (int32_t i = 0; i < pRsp->numOfRetensions; ++i) { SRetention *pRetension = taosArrayGet(pRsp->pRetensions, i); @@ -2733,7 +2783,7 @@ int32_t tDeserializeSDbCfgRsp(void *buf, int32_t bufLen, SDbCfgRsp *pRsp) { if (tDecodeI8(&decoder, &pRsp->compression) < 0) return -1; if (tDecodeI8(&decoder, &pRsp->replications) < 0) return -1; if (tDecodeI8(&decoder, &pRsp->strict) < 0) return -1; - if (tDecodeI8(&decoder, &pRsp->cacheLastRow) < 0) return -1; + if (tDecodeI8(&decoder, &pRsp->cacheLast) < 0) return -1; if (tDecodeI32(&decoder, &pRsp->numOfRetensions) < 0) return -1; if (pRsp->numOfRetensions > 0) { pRsp->pRetensions = taosArrayInit(pRsp->numOfRetensions, sizeof(SRetention)); @@ -3644,7 +3694,7 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR if (tEncodeI32(&encoder, pReq->buffer) < 0) return -1; if (tEncodeI32(&encoder, pReq->pageSize) < 0) return -1; if (tEncodeI32(&encoder, pReq->pages) < 0) return -1; - if (tEncodeI32(&encoder, pReq->lastRowMem) < 0) return -1; + if (tEncodeI32(&encoder, pReq->cacheLastSize) < 0) return -1; if (tEncodeI32(&encoder, pReq->daysPerFile) < 0) return -1; if (tEncodeI32(&encoder, pReq->daysToKeep0) < 0) return -1; if (tEncodeI32(&encoder, pReq->daysToKeep1) < 0) return -1; @@ -3659,7 +3709,7 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR if (tEncodeI8(&encoder, pReq->precision) < 0) return -1; if (tEncodeI8(&encoder, pReq->compression) < 0) return -1; if (tEncodeI8(&encoder, pReq->strict) < 0) return -1; - if (tEncodeI8(&encoder, pReq->cacheLastRow) < 0) return -1; + if (tEncodeI8(&encoder, pReq->cacheLast) < 0) return -1; if (tEncodeI8(&encoder, pReq->standby) < 0) return -1; if (tEncodeI8(&encoder, pReq->replica) < 0) return -1; if (tEncodeI8(&encoder, pReq->selfIndex) < 0) return -1; @@ -3702,7 +3752,7 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq * if (tDecodeI32(&decoder, &pReq->buffer) < 0) return -1; if (tDecodeI32(&decoder, &pReq->pageSize) < 0) return -1; if (tDecodeI32(&decoder, &pReq->pages) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->lastRowMem) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->cacheLastSize) < 0) return -1; if (tDecodeI32(&decoder, &pReq->daysPerFile) < 0) return -1; if (tDecodeI32(&decoder, &pReq->daysToKeep0) < 0) return -1; if (tDecodeI32(&decoder, &pReq->daysToKeep1) < 0) return -1; @@ -3717,7 +3767,7 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq * if (tDecodeI8(&decoder, &pReq->precision) < 0) return -1; if (tDecodeI8(&decoder, &pReq->compression) < 0) return -1; if (tDecodeI8(&decoder, &pReq->strict) < 0) return -1; - if (tDecodeI8(&decoder, &pReq->cacheLastRow) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->cacheLast) < 0) return -1; if (tDecodeI8(&decoder, &pReq->standby) < 0) return -1; if (tDecodeI8(&decoder, &pReq->replica) < 0) return -1; if (tDecodeI8(&decoder, &pReq->selfIndex) < 0) return -1; @@ -3827,7 +3877,7 @@ int32_t tSerializeSAlterVnodeReq(void *buf, int32_t bufLen, SAlterVnodeReq *pReq if (tEncodeI32(&encoder, pReq->buffer) < 0) return -1; if (tEncodeI32(&encoder, pReq->pageSize) < 0) return -1; if (tEncodeI32(&encoder, pReq->pages) < 0) return -1; - if (tEncodeI32(&encoder, pReq->lastRowMem) < 0) return -1; + if (tEncodeI32(&encoder, pReq->cacheLastSize) < 0) return -1; if (tEncodeI32(&encoder, pReq->daysPerFile) < 0) return -1; if (tEncodeI32(&encoder, pReq->daysToKeep0) < 0) return -1; if (tEncodeI32(&encoder, pReq->daysToKeep1) < 0) return -1; @@ -3835,7 +3885,7 @@ int32_t tSerializeSAlterVnodeReq(void *buf, int32_t bufLen, SAlterVnodeReq *pReq if (tEncodeI32(&encoder, pReq->fsyncPeriod) < 0) return -1; if (tEncodeI8(&encoder, pReq->walLevel) < 0) return -1; if (tEncodeI8(&encoder, pReq->strict) < 0) return -1; - if (tEncodeI8(&encoder, pReq->cacheLastRow) < 0) return -1; + if (tEncodeI8(&encoder, pReq->cacheLast) < 0) return -1; if (tEncodeI8(&encoder, pReq->selfIndex) < 0) return -1; if (tEncodeI8(&encoder, pReq->replica) < 0) return -1; for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { @@ -3858,7 +3908,7 @@ int32_t tDeserializeSAlterVnodeReq(void *buf, int32_t bufLen, SAlterVnodeReq *pR if (tDecodeI32(&decoder, &pReq->buffer) < 0) return -1; if (tDecodeI32(&decoder, &pReq->pageSize) < 0) return -1; if (tDecodeI32(&decoder, &pReq->pages) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->lastRowMem) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->cacheLastSize) < 0) return -1; if (tDecodeI32(&decoder, &pReq->daysPerFile) < 0) return -1; if (tDecodeI32(&decoder, &pReq->daysToKeep0) < 0) return -1; if (tDecodeI32(&decoder, &pReq->daysToKeep1) < 0) return -1; @@ -3866,7 +3916,7 @@ int32_t tDeserializeSAlterVnodeReq(void *buf, int32_t bufLen, SAlterVnodeReq *pR if (tDecodeI32(&decoder, &pReq->fsyncPeriod) < 0) return -1; if (tDecodeI8(&decoder, &pReq->walLevel) < 0) return -1; if (tDecodeI8(&decoder, &pReq->strict) < 0) return -1; - if (tDecodeI8(&decoder, &pReq->cacheLastRow) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->cacheLast) < 0) return -1; if (tDecodeI8(&decoder, &pReq->selfIndex) < 0) return -1; if (tDecodeI8(&decoder, &pReq->replica) < 0) return -1; for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 4e3449feb7..b9a02728fc 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -171,6 +171,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_USE_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_COMPACT_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_TRIM_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_GET_DB_CFG, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_VGROUP_LIST, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_REDISTRIBUTE_VGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c index 012e61d239..84491a82b0 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c @@ -150,6 +150,7 @@ static void mmStop(SMnodeMgmt *pMgmt) { dDebug("mnode-mgmt start to stop"); taosThreadRwlockWrlock(&pMgmt->lock); pMgmt->stopped = 1; + mndPreClose(pMgmt->pMnode); taosThreadRwlockUnlock(&pMgmt->lock); mndStop(pMgmt->pMnode); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 5ffddd0127..7e7139ba53 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -138,6 +138,8 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { pCfg->dbId = pCreate->dbUid; pCfg->szPage = pCreate->pageSize * 1024; pCfg->szCache = pCreate->pages; + pCfg->cacheLast = pCreate->cacheLast; + pCfg->cacheLastSize = pCreate->cacheLastSize; pCfg->szBuf = (uint64_t)pCreate->buffer * 1024 * 1024; pCfg->isWeak = true; pCfg->isTsma = pCreate->isTsma; @@ -210,7 +212,8 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return -1; } - dDebug("vgId:%d, start to create vnode, tsma:%d standby:%d", createReq.vgId, createReq.isTsma, createReq.standby); + dDebug("vgId:%d, start to create vnode, tsma:%d standby:%d cacheLast:%d cacheLastSize:%d", createReq.vgId, + createReq.isTsma, createReq.standby, createReq.cacheLast, createReq.cacheLastSize); vmGenerateVnodeCfg(&createReq, &vnodeCfg); if (vmTsmaAdjustDays(&vnodeCfg, &createReq) < 0) { @@ -370,6 +373,7 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIRM, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_HASHRANGE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TRIM, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index 051e5defb0..eac9052289 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -75,11 +75,13 @@ int32_t vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) { void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { char path[TSDB_FILENAME_LEN] = {0}; + vnodePreClose(pVnode->pImpl); + taosThreadRwlockWrlock(&pMgmt->lock); taosHashRemove(pMgmt->hash, &pVnode->vgId, sizeof(int32_t)); taosThreadRwlockUnlock(&pMgmt->lock); - vmReleaseVnode(pMgmt, pVnode); + while (pVnode->refCount > 0) taosMsleep(10); dTrace("vgId:%d, wait for vnode queue is empty", pVnode->vgId); diff --git a/source/dnode/mgmt/test/vnode/vnode.cpp b/source/dnode/mgmt/test/vnode/vnode.cpp index 8aba4f81b5..520d844dbd 100644 --- a/source/dnode/mgmt/test/vnode/vnode.cpp +++ b/source/dnode/mgmt/test/vnode/vnode.cpp @@ -45,7 +45,7 @@ TEST_F(DndTestVnode, 01_Create_Vnode) { createReq.compression = 2; createReq.replica = 1; createReq.strict = 1; - createReq.cacheLastRow = 0; + createReq.cacheLast = 0; createReq.selfIndex = 0; for (int r = 0; r < createReq.replica; ++r) { SReplica* pReplica = &createReq.replicas[r]; @@ -80,7 +80,7 @@ TEST_F(DndTestVnode, 02_Alter_Vnode) { alterReq.walLevel = 1; alterReq.replica = 1; alterReq.strict = 1; - alterReq.cacheLastRow = 0; + alterReq.cacheLast = 0; alterReq.selfIndex = 0; for (int r = 0; r < alterReq.replica; ++r) { SReplica* pReplica = &alterReq.replicas[r]; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index f39a848992..7ac991451e 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -246,7 +246,7 @@ typedef struct { int32_t buffer; int32_t pageSize; int32_t pages; - int32_t lastRowMem; + int32_t cacheLastSize; int32_t daysPerFile; int32_t daysToKeep0; int32_t daysToKeep1; @@ -260,7 +260,7 @@ typedef struct { int8_t replications; int8_t strict; int8_t hashMethod; // default is 1 - int8_t cacheLastRow; + int8_t cacheLast; int32_t numOfRetensions; SArray* pRetensions; int8_t schemaless; diff --git a/source/dnode/mnode/impl/inc/mndPrivilege.h b/source/dnode/mnode/impl/inc/mndPrivilege.h index 0c56f6f155..a149c0f0e2 100644 --- a/source/dnode/mnode/impl/inc/mndPrivilege.h +++ b/source/dnode/mnode/impl/inc/mndPrivilege.h @@ -54,6 +54,7 @@ typedef enum { MND_OPER_ALTER_DB, MND_OPER_DROP_DB, MND_OPER_COMPACT_DB, + MND_OPER_TRIM_DB, MND_OPER_USE_DB, MND_OPER_WRITE_DB, MND_OPER_READ_DB, diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 156afb09fc..b1c2b0e277 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -42,6 +42,7 @@ static int32_t mndProcessAlterDbReq(SRpcMsg *pReq); static int32_t mndProcessDropDbReq(SRpcMsg *pReq); static int32_t mndProcessUseDbReq(SRpcMsg *pReq); static int32_t mndProcessCompactDbReq(SRpcMsg *pReq); +static int32_t mndProcessTrimDbReq(SRpcMsg *pReq); static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity); static void mndCancelGetNextDb(SMnode *pMnode, void *pIter); static int32_t mndProcessGetDbCfgReq(SRpcMsg *pReq); @@ -62,6 +63,7 @@ int32_t mndInitDb(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_DROP_DB, mndProcessDropDbReq); mndSetMsgHandle(pMnode, TDMT_MND_USE_DB, mndProcessUseDbReq); mndSetMsgHandle(pMnode, TDMT_MND_COMPACT_DB, mndProcessCompactDbReq); + mndSetMsgHandle(pMnode, TDMT_MND_TRIM_DB, mndProcessTrimDbReq); mndSetMsgHandle(pMnode, TDMT_MND_GET_DB_CFG, mndProcessGetDbCfgReq); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_DB, mndRetrieveDbs); @@ -93,7 +95,7 @@ static SSdbRaw *mndDbActionEncode(SDbObj *pDb) { SDB_SET_INT32(pRaw, dataPos, pDb->cfg.buffer, _OVER) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.pageSize, _OVER) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.pages, _OVER) - SDB_SET_INT32(pRaw, dataPos, pDb->cfg.lastRowMem, _OVER) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.cacheLastSize, _OVER) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysPerFile, _OVER) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysToKeep0, _OVER) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysToKeep1, _OVER) @@ -106,7 +108,7 @@ static SSdbRaw *mndDbActionEncode(SDbObj *pDb) { SDB_SET_INT8(pRaw, dataPos, pDb->cfg.compression, _OVER) SDB_SET_INT8(pRaw, dataPos, pDb->cfg.replications, _OVER) SDB_SET_INT8(pRaw, dataPos, pDb->cfg.strict, _OVER) - SDB_SET_INT8(pRaw, dataPos, pDb->cfg.cacheLastRow, _OVER) + SDB_SET_INT8(pRaw, dataPos, pDb->cfg.cacheLast, _OVER) SDB_SET_INT8(pRaw, dataPos, pDb->cfg.hashMethod, _OVER) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.numOfRetensions, _OVER) for (int32_t i = 0; i < pDb->cfg.numOfRetensions; ++i) { @@ -166,7 +168,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.buffer, _OVER) SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.pageSize, _OVER) SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.pages, _OVER) - SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.lastRowMem, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.cacheLastSize, _OVER) SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.daysPerFile, _OVER) SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.daysToKeep0, _OVER) SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.daysToKeep1, _OVER) @@ -179,7 +181,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) { SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.compression, _OVER) SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.replications, _OVER) SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.strict, _OVER) - SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.cacheLastRow, _OVER) + SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.cacheLast, _OVER) SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.hashMethod, _OVER) SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.numOfRetensions, _OVER) if (pDb->cfg.numOfRetensions > 0) { @@ -234,7 +236,7 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) { pOld->cfg.buffer = pNew->cfg.buffer; pOld->cfg.pageSize = pNew->cfg.pageSize; pOld->cfg.pages = pNew->cfg.pages; - pOld->cfg.lastRowMem = pNew->cfg.lastRowMem; + pOld->cfg.cacheLastSize = pNew->cfg.cacheLastSize; pOld->cfg.daysPerFile = pNew->cfg.daysPerFile; pOld->cfg.daysToKeep0 = pNew->cfg.daysToKeep0; pOld->cfg.daysToKeep1 = pNew->cfg.daysToKeep1; @@ -242,7 +244,7 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) { pOld->cfg.fsyncPeriod = pNew->cfg.fsyncPeriod; pOld->cfg.walLevel = pNew->cfg.walLevel; pOld->cfg.strict = pNew->cfg.strict; - pOld->cfg.cacheLastRow = pNew->cfg.cacheLastRow; + pOld->cfg.cacheLast = pNew->cfg.cacheLast; pOld->cfg.replications = pNew->cfg.replications; taosWUnLockLatch(&pOld->lock); return 0; @@ -291,7 +293,7 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { if (pCfg->buffer < TSDB_MIN_BUFFER_PER_VNODE || pCfg->buffer > TSDB_MAX_BUFFER_PER_VNODE) return -1; if (pCfg->pageSize < TSDB_MIN_PAGESIZE_PER_VNODE || pCfg->pageSize > TSDB_MAX_PAGESIZE_PER_VNODE) return -1; if (pCfg->pages < TSDB_MIN_PAGES_PER_VNODE || pCfg->pages > TSDB_MAX_PAGES_PER_VNODE) return -1; - if (pCfg->lastRowMem < TSDB_MIN_DB_LAST_ROW_MEM || pCfg->lastRowMem > TSDB_MAX_DB_LAST_ROW_MEM) return -1; + if (pCfg->cacheLastSize < TSDB_MIN_DB_CACHE_LAST_SIZE || pCfg->cacheLastSize > TSDB_MAX_DB_CACHE_LAST_SIZE) return -1; if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) return -1; if (pCfg->daysToKeep0 < TSDB_MIN_KEEP || pCfg->daysToKeep0 > TSDB_MAX_KEEP) return -1; if (pCfg->daysToKeep1 < TSDB_MIN_KEEP || pCfg->daysToKeep1 > TSDB_MAX_KEEP) return -1; @@ -310,7 +312,7 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { if (pCfg->replications != 1 && pCfg->replications != 3) return -1; if (pCfg->strict < TSDB_DB_STRICT_OFF || pCfg->strict > TSDB_DB_STRICT_ON) return -1; if (pCfg->schemaless < TSDB_DB_SCHEMALESS_OFF || pCfg->schemaless > TSDB_DB_SCHEMALESS_ON) return -1; - if (pCfg->cacheLastRow < TSDB_MIN_DB_CACHE_LAST_ROW || pCfg->cacheLastRow > TSDB_MAX_DB_CACHE_LAST_ROW) return -1; + if (pCfg->cacheLast < TSDB_MIN_DB_CACHE_LAST || pCfg->cacheLast > TSDB_MAX_DB_CACHE_LAST) return -1; if (pCfg->hashMethod != 1) return -1; if (pCfg->replications > mndGetDnodeSize(pMnode)) { terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; @@ -339,8 +341,8 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->compression < 0) pCfg->compression = TSDB_DEFAULT_COMP_LEVEL; if (pCfg->replications < 0) pCfg->replications = TSDB_DEFAULT_DB_REPLICA; if (pCfg->strict < 0) pCfg->strict = TSDB_DEFAULT_DB_STRICT; - if (pCfg->cacheLastRow < 0) pCfg->cacheLastRow = TSDB_DEFAULT_CACHE_LAST_ROW; - if (pCfg->lastRowMem <= 0) pCfg->lastRowMem = TSDB_DEFAULT_LAST_ROW_MEM; + if (pCfg->cacheLast < 0) pCfg->cacheLast = TSDB_DEFAULT_CACHE_LAST; + if (pCfg->cacheLastSize <= 0) pCfg->cacheLastSize = TSDB_DEFAULT_CACHE_LAST_SIZE; if (pCfg->numOfRetensions < 0) pCfg->numOfRetensions = 0; if (pCfg->schemaless < 0) pCfg->schemaless = TSDB_DB_SCHEMALESS_OFF; } @@ -439,7 +441,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate, .buffer = pCreate->buffer, .pageSize = pCreate->pageSize, .pages = pCreate->pages, - .lastRowMem = pCreate->lastRowMem, + .cacheLastSize = pCreate->cacheLastSize, .daysPerFile = pCreate->daysPerFile, .daysToKeep0 = pCreate->daysToKeep0, .daysToKeep1 = pCreate->daysToKeep1, @@ -452,7 +454,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate, .compression = pCreate->compression, .replications = pCreate->replications, .strict = pCreate->strict, - .cacheLastRow = pCreate->cacheLastRow, + .cacheLast = pCreate->cacheLast, .hashMethod = 1, .schemaless = pCreate->schemaless, }; @@ -623,13 +625,13 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) { #endif } - if (pAlter->cacheLastRow >= 0 && pAlter->cacheLastRow != pDb->cfg.cacheLastRow) { - pDb->cfg.cacheLastRow = pAlter->cacheLastRow; + if (pAlter->cacheLast >= 0 && pAlter->cacheLast != pDb->cfg.cacheLast) { + pDb->cfg.cacheLast = pAlter->cacheLast; terrno = 0; } - if (pAlter->lastRowMem > 0 && pAlter->lastRowMem != pDb->cfg.lastRowMem) { - pDb->cfg.lastRowMem = pAlter->lastRowMem; + if (pAlter->cacheLastSize > 0 && pAlter->cacheLastSize != pDb->cfg.cacheLastSize) { + pDb->cfg.cacheLastSize = pAlter->cacheLastSize; terrno = 0; } @@ -801,7 +803,7 @@ static int32_t mndProcessGetDbCfgReq(SRpcMsg *pReq) { cfgRsp.compression = pDb->cfg.compression; cfgRsp.replications = pDb->cfg.replications; cfgRsp.strict = pDb->cfg.strict; - cfgRsp.cacheLastRow = pDb->cfg.cacheLastRow; + cfgRsp.cacheLast = pDb->cfg.cacheLast; cfgRsp.numOfRetensions = pDb->cfg.numOfRetensions; cfgRsp.pRetensions = pDb->cfg.pRetensions; cfgRsp.schemaless = pDb->cfg.schemaless; @@ -1268,6 +1270,8 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, return 0; } +static int32_t mndCompactDb(SMnode *pMnode, SDbObj *pDb) { return 0; } + static int32_t mndProcessCompactDbReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; @@ -1279,7 +1283,7 @@ static int32_t mndProcessCompactDbReq(SRpcMsg *pReq) { goto _OVER; } - mDebug("db:%s, start to sync", compactReq.db); + mDebug("db:%s, start to compact", compactReq.db); pDb = mndAcquireDb(pMnode, compactReq.db); if (pDb == NULL) { @@ -1290,7 +1294,7 @@ static int32_t mndProcessCompactDbReq(SRpcMsg *pReq) { goto _OVER; } - // code = mndCompactDb(); + code = mndCompactDb(pMnode, pDb); _OVER: if (code != 0) { @@ -1301,6 +1305,75 @@ _OVER: return code; } +static int32_t mndTrimDb(SMnode *pMnode, SDbObj *pDb) { + SSdb *pSdb = pMnode->pSdb; + SVgObj *pVgroup = NULL; + void *pIter = NULL; + SVTrimDbReq trimReq = {.timestamp = taosGetTimestampSec()}; + int32_t reqLen = tSerializeSVTrimDbReq(NULL, 0, &trimReq); + int32_t contLen = reqLen + sizeof(SMsgHead); + + while (1) { + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + + SMsgHead *pHead = rpcMallocCont(contLen); + if (pHead == NULL) { + sdbCancelFetch(pSdb, pVgroup); + sdbRelease(pSdb, pVgroup); + continue; + } + pHead->contLen = htonl(contLen); + pHead->vgId = htonl(pVgroup->vgId); + tSerializeSVTrimDbReq((char *)pHead + sizeof(SMsgHead), contLen, &trimReq); + + SRpcMsg rpcMsg = {.msgType = TDMT_VND_TRIM, .pCont = pHead, .contLen = contLen}; + SEpSet epSet = mndGetVgroupEpset(pMnode, pVgroup); + int32_t code = tmsgSendReq(&epSet, &rpcMsg); + if (code != 0) { + mError("vgId:%d, failed to send vnode-trim request to vnode since 0x%x", pVgroup->vgId, code); + } else { + mDebug("vgId:%d, send vnode-trim request to vnode, time:%d", pVgroup->vgId, trimReq.timestamp); + } + sdbRelease(pSdb, pVgroup); + } + + return 0; +} + +static int32_t mndProcessTrimDbReq(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; + int32_t code = -1; + SDbObj *pDb = NULL; + STrimDbReq trimReq = {0}; + + if (tDeserializeSTrimDbReq(pReq->pCont, pReq->contLen, &trimReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto _OVER; + } + + mDebug("db:%s, start to trim", trimReq.db); + + pDb = mndAcquireDb(pMnode, trimReq.db); + if (pDb == NULL) { + goto _OVER; + } + + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_TRIM_DB, pDb) != 0) { + goto _OVER; + } + + code = mndTrimDb(pMnode, pDb); + +_OVER: + if (code != 0) { + mError("db:%s, failed to process trim db req since %s", trimReq.db, terrstr()); + } + + mndReleaseDb(pMnode, pDb); + return code; +} + const char *mndGetDbStr(const char *src) { char *pos = strstr(src, TS_PATH_DELIMITER); if (pos != NULL) ++pos; @@ -1467,7 +1540,7 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.compression, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.cacheLastRow, false); + colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.cacheLast, false); const char *precStr = NULL; switch (pDb->cfg.precision) { diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index f18f3c983e..df8dc42d17 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -366,6 +366,12 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) { return pMnode; } +void mndPreClose(SMnode *pMnode) { + if (pMnode != NULL) { + syncLeaderTransfer(pMnode->syncMgmt.sync); + } +} + void mndClose(SMnode *pMnode) { if (pMnode != NULL) { mDebug("start to close mnode"); @@ -531,8 +537,7 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) { if (!IsReq(pMsg)) return 0; if (pMsg->msgType == TDMT_SCH_QUERY || pMsg->msgType == TDMT_SCH_MERGE_QUERY || pMsg->msgType == TDMT_SCH_QUERY_CONTINUE || pMsg->msgType == TDMT_SCH_QUERY_HEARTBEAT || - pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_SCH_MERGE_FETCH || - pMsg->msgType == TDMT_SCH_DROP_TASK) { + pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_SCH_MERGE_FETCH || pMsg->msgType == TDMT_SCH_DROP_TASK) { return 0; } if (mndAcquireRpcRef(pMsg->info.node) == 0) return 0; diff --git a/source/dnode/mnode/impl/src/mndPrivilege.c b/source/dnode/mnode/impl/src/mndPrivilege.c index dc321e38d1..e4422c480f 100644 --- a/source/dnode/mnode/impl/src/mndPrivilege.c +++ b/source/dnode/mnode/impl/src/mndPrivilege.c @@ -155,7 +155,8 @@ int32_t mndCheckDbPrivilege(SMnode *pMnode, const char *user, EOperType operType if (pUser->sysInfo) goto _OVER; } - if (operType == MND_OPER_ALTER_DB || operType == MND_OPER_DROP_DB || operType == MND_OPER_COMPACT_DB) { + if (operType == MND_OPER_ALTER_DB || operType == MND_OPER_DROP_DB || operType == MND_OPER_COMPACT_DB || + operType == MND_OPER_TRIM_DB) { if (strcmp(pUser->user, pDb->createUser) == 0 && pUser->sysInfo) goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index c1513cd92f..c040f0d05b 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -204,6 +204,8 @@ _OVER: mError("sma:%s, failed to decode from raw:%p since %s", pSma->name, pRaw, terrstr()); taosMemoryFreeClear(pSma->expr); taosMemoryFreeClear(pSma->tagsFilter); + taosMemoryFreeClear(pSma->sql); + taosMemoryFreeClear(pSma->ast); taosMemoryFreeClear(pRow); return NULL; } @@ -221,6 +223,8 @@ static int32_t mndSmaActionDelete(SSdb *pSdb, SSmaObj *pSma) { mTrace("sma:%s, perform delete action, row:%p", pSma->name, pSma); taosMemoryFreeClear(pSma->tagsFilter); taosMemoryFreeClear(pSma->expr); + taosMemoryFreeClear(pSma->sql); + taosMemoryFreeClear(pSma->ast); return 0; } diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 8ce22c2b2e..1e57625028 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -813,16 +813,18 @@ int32_t mndAddStbToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *p } static int32_t mndProcessTtlTimer(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; - SSdb *pSdb = pMnode->pSdb; - SVgObj *pVgroup = NULL; - void *pIter = NULL; + SMnode *pMnode = pReq->info.node; + SSdb *pSdb = pMnode->pSdb; + SVgObj *pVgroup = NULL; + void *pIter = NULL; + SVDropTtlTableReq ttlReq = {.timestamp = taosGetTimestampSec()}; + int32_t reqLen = tSerializeSVDropTtlTableReq(NULL, 0, &ttlReq); + int32_t contLen = reqLen + sizeof(SMsgHead); while (1) { pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); if (pIter == NULL) break; - int32_t contLen = sizeof(SMsgHead) + sizeof(int32_t); SMsgHead *pHead = rpcMallocCont(contLen); if (pHead == NULL) { sdbCancelFetch(pSdb, pVgroup); @@ -831,17 +833,15 @@ static int32_t mndProcessTtlTimer(SRpcMsg *pReq) { } pHead->contLen = htonl(contLen); pHead->vgId = htonl(pVgroup->vgId); - - int32_t t = taosGetTimestampSec(); - *(int32_t *)((char *)pHead + sizeof(SMsgHead)) = htonl(t); + tSerializeSVDropTtlTableReq((char *)pHead + sizeof(SMsgHead), contLen, &ttlReq); SRpcMsg rpcMsg = {.msgType = TDMT_VND_DROP_TTL_TABLE, .pCont = pHead, .contLen = contLen}; SEpSet epSet = mndGetVgroupEpset(pMnode, pVgroup); int32_t code = tmsgSendReq(&epSet, &rpcMsg); if (code != 0) { - mError("failed to send ttl time seed, code:0x%x", code); + mError("vgId:%d, failed to send drop ttl table request to vnode since 0x%x", pVgroup->vgId, code); } else { - mDebug("send ttl time seed success, time:%d", t); + mDebug("vgId:%d, send drop ttl table request to vnode, time:%d", pVgroup->vgId, ttlReq.timestamp); } sdbRelease(pSdb, pVgroup); } @@ -2109,7 +2109,7 @@ static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&pStb->updateTime, false); // number of tables - pColInfo = taosArrayGet(pBlock->pDataBlock, cols); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); if (pStb->commentLen > 0) { char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(comment, pStb->comment); @@ -2122,6 +2122,34 @@ static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc colDataAppendNULL(pColInfo, numOfRows); } + char watermark[64 + VARSTR_HEADER_SIZE] = {0}; + sprintf(varDataVal(watermark), "%" PRId64 "a,%" PRId64 "a", pStb->watermark[0], pStb->watermark[1]); + varDataSetLen(watermark, strlen(varDataVal(watermark))); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)watermark, false); + + char maxDelay[64 + VARSTR_HEADER_SIZE] = {0}; + sprintf(varDataVal(maxDelay), "%" PRId64 "a,%" PRId64 "a", pStb->maxdelay[0], pStb->maxdelay[1]); + varDataSetLen(maxDelay, strlen(varDataVal(maxDelay))); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)maxDelay, false); + + char rollup[128 + VARSTR_HEADER_SIZE] = {0}; + int32_t rollupNum = (int32_t)taosArrayGetSize(pStb->pFuncs); + for (int32_t i = 0; i < rollupNum; ++i) { + char *funcName = taosArrayGet(pStb->pFuncs, i); + if (i) { + strcat(varDataVal(rollup), ", "); + } + strcat(varDataVal(rollup), funcName); + } + varDataSetLen(rollup, strlen(varDataVal(rollup))); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)rollup, false); + numOfRows++; sdbRelease(pSdb, pStb); } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 85f1ce6843..0b24b570a5 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -207,7 +207,7 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg createReq.buffer = pDb->cfg.buffer; createReq.pageSize = pDb->cfg.pageSize; createReq.pages = pDb->cfg.pages; - createReq.lastRowMem = pDb->cfg.lastRowMem; + createReq.cacheLastSize = pDb->cfg.cacheLastSize; createReq.daysPerFile = pDb->cfg.daysPerFile; createReq.daysToKeep0 = pDb->cfg.daysToKeep0; createReq.daysToKeep1 = pDb->cfg.daysToKeep1; @@ -219,7 +219,7 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg createReq.precision = pDb->cfg.precision; createReq.compression = pDb->cfg.compression; createReq.strict = pDb->cfg.strict; - createReq.cacheLastRow = pDb->cfg.cacheLastRow; + createReq.cacheLast = pDb->cfg.cacheLast; createReq.replica = pVgroup->replica; createReq.selfIndex = -1; createReq.hashBegin = pVgroup->hashBegin; @@ -277,7 +277,7 @@ void *mndBuildAlterVnodeReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup, int32_ alterReq.buffer = pDb->cfg.buffer; alterReq.pageSize = pDb->cfg.pageSize; alterReq.pages = pDb->cfg.pages; - alterReq.lastRowMem = pDb->cfg.lastRowMem; + alterReq.cacheLastSize = pDb->cfg.cacheLastSize; alterReq.daysPerFile = pDb->cfg.daysPerFile; alterReq.daysToKeep0 = pDb->cfg.daysToKeep0; alterReq.daysToKeep1 = pDb->cfg.daysToKeep1; @@ -285,7 +285,7 @@ void *mndBuildAlterVnodeReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup, int32_ alterReq.fsyncPeriod = pDb->cfg.fsyncPeriod; alterReq.walLevel = pDb->cfg.walLevel; alterReq.strict = pDb->cfg.strict; - alterReq.cacheLastRow = pDb->cfg.cacheLastRow; + alterReq.cacheLast = pDb->cfg.cacheLast; alterReq.replica = pVgroup->replica; for (int32_t v = 0; v < pVgroup->replica; ++v) { @@ -307,7 +307,7 @@ void *mndBuildAlterVnodeReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup, int32_ terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - contLen += +sizeof(SMsgHead); + contLen += sizeof(SMsgHead); void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { @@ -742,8 +742,8 @@ int64_t mndGetVgroupMemory(SMnode *pMnode, SDbObj *pDbInput, SVgObj *pVgroup) { int64_t vgroupMemroy = 0; if (pDb != NULL) { vgroupMemroy = (int64_t)pDb->cfg.buffer * 1024 * 1024 + (int64_t)pDb->cfg.pages * pDb->cfg.pageSize * 1024; - if (pDb->cfg.cacheLastRow > 0) { - vgroupMemroy += (int64_t)pDb->cfg.lastRowMem * 1024 * 1024; + if (pDb->cfg.cacheLast > 0) { + vgroupMemroy += (int64_t)pDb->cfg.cacheLastSize * 1024 * 1024; } } diff --git a/source/dnode/mnode/impl/test/db/db.cpp b/source/dnode/mnode/impl/test/db/db.cpp index a3d129c7c4..0fb8e9d530 100644 --- a/source/dnode/mnode/impl/test/db/db.cpp +++ b/source/dnode/mnode/impl/test/db/db.cpp @@ -50,7 +50,7 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) { createReq.compression = 2; createReq.replications = 1; createReq.strict = 1; - createReq.cacheLastRow = 0; + createReq.cacheLast = 0; createReq.ignoreExist = 1; createReq.numOfStables = 0; createReq.numOfRetensions = 0; @@ -84,7 +84,7 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) { alterdbReq.fsyncPeriod = 4000; alterdbReq.walLevel = 2; alterdbReq.strict = 1; - alterdbReq.cacheLastRow = 1; + alterdbReq.cacheLast = 1; alterdbReq.replications = 1; int32_t contLen = tSerializeSAlterDbReq(NULL, 0, &alterdbReq); @@ -146,7 +146,7 @@ TEST_F(MndTestDb, 03_Create_Use_Restart_Use_Db) { createReq.compression = 2; createReq.replications = 1; createReq.strict = 1; - createReq.cacheLastRow = 0; + createReq.cacheLast = 0; createReq.ignoreExist = 1; createReq.numOfStables = 0; createReq.numOfRetensions = 0; diff --git a/source/dnode/mnode/impl/test/dnode/mdnode.cpp b/source/dnode/mnode/impl/test/dnode/mdnode.cpp index 0b42b28219..8e4e728416 100644 --- a/source/dnode/mnode/impl/test/dnode/mdnode.cpp +++ b/source/dnode/mnode/impl/test/dnode/mdnode.cpp @@ -288,7 +288,7 @@ TEST_F(MndTestDnode, 05_Create_Drop_Restart_Dnode) { createReq.compression = 2; createReq.replications = 1; createReq.strict = 1; - createReq.cacheLastRow = 0; + createReq.cacheLast = 0; createReq.ignoreExist = 1; createReq.numOfStables = 0; createReq.numOfRetensions = 0; @@ -319,7 +319,7 @@ TEST_F(MndTestDnode, 05_Create_Drop_Restart_Dnode) { alterdbReq.fsyncPeriod = 4000; alterdbReq.walLevel = 2; alterdbReq.strict = 1; - alterdbReq.cacheLastRow = 1; + alterdbReq.cacheLast = 1; alterdbReq.replications = 3; int32_t contLen = tSerializeSAlterDbReq(NULL, 0, &alterdbReq); @@ -345,7 +345,7 @@ TEST_F(MndTestDnode, 05_Create_Drop_Restart_Dnode) { alterdbReq.fsyncPeriod = 4000; alterdbReq.walLevel = 2; alterdbReq.strict = 1; - alterdbReq.cacheLastRow = 1; + alterdbReq.cacheLast = 1; alterdbReq.replications = 1; int32_t contLen = tSerializeSAlterDbReq(NULL, 0, &alterdbReq); diff --git a/source/dnode/mnode/impl/test/sma/sma.cpp b/source/dnode/mnode/impl/test/sma/sma.cpp index d795816f57..ce6954279f 100644 --- a/source/dnode/mnode/impl/test/sma/sma.cpp +++ b/source/dnode/mnode/impl/test/sma/sma.cpp @@ -55,7 +55,7 @@ void* MndTestSma::BuildCreateDbReq(const char* dbname, int32_t* pContLen) { createReq.compression = 2; createReq.replications = 1; createReq.strict = 1; - createReq.cacheLastRow = 0; + createReq.cacheLast = 0; createReq.ignoreExist = 1; int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq); diff --git a/source/dnode/mnode/impl/test/stb/stb.cpp b/source/dnode/mnode/impl/test/stb/stb.cpp index 63bb1bf540..dfdd8f3a49 100644 --- a/source/dnode/mnode/impl/test/stb/stb.cpp +++ b/source/dnode/mnode/impl/test/stb/stb.cpp @@ -56,7 +56,7 @@ void* MndTestStb::BuildCreateDbReq(const char* dbname, int32_t* pContLen) { createReq.compression = 2; createReq.replications = 1; createReq.strict = 1; - createReq.cacheLastRow = 0; + createReq.cacheLast = 0; createReq.ignoreExist = 1; int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq); diff --git a/source/dnode/mnode/impl/test/topic/topic.cpp b/source/dnode/mnode/impl/test/topic/topic.cpp index 433a0ab5cc..353cedf636 100644 --- a/source/dnode/mnode/impl/test/topic/topic.cpp +++ b/source/dnode/mnode/impl/test/topic/topic.cpp @@ -48,7 +48,7 @@ void* MndTestTopic::BuildCreateDbReq(const char* dbname, int32_t* pContLen) { createReq.compression = 2; createReq.replications = 1; createReq.strict = 1; - createReq.cacheLastRow = 0; + createReq.cacheLast = 0; createReq.ignoreExist = 1; int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq); diff --git a/source/dnode/mnode/impl/test/user/user.cpp b/source/dnode/mnode/impl/test/user/user.cpp index 3b1a5fa3c5..d8b6964114 100644 --- a/source/dnode/mnode/impl/test/user/user.cpp +++ b/source/dnode/mnode/impl/test/user/user.cpp @@ -315,7 +315,7 @@ TEST_F(MndTestUser, 03_Alter_User) { createReq.compression = 2; createReq.replications = 1; createReq.strict = 1; - createReq.cacheLastRow = 0; + createReq.cacheLast = 0; createReq.ignoreExist = 1; int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq); diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index b42b0f2b44..8f2d3bde09 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -51,6 +51,7 @@ void vnodeCleanup(); int32_t vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs); void vnodeDestroy(const char *path, STfs *pTfs); SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb); +void vnodePreClose(SVnode *pVnode); void vnodeClose(SVnode *pVnode); int32_t vnodeStart(SVnode *pVnode); @@ -140,7 +141,10 @@ int32_t tsdbLastRowReaderOpen(void *pVnode, int32_t type, SArray *pTableIdList, void **pReader); int32_t tsdbRetrieveLastRow(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds); int32_t tsdbLastrowReaderClose(void *pReader); -int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int64_t* suid); +int32_t tsdbGetTableSchema(SVnode *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid); + +void tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity); +size_t tsdbCacheGetCapacity(SVnode *pVnode); // tq @@ -210,11 +214,13 @@ struct SVnodeCfg { int32_t vgId; char dbname[TSDB_DB_FNAME_LEN]; uint64_t dbId; + int32_t cacheLastSize; int32_t szPage; int32_t szCache; uint64_t szBuf; bool isHeap; bool isWeak; + int8_t cacheLast; int8_t isTsma; int8_t isRsma; int8_t hashMethod; diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index d5b719dfb9..e767d94ebd 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -67,7 +67,6 @@ struct SRSmaStat { int64_t submitVer; int64_t refId; // shared by fetch tasks int8_t triggerStat; // shared by fetch tasks - int8_t runningStat; // for persistence task SHashObj *rsmaInfoHash; // key: stbUid, value: SRSmaInfo; }; @@ -83,7 +82,6 @@ struct SSmaStat { #define SMA_RSMA_STAT(s) (&(s)->rsmaStat) #define RSMA_INFO_HASH(r) ((r)->rsmaInfoHash) #define RSMA_TRIGGER_STAT(r) (&(r)->triggerStat) -#define RSMA_RUNNING_STAT(r) (&(r)->runningStat) #define RSMA_REF_ID(r) ((r)->refId) #define RSMA_SUBMIT_VER(r) ((r)->submitVer) @@ -93,7 +91,7 @@ enum { TASK_TRIGGER_STAT_INACTIVE = 2, TASK_TRIGGER_STAT_PAUSED = 3, TASK_TRIGGER_STAT_CANCELLED = 4, - TASK_TRIGGER_STAT_FINISHED = 5, + TASK_TRIGGER_STAT_DROPPED = 5, }; void tdDestroySmaEnv(SSmaEnv *pSmaEnv); diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index bde9e578a7..4df9f96514 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -32,39 +32,38 @@ extern "C" { #define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TSDB ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0) // clang-format on -typedef struct TSDBROW TSDBROW; -typedef struct TABLEID TABLEID; -typedef struct TSDBKEY TSDBKEY; -typedef struct SDelData SDelData; -typedef struct SDelIdx SDelIdx; -typedef struct STbData STbData; -typedef struct SMemTable SMemTable; -typedef struct STbDataIter STbDataIter; -typedef struct STable STable; -typedef struct SMapData SMapData; -typedef struct SBlockIdx SBlockIdx; -typedef struct SBlock SBlock; -typedef struct SBlockStatis SBlockStatis; -typedef struct SAggrBlkCol SAggrBlkCol; -typedef struct SColData SColData; -typedef struct SBlockDataHdr SBlockDataHdr; -typedef struct SBlockData SBlockData; -typedef struct SDelFile SDelFile; -typedef struct STsdbCacheFile STsdbCacheFile; -typedef struct SHeadFile SHeadFile; -typedef struct SDataFile SDataFile; -typedef struct SLastFile SLastFile; -typedef struct SSmaFile SSmaFile; -typedef struct SDFileSet SDFileSet; -typedef struct SDataFWriter SDataFWriter; -typedef struct SDataFReader SDataFReader; -typedef struct SDelFWriter SDelFWriter; -typedef struct SDelFReader SDelFReader; -typedef struct SRowIter SRowIter; -typedef struct STsdbFS STsdbFS; -typedef struct SRowMerger SRowMerger; -typedef struct STsdbFSState STsdbFSState; -typedef struct STsdbSnapHdr STsdbSnapHdr; +typedef struct TSDBROW TSDBROW; +typedef struct TABLEID TABLEID; +typedef struct TSDBKEY TSDBKEY; +typedef struct SDelData SDelData; +typedef struct SDelIdx SDelIdx; +typedef struct STbData STbData; +typedef struct SMemTable SMemTable; +typedef struct STbDataIter STbDataIter; +typedef struct STable STable; +typedef struct SMapData SMapData; +typedef struct SBlockIdx SBlockIdx; +typedef struct SBlock SBlock; +typedef struct SBlockStatis SBlockStatis; +typedef struct SAggrBlkCol SAggrBlkCol; +typedef struct SColData SColData; +typedef struct SBlockDataHdr SBlockDataHdr; +typedef struct SBlockData SBlockData; +typedef struct SDelFile SDelFile; +typedef struct SHeadFile SHeadFile; +typedef struct SDataFile SDataFile; +typedef struct SLastFile SLastFile; +typedef struct SSmaFile SSmaFile; +typedef struct SDFileSet SDFileSet; +typedef struct SDataFWriter SDataFWriter; +typedef struct SDataFReader SDataFReader; +typedef struct SDelFWriter SDelFWriter; +typedef struct SDelFReader SDelFReader; +typedef struct SRowIter SRowIter; +typedef struct STsdbFS STsdbFS; +typedef struct SRowMerger SRowMerger; +typedef struct STsdbFSState STsdbFSState; +typedef struct STsdbSnapHdr STsdbSnapHdr; #define TSDB_MAX_SUBBLOCKS 8 #define TSDB_FHDR_SIZE 512 @@ -163,6 +162,7 @@ int32_t tGetMapData(uint8_t *p, SMapData *pMapData); // other int32_t tsdbKeyFid(TSKEY key, int32_t minutes, int8_t precision); void tsdbFidKeyRange(int32_t fid, int32_t minutes, int8_t precision, TSKEY *minKey, TSKEY *maxKey); +int32_t tsdbFidLevel(int32_t fid, STsdbKeepCfg *pKeepCfg, int64_t now); int32_t tsdbBuildDeleteSkyline(SArray *aDelData, int32_t sidx, int32_t eidx, SArray *aSkyline); void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg); // tsdbMemTable ============================================================================================== @@ -200,6 +200,7 @@ int32_t tsdbFSRollback(STsdbFS *pFS); int32_t tsdbFSStateUpsertDelFile(STsdbFSState *pState, SDelFile *pDelFile); int32_t tsdbFSStateUpsertDFileSet(STsdbFSState *pState, SDFileSet *pSet); +void tsdbFSStateDeleteDFileSet(STsdbFSState *pState, int32_t fid); SDelFile *tsdbFSStateGetDelFile(STsdbFSState *pState); SDFileSet *tsdbFSStateGetDFileSet(STsdbFSState *pState, int32_t fid); // tsdbReaderWriter.c ============================================================================================== @@ -213,6 +214,7 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, uint8_ SBlockIdx *pBlockIdx, SBlock *pBlock, int8_t cmprAlg); SDFileSet *tsdbDataFWriterGetWSet(SDataFWriter *pWriter); +int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo); // SDataFReader int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet); int32_t tsdbDataFReaderClose(SDataFReader **ppReader); @@ -235,6 +237,10 @@ int32_t tsdbDelFReaderClose(SDelFReader **ppReader); int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData, uint8_t **ppBuf); int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx, uint8_t **ppBuf); +#define TSDB_CACHE_NO(c) ((c).cacheLast == 0) +#define TSDB_CACHE_LAST_ROW(c) (((c).cacheLast & 1) > 0) +#define TSDB_CACHE_LAST(c) (((c).cacheLast & 2) > 0) + // tsdbCache int32_t tsdbOpenCache(STsdb *pTsdb); void tsdbCloseCache(SLRUCache *pCache); @@ -244,8 +250,13 @@ int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUHand int32_t tsdbCacheGetLastrowH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUHandle **h); int32_t tsdbCacheRelease(SLRUCache *pCache, LRUHandle *h); +int32_t tsdbCacheDeleteLastrow(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey); +int32_t tsdbCacheDeleteLast(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey); int32_t tsdbCacheDelete(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey); +void tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity); +size_t tsdbCacheGetCapacity(SVnode *pVnode); + int32_t tsdbCacheLastArray2Row(SArray *pLastArray, STSRow **ppRow, STSchema *pSchema); // structs ======================= diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 0c386babde..7b298ba830 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -122,6 +122,7 @@ int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepC int tsdbClose(STsdb** pTsdb); int32_t tsdbBegin(STsdb* pTsdb); int32_t tsdbCommit(STsdb* pTsdb); +int32_t tsdbDoRetention(STsdb* pTsdb, int64_t now); int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq* pMsg); int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp); int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, @@ -171,8 +172,9 @@ int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg); int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg); int64_t tdRSmaGetMaxSubmitVer(SSma* pSma, int8_t level); -int32_t tdProcessRSmaCreate(SVnode* pVnode, SVCreateStbReq* pReq); +int32_t tdProcessRSmaCreate(SSma* pSma, SVCreateStbReq* pReq); int32_t tdProcessRSmaSubmit(SSma* pSma, void* pMsg, int32_t inputType); +int32_t tdProcessRSmaDrop(SSma* pSma, SVDropStbReq* pReq); int32_t tdFetchTbUidList(SSma* pSma, STbUidStore** ppStore, tb_uid_t suid, tb_uid_t uid); int32_t tdUpdateTbUidList(SSma* pSma, STbUidStore* pUidStore); void tdUidStoreDestory(STbUidStore* pStore); diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index 5eb9665326..2cf4fd51a9 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -254,26 +254,7 @@ static void tdDestroyRSmaStat(void *pRSmaStat) { // step 1: set rsma trigger stat cancelled atomic_store_8(RSMA_TRIGGER_STAT(pStat), TASK_TRIGGER_STAT_CANCELLED); - // step 2: wait the persistence thread to finish - int32_t nLoops = 0; - if (atomic_load_8(RSMA_RUNNING_STAT(pStat)) == 1) { - while (1) { - if (atomic_load_8(RSMA_TRIGGER_STAT(pStat)) == TASK_TRIGGER_STAT_FINISHED) { - smaDebug("vgId:%d, rsma persist task finished already", SMA_VID(pSma)); - break; - } else { - smaDebug("vgId:%d, rsma persist task not finished yet since rsma stat in %" PRIi8, SMA_VID(pSma), - atomic_load_8(RSMA_TRIGGER_STAT(pStat))); - } - ++nLoops; - if (nLoops > 1000) { - sched_yield(); - nLoops = 0; - } - } - } - - // step 3: destroy the rsma info and associated fetch tasks + // step 2: destroy the rsma info and associated fetch tasks // TODO: use taosHashSetFreeFp when taosHashSetFreeFp is ready. if (taosHashGetSize(RSMA_INFO_HASH(pStat)) > 0) { void *infoHash = taosHashIterate(RSMA_INFO_HASH(pStat), NULL); @@ -285,8 +266,8 @@ static void tdDestroyRSmaStat(void *pRSmaStat) { } taosHashCleanup(RSMA_INFO_HASH(pStat)); - // step 5: wait all triggered fetch tasks finished - nLoops = 0; + // step 3: wait all triggered fetch tasks finished + int32_t nLoops = 0; while (1) { if (T_REF_VAL_GET((SSmaStat *)pStat) == 0) { smaDebug("vgId:%d, rsma fetch tasks all finished", SMA_VID(pSma)); diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index efa2886e48..06ffb639de 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -37,8 +37,6 @@ static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid); static int32_t tdRSmaFetchAndSubmitResult(SRSmaInfoItem *pItem, STSchema *pTSchema, int64_t suid, SRSmaStat *pStat, int8_t blkType); static void tdRSmaFetchTrigger(void *param, void *tmrId); -static void tdRSmaPersistTrigger(void *param, void *tmrId); -static void *tdRSmaPersistExec(void *param); static void tdRSmaQTaskInfoGetFName(int32_t vid, int64_t version, char *outputName); static int32_t tdRSmaQTaskInfoIterInit(SRSmaQTaskInfoIter *pIter, STFile *pTFile); @@ -67,9 +65,12 @@ struct SRSmaInfo { static SRSmaInfo *tdGetRSmaInfoByItem(SRSmaInfoItem *pItem) { // adapt accordingly if definition of SRSmaInfo update - int32_t rsmaInfoHeadLen = sizeof(int64_t) + sizeof(STSchema *); - ASSERT(pItem->level == 1 || pItem->level == 2); - return (SRSmaInfo *)POINTER_SHIFT(pItem, -sizeof(SRSmaInfoItem) * (pItem->level - 1) - rsmaInfoHeadLen); + SRSmaInfo *pResult = NULL; + int32_t rsmaInfoHeadLen = sizeof(int64_t) + sizeof(STSchema *); + ASSERT(pItem->level == TSDB_RETENTION_L1 || pItem->level == TSDB_RETENTION_L2); + pResult = (SRSmaInfo *)POINTER_SHIFT(pItem, -(sizeof(SRSmaInfoItem) * (pItem->level - 1) + rsmaInfoHeadLen)); + ASSERT(pResult->pTSchema->numOfCols > 1); + return pResult; } struct SRSmaQTaskInfoItem { @@ -278,7 +279,7 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat if (pItem->maxDelay > TSDB_MAX_ROLLUP_MAX_DELAY) { pItem->maxDelay = TSDB_MAX_ROLLUP_MAX_DELAY; } - pItem->level = (idx == 0 ? TSDB_RETENTION_L1 : TSDB_RETENTION_L2); + pItem->level = idx == 0 ? TSDB_RETENTION_L1 : TSDB_RETENTION_L2; smaInfo("vgId:%d table:%" PRIi64 " level:%" PRIi8 " maxdelay:%" PRIi64 " watermark:%" PRIi64 ", finally maxdelay:%" PRIi32, SMA_VID(pSma), pRSmaInfo->suid, idx + 1, param->maxdelay[idx], param->watermark[idx], pItem->maxDelay); @@ -375,20 +376,48 @@ _err: /** * @brief Check and init qTaskInfo_t, only applicable to stable with SRSmaParam currently * - * @param pVnode + * @param pSma * @param pReq * @return int32_t */ -int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) { - SSma *pSma = pVnode->pSma; +int32_t tdProcessRSmaCreate(SSma *pSma, SVCreateStbReq *pReq) { + SVnode *pVnode = pSma->pVnode; if (!pReq->rollup) { - smaTrace("vgId:%d, return directly since no rollup for stable %s %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid); + smaTrace("vgId:%d, not create rsma for stable %s %" PRIi64 " since no rollup in req", TD_VID(pVnode), pReq->name, + pReq->suid); + return TSDB_CODE_SUCCESS; + } + + if (!VND_IS_RSMA(pVnode)) { + smaTrace("vgId:%d, not create rsma for stable %s %" PRIi64 " since vnd is not rsma", TD_VID(pVnode), pReq->name, + pReq->suid); return TSDB_CODE_SUCCESS; } return tdProcessRSmaCreateImpl(pSma, &pReq->rsmaParam, pReq->suid, pReq->name); } +/** + * @brief drop cache for stb + * + * @param pSma + * @param pReq + * @return int32_t + */ +int32_t tdProcessRSmaDrop(SSma *pSma, SVDropStbReq *pReq) { + SVnode *pVnode = pSma->pVnode; + if (!VND_IS_RSMA(pVnode)) { + smaTrace("vgId:%d, not create rsma for stable %s %" PRIi64 " since vnd is not rsma", TD_VID(pVnode), pReq->name, + pReq->suid); + return TSDB_CODE_SUCCESS; + } + + + + smaDebug("vgId:%d, drop rsma for table %" PRIi64 " succeed", TD_VID(pVnode), pReq->suid); + return TSDB_CODE_SUCCESS; + } + /** * @brief store suid/[uids], prefer to use array and then hash * @@ -1174,123 +1203,6 @@ _err: return TSDB_CODE_FAILED; } -static void *tdRSmaPersistExec(void *param) { - setThreadName("rsma-task-persist"); - SRSmaStat *pRSmaStat = param; - SSma *pSma = pRSmaStat->pSma; - - int8_t triggerStat = atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat)); - - if (TASK_TRIGGER_STAT_CANCELLED == triggerStat || TASK_TRIGGER_STAT_PAUSED == triggerStat) { - goto _end; - } - - // execution - tdRSmaPersistExecImpl(pRSmaStat); - -_end: - if (TASK_TRIGGER_STAT_INACTIVE == atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), - TASK_TRIGGER_STAT_INACTIVE, - TASK_TRIGGER_STAT_ACTIVE)) { - smaDebug("vgId:%d, rsma persist task is active again", SMA_VID(pSma)); - } else if (TASK_TRIGGER_STAT_CANCELLED == atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), - TASK_TRIGGER_STAT_CANCELLED, - TASK_TRIGGER_STAT_FINISHED)) { - smaDebug("vgId:%d, rsma persist task is cancelled", SMA_VID(pSma)); - } else { - smaWarn("vgId:%d, rsma persist task in stat %" PRIi8, SMA_VID(pSma), atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat))); - } - - atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0); - smaDebug("vgId:%d, release rsetId rsetId:%" PRIi64 " refId:%d", SMA_VID(pSma), smaMgmt.rsetId, pRSmaStat->refId); - tdReleaseSmaRef(smaMgmt.rsetId, pRSmaStat->refId, __func__, __LINE__); - taosThreadExit(NULL); - return NULL; -} - -static void tdRSmaPersistTask(SRSmaStat *pRSmaStat) { - TdThreadAttr thAttr; - taosThreadAttrInit(&thAttr); - taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_DETACHED); - TdThread tid; - - if (taosThreadCreate(&tid, &thAttr, tdRSmaPersistExec, pRSmaStat) != 0) { - if (TASK_TRIGGER_STAT_INACTIVE == atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), - TASK_TRIGGER_STAT_INACTIVE, - TASK_TRIGGER_STAT_ACTIVE)) { - smaDebug("vgId:%d, persist task is active again", SMA_VID(pRSmaStat->pSma)); - } else if (TASK_TRIGGER_STAT_CANCELLED == atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), - TASK_TRIGGER_STAT_CANCELLED, - TASK_TRIGGER_STAT_FINISHED)) { - smaDebug("vgId:%d, persist task is cancelled and set finished", SMA_VID(pRSmaStat->pSma)); - } else { - smaWarn("vgId:%d, persist task in abnormal stat %" PRIi8, SMA_VID(pRSmaStat->pSma), - atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat))); - } - atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0); - smaDebug("vgId:%d, release rsetId rsetId:%" PRIi64 " refId:%d)", SMA_VID(pRSmaStat->pSma), smaMgmt.rsetId, - pRSmaStat->refId); - tdReleaseSmaRef(smaMgmt.rsetId, pRSmaStat->refId, __func__, __LINE__); - } - - taosThreadAttrDestroy(&thAttr); -} - -/** - * @brief trigger to persist rsma qTaskInfo - * - * @param param - * @param tmrId - */ -static void tdRSmaPersistTrigger(void *param, void *tmrId) { - SRSmaStat *rsmaStat = param; - SRSmaStat *pRSmaStat = (SRSmaStat *)taosAcquireRef(smaMgmt.rsetId, rsmaStat->refId); - ASSERT(0); - if (!pRSmaStat) { - smaDebug("rsma persistence task not start since already destroyed"); - return; - } - - int8_t tmrStat = - atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE); - switch (tmrStat) { - case TASK_TRIGGER_STAT_ACTIVE: { - atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 1); - if (TASK_TRIGGER_STAT_CANCELLED != atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), - TASK_TRIGGER_STAT_CANCELLED, - TASK_TRIGGER_STAT_FINISHED)) { - smaDebug("vgId:%d, rsma persistence start since active", SMA_VID(pRSmaStat->pSma)); - - // start persist task - tdRSmaPersistTask(pRSmaStat); - - // taosTmrReset(tdRSmaPersistTrigger, 5000, pRSmaStat, pRSmaStat->tmrHandle, - // RSMA_TMR_ID(pRSmaStat)); - } else { - atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0); - } - return; - } break; - case TASK_TRIGGER_STAT_CANCELLED: { - atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_FINISHED); - smaDebug("rsma persistence not start since cancelled and finished"); - } break; - case TASK_TRIGGER_STAT_PAUSED: { - smaDebug("rsma persistence not start since paused"); - } break; - case TASK_TRIGGER_STAT_INACTIVE: { - smaDebug("rsma persistence not start since inactive"); - } break; - case TASK_TRIGGER_STAT_INIT: { - smaDebug("rsma persistence not start since init"); - } break; - default: { - smaWarn("rsma persistence not start since unknown stat %" PRIi8, tmrStat); - } break; - } - taosReleaseRef(smaMgmt.rsetId, rsmaStat->refId); -} - /** * @brief trigger to get rsma result * @@ -1314,8 +1226,7 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { int8_t rsmaTriggerStat = atomic_load_8(RSMA_TRIGGER_STAT(pStat)); switch (rsmaTriggerStat) { case TASK_TRIGGER_STAT_PAUSED: - case TASK_TRIGGER_STAT_CANCELLED: - case TASK_TRIGGER_STAT_FINISHED: { + case TASK_TRIGGER_STAT_CANCELLED: { tdReleaseSmaRef(smaMgmt.rsetId, pItem->refId, __func__, __LINE__); smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data since stat is %" PRIi8 ", rsetId rsetId:%" PRIi64 " refId:%d", @@ -1328,8 +1239,6 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { SRSmaInfo *pRSmaInfo = tdGetRSmaInfoByItem(pItem); - ASSERT(pRSmaInfo->suid > 0); - int8_t fetchTriggerStat = atomic_val_compare_exchange_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE); switch (fetchTriggerStat) { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 3873073f03..fbb972fafe 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -394,7 +394,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { } else { ASSERT(pHandle->fetchMeta); ASSERT(IS_META_MSG(pHead->msgType)); - tqInfo("fetch meta msg, ver:%" PRId64 ", type:%d", pHead->version, pHead->msgType); + tqDebug("fetch meta msg, ver:%" PRId64 ", type:%d", pHead->version, pHead->msgType); SMqMetaRsp metaRsp = {0}; /*metaRsp.reqOffset = pReq->reqOffset.version;*/ /*metaRsp.rspOffset = fetchVer;*/ diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index e4b322d0b8..605f4bf3d5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -15,11 +15,15 @@ #include "tsdb.h" +typedef struct { + TSKEY ts; + SColVal colVal; +} SLastCol; + int32_t tsdbOpenCache(STsdb *pTsdb) { int32_t code = 0; SLRUCache *pCache = NULL; - // TODO: get cfg from vnode config: pTsdb->pVnode->config.lruCapacity - size_t cfgCapacity = 1024 * 1024; + size_t cfgCapacity = pTsdb->pVnode->config.cacheLastSize * 1024 * 1024; pCache = taosLRUCacheInit(cfgCapacity, -1, .5); if (pCache == NULL) { @@ -61,10 +65,11 @@ static void deleteTableCacheLastrow(const void *key, size_t keyLen, void *value) static void deleteTableCacheLast(const void *key, size_t keyLen, void *value) { taosArrayDestroy(value); } -static int32_t tsdbCacheDeleteLastrow(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey) { +int32_t tsdbCacheDeleteLastrow(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey) { int32_t code = 0; - char key[32] = {0}; - int keyLen = 0; + + char key[32] = {0}; + int keyLen = 0; // getTableCacheKey(uid, "lr", key, &keyLen); getTableCacheKey(uid, 0, key, &keyLen); @@ -83,18 +88,79 @@ static int32_t tsdbCacheDeleteLastrow(SLRUCache *pCache, tb_uid_t uid, TSKEY eKe return code; } -static int32_t tsdbCacheDeleteLast(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey) { +int32_t tsdbCacheDeleteLast(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey) { int32_t code = 0; - char key[32] = {0}; - int keyLen = 0; + + char key[32] = {0}; + int keyLen = 0; // getTableCacheKey(uid, "l", key, &keyLen); getTableCacheKey(uid, 1, key, &keyLen); LRUHandle *h = taosLRUCacheLookup(pCache, key, keyLen); if (h) { - // clear last cache anyway, no matter where eKey ends. - taosLRUCacheRelease(pCache, h, true); + SArray *pLast = (SArray *)taosLRUCacheValue(pCache, h); + bool invalidate = false; + int16_t nCol = taosArrayGetSize(pLast); + for (int16_t iCol = 0; iCol < nCol; ++iCol) { + SLastCol *tTsVal = (SLastCol *)taosArrayGet(pLast, iCol); + if (eKey >= tTsVal->ts) { + invalidate = true; + break; + } + } + + if (invalidate) { + taosLRUCacheRelease(pCache, h, true); + } else { + taosLRUCacheRelease(pCache, h, false); + } + // void taosLRUCacheErase(SLRUCache * cache, const void *key, size_t keyLen); + } + + return code; +} + +int32_t tsdbCacheDelete(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey) { + int32_t code = 0; + char key[32] = {0}; + int keyLen = 0; + + // getTableCacheKey(uid, "lr", key, &keyLen); + getTableCacheKey(uid, 0, key, &keyLen); + LRUHandle *h = taosLRUCacheLookup(pCache, key, keyLen); + if (h) { + STSRow *pRow = (STSRow *)taosLRUCacheValue(pCache, h); + if (pRow->ts <= eKey) { + taosLRUCacheRelease(pCache, h, true); + } else { + taosLRUCacheRelease(pCache, h, false); + } + + // void taosLRUCacheErase(SLRUCache * cache, const void *key, size_t keyLen); + } + + // getTableCacheKey(uid, "l", key, &keyLen); + getTableCacheKey(uid, 1, key, &keyLen); + h = taosLRUCacheLookup(pCache, key, keyLen); + if (h) { + SArray *pLast = (SArray *)taosLRUCacheValue(pCache, h); + bool invalidate = false; + int16_t nCol = taosArrayGetSize(pLast); + + for (int16_t iCol = 0; iCol < nCol; ++iCol) { + SLastCol *tTsVal = (SLastCol *)taosArrayGet(pLast, iCol); + if (eKey >= tTsVal->ts) { + invalidate = true; + break; + } + } + + if (invalidate) { + taosLRUCacheRelease(pCache, h, true); + } else { + taosLRUCacheRelease(pCache, h, false); + } // void taosLRUCacheErase(SLRUCache * cache, const void *key, size_t keyLen); } @@ -173,11 +239,6 @@ int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, ST return code; } -typedef struct { - TSKEY ts; - SColVal colVal; -} SLastCol; - int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb *pTsdb) { int32_t code = 0; STSRow *cacheRow = NULL; @@ -405,12 +466,13 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { case SFSNEXTROW_FS: state->aDFileSet = state->pTsdb->fs->cState->aDFileSet; state->nFileSet = taosArrayGetSize(state->aDFileSet); - state->iFileSet = state->nFileSet - 1; + state->iFileSet = state->nFileSet; state->pBlockData = NULL; case SFSNEXTROW_FILESET: { SDFileSet *pFileSet = NULL; + _next_fileset: if (--state->iFileSet >= 0) { pFileSet = (SDFileSet *)taosArrayGet(state->aDFileSet, state->iFileSet); } else { @@ -447,6 +509,10 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { state->pBlockIdx = taosArraySearch(state->aBlockIdx, state->pBlockIdxExp, tCmprBlockIdx, TD_EQ); if (code) goto _err; + if (!state->pBlockIdx) { + goto _next_fileset; + } + tMapDataReset(&state->blockMap); code = tsdbReadBlock(state->pDataFReader, state->pBlockIdx, &state->blockMap, NULL); /* code = tsdbReadBlock(state->pDataFReader, &state->blockIdx, &state->blockMap, NULL); */ @@ -1679,52 +1745,6 @@ int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUHand return code; } -int32_t tsdbCacheDelete(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey) { - int32_t code = 0; - char key[32] = {0}; - int keyLen = 0; - - // getTableCacheKey(uid, "lr", key, &keyLen); - getTableCacheKey(uid, 0, key, &keyLen); - LRUHandle *h = taosLRUCacheLookup(pCache, key, keyLen); - if (h) { - STSRow *pRow = (STSRow *)taosLRUCacheValue(pCache, h); - if (pRow->ts <= eKey) { - taosLRUCacheRelease(pCache, h, true); - } else { - taosLRUCacheRelease(pCache, h, false); - } - - // void taosLRUCacheErase(SLRUCache * cache, const void *key, size_t keyLen); - } - - // getTableCacheKey(uid, "l", key, &keyLen); - getTableCacheKey(uid, 1, key, &keyLen); - h = taosLRUCacheLookup(pCache, key, keyLen); - if (h) { - SArray *pLast = (SArray *)taosLRUCacheValue(pCache, h); - bool invalidate = false; - int16_t nCol = taosArrayGetSize(pLast); - - for (int16_t iCol = 0; iCol < nCol; ++iCol) { - SLastCol *tTsVal = (SLastCol *)taosArrayGet(pLast, iCol); - if (eKey >= tTsVal->ts) { - invalidate = true; - break; - } - } - - if (invalidate) { - taosLRUCacheRelease(pCache, h, true); - } else { - taosLRUCacheRelease(pCache, h, false); - } - // void taosLRUCacheErase(SLRUCache * cache, const void *key, size_t keyLen); - } - - return code; -} - int32_t tsdbCacheRelease(SLRUCache *pCache, LRUHandle *h) { int32_t code = 0; @@ -1732,3 +1752,9 @@ int32_t tsdbCacheRelease(SLRUCache *pCache, LRUHandle *h) { return code; } + +void tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity) { + taosLRUCacheSetCapacity(pVnode->pTsdb->lruCache, capacity); +} + +size_t tsdbCacheGetCapacity(SVnode *pVnode) { return taosLRUCacheGetCapacity(pVnode->pTsdb->lruCache); } diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 4a33dab08c..53b6735c30 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -688,6 +688,14 @@ _exit: return code; } +void tsdbFSStateDeleteDFileSet(STsdbFSState *pState, int32_t fid) { + int32_t idx; + + idx = taosArraySearchIdx(pState->aDFileSet, &(SDFileSet){.fid = fid}, tDFileSetCmprFn, TD_EQ); + ASSERT(idx >= 0); + taosArrayRemove(pState->aDFileSet, idx); +} + SDelFile *tsdbFSStateGetDelFile(STsdbFSState *pState) { return pState->pDelFile; } SDFileSet *tsdbFSStateGetDFileSet(STsdbFSState *pState, int32_t fid) { diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 5186f8288f..82de931872 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -181,8 +181,12 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid pMemTable->maxVersion = TMAX(pMemTable->maxVersion, version); pMemTable->nDel++; - if (tsdbKeyCmprFn(&lastKey, &pTbData->maxKey) >= 0) { - tsdbCacheDelete(pTsdb->lruCache, pTbData->uid, eKey); + if (TSDB_CACHE_LAST_ROW(pMemTable->pTsdb->pVnode->config) && tsdbKeyCmprFn(&lastKey, &pTbData->maxKey) >= 0) { + tsdbCacheDeleteLastrow(pTsdb->lruCache, pTbData->uid, eKey); + } + + if (TSDB_CACHE_LAST(pMemTable->pTsdb->pVnode->config)) { + tsdbCacheDeleteLast(pTsdb->lruCache, pTbData->uid, eKey); } tsdbError("vgId:%d, delete data from table suid:%" PRId64 " uid:%" PRId64 " skey:%" PRId64 " eKey:%" PRId64 @@ -556,12 +560,14 @@ static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, STbData *pTbData, i pTbData->maxKey = key.ts; } - if (pLastRow != NULL) { + if (TSDB_CACHE_LAST_ROW(pMemTable->pTsdb->pVnode->config) && pLastRow != NULL) { tsdbCacheInsertLastrow(pMemTable->pTsdb->lruCache, pMemTable->pTsdb, pTbData->uid, pLastRow, true); } } - tsdbCacheInsertLast(pMemTable->pTsdb->lruCache, pTbData->uid, pLastRow, pMemTable->pTsdb); + if (TSDB_CACHE_LAST(pMemTable->pTsdb->pVnode->config)) { + tsdbCacheInsertLast(pMemTable->pTsdb->lruCache, pTbData->uid, pLastRow, pMemTable->pTsdb); + } pTbData->minVersion = TMIN(pTbData->minVersion, version); pTbData->maxVersion = TMAX(pTbData->maxVersion, version); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 5f796bbab9..6e68d30a08 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -400,7 +400,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd pReader->idStr = (idstr != NULL) ? strdup(idstr) : NULL; pReader->verRange = getQueryVerRange(pVnode, pCond, level); pReader->type = pCond->type; - pReader->window = updateQueryTimeWindow(pVnode->pTsdb, &pCond->twindows); + pReader->window = updateQueryTimeWindow(pReader->pTsdb, &pCond->twindows); ASSERT(pCond->numOfCols > 0); @@ -2203,15 +2203,15 @@ static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* ret if (level == TSDB_RETENTION_L0) { *pLevel = TSDB_RETENTION_L0; - tsdbDebug("vgId:%d, read handle %p rsma level %d is selected to query %s", vgId, TSDB_RETENTION_L0, str); + tsdbDebug("vgId:%d, rsma level %d is selected to query %s", vgId, TSDB_RETENTION_L0, str); return VND_RSMA0(pVnode); } else if (level == TSDB_RETENTION_L1) { *pLevel = TSDB_RETENTION_L1; - tsdbDebug("vgId:%d, read handle %p rsma level %d is selected to query %s", vgId, TSDB_RETENTION_L1, str); + tsdbDebug("vgId:%d, rsma level %d is selected to query %s", vgId, TSDB_RETENTION_L1, str); return VND_RSMA1(pVnode); } else { *pLevel = TSDB_RETENTION_L2; - tsdbDebug("vgId:%d, read handle %p rsma level %d is selected to query %s", vgId, TSDB_RETENTION_L2, str); + tsdbDebug("vgId:%d, rsma level %d is selected to query %s", vgId, TSDB_RETENTION_L2, str); return VND_RSMA2(pVnode); } } diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index e96ee03b03..4e7a9d3b04 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -1913,3 +1913,114 @@ _err: taosArrayDestroy(aBlockCol); return code; } + +int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { + int32_t code = 0; + int64_t n; + int64_t size; + TdFilePtr pOutFD = NULL; // TODO + TdFilePtr PInFD = NULL; // TODO + char fNameFrom[TSDB_FILENAME_LEN]; + char fNameTo[TSDB_FILENAME_LEN]; + + // head + tsdbDataFileName(pTsdb, pSetFrom, TSDB_HEAD_FILE, fNameFrom); + tsdbDataFileName(pTsdb, pSetTo, TSDB_HEAD_FILE, fNameTo); + + pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + if (pOutFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + PInFD = taosOpenFile(fNameFrom, TD_FILE_READ); + if (PInFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->fHead.size); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + taosCloseFile(&pOutFD); + taosCloseFile(&PInFD); + + // data + tsdbDataFileName(pTsdb, pSetFrom, TSDB_DATA_FILE, fNameFrom); + tsdbDataFileName(pTsdb, pSetTo, TSDB_DATA_FILE, fNameTo); + + pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + if (pOutFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + PInFD = taosOpenFile(fNameFrom, TD_FILE_READ); + if (PInFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->fData.size); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + taosCloseFile(&pOutFD); + taosCloseFile(&PInFD); + + // last + tsdbDataFileName(pTsdb, pSetFrom, TSDB_LAST_FILE, fNameFrom); + tsdbDataFileName(pTsdb, pSetTo, TSDB_LAST_FILE, fNameTo); + pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + if (pOutFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + PInFD = taosOpenFile(fNameFrom, TD_FILE_READ); + if (PInFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->fLast.size); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + taosCloseFile(&pOutFD); + taosCloseFile(&PInFD); + + // sma + tsdbDataFileName(pTsdb, pSetFrom, TSDB_SMA_FILE, fNameFrom); + tsdbDataFileName(pTsdb, pSetTo, TSDB_SMA_FILE, fNameTo); + + pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + if (pOutFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + PInFD = taosOpenFile(fNameFrom, TD_FILE_READ); + if (PInFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->fSma.size); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + taosCloseFile(&pOutFD); + taosCloseFile(&PInFD); + + return code; + +_err: + tsdbError("vgId:%d tsdb DFileSet copy failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + return code; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index e73f3f947c..2ae646a571 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -15,19 +15,80 @@ #include "tsdb.h" +static int32_t tsdbDoRetentionImpl(STsdb *pTsdb, int64_t now, int8_t try, int8_t *canDo) { + int32_t code = 0; + STsdbFSState *pState; + + if (try) { + pState = pTsdb->fs->cState; + *canDo = 0; + } else { + pState = pTsdb->fs->nState; + } + + for (int32_t iSet = 0; iSet < taosArrayGetSize(pState->aDFileSet); iSet++) { + SDFileSet *pDFileSet = (SDFileSet *)taosArrayGet(pState->aDFileSet, iSet); + int32_t expLevel = tsdbFidLevel(pDFileSet->fid, &pTsdb->keepCfg, now); + SDiskID did; + + // check + if (expLevel == pDFileSet->diskId.id) continue; + + // delete or move + if (expLevel < 0) { + if (try) { + *canDo = 1; + } else { + tsdbFSStateDeleteDFileSet(pState, pDFileSet->fid); + iSet--; + } + } else { + // alloc + if (tfsAllocDisk(pTsdb->pVnode->pTfs, expLevel, &did) < 0) { + code = terrno; + goto _exit; + } + + if (did.level == pDFileSet->diskId.level) continue; + + if (try) { + *canDo = 1; + } else { + // copy the file to new disk + + SDFileSet nDFileSet = *pDFileSet; + nDFileSet.diskId = did; + + tfsMkdirRecurAt(pTsdb->pVnode->pTfs, pTsdb->path, did); + + code = tsdbDFileSetCopy(pTsdb, pDFileSet, &nDFileSet); + if (code) goto _exit; + + code = tsdbFSStateUpsertDFileSet(pState, &nDFileSet); + if (code) goto _exit; + } + } + } + +_exit: + return code; +} + int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now) { int32_t code = 0; + int8_t canDo; + + // try + tsdbDoRetentionImpl(pTsdb, now, 1, &canDo); + if (!canDo) goto _exit; // begin code = tsdbFSBegin(pTsdb->fs); if (code) goto _err; // do retention - for (int32_t iSet = 0; iSet < taosArrayGetSize(pTsdb->fs->nState->aDFileSet); iSet++) { - SDFileSet *pDFileSet = (SDFileSet *)taosArrayGet(pTsdb->fs->nState->aDFileSet, iSet); - - // TODO - } + code = tsdbDoRetentionImpl(pTsdb, now, 0, NULL); + if (code) goto _err; // commit code = tsdbFSCommit(pTsdb->fs); @@ -38,5 +99,6 @@ _exit: _err: tsdbError("vgId:%d tsdb do retention failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + tsdbFSRollback(pTsdb->fs); return code; } \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 54087a7871..06d4a86116 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -312,6 +312,9 @@ struct STsdbSnapWriter { // config int32_t minutes; int8_t precision; + int32_t minRow; + int32_t maxRow; + int8_t cmprAlg; // for data file int32_t fid; @@ -321,14 +324,18 @@ struct STsdbSnapWriter { SBlockIdx* pBlockIdx; SMapData mBlock; int32_t iBlock; + SBlock* pBlock; + SBlock block; SBlockData blockData; int32_t iRow; SDataFWriter* pDataFWriter; SArray* aBlockIdxN; + SBlockIdx* pBlockIdxN; SBlockIdx blockIdx; SMapData mBlockN; - SBlock block; + SBlock* pBlockN; + SBlock blockN; SBlockData nBlockData; // for del file @@ -394,13 +401,114 @@ _err: return code; } +static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWrite) { + int32_t code = 0; + // TODO + return code; +} + +static int32_t tsdbSnapWriteTableData(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { + int32_t code = 0; + TABLEID id = {0}; // TODO + + // skip + while (pWriter->pBlockIdx && tTABLEIDCmprFn(&id, pWriter->pBlockIdx) < 0) { + code = tsdbSnapWriteTableDataEnd(pWriter); + if (code) goto _err; + + pWriter->iBlockIdx++; + if (pWriter->iBlockIdx < taosArrayGetSize(pWriter->aBlockIdx)) { + pWriter->pBlockIdx = (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlockIdx); + } else { + pWriter->pBlockIdx = NULL; + } + } + + // new or merge + if (pWriter->pBlockIdx == NULL || tTABLEIDCmprFn(&id, pWriter->pBlockIdx) < 0) { + int32_t c; + + if (pWriter->pBlockIdxN && ((c = tTABLEIDCmprFn(&id, pWriter->pBlockIdxN)) != 0)) { + ASSERT(c > 0); + + code = tsdbSnapWriteTableDataEnd(pWriter); + if (code) goto _err; + } + + if (pWriter->pBlockIdxN == NULL) { + pWriter->pBlockIdx = &pWriter->blockIdx; + pWriter->pBlockIdx->suid = id.suid; + pWriter->pBlockIdx->uid = id.uid; + } + + // loop to write the data + TSDBROW* pRow = NULL; // todo + int32_t nRow = 0; // todo + SBlockData* pBlockData = NULL; // todo + for (int32_t iRow = 0; iRow < nRow; iRow++) { + code = tBlockDataAppendRow(&pWriter->nBlockData, &tsdbRowFromBlockData(pBlockData, iRow), NULL); + if (code) goto _err; + + if (pWriter->nBlockData.nRow > pWriter->maxRow * 4 / 5) { + code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->nBlockData, NULL, NULL, pWriter->pBlockIdxN, + pWriter->pBlockN, pWriter->cmprAlg); + if (code) goto _err; + } + } + } else { + // skip + while (true) { + if (pWriter->pBlock == NULL) break; + if (pWriter->pBlock->last) break; + if (tBlockCmprFn(&(SBlock){.minKey = {0}, .maxKey = {0}}, pWriter->pBlock) >= 0) break; + + code = tMapDataPutItem(&pWriter->mBlockN, pWriter->pBlock, tPutBlock); + if (code) goto _err; + } + + if (pWriter->pBlock) { + if (pWriter->pBlock->last) { + // load the last block and merge with the data (todo) + } else { + int32_t c = tBlockCmprFn(&(SBlock){0 /*TODO*/}, pWriter->pBlock); + + if (c > 0) { + // commit until pWriter->pBlock (todo) + } else { + // load the block and merge with the data (todo) + } + } + } else { + int32_t nRow = 0; + SBlockData* pBlockData = NULL; + + for (int32_t iRow = 0; iRow < nRow; iRow++) { + code = tBlockDataAppendRow(&pWriter->nBlockData, &tsdbRowFromBlockData(pBlockData, iRow), NULL); + if (code) goto _err; + + if (pWriter->nBlockData.nRow >= pWriter->maxRow * 4 / 5) { + code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->nBlockData, NULL, NULL, pWriter->pBlockIdxN, + pWriter->pBlockN, pWriter->cmprAlg); + if (code) goto _err; + + tBlockDataClearData(&pWriter->nBlockData); + } + } + } + } + + return code; + +_err: + tsdbError("vgId:%d tsdb snapshot write table data failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); + return code; +} + static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { int32_t code = 0; STsdb* pTsdb = pWriter->pTsdb; - int64_t suid = 0; // todo - int64_t uid = 0; // todo - int64_t skey; // todo - int64_t ekey; // todo + int64_t skey; // todo + int64_t ekey; // todo int32_t fid = tsdbKeyFid(skey, pWriter->minutes, pWriter->precision); ASSERT(fid == tsdbKeyFid(ekey, pWriter->minutes, pWriter->precision)); @@ -440,47 +548,8 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3 taosArrayClear(pWriter->aBlockIdxN); } - // process - TABLEID id = {0}; // TODO - TSKEY minKey = 0; // TODO - TSKEY maxKey = 0; // TODO - - while (true) { - if (pWriter->pBlockIdx) { - int32_t c = tTABLEIDCmprFn(&id, pWriter->pBlockIdx); - - if (c == 0) { - } else if (c < 0) { - // keep merge - } else { - // code = tsdbSnapWriteTableDataEnd(pWriter); - if (code) goto _err; - - pWriter->iBlockIdx++; - if (pWriter->iBlockIdx < taosArrayGetSize(pWriter->aBlockIdx)) { - pWriter->pBlockIdx = (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlockIdx); - } else { - pWriter->pBlockIdx = NULL; - } - - if (pWriter->pBlockIdx) { - code = tsdbReadBlock(pWriter->pDataFReader, pWriter->pBlockIdx, &pWriter->mBlock, NULL); - if (code) goto _err; - } - } - } else { - int32_t c = tTABLEIDCmprFn(&id, &pWriter->blockIdx); - - if (c == 0) { - // merge commit the block data - } else if (c > 0) { - // code = tsdbSnapWriteTableDataEnd(pWriter); - if (code) goto _err; - } else { - ASSERT(0); - } - } - } + code = tsdbSnapWriteTableData(pWriter, pData, nData); + if (code) goto _err; return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 415a674737..dc6c823368 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -465,17 +465,37 @@ void tsdbFidKeyRange(int32_t fid, int32_t minutes, int8_t precision, TSKEY *minK *maxKey = *minKey + minutes * tsTickPerMin[precision] - 1; } -// int tsdFidLevel(int fid, TSKEY now, minute) { -// if (fid >= pRtn->maxFid) { -// return 0; -// } else if (fid >= pRtn->midFid) { -// return 1; -// } else if (fid >= pRtn->minFid) { -// return 2; -// } else { -// return -1; -// } -// } +int32_t tsdbFidLevel(int32_t fid, STsdbKeepCfg *pKeepCfg, int64_t now) { + int32_t aFid[3]; + TSKEY key; + + if (pKeepCfg->precision == TSDB_TIME_PRECISION_MILLI) { + now = now * 1000; + } else if (pKeepCfg->precision == TSDB_TIME_PRECISION_MICRO) { + now = now * 1000000l; + } else if (pKeepCfg->precision == TSDB_TIME_PRECISION_NANO) { + now = now * 1000000000l; + } else { + ASSERT(0); + } + + key = now - pKeepCfg->keep0 * tsTickPerMin[pKeepCfg->precision]; + aFid[0] = tsdbKeyFid(key, pKeepCfg->days, pKeepCfg->precision); + key = now - pKeepCfg->keep1 * tsTickPerMin[pKeepCfg->precision]; + aFid[1] = tsdbKeyFid(key, pKeepCfg->days, pKeepCfg->precision); + key = now - pKeepCfg->keep2 * tsTickPerMin[pKeepCfg->precision]; + aFid[2] = tsdbKeyFid(key, pKeepCfg->days, pKeepCfg->precision); + + if (fid >= aFid[0]) { + return 0; + } else if (fid >= aFid[1]) { + return 1; + } else if (fid >= aFid[2]) { + return 2; + } else { + return -1; + } +} // TSDBROW ====================================================== void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index 20ac56617f..eac1fd1a74 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -20,6 +20,8 @@ const SVnodeCfg vnodeCfgDefault = {.vgId = -1, .dbId = 0, .szPage = 4096, .szCache = 256, + .cacheLast = 3, + .cacheLastSize = 8, .szBuf = 96 * 1024 * 1024, .isHeap = false, .isWeak = 0, @@ -60,6 +62,8 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) { if (tjsonAddIntegerToObject(pJson, "dbId", pCfg->dbId) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "szPage", pCfg->szPage) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "szCache", pCfg->szCache) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "cacheLast", pCfg->cacheLast) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "cacheLastSize", pCfg->cacheLastSize) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "szBuf", pCfg->szBuf) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "isHeap", pCfg->isHeap) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "isWeak", pCfg->isWeak) < 0) return -1; @@ -133,6 +137,10 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { if (code < 0) return -1; tjsonGetNumberValue(pJson, "szCache", pCfg->szCache, code); if (code < 0) return -1; + tjsonGetNumberValue(pJson, "cacheLast", pCfg->cacheLast, code); + if (code < 0) return -1; + tjsonGetNumberValue(pJson, "cacheLastSize", pCfg->cacheLastSize, code); + if (code < 0) return -1; tjsonGetNumberValue(pJson, "szBuf", pCfg->szBuf, code); if (code < 0) return -1; tjsonGetNumberValue(pJson, "isHeap", pCfg->isHeap, code); diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index fe26bd1090..0914827950 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -28,7 +28,7 @@ int vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) { } // create vnode env - if (tfsMkdir(pTfs, path) < 0) { + if (tfsMkdirAt(pTfs, path, (SDiskID){0}) < 0) { vError("vgId:%d, failed to create vnode since: %s", pCfg->vgId, tstrerror(terrno)); return -1; } @@ -175,6 +175,12 @@ _err: return NULL; } +void vnodePreClose(SVnode *pVnode) { + if (pVnode) { + syncLeaderTransfer(pVnode->sync); + } +} + void vnodeClose(SVnode *pVnode) { if (pVnode) { vnodeCommit(pVnode); diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index dceeb4c282..d25ae817c7 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -24,8 +24,10 @@ static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessAlterHasnRangeReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessAlterHashRangeReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { @@ -172,9 +174,12 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp case TDMT_VND_DROP_TTL_TABLE: if (vnodeProcessDropTtlTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; break; - case TDMT_VND_CREATE_SMA: { + case TDMT_VND_TRIM: + if (vnodeProcessTrimReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; + break; + case TDMT_VND_CREATE_SMA: if (vnodeProcessCreateTSmaReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; - } break; + break; /* TSDB */ case TDMT_VND_SUBMIT: if (vnodeProcessSubmitReq(pVnode, version, pMsg->pCont, pMsg->contLen, pRsp) < 0) goto _err; @@ -215,9 +220,10 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp vnodeProcessAlterConfirmReq(pVnode, version, pReq, len, pRsp); break; case TDMT_VND_ALTER_HASHRANGE: - vnodeProcessAlterHasnRangeReq(pVnode, version, pReq, len, pRsp); + vnodeProcessAlterHashRangeReq(pVnode, version, pReq, len, pRsp); break; case TDMT_VND_ALTER_CONFIG: + vnodeProcessAlterConfigReq(pVnode, version, pReq, len, pRsp); break; case TDMT_VND_COMMIT: goto _do_commit; @@ -345,13 +351,38 @@ void vnodeUpdateMetaRsp(SVnode *pVnode, STableMetaRsp *pMetaRsp) { pMetaRsp->precision = pVnode->config.tsdbCfg.precision; } +static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { + int32_t code = 0; + SVTrimDbReq trimReq = {0}; + + vInfo("vgId:%d, trim vnode request will be processed, time:%d", pVnode->config.vgId, trimReq.timestamp); + + // decode + if (tDeserializeSVTrimDbReq(pReq, len, &trimReq) != 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + // process + code = tsdbDoRetention(pVnode->pTsdb, trimReq.timestamp); + if (code) goto _exit; + +_exit: + return code; +} + static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { SArray *tbUids = taosArrayInit(8, sizeof(int64_t)); if (tbUids == NULL) return TSDB_CODE_OUT_OF_MEMORY; - int32_t t = ntohl(*(int32_t *)pReq); - vDebug("rec ttl time:%d", t); - int32_t ret = metaTtlDropTable(pVnode->pMeta, t, tbUids); + SVDropTtlTableReq ttlReq = {0}; + if (tDeserializeSVDropTtlTableReq(pReq, len, &ttlReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto end; + } + + vInfo("vgId:%d, drop ttl table req will be processed, time:%d", pVnode->config.vgId, ttlReq.timestamp); + int32_t ret = metaTtlDropTable(pVnode->pMeta, ttlReq.timestamp, tbUids); if (ret != 0) { goto end; } @@ -386,7 +417,7 @@ static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *p goto _err; } - if (tdProcessRSmaCreate(pVnode, &req) < 0) { + if (tdProcessRSmaCreate(pVnode->pSma, &req) < 0) { pRsp->code = terrno; goto _err; } @@ -542,6 +573,11 @@ static int32_t vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pRe goto _exit; } + if (tdProcessRSmaDrop(pVnode->pSma, &req) < 0) { + rcode = terrno; + goto _exit; + } + // return rsp _exit: pRsp->code = rcode; @@ -886,7 +922,7 @@ static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t version, void return 0; } -static int32_t vnodeProcessAlterHasnRangeReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessAlterHashRangeReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { vInfo("vgId:%d, alter hashrange msg will be processed", TD_VID(pVnode)); // todo @@ -896,6 +932,23 @@ static int32_t vnodeProcessAlterHasnRangeReq(SVnode *pVnode, int64_t version, vo return 0; } +static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { + SAlterVnodeReq alterReq = {0}; + if (tDeserializeSAlterVnodeReq(pReq, len, &alterReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return TSDB_CODE_INVALID_MSG; + } + + vInfo("vgId:%d, start to alter vnode config, cacheLast:%d cacheLastSize:%d", TD_VID(pVnode), alterReq.cacheLast, + alterReq.cacheLastSize); + if (pVnode->config.cacheLastSize != alterReq.cacheLastSize) { + pVnode->config.cacheLastSize = alterReq.cacheLastSize; + // TODO: save config + tsdbCacheSetCapacity(pVnode, (size_t)pVnode->config.cacheLastSize * 1024 * 1024); + } + return 0; +} + static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { int32_t code = 0; SDecoder *pCoder = &(SDecoder){0}; diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 598a754c50..dce7adfea9 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -482,33 +482,33 @@ typedef struct SCtgOperation { #define CTG_LOCK(type, _lock) do { \ if (CTG_READ == (type)) { \ - assert(atomic_load_64((_lock)) >= 0); \ - CTG_LOCK_DEBUG("CTG RLOCK%p:%" PRIx64 ", %s:%d B", (_lock), atomic_load_64(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32((_lock)) >= 0); \ + CTG_LOCK_DEBUG("CTG RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosRLockLatch(_lock); \ - CTG_LOCK_DEBUG("CTG RLOCK%p:%" PRIx64 ", %s:%d E", (_lock), atomic_load_64(_lock), __FILE__, __LINE__); \ - assert(atomic_load_64((_lock)) > 0); \ + CTG_LOCK_DEBUG("CTG RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32((_lock)) > 0); \ } else { \ - assert(atomic_load_64((_lock)) >= 0); \ - CTG_LOCK_DEBUG("CTG WLOCK%p:%" PRIx64 ", %s:%d B", (_lock), atomic_load_64(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32((_lock)) >= 0); \ + CTG_LOCK_DEBUG("CTG WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosWLockLatch(_lock); \ - CTG_LOCK_DEBUG("CTG WLOCK%p:%" PRIx64 ", %s:%d E", (_lock), atomic_load_64(_lock), __FILE__, __LINE__); \ - assert(atomic_load_64((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY); \ + CTG_LOCK_DEBUG("CTG WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY); \ } \ } while (0) #define CTG_UNLOCK(type, _lock) do { \ if (CTG_READ == (type)) { \ - assert(atomic_load_64((_lock)) > 0); \ - CTG_LOCK_DEBUG("CTG RULOCK%p:%" PRIx64 ", %s:%d B", (_lock), atomic_load_64(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32((_lock)) > 0); \ + CTG_LOCK_DEBUG("CTG RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosRUnLockLatch(_lock); \ - CTG_LOCK_DEBUG("CTG RULOCK%p:%" PRIx64 ", %s:%d E", (_lock), atomic_load_64(_lock), __FILE__, __LINE__); \ - assert(atomic_load_64((_lock)) >= 0); \ + CTG_LOCK_DEBUG("CTG RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32((_lock)) >= 0); \ } else { \ - assert(atomic_load_64((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY); \ - CTG_LOCK_DEBUG("CTG WULOCK%p:%" PRIx64 ", %s:%d B", (_lock), atomic_load_64(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY); \ + CTG_LOCK_DEBUG("CTG WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosWUnLockLatch(_lock); \ - CTG_LOCK_DEBUG("CTG WULOCK%p:%" PRIx64 ", %s:%d E", (_lock), atomic_load_64(_lock), __FILE__, __LINE__); \ - assert(atomic_load_64((_lock)) >= 0); \ + CTG_LOCK_DEBUG("CTG WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32((_lock)) >= 0); \ } \ } while (0) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 3a7ad4a2d6..ca50702894 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -1293,6 +1293,7 @@ void catalogDestroy(void) { if (!taosCheckCurrentInDll()) { ctgClearCacheEnqueue(NULL, true, true, true); + taosThreadJoin(gCtgMgmt.updateThread, NULL); } taosHashCleanup(gCtgMgmt.pCluster); diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 3245fcd16a..51b721f818 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -109,7 +109,7 @@ void sendCreateDbMsg(void *shandle, SEpSet *pEpSet) { createReq.compression = 2; createReq.replications = 1; createReq.strict = 1; - createReq.cacheLastRow = 0; + createReq.cacheLast = 0; createReq.ignoreExist = 1; int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq); diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 7f70a78b12..a2816209a9 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -223,7 +223,7 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbFName, S "CREATE DATABASE `%s` BUFFER %d CACHELAST %d COMP %d DURATION %dm " "FSYNC %d MAXROWS %d MINROWS %d KEEP %dm,%dm,%dm PAGES %d PAGESIZE %d PRECISION '%s' REPLICA %d " "STRICT %d WAL %d VGROUPS %d SINGLE_STABLE %d", - dbFName, pCfg->buffer, pCfg->cacheLastRow, pCfg->compression, pCfg->daysPerFile, pCfg->fsyncPeriod, + dbFName, pCfg->buffer, pCfg->cacheLast, pCfg->compression, pCfg->daysPerFile, pCfg->fsyncPeriod, pCfg->maxRows, pCfg->minRows, pCfg->daysToKeep0, pCfg->daysToKeep1, pCfg->daysToKeep2, pCfg->pages, pCfg->pageSize, prec, pCfg->replications, pCfg->strict, pCfg->walLevel, pCfg->numOfVgroups, 1 == pCfg->numOfStables); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 456f4a7470..69ba88916a 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -779,6 +779,8 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo int32_t compLen, int32_t numOfOutput, int64_t startTs, uint64_t* total, SArray* pColList); void getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key, STimeWindow* win); +STimeWindow getFirstQualifiedTimeWindow(int64_t ts, STimeWindow* pWindow, SInterval* pInterval, int32_t order); + int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t *order, int32_t* scanFlag); int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz); diff --git a/source/libs/executor/inc/tfill.h b/source/libs/executor/inc/tfill.h index 3b80b262ca..0349632b9a 100644 --- a/source/libs/executor/inc/tfill.h +++ b/source/libs/executor/inc/tfill.h @@ -42,6 +42,7 @@ typedef struct SFillInfo { TSKEY start; // start timestamp TSKEY end; // endKey for fill TSKEY currentKey; // current active timestamp, the value may be changed during the fill procedure. + int32_t tsSlotId; // primary time stamp slot id int32_t order; // order [TSDB_ORDER_ASC|TSDB_ORDER_DESC] int32_t type; // fill type int32_t numOfRows; // number of rows in the input data block @@ -74,8 +75,8 @@ struct SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfOutput, co bool taosFillHasMoreResults(struct SFillInfo* pFillInfo); SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols, - SInterval* pInterval, int32_t fillType, - struct SFillColInfo* pCol, const char* id); + SInterval* pInterval, int32_t fillType, struct SFillColInfo* pCol, int32_t slotId, + const char* id); void* taosDestroyFillInfo(struct SFillInfo *pFillInfo); int64_t taosFillResultDataBlock(struct SFillInfo* pFillInfo, SSDataBlock* p, int32_t capacity); diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c new file mode 100644 index 0000000000..b6ad3b6cc0 --- /dev/null +++ b/source/libs/executor/src/cachescanoperator.c @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" +#include "function.h" +#include "tname.h" + +#include "tdatablock.h" +#include "tmsg.h" + +#include "executorimpl.h" +#include "tcompare.h" +#include "thash.h" +#include "ttypes.h" +#include "executorInt.h" + +static SSDataBlock* doScanLastrow(SOperatorInfo* pOperator); +static void destroyLastrowScanOperator(void* param, int32_t numOfOutput); +static int32_t extractTargetSlotId(const SArray* pColMatchInfo, SExecTaskInfo* pTaskInfo, int32_t** pSlotIds); + +SOperatorInfo* createLastrowScanOperator(SLastRowScanPhysiNode* pScanNode, SReadHandle* readHandle, SArray* pTableList, + SExecTaskInfo* pTaskInfo) { + SLastrowScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SLastrowScanInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + pInfo->pTableList = pTableList; + pInfo->readHandle = *readHandle; + pInfo->pRes = createResDataBlock(pScanNode->node.pOutputDataBlockDesc); + + int32_t numOfCols = 0; + pInfo->pColMatchInfo = extractColMatchInfo(pScanNode->pScanCols, pScanNode->node.pOutputDataBlockDesc, &numOfCols, + COL_MATCH_FROM_COL_ID); + int32_t* pCols = taosMemoryMalloc(numOfCols * sizeof(int32_t)); + for (int32_t i = 0; i < numOfCols; ++i) { + SColMatchInfo* pColMatch = taosArrayGet(pInfo->pColMatchInfo, i); + pCols[i] = pColMatch->colId; + } + + int32_t code = extractTargetSlotId(pInfo->pColMatchInfo, pTaskInfo, &pInfo->pSlotIds); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + tsdbLastRowReaderOpen(readHandle->vnode, LASTROW_RETRIEVE_TYPE_ALL, pTableList, pCols, numOfCols, + &pInfo->pLastrowReader); + taosMemoryFree(pCols); + + pOperator->name = "LastrowScanOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->pTaskInfo = pTaskInfo; + pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); + + initResultSizeInfo(pOperator, 1024); + blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); + + pOperator->fpSet = + createOperatorFpSet(operatorDummyOpenFn, doScanLastrow, NULL, NULL, destroyLastrowScanOperator, NULL, NULL, NULL); + pOperator->cost.openCost = 0; + return pOperator; + + _error: + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(pInfo); + taosMemoryFree(pOperator); + return NULL; +} + +SSDataBlock* doScanLastrow(SOperatorInfo* pOperator) { + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SLastrowScanInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + int32_t size = taosArrayGetSize(pInfo->pTableList); + if (size == 0) { + setTaskStatus(pTaskInfo, TASK_COMPLETED); + return NULL; + } + + // check if it is a group by tbname + if (size == taosArrayGetSize(pInfo->pTableList)) { + blockDataCleanup(pInfo->pRes); + tsdbRetrieveLastRow(pInfo->pLastrowReader, pInfo->pRes, pInfo->pSlotIds); + return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; + } else { + // todo fetch the result for each group + } + + return pInfo->pRes->info.rows == 0 ? NULL : pInfo->pRes; +} + +void destroyLastrowScanOperator(void* param, int32_t numOfOutput) { + SLastrowScanInfo* pInfo = (SLastrowScanInfo*)param; + blockDataDestroy(pInfo->pRes); + tsdbLastrowReaderClose(pInfo->pLastrowReader); + + taosMemoryFreeClear(param); +} + +int32_t extractTargetSlotId(const SArray* pColMatchInfo, SExecTaskInfo* pTaskInfo, int32_t** pSlotIds) { + size_t numOfCols = taosArrayGetSize(pColMatchInfo); + + *pSlotIds = taosMemoryMalloc(numOfCols * sizeof(int32_t)); + if (*pSlotIds == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < numOfCols; ++i) { + SColMatchInfo* pColMatch = taosArrayGet(pColMatchInfo, i); + for (int32_t j = 0; j < pTaskInfo->schemaVer.sw->nCols; ++j) { + if (pColMatch->colId == pTaskInfo->schemaVer.sw->pSchema[j].colId && + pColMatch->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + (*pSlotIds)[pColMatch->targetSlotId] = -1; + break; + } + + if (pColMatch->colId == pTaskInfo->schemaVer.sw->pSchema[j].colId) { + (*pSlotIds)[pColMatch->targetSlotId] = j; + break; + } + } + } + + return TSDB_CODE_SUCCESS; +} \ No newline at end of file diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index adbfa2c4ba..2da8811e5e 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -869,7 +869,7 @@ static STimeWindow doCalculateTimeWindow(int64_t ts, SInterval* pInterval) { return w; } -static STimeWindow getFirstQualifiedTimeWindow(int64_t ts, STimeWindow* pWindow, SInterval* pInterval, int32_t order) { +STimeWindow getFirstQualifiedTimeWindow(int64_t ts, STimeWindow* pWindow, SInterval* pInterval, int32_t order) { int32_t factor = (order == TSDB_ORDER_ASC)? -1:1; STimeWindow win = *pWindow; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 86467f1f68..e9a244b573 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -545,9 +545,7 @@ static int32_t doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SqlFunct if (pCtx[k].fpSet.process == NULL) { continue; } -#ifdef BUF_PAGE_DEBUG - qDebug("page_process"); -#endif + int32_t code = pCtx[k].fpSet.process(&pCtx[k]); if (code != TSDB_CODE_SUCCESS) { qError("%s aggregate function error happens, code: %s", GET_TASKID(pOperator->pTaskInfo), tstrerror(code)); @@ -578,14 +576,15 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc int32_t numOfRows = 0; for (int32_t k = 0; k < numOfOutput; ++k) { - int32_t outputSlotId = pExpr[k].base.resSchema.slotId; - SqlFunctionCtx* pfCtx = &pCtx[k]; + int32_t outputSlotId = pExpr[k].base.resSchema.slotId; + SqlFunctionCtx* pfCtx = &pCtx[k]; SInputColumnInfoData* pInputData = &pfCtx->input; if (pExpr[k].pExpr->nodeType == QUERY_NODE_COLUMN) { // it is a project query SColumnInfoData* pColInfoData = taosArrayGet(pResult->pDataBlock, outputSlotId); if (pResult->info.rows > 0 && !createNewColModel) { - colDataMergeCol(pColInfoData, pResult->info.rows, &pResult->info.capacity, pInputData->pData[0], pInputData->numOfRows); + colDataMergeCol(pColInfoData, pResult->info.rows, &pResult->info.capacity, pInputData->pData[0], + pInputData->numOfRows); } else { colDataAssign(pColInfoData, pInputData->pData[0], pInputData->numOfRows, &pResult->info); } @@ -643,11 +642,11 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc } else if (fmIsAggFunc(pfCtx->functionId)) { // _group_key function for "partition by tbname" + csum(col_name) query SColumnInfoData* pOutput = taosArrayGet(pResult->pDataBlock, outputSlotId); - int32_t slotId = pfCtx->param[0].pCol->slotId; + int32_t slotId = pfCtx->param[0].pCol->slotId; // todo handle the json tag SColumnInfoData* pInput = taosArrayGet(pSrcBlock->pDataBlock, slotId); - for(int32_t f = 0; f < pSrcBlock->info.rows; ++f) { + for (int32_t f = 0; f < pSrcBlock->info.rows; ++f) { bool isNull = colDataIsNull_s(pInput, f); if (isNull) { colDataAppendNULL(pOutput, pResult->info.rows + f); @@ -2443,7 +2442,6 @@ _error: doDestroyExchangeOperatorInfo(pInfo); } - taosMemoryFreeClear(pInfo); taosMemoryFreeClear(pOperator); pTaskInfo->code = code; return NULL; @@ -3393,6 +3391,8 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { assert(pBlock != NULL); } + blockDataUpdateTsWindow(pBlock, pInfo->primaryTsCol); + if (*newgroup && pInfo->totalInputRows > 0) { // there are already processed current group data block pInfo->existNewGroupBlock = pBlock; *newgroup = false; @@ -3504,11 +3504,7 @@ static void destroyOperatorInfo(SOperatorInfo* pOperator) { pOperator->numOfDownstream = 0; } - if (pOperator->exprSupp.pExprInfo != NULL) { - destroyExprInfo(pOperator->exprSupp.pExprInfo, pOperator->exprSupp.numOfExprs); - } - - taosMemoryFreeClear(pOperator->exprSupp.pExprInfo); + cleanupExprSupp(&pOperator->exprSupp); taosMemoryFreeClear(pOperator); } @@ -3587,6 +3583,25 @@ void initBasicInfo(SOptrBasicInfo* pInfo, SSDataBlock* pBlock) { initResultRowInfo(&pInfo->resultRowInfo); } +static void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) { + if (pCtx == NULL) { + return NULL; + } + + for (int32_t i = 0; i < numOfOutput; ++i) { + for (int32_t j = 0; j < pCtx[i].numOfParams; ++j) { + taosVariantDestroy(&pCtx[i].param[j].param); + } + + taosMemoryFreeClear(pCtx[i].subsidiaries.pCtx); + taosMemoryFree(pCtx[i].input.pData); + taosMemoryFree(pCtx[i].input.pColumnDataAgg); + } + + taosMemoryFreeClear(pCtx); + return NULL; +} + int32_t initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr) { pSup->pExprInfo = pExprInfo; pSup->numOfExprs = numOfExpr; @@ -3600,6 +3615,16 @@ int32_t initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr) { return TSDB_CODE_SUCCESS; } +void cleanupExprSupp(SExprSupp* pSupp) { + destroySqlFunctionCtx(pSupp->pCtx, pSupp->numOfExprs); + if (pSupp->pExprInfo != NULL) { + destroyExprInfo(pSupp->pExprInfo, pSupp->numOfExprs); + } + + taosMemoryFreeClear(pSupp->pExprInfo); + taosMemoryFree(pSupp->rowEntryInfoOffset); +} + SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SNode* pCondition, SExprInfo* pScalarExprInfo, int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo) { @@ -3650,25 +3675,6 @@ _error: return NULL; } -static void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) { - if (pCtx == NULL) { - return NULL; - } - - for (int32_t i = 0; i < numOfOutput; ++i) { - for (int32_t j = 0; j < pCtx[i].numOfParams; ++j) { - taosVariantDestroy(&pCtx[i].param[j].param); - } - - taosMemoryFreeClear(pCtx[i].subsidiaries.pCtx); - taosMemoryFree(pCtx[i].input.pData); - taosMemoryFree(pCtx[i].input.pColumnDataAgg); - } - - taosMemoryFreeClear(pCtx); - return NULL; -} - void cleanupBasicInfo(SOptrBasicInfo* pInfo) { assert(pInfo != NULL); cleanupResultRowInfo(&pInfo->resultRowInfo); @@ -3710,13 +3716,6 @@ static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput) { taosMemoryFreeClear(param); } -void cleanupExprSupp(SExprSupp* pSupp) { - destroySqlFunctionCtx(pSupp->pCtx, pSupp->numOfExprs); - destroyExprInfo(pSupp->pExprInfo, pSupp->numOfExprs); - - taosMemoryFree(pSupp->rowEntryInfoOffset); -} - static void destroyIndefinitOperatorInfo(void* param, int32_t numOfOutput) { SIndefOperatorInfo* pInfo = (SIndefOperatorInfo*)param; cleanupBasicInfo(&pInfo->binfo); @@ -3821,7 +3820,8 @@ _error: return NULL; } -static void doHandleDataBlock(SOperatorInfo* pOperator, SSDataBlock* pBlock, SOperatorInfo* downstream, SExecTaskInfo* pTaskInfo) { +static void doHandleDataBlock(SOperatorInfo* pOperator, SSDataBlock* pBlock, SOperatorInfo* downstream, + SExecTaskInfo* pTaskInfo) { int32_t order = 0; int32_t scanFlag = 0; @@ -3876,9 +3876,9 @@ static SSDataBlock* doApplyIndefinitFunction(SOperatorInfo* pOperator) { SOperatorInfo* downstream = pOperator->pDownstream[0]; - while(1) { + while (1) { // here we need to handle the existsed group results - if (pIndefInfo->pNextGroupRes != NULL) { // todo extract method + if (pIndefInfo->pNextGroupRes != NULL) { // todo extract method for (int32_t k = 0; k < pSup->numOfExprs; ++k) { SqlFunctionCtx* pCtx = &pSup->pCtx[k]; @@ -3976,15 +3976,15 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy setFunctionResultOutput(pOperator, &pInfo->binfo, &pInfo->aggSup, MAIN_SCAN, numOfExpr); - pInfo->binfo.pRes = pResBlock; - pInfo->pCondition = pPhyNode->node.pConditions; - pInfo->pPseudoColInfo= setRowTsColumnOutputInfo(pSup->pCtx, numOfExpr); + pInfo->binfo.pRes = pResBlock; + pInfo->pCondition = pPhyNode->node.pConditions; + pInfo->pPseudoColInfo = setRowTsColumnOutputInfo(pSup->pCtx, numOfExpr); - pOperator->name = "IndefinitOperator"; + pOperator->name = "IndefinitOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doApplyIndefinitFunction, NULL, NULL, @@ -4010,12 +4010,15 @@ static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t STimeWindow w = TSWINDOW_INITIALIZER; getAlignQueryTimeWindow(pInterval, pInterval->precision, win.skey, &w); + w = getFirstQualifiedTimeWindow(win.skey, &w, pInterval, TSDB_ORDER_ASC); int32_t order = TSDB_ORDER_ASC; - pInfo->pFillInfo = taosCreateFillInfo(order, w.skey, 0, capacity, numOfCols, pInterval, fillType, pColInfo, id); + pInfo->pFillInfo = taosCreateFillInfo(order, w.skey, 0, capacity, numOfCols, pInterval, + fillType, pColInfo, pInfo->primaryTsCol, id); pInfo->win = win; pInfo->p = taosMemoryCalloc(numOfCols, POINTER_BYTES); + if (pInfo->pFillInfo == NULL || pInfo->p == NULL) { taosMemoryFree(pInfo->pFillInfo); taosMemoryFree(pInfo->p); @@ -4048,8 +4051,8 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pInfo->primaryTsCol = ((SColumnNode*)pPhyFillNode->pWStartTs)->slotId; int32_t numOfOutputCols = 0; - SArray* pColMatchColInfo = - extractColMatchInfo(pPhyFillNode->pTargets, pPhyFillNode->node.pOutputDataBlockDesc, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID); + SArray* pColMatchColInfo = extractColMatchInfo(pPhyFillNode->pTargets, pPhyFillNode->node.pOutputDataBlockDesc, + &numOfOutputCols, COL_MATCH_FROM_SLOT_ID); int32_t code = initFillInfo(pInfo, pExprInfo, num, (SNodeListNode*)pPhyFillNode->pValues, pPhyFillNode->timeRange, pResultInfo->capacity, pTaskInfo->id.str, pInterval, type); @@ -4057,18 +4060,18 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* goto _error; } - pInfo->pRes = pResBlock; - pInfo->multigroupResult = multigroupResult; - pInfo->pCondition = pPhyFillNode->node.pConditions; - pInfo->pColMatchColInfo = pColMatchColInfo; - pOperator->name = "FillOperator"; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_FILL; - pOperator->exprSupp.pExprInfo = pExprInfo; + pInfo->pRes = pResBlock; + pInfo->multigroupResult = multigroupResult; + pInfo->pCondition = pPhyFillNode->node.pConditions; + pInfo->pColMatchColInfo = pColMatchColInfo; + pOperator->name = "FillOperator"; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_FILL; + pOperator->exprSupp.pExprInfo = pExprInfo; pOperator->exprSupp.numOfExprs = num; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; + pOperator->info = pInfo; + pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doFill, NULL, NULL, destroySFillOperatorInfo, NULL, NULL, NULL); @@ -4118,6 +4121,8 @@ int32_t extractTableSchemaInfo(SReadHandle* pHandle, uint64_t uid, SExecTaskInfo pTaskInfo->schemaVer.sw = tCloneSSchemaWrapper(&mr.me.stbEntry.schemaRow); pTaskInfo->schemaVer.tversion = mr.me.stbEntry.schemaTag.version; } else if (mr.me.type == TSDB_CHILD_TABLE) { + tDecoderClear(&mr.coder); + tb_uid_t suid = mr.me.ctbEntry.suid; metaGetTableEntryByUid(&mr, suid); pTaskInfo->schemaVer.sw = tCloneSSchemaWrapper(&mr.me.stbEntry.schemaRow); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 0194cd78dc..6ab3effc85 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -578,6 +578,7 @@ static void destroyTableScanOperatorInfo(void* param, int32_t numOfOutput) { taosArrayDestroy(pTableScanInfo->pColMatchInfo); } + cleanupExprSupp(&pTableScanInfo->pseudoSup); taosMemoryFreeClear(param); } diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index b0e2166baf..c008c7c4a9 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -14,6 +14,7 @@ */ #include "os.h" +#include "query.h" #include "taosdef.h" #include "tmsg.h" #include "ttypes.h" @@ -48,14 +49,15 @@ static void setTagsValue(SFillInfo* pFillInfo, void** data, int32_t genRows) { } } -static void setNullRow(SSDataBlock* pBlock, int32_t numOfCol, int32_t rowIndex) { +static void setNullRow(SSDataBlock* pBlock, int64_t ts, int32_t rowIndex) { // the first are always the timestamp column, so start from the second column. for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) { SColumnInfoData* p = taosArrayGet(pBlock->pDataBlock, i); - if (p->info.type == TSDB_DATA_TYPE_TIMESTAMP && i == 0) { - continue; + if (p->info.type == TSDB_DATA_TYPE_TIMESTAMP) { + colDataAppend(p, rowIndex, (const char*)&ts, false); + } else { + colDataAppendNULL(p, rowIndex); } - colDataAppendNULL(p, rowIndex); } } @@ -64,16 +66,17 @@ static void setNullRow(SSDataBlock* pBlock, int32_t numOfCol, int32_t rowIndex) static void doSetVal(SColumnInfoData* pDstColInfoData, int32_t rowIndex, const SGroupKeys* pKey); -static void doFillOneRowResult(SFillInfo* pFillInfo, SSDataBlock* pBlock, SSDataBlock* pSrcBlock, int64_t ts, - bool outOfBound) { +static void doFillOneRow(SFillInfo* pFillInfo, SSDataBlock* pBlock, SSDataBlock* pSrcBlock, int64_t ts, + bool outOfBound) { SPoint point1, point2, point; int32_t step = GET_FORWARD_DIRECTION_FACTOR(pFillInfo->order); // set the primary timestamp column value int32_t index = pFillInfo->numOfCurrent; - SColumnInfoData* pCol0 = taosArrayGet(pBlock->pDataBlock, 0); + SColumnInfoData* pCol0 = taosArrayGet(pBlock->pDataBlock, pFillInfo->tsSlotId); char* val = colDataGetData(pCol0, index); + // set the primary timestamp value *(TSKEY*)val = pFillInfo->currentKey; // set the other values @@ -92,7 +95,7 @@ static void doFillOneRowResult(SFillInfo* pFillInfo, SSDataBlock* pBlock, SSData } } else if (pFillInfo->type == TSDB_FILL_NEXT) { SArray* p = FILL_IS_ASC_FILL(pFillInfo) ? pFillInfo->next : pFillInfo->prev; - + // todo refactor: start from 0 not 1 for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) { SFillColInfo* pCol = &pFillInfo->pFillCol[i]; if (TSDB_COL_IS_TAG(pCol->flag)) { @@ -106,7 +109,7 @@ static void doFillOneRowResult(SFillInfo* pFillInfo, SSDataBlock* pBlock, SSData } else if (pFillInfo->type == TSDB_FILL_LINEAR) { // TODO : linear interpolation supports NULL value if (outOfBound) { - setNullRow(pBlock, pFillInfo->numOfCols, index); + setNullRow(pBlock, pFillInfo->currentKey, index); } else { for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) { SFillColInfo* pCol = &pFillInfo->pFillCol[i]; @@ -143,7 +146,7 @@ static void doFillOneRowResult(SFillInfo* pFillInfo, SSDataBlock* pBlock, SSData } } } else if (pFillInfo->type == TSDB_FILL_NULL) { // fill with NULL - setNullRow(pBlock, pFillInfo->numOfCols, index); + setNullRow(pBlock, pFillInfo->currentKey, index); } else { // fill with user specified value for each column for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) { SFillColInfo* pCol = &pFillInfo->pFillCol[i]; @@ -166,6 +169,8 @@ static void doFillOneRowResult(SFillInfo* pFillInfo, SSDataBlock* pBlock, SSData int64_t v = 0; GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i); colDataAppend(pDst, index, (char*)&v, false); + } else if (pDst->info.type == TSDB_DATA_TYPE_TIMESTAMP) { + colDataAppend(pDst, index, (const char*)&pFillInfo->currentKey, false); } } } @@ -247,7 +252,7 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t // fill the gap between two input rows while (((pFillInfo->currentKey < ts && ascFill) || (pFillInfo->currentKey > ts && !ascFill)) && pFillInfo->numOfCurrent < outputRows) { - doFillOneRowResult(pFillInfo, pBlock, pFillInfo->pSrcBlock, ts, false); + doFillOneRow(pFillInfo, pBlock, pFillInfo->pSrcBlock, ts, false); } // output buffer is full, abort @@ -343,7 +348,7 @@ static int64_t appendFilledResult(SFillInfo* pFillInfo, SSDataBlock* pBlock, int */ pFillInfo->numOfCurrent = 0; while (pFillInfo->numOfCurrent < resultCapacity) { - doFillOneRowResult(pFillInfo, pBlock, pFillInfo->pSrcBlock, pFillInfo->start, true); + doFillOneRow(pFillInfo, pBlock, pFillInfo->pSrcBlock, pFillInfo->start, true); } pFillInfo->numOfTotal += pFillInfo->numOfCurrent; @@ -408,7 +413,7 @@ static int32_t taosNumOfRemainRows(SFillInfo* pFillInfo) { } struct SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols, - SInterval* pInterval, int32_t fillType, struct SFillColInfo* pCol, + SInterval* pInterval, int32_t fillType, struct SFillColInfo* pCol, int32_t primaryTsSlotId, const char* id) { if (fillType == TSDB_FILL_NONE) { return NULL; @@ -420,6 +425,8 @@ struct SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTag return NULL; } + pFillInfo->tsSlotId = primaryTsSlotId; + taosResetFillInfo(pFillInfo, skey); pFillInfo->order = order; @@ -589,11 +596,10 @@ int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, SSDataBlock* p, int32_t ca assert(numOfRes == pFillInfo->numOfCurrent); } - // qDebug("fill:%p, generated fill result, src block:%d, index:%d, brange:%"PRId64"-%"PRId64", currentKey:%"PRId64", - // current:%d, total:%d, %p", - // pFillInfo, pFillInfo->numOfRows, pFillInfo->index, pFillInfo->start, pFillInfo->end, pFillInfo->currentKey, - // pFillInfo->numOfCurrent, - // pFillInfo->numOfTotal, pFillInfo->handle); + qDebug("fill:%p, generated fill result, src block:%d, index:%d, brange:%" PRId64 "-%" PRId64 ", currentKey:%" PRId64 + ", current : % d, total : % d, %s", pFillInfo, + pFillInfo->numOfRows, pFillInfo->index, pFillInfo->start, pFillInfo->end, pFillInfo->currentKey, + pFillInfo->numOfCurrent, pFillInfo->numOfTotal, pFillInfo->id); return numOfRes; } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 947d10dcb4..78775073a4 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -4502,7 +4502,6 @@ static SSDataBlock* doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) { } size_t rows = pRes->info.rows; - blockDataUpdateTsWindow(pRes, iaInfo->primaryTsIndex); pOperator->resultInfo.totalRows += rows; return (rows == 0) ? NULL : pRes; } diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 06944edadd..e8e37e5c6b 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -157,6 +157,7 @@ int32_t elapsedCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); bool getHistogramFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool histogramFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); int32_t histogramFunction(SqlFunctionCtx* pCtx); +int32_t histogramFunctionPartial(SqlFunctionCtx* pCtx); int32_t histogramFunctionMerge(SqlFunctionCtx* pCtx); int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t histogramPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index fc87ba964a..a735edafab 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1427,9 +1427,12 @@ static int32_t translateIrate(SFunctionNode* pFunc, char* pErrBuf, int32_t len) } static int32_t translateFirstLast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { - // first(col_list) will be rewritten as first(col) - if (1 != LIST_LENGTH(pFunc->pParameterList)) { - return TSDB_CODE_SUCCESS; + int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList); + for (int32_t i = 0; i < numOfParams; ++i) { + SNode* pParamNode = nodesListGetNode(pFunc->pParameterList, i); + if (QUERY_NODE_VALUE == nodeType(pParamNode)) { + return invaildFuncParaValueErrMsg(pErrBuf, len, pFunc->functionName); + } } pFunc->node.resType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType; @@ -2323,7 +2326,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .translateFunc = translateHistogramPartial, .getEnvFunc = getHistogramFuncEnv, .initFunc = histogramFunctionSetup, - .processFunc = histogramFunction, + .processFunc = histogramFunctionPartial, .finalizeFunc = histogramPartialFinalize, .invertFunc = NULL, .combineFunc = histogramCombine, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 932bfb8793..cd550b39cf 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -338,6 +338,104 @@ typedef struct SGroupKeyInfo { } \ } while (0) +#define LIST_ADD_N(_res, _col, _start, _rows, _t, numOfElem) \ + do { \ + _t* d = (_t*)(_col->pData); \ + for (int32_t i = (_start); i < (_rows) + (_start); ++i) { \ + if (((_col)->hasNull) && colDataIsNull_f((_col)->nullbitmap, i)) { \ + continue; \ + }; \ + (_res) += (d)[i]; \ + (numOfElem)++; \ + } \ + } while (0) + +#define LIST_SUB_N(_res, _col, _start, _rows, _t, numOfElem) \ + do { \ + _t* d = (_t*)(_col->pData); \ + for (int32_t i = (_start); i < (_rows) + (_start); ++i) { \ + if (((_col)->hasNull) && colDataIsNull_f((_col)->nullbitmap, i)) { \ + continue; \ + }; \ + (_res) -= (d)[i]; \ + (numOfElem)++; \ + } \ + } while (0) + +#define LIST_AVG_N(sumT, T) \ + do { \ + T* plist = (T*)pCol->pData; \ + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { \ + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { \ + continue; \ + } \ + \ + numOfElem += 1; \ + pAvgRes->count -= 1; \ + sumT -= plist[i]; \ + } \ + } while (0) + +#define LIST_STDDEV_SUB_N(sumT, T) \ + do { \ + T* plist = (T*)pCol->pData; \ + for (int32_t i = start; i < numOfRows + start; ++i) { \ + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { \ + continue; \ + } \ + numOfElem += 1; \ + pStddevRes->count -= 1; \ + sumT -= plist[i]; \ + pStddevRes->quadraticISum -= plist[i] * plist[i]; \ + } \ + } while (0) + +#define LEASTSQR_CAL(p, x, y, index, step) \ + do { \ + (p)[0][0] += (double)(x) * (x); \ + (p)[0][1] += (double)(x); \ + (p)[0][2] += (double)(x) * (y)[index]; \ + (p)[1][2] += (y)[index]; \ + (x) += step; \ + } while (0) + + +#define STATE_COMP(_op, _lval, _param) STATE_COMP_IMPL(_op, _lval, GET_STATE_VAL(_param)) + +#define GET_STATE_VAL(param) ((param.nType == TSDB_DATA_TYPE_BIGINT) ? (param.i) : (param.d)) + +#define STATE_COMP_IMPL(_op, _lval, _rval) \ + do { \ + switch (_op) { \ + case STATE_OPER_LT: \ + return ((_lval) < (_rval)); \ + break; \ + case STATE_OPER_GT: \ + return ((_lval) > (_rval)); \ + break; \ + case STATE_OPER_LE: \ + return ((_lval) <= (_rval)); \ + break; \ + case STATE_OPER_GE: \ + return ((_lval) >= (_rval)); \ + break; \ + case STATE_OPER_NE: \ + return ((_lval) != (_rval)); \ + break; \ + case STATE_OPER_EQ: \ + return ((_lval) == (_rval)); \ + break; \ + default: \ + break; \ + } \ + } while (0) + +#define INIT_INTP_POINT(_p, _k, _v) \ + do { \ + (_p).key = (_k); \ + (_p).val = (_v); \ + } while (0) + bool dummyGetEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* UNUSED_PARAM(pEnv)) { return true; } bool dummyInit(SqlFunctionCtx* UNUSED_PARAM(pCtx), SResultRowEntryInfo* UNUSED_PARAM(pResultInfo)) { return true; } @@ -499,30 +597,6 @@ int32_t combineFunction(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { return TSDB_CODE_SUCCESS; } -#define LIST_ADD_N(_res, _col, _start, _rows, _t, numOfElem) \ - do { \ - _t* d = (_t*)(_col->pData); \ - for (int32_t i = (_start); i < (_rows) + (_start); ++i) { \ - if (((_col)->hasNull) && colDataIsNull_f((_col)->nullbitmap, i)) { \ - continue; \ - }; \ - (_res) += (d)[i]; \ - (numOfElem)++; \ - } \ - } while (0) - -#define LIST_SUB_N(_res, _col, _start, _rows, _t, numOfElem) \ - do { \ - _t* d = (_t*)(_col->pData); \ - for (int32_t i = (_start); i < (_rows) + (_start); ++i) { \ - if (((_col)->hasNull) && colDataIsNull_f((_col)->nullbitmap, i)) { \ - continue; \ - }; \ - (_res) -= (d)[i]; \ - (numOfElem)++; \ - } \ - } while (0) - int32_t sumFunction(SqlFunctionCtx* pCtx) { int32_t numOfElem = 0; @@ -907,31 +981,19 @@ int32_t avgFunctionMerge(SqlFunctionCtx* pCtx) { SAvgRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - int32_t start = pInput->startRowIndex; - char* data = colDataGetData(pCol, start); - SAvgRes* pInputInfo = (SAvgRes*)varDataVal(data); + int32_t start = pInput->startRowIndex; - avgTransferInfo(pInputInfo, pInfo); + for(int32_t i = start; i < start + pInput->numOfRows; ++i) { + char* data = colDataGetData(pCol, i); + SAvgRes* pInputInfo = (SAvgRes*)varDataVal(data); + avgTransferInfo(pInputInfo, pInfo); + } SET_VAL(GET_RES_INFO(pCtx), 1, 1); return TSDB_CODE_SUCCESS; } -#define LIST_AVG_N(sumT, T) \ - do { \ - T* plist = (T*)pCol->pData; \ - for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { \ - if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { \ - continue; \ - } \ - \ - numOfElem += 1; \ - pAvgRes->count -= 1; \ - sumT -= plist[i]; \ - } \ - } while (0) - int32_t avgInvertFunction(SqlFunctionCtx* pCtx) { int32_t numOfElem = 0; @@ -1882,20 +1944,6 @@ int32_t stddevFunctionMerge(SqlFunctionCtx* pCtx) { return TSDB_CODE_SUCCESS; } -#define LIST_STDDEV_SUB_N(sumT, T) \ - do { \ - T* plist = (T*)pCol->pData; \ - for (int32_t i = start; i < numOfRows + start; ++i) { \ - if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { \ - continue; \ - } \ - numOfElem += 1; \ - pStddevRes->count -= 1; \ - sumT -= plist[i]; \ - pStddevRes->quadraticISum -= plist[i] * plist[i]; \ - } \ - } while (0) - int32_t stddevInvertFunction(SqlFunctionCtx* pCtx) { int32_t numOfElem = 0; @@ -2044,15 +2092,6 @@ bool leastSQRFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInf return true; } -#define LEASTSQR_CAL(p, x, y, index, step) \ - do { \ - (p)[0][0] += (double)(x) * (x); \ - (p)[0][1] += (double)(x); \ - (p)[0][2] += (double)(x) * (y)[index]; \ - (p)[1][2] += (y)[index]; \ - (x) += step; \ - } while (0) - int32_t leastSQRFunction(SqlFunctionCtx* pCtx) { int32_t numOfElem = 0; @@ -2512,11 +2551,13 @@ int32_t apercentileFunctionMerge(SqlFunctionCtx* pCtx) { SAPercentileInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); - int32_t start = pInput->startRowIndex; - char* data = colDataGetData(pCol, start); - SAPercentileInfo* pInputInfo = (SAPercentileInfo*)varDataVal(data); + int32_t start = pInput->startRowIndex; - apercentileTransferInfo(pInputInfo, pInfo); + for(int32_t i = start; i < start + pInput->numOfRows; ++i) { + char* data = colDataGetData(pCol, i); + SAPercentileInfo* pInputInfo = (SAPercentileInfo*)varDataVal(data); + apercentileTransferInfo(pInputInfo, pInfo); + } SET_VAL(pResInfo, 1, 1); return TSDB_CODE_SUCCESS; @@ -2877,13 +2918,17 @@ static int32_t firstLastFunctionMergeImpl(SqlFunctionCtx* pCtx, bool isFirstQuer SFirstLastRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - int32_t start = pInput->startRowIndex; - char* data = colDataGetData(pCol, start); - SFirstLastRes* pInputInfo = (SFirstLastRes*)varDataVal(data); + int32_t start = pInput->startRowIndex; + int32_t numOfElems = 0; - firstLastTransferInfo(pCtx, pInputInfo, pInfo, isFirstQuery); - - int32_t numOfElems = pInputInfo->hasResult ? 1 : 0; + for(int32_t i = start; i < start + pInput->numOfRows; ++i) { + char* data = colDataGetData(pCol, i); + SFirstLastRes* pInputInfo = (SFirstLastRes*)varDataVal(data); + firstLastTransferInfo(pCtx, pInputInfo, pInfo, isFirstQuery); + if (!numOfElems) { + numOfElems = pInputInfo->hasResult ? 1 : 0; + } + } SET_VAL(GET_RES_INFO(pCtx), numOfElems, 1); @@ -3703,11 +3748,13 @@ int32_t spreadFunctionMerge(SqlFunctionCtx* pCtx) { SSpreadInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - int32_t start = pInput->startRowIndex; - char* data = colDataGetData(pCol, start); - SSpreadInfo* pInputInfo = (SSpreadInfo*)varDataVal(data); + int32_t start = pInput->startRowIndex; - spreadTransferInfo(pInputInfo, pInfo); + for(int32_t i = start; i < start + pInput->numOfRows; ++i) { + char* data = colDataGetData(pCol, i); + SSpreadInfo* pInputInfo = (SSpreadInfo*)varDataVal(data); + spreadTransferInfo(pInputInfo, pInfo); + } SET_VAL(GET_RES_INFO(pCtx), 1, 1); @@ -3873,11 +3920,13 @@ int32_t elapsedFunctionMerge(SqlFunctionCtx* pCtx) { SElapsedInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - int32_t start = pInput->startRowIndex; - char* data = colDataGetData(pCol, start); - SElapsedInfo* pInputInfo = (SElapsedInfo*)varDataVal(data); + int32_t start = pInput->startRowIndex; - elapsedTransferInfo(pInputInfo, pInfo); + for(int32_t i = start; i < start + pInput->numOfRows; ++i) { + char* data = colDataGetData(pCol, i); + SElapsedInfo* pInputInfo = (SElapsedInfo*)varDataVal(data); + elapsedTransferInfo(pInputInfo, pInfo); + } SET_VAL(GET_RES_INFO(pCtx), 1, 1); return TSDB_CODE_SUCCESS; @@ -4098,7 +4147,7 @@ bool histogramFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultIn return true; } -int32_t histogramFunction(SqlFunctionCtx* pCtx) { +static int32_t histogramFunctionImpl(SqlFunctionCtx* pCtx, bool isPartial) { SHistoFuncInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); SInputColumnInfoData* pInput = &pCtx->input; @@ -4130,10 +4179,22 @@ int32_t histogramFunction(SqlFunctionCtx* pCtx) { } } - SET_VAL(GET_RES_INFO(pCtx), numOfElems, pInfo->numOfBins); + if (!isPartial) { + SET_VAL(GET_RES_INFO(pCtx), numOfElems, pInfo->numOfBins); + } else { + SET_VAL(GET_RES_INFO(pCtx), numOfElems, 1); + } return TSDB_CODE_SUCCESS; } +int32_t histogramFunction(SqlFunctionCtx* pCtx) { + return histogramFunctionImpl(pCtx, false); +} + +int32_t histogramFunctionPartial(SqlFunctionCtx* pCtx) { + return histogramFunctionImpl(pCtx, true); +} + static void histogramTransferInfo(SHistoFuncInfo* pInput, SHistoFuncInfo* pOutput) { pOutput->normalized = pInput->normalized; pOutput->numOfBins = pInput->numOfBins; @@ -4152,11 +4213,13 @@ int32_t histogramFunctionMerge(SqlFunctionCtx* pCtx) { SHistoFuncInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - int32_t start = pInput->startRowIndex; - char* data = colDataGetData(pCol, start); - SHistoFuncInfo* pInputInfo = (SHistoFuncInfo*)varDataVal(data); + int32_t start = pInput->startRowIndex; - histogramTransferInfo(pInputInfo, pInfo); + for(int32_t i = start; i < start + pInput->numOfRows; ++i) { + char* data = colDataGetData(pCol, i); + SHistoFuncInfo* pInputInfo = (SHistoFuncInfo*)varDataVal(data); + histogramTransferInfo(pInputInfo, pInfo); + } SET_VAL(GET_RES_INFO(pCtx), pInfo->numOfBins, pInfo->numOfBins); return TSDB_CODE_SUCCESS; @@ -4199,6 +4262,7 @@ int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { } int32_t histogramPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SHistoFuncInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); int32_t resultBytes = getHistogramInfoSize(); char* res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char)); @@ -4212,7 +4276,7 @@ int32_t histogramPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { colDataAppend(pCol, pBlock->info.rows, res, false); taosMemoryFree(res); - return 1; + return pResInfo->numOfRes; } int32_t histogramCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { @@ -4370,11 +4434,13 @@ int32_t hllFunctionMerge(SqlFunctionCtx* pCtx) { SHLLInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - int32_t start = pInput->startRowIndex; - char* data = colDataGetData(pCol, start); - SHLLInfo* pInputInfo = (SHLLInfo*)varDataVal(data); + int32_t start = pInput->startRowIndex; - hllTransferInfo(pInputInfo, pInfo); + for(int32_t i = start; i < start + pInput->numOfRows; ++i) { + char* data = colDataGetData(pCol, i); + SHLLInfo* pInputInfo = (SHLLInfo*)varDataVal(data); + hllTransferInfo(pInputInfo, pInfo); + } SET_VAL(GET_RES_INFO(pCtx), 1, 1); return TSDB_CODE_SUCCESS; @@ -4448,36 +4514,6 @@ static int8_t getStateOpType(char* opStr) { return opType; } -#define GET_STATE_VAL(param) ((param.nType == TSDB_DATA_TYPE_BIGINT) ? (param.i) : (param.d)) - -#define STATE_COMP(_op, _lval, _param) STATE_COMP_IMPL(_op, _lval, GET_STATE_VAL(_param)) - -#define STATE_COMP_IMPL(_op, _lval, _rval) \ - do { \ - switch (_op) { \ - case STATE_OPER_LT: \ - return ((_lval) < (_rval)); \ - break; \ - case STATE_OPER_GT: \ - return ((_lval) > (_rval)); \ - break; \ - case STATE_OPER_LE: \ - return ((_lval) <= (_rval)); \ - break; \ - case STATE_OPER_GE: \ - return ((_lval) >= (_rval)); \ - break; \ - case STATE_OPER_NE: \ - return ((_lval) != (_rval)); \ - break; \ - case STATE_OPER_EQ: \ - return ((_lval) == (_rval)); \ - break; \ - default: \ - break; \ - } \ - } while (0) - static bool checkStateOp(int8_t op, SColumnInfoData* pCol, int32_t index, SVariant param) { char* data = colDataGetData(pCol, index); switch (pCol->info.type) { @@ -5185,12 +5221,6 @@ static double twa_get_area(SPoint1 s, SPoint1 e) { return val; } -#define INIT_INTP_POINT(_p, _k, _v) \ - do { \ - (_p).key = (_k); \ - (_p).val = (_v); \ - } while (0) - int32_t twaFunction(SqlFunctionCtx* pCtx) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pInputCol = pInput->pData[0]; diff --git a/source/libs/parser/inc/parInsertData.h b/source/libs/parser/inc/parInsertData.h index 167970838b..ea78735d5e 100644 --- a/source/libs/parser/inc/parInsertData.h +++ b/source/libs/parser/inc/parInsertData.h @@ -20,6 +20,7 @@ #include "os.h" #include "tname.h" #include "ttypes.h" +#include "query.h" #define IS_DATA_COL_ORDERED(spd) ((spd->orderStatus) == (int8_t)ORDER_STATUS_ORDERED) diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 7b2cd78711..c60e5541e0 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -759,8 +759,8 @@ SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt) { SDatabaseOptions* pOptions = (SDatabaseOptions*)nodesMakeNode(QUERY_NODE_DATABASE_OPTIONS); CHECK_OUT_OF_MEM(pOptions); pOptions->buffer = TSDB_DEFAULT_BUFFER_PER_VNODE; - pOptions->cacheLast = TSDB_DEFAULT_CACHE_LAST_ROW; - pOptions->cacheLastSize = TSDB_DEFAULT_LAST_ROW_MEM; + pOptions->cacheLast = TSDB_DEFAULT_CACHE_LAST; + pOptions->cacheLastSize = TSDB_DEFAULT_CACHE_LAST_SIZE; pOptions->compressionLevel = TSDB_DEFAULT_COMP_LEVEL; pOptions->daysPerFile = TSDB_DEFAULT_DAYS_PER_FILE; pOptions->fsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD; diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index 84bcef7185..290c65de12 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -630,6 +630,17 @@ int32_t qCloneStmtDataBlock(void** pDst, void* pSrc) { memcpy(*pDst, pSrc, sizeof(STableDataBlocks)); ((STableDataBlocks*)(*pDst))->cloned = true; + STableDataBlocks* pBlock = (STableDataBlocks*)(*pDst); + if (pBlock->pTableMeta) { + void *pNewMeta = taosMemoryMalloc(TABLE_META_SIZE(pBlock->pTableMeta)); + if (NULL == pNewMeta) { + taosMemoryFreeClear(*pDst); + return TSDB_CODE_OUT_OF_MEMORY; + } + memcpy(pNewMeta, pBlock->pTableMeta, TABLE_META_SIZE(pBlock->pTableMeta)); + pBlock->pTableMeta = pNewMeta; + } + return qResetStmtDataBlock(*pDst, false); } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 16c208f984..f417f0e084 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2836,8 +2836,8 @@ static int32_t buildCreateDbReq(STranslateContext* pCxt, SCreateDatabaseStmt* pS pReq->compression = pStmt->pOptions->compressionLevel; pReq->replications = pStmt->pOptions->replica; pReq->strict = pStmt->pOptions->strict; - pReq->cacheLastRow = pStmt->pOptions->cacheLast; - pReq->lastRowMem = pStmt->pOptions->cacheLastSize; + pReq->cacheLast = pStmt->pOptions->cacheLast; + pReq->cacheLastSize = pStmt->pOptions->cacheLastSize; pReq->schemaless = pStmt->pOptions->schemaless; pReq->ignoreExist = pStmt->ignoreExists; return buildCreateDbRetentions(pStmt->pOptions->pRetentions, pReq); @@ -2998,12 +2998,12 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName int32_t code = checkRangeOption(pCxt, "buffer", pOptions->buffer, TSDB_MIN_BUFFER_PER_VNODE, TSDB_MAX_BUFFER_PER_VNODE); if (TSDB_CODE_SUCCESS == code) { - code = checkRangeOption(pCxt, "cacheLast", pOptions->cacheLast, TSDB_MIN_DB_CACHE_LAST_ROW, - TSDB_MAX_DB_CACHE_LAST_ROW); + code = checkRangeOption(pCxt, "cacheLast", pOptions->cacheLast, TSDB_MIN_DB_CACHE_LAST, + TSDB_MAX_DB_CACHE_LAST); } if (TSDB_CODE_SUCCESS == code) { - code = checkRangeOption(pCxt, "cacheLastSize", pOptions->cacheLastSize, TSDB_MIN_DB_LAST_ROW_MEM, - TSDB_MAX_DB_LAST_ROW_MEM); + code = checkRangeOption(pCxt, "cacheLastSize", pOptions->cacheLastSize, TSDB_MIN_DB_CACHE_LAST_SIZE, + TSDB_MAX_DB_CACHE_LAST_SIZE); } if (TSDB_CODE_SUCCESS == code) { code = checkRangeOption(pCxt, "compression", pOptions->compressionLevel, TSDB_MIN_COMP_LEVEL, TSDB_MAX_COMP_LEVEL); @@ -3116,7 +3116,7 @@ static void buildAlterDbReq(STranslateContext* pCxt, SAlterDatabaseStmt* pStmt, pReq->buffer = pStmt->pOptions->buffer; pReq->pageSize = -1; pReq->pages = pStmt->pOptions->pages; - pReq->lastRowMem = -1; + pReq->cacheLastSize = -1; pReq->daysPerFile = -1; pReq->daysToKeep0 = pStmt->pOptions->keep[0]; pReq->daysToKeep1 = pStmt->pOptions->keep[1]; @@ -3124,8 +3124,8 @@ static void buildAlterDbReq(STranslateContext* pCxt, SAlterDatabaseStmt* pStmt, pReq->fsyncPeriod = pStmt->pOptions->fsyncPeriod; pReq->walLevel = pStmt->pOptions->walLevel; pReq->strict = pStmt->pOptions->strict; - pReq->cacheLastRow = pStmt->pOptions->cacheLast; - pReq->lastRowMem = pStmt->pOptions->cacheLastSize; + pReq->cacheLast = pStmt->pOptions->cacheLast; + pReq->cacheLastSize = pStmt->pOptions->cacheLastSize; pReq->replications = pStmt->pOptions->replica; return; } diff --git a/source/libs/parser/test/parInitialCTest.cpp b/source/libs/parser/test/parInitialCTest.cpp index b39a066ba1..e9c8fb5326 100644 --- a/source/libs/parser/test/parInitialCTest.cpp +++ b/source/libs/parser/test/parInitialCTest.cpp @@ -76,8 +76,8 @@ TEST_F(ParserInitialCTest, createDatabase) { expect.db[len] = '\0'; expect.ignoreExist = igExists; expect.buffer = TSDB_DEFAULT_BUFFER_PER_VNODE; - expect.cacheLastRow = TSDB_DEFAULT_CACHE_LAST_ROW; - expect.lastRowMem = TSDB_DEFAULT_LAST_ROW_MEM; + expect.cacheLast = TSDB_DEFAULT_CACHE_LAST; + expect.cacheLastSize = TSDB_DEFAULT_CACHE_LAST_SIZE; expect.compression = TSDB_DEFAULT_COMP_LEVEL; expect.daysPerFile = TSDB_DEFAULT_DAYS_PER_FILE; expect.fsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD; @@ -98,8 +98,8 @@ TEST_F(ParserInitialCTest, createDatabase) { }; auto setDbBufferFunc = [&](int32_t buffer) { expect.buffer = buffer; }; - auto setDbCachelastFunc = [&](int8_t cachelast) { expect.cacheLastRow = cachelast; }; - auto setDbCachelastSize = [&](int8_t cachelastSize) { expect.lastRowMem = cachelastSize; }; + auto setDbCachelastFunc = [&](int8_t cachelast) { expect.cacheLast = cachelast; }; + auto setDbCachelastSize = [&](int8_t cachelastSize) { expect.cacheLastSize = cachelastSize; }; auto setDbCompressionFunc = [&](int8_t compressionLevel) { expect.compression = compressionLevel; }; auto setDbDaysFunc = [&](int32_t daysPerFile) { expect.daysPerFile = daysPerFile; }; auto setDbFsyncFunc = [&](int32_t fsyncPeriod) { expect.fsyncPeriod = fsyncPeriod; }; @@ -155,8 +155,8 @@ TEST_F(ParserInitialCTest, createDatabase) { ASSERT_EQ(req.compression, expect.compression); ASSERT_EQ(req.replications, expect.replications); ASSERT_EQ(req.strict, expect.strict); - ASSERT_EQ(req.cacheLastRow, expect.cacheLastRow); - ASSERT_EQ(req.lastRowMem, expect.lastRowMem); + ASSERT_EQ(req.cacheLast, expect.cacheLast); + ASSERT_EQ(req.cacheLastSize, expect.cacheLastSize); // ASSERT_EQ(req.schemaless, expect.schemaless); ASSERT_EQ(req.ignoreExist, expect.ignoreExist); ASSERT_EQ(req.numOfRetensions, expect.numOfRetensions); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index cb38e1fc18..4d4756c3ac 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -162,12 +162,11 @@ static EScanType getScanType(SLogicPlanContext* pCxt, SNodeList* pScanPseudoCols } if (NULL == pScanCols) { - // select count(*) from t return NULL == pScanPseudoCols ? SCAN_TYPE_TABLE : ((FUNCTION_TYPE_BLOCK_DIST_INFO == ((SFunctionNode*)nodesListGetNode(pScanPseudoCols, 0))->funcType) ? SCAN_TYPE_BLOCK_INFO - : SCAN_TYPE_TAG); + : SCAN_TYPE_TABLE); } if (TSDB_SYSTEM_TABLE == tableType) { @@ -181,7 +180,7 @@ static EScanType getScanType(SLogicPlanContext* pCxt, SNodeList* pScanPseudoCols } } - return SCAN_TYPE_TAG; + return SCAN_TYPE_TABLE; } static SNode* createPrimaryKeyCol(uint64_t tableId) { @@ -476,12 +475,12 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, int32_t code = TSDB_CODE_SUCCESS; // set grouyp keys, agg funcs and having conditions - if (TSDB_CODE_SUCCESS == code && pSelect->hasAggFuncs) { + if (TSDB_CODE_SUCCESS == code) { code = nodesCollectFuncs(pSelect, SQL_CLAUSE_GROUP_BY, fmIsAggFunc, &pAgg->pAggFuncs); } // rewrite the expression in subsequent clauses - if (TSDB_CODE_SUCCESS == code) { + if (TSDB_CODE_SUCCESS == code && NULL != pAgg->pAggFuncs) { code = rewriteExprsForSelect(pAgg->pAggFuncs, pSelect, SQL_CLAUSE_GROUP_BY); } diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index dee7bd49db..855427f26e 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1364,9 +1364,9 @@ static EDealRes partTagsOptHasColImpl(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } -static bool partTagsOptHasCol(SNodeList* pPartKeys) { +static bool planOptNodeListHasCol(SNodeList* pKeys) { bool hasCol = false; - nodesWalkExprs(pPartKeys, partTagsOptHasColImpl, &hasCol); + nodesWalkExprs(pKeys, partTagsOptHasColImpl, &hasCol); return hasCol; } @@ -1409,7 +1409,7 @@ static bool partTagsOptMayBeOptimized(SLogicNode* pNode) { return false; } - return !partTagsOptHasCol(partTagsGetPartKeys(pNode)) && partTagsOptAreSupportedFuncs(partTagsGetFuncs(pNode)); + return !planOptNodeListHasCol(partTagsGetPartKeys(pNode)) && partTagsOptAreSupportedFuncs(partTagsGetFuncs(pNode)); } static int32_t partTagsOptRebuildTbanme(SNodeList* pPartKeys) { @@ -1986,7 +1986,8 @@ static bool lastRowScanOptMayBeOptimized(SLogicNode* pNode) { SNode* pFunc = NULL; FOREACH(pFunc, ((SAggLogicNode*)pNode)->pAggFuncs) { - if (FUNCTION_TYPE_LAST_ROW != ((SFunctionNode*)pFunc)->funcType) { + if (FUNCTION_TYPE_LAST_ROW != ((SFunctionNode*)pFunc)->funcType && + FUNCTION_TYPE_SELECT_VALUE != ((SFunctionNode*)pFunc)->funcType) { return false; } } @@ -2095,6 +2096,37 @@ static int32_t mergeProjectsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLog return mergeProjectsOptimizeImpl(pCxt, pLogicSubplan, pProjectNode); } +static bool tagScanMayBeOptimized(SLogicNode* pNode) { + if (QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(pNode) || (SCAN_TYPE_TAG == ((SScanLogicNode*)pNode)->scanType)) { + return false; + } + SScanLogicNode *pScan = (SScanLogicNode*)pNode; + if (NULL != pScan->pScanCols) { + return false; + } + if (NULL == pNode->pParent || QUERY_NODE_LOGIC_PLAN_AGG != nodeType(pNode->pParent) || 1 != LIST_LENGTH(pNode->pParent->pChildren)) { + return false; + } + + SAggLogicNode* pAgg = (SAggLogicNode*)(pNode->pParent); + if (NULL == pAgg->pGroupKeys || NULL != pAgg->pAggFuncs || planOptNodeListHasCol(pAgg->pGroupKeys)) { + return false; + } + + return true; +} + +static int32_t tagScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { + SScanLogicNode* pScanNode = (SScanLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, tagScanMayBeOptimized); + if (NULL == pScanNode) { + return TSDB_CODE_SUCCESS; + } + + pScanNode->scanType = SCAN_TYPE_TAG; + pCxt->optimized = true; + return TSDB_CODE_SUCCESS; +} + // clang-format off static const SOptimizeRule optimizeRuleSet[] = { {.pName = "ScanPath", .optimizeFunc = scanPathOptimize}, @@ -2107,7 +2139,8 @@ static const SOptimizeRule optimizeRuleSet[] = { {.pName = "EliminateSetOperator", .optimizeFunc = eliminateSetOpOptimize}, {.pName = "RewriteTail", .optimizeFunc = rewriteTailOptimize}, {.pName = "RewriteUnique", .optimizeFunc = rewriteUniqueOptimize}, - {.pName = "LastRowScan", .optimizeFunc = lastRowScanOptimize} + {.pName = "LastRowScan", .optimizeFunc = lastRowScanOptimize}, + {.pName = "TagScan", .optimizeFunc = tagScanOptimize} }; // clang-format on diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index d10fe1ce0c..8c9c06be35 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -1519,6 +1519,7 @@ static SSubplan* makeSubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogicSubpl static int32_t buildInsertValuesSubplan(SPhysiPlanContext* pCxt, SVnodeModifyLogicNode* pModify, SSubplan* pSubplan) { pSubplan->msgType = pModify->msgType; + pSubplan->execNode.nodeId = pModify->pVgDataBlocks->vg.vgId; pSubplan->execNode.epSet = pModify->pVgDataBlocks->vg.epSet; return createDataInserter(pCxt, pModify->pVgDataBlocks, &pSubplan->pDataSink); } diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index d7eccf4b8e..987da0dd17 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -877,7 +877,7 @@ static int32_t stbSplSplitSortNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) return code; } -static int32_t stbSplSplitScanNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) { +static int32_t stbSplSplitScanNodeWithoutPartTags(SSplitContext* pCxt, SStableSplitInfo* pInfo) { int32_t code = splCreateExchangeNodeForSubplan(pCxt, pInfo->pSubplan, pInfo->pSplitNode, SUBPLAN_TYPE_MERGE); if (TSDB_CODE_SUCCESS == code) { code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren, @@ -887,6 +887,25 @@ static int32_t stbSplSplitScanNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) return code; } +static int32_t stbSplSplitScanNodeWithPartTags(SSplitContext* pCxt, SStableSplitInfo* pInfo) { + int32_t code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pInfo->pSplitNode, NULL, pInfo->pSplitNode, true); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren, + (SNode*)splCreateScanSubplan(pCxt, pInfo->pSplitNode, SPLIT_FLAG_STABLE_SPLIT)); + } + pInfo->pSubplan->subplanType = SUBPLAN_TYPE_MERGE; + ++(pCxt->groupId); + return code; +} + +static int32_t stbSplSplitScanNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) { + SScanLogicNode* pScan = (SScanLogicNode*)pInfo->pSplitNode; + if (NULL != pScan->pGroupTags) { + return stbSplSplitScanNodeWithPartTags(pCxt, pInfo); + } + return stbSplSplitScanNodeWithoutPartTags(pCxt, pInfo); +} + static SNode* stbSplFindPrimaryKeyFromScan(SScanLogicNode* pScan) { SNode* pCol = NULL; FOREACH(pCol, pScan->pScanCols) { diff --git a/source/libs/planner/test/planOptimizeTest.cpp b/source/libs/planner/test/planOptimizeTest.cpp index ef056c5ec8..316148df27 100644 --- a/source/libs/planner/test/planOptimizeTest.cpp +++ b/source/libs/planner/test/planOptimizeTest.cpp @@ -87,4 +87,11 @@ TEST_F(PlanOptimizeTest, eliminateProjection) { TEST_F(PlanOptimizeTest, pushDownProjectCond) { useDb("root", "test"); run("select 1-abs(c1) from (select unique(c1) c1 from st1s3) where 1-c1>5 order by 1 nulls first"); +} + +TEST_F(PlanOptimizeTest, tagScan) { + useDb("root", "test"); + run("select tag1 from st1 group by tag1"); + run("select distinct tag1 from st1"); + run("select tag1*tag1 from st1 group by tag1*tag1"); } \ No newline at end of file diff --git a/source/libs/planner/test/planPartByTest.cpp b/source/libs/planner/test/planPartByTest.cpp index 48a4c12577..f1dd9403dd 100644 --- a/source/libs/planner/test/planPartByTest.cpp +++ b/source/libs/planner/test/planPartByTest.cpp @@ -59,4 +59,6 @@ TEST_F(PlanPartitionByTest, withGroupBy) { useDb("root", "test"); run("select count(*) from t1 partition by c1 group by c2"); + + run("SELECT TBNAME, c1 FROM st1 PARTITION BY TBNAME GROUP BY c1"); } diff --git a/source/libs/qworker/inc/qwInt.h b/source/libs/qworker/inc/qwInt.h index b35e0e2fc4..539643c390 100644 --- a/source/libs/qworker/inc/qwInt.h +++ b/source/libs/qworker/inc/qwInt.h @@ -316,34 +316,34 @@ typedef struct SQWorkerMgmt { #define QW_LOCK(type, _lock) \ do { \ if (QW_READ == (type)) { \ - assert(atomic_load_64((_lock)) >= 0); \ - QW_LOCK_DEBUG("QW RLOCK%p:%" PRIx64 ", %s:%d B", (_lock), atomic_load_64(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32((_lock)) >= 0); \ + QW_LOCK_DEBUG("QW RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosRLockLatch(_lock); \ - QW_LOCK_DEBUG("QW RLOCK%p:%" PRIx64 ", %s:%d E", (_lock), atomic_load_64(_lock), __FILE__, __LINE__); \ - assert(atomic_load_64((_lock)) > 0); \ + QW_LOCK_DEBUG("QW RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32((_lock)) > 0); \ } else { \ - assert(atomic_load_64((_lock)) >= 0); \ - QW_LOCK_DEBUG("QW WLOCK%p:%" PRIx64 ", %s:%d B", (_lock), atomic_load_64(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32((_lock)) >= 0); \ + QW_LOCK_DEBUG("QW WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosWLockLatch(_lock); \ - QW_LOCK_DEBUG("QW WLOCK%p:%" PRIx64 ", %s:%d E", (_lock), atomic_load_64(_lock), __FILE__, __LINE__); \ - assert(atomic_load_64((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY); \ + QW_LOCK_DEBUG("QW WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY); \ } \ } while (0) #define QW_UNLOCK(type, _lock) \ do { \ if (QW_READ == (type)) { \ - assert(atomic_load_64((_lock)) > 0); \ - QW_LOCK_DEBUG("QW RULOCK%p:%" PRIx64 ", %s:%d B", (_lock), atomic_load_64(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32((_lock)) > 0); \ + QW_LOCK_DEBUG("QW RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosRUnLockLatch(_lock); \ - QW_LOCK_DEBUG("QW RULOCK%p:%" PRIx64 ", %s:%d E", (_lock), atomic_load_64(_lock), __FILE__, __LINE__); \ - assert(atomic_load_64((_lock)) >= 0); \ + QW_LOCK_DEBUG("QW RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32((_lock)) >= 0); \ } else { \ - assert(atomic_load_64((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY); \ - QW_LOCK_DEBUG("QW WULOCK%p:%" PRIx64 ", %s:%d B", (_lock), atomic_load_64(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY); \ + QW_LOCK_DEBUG("QW WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosWUnLockLatch(_lock); \ - QW_LOCK_DEBUG("QW WULOCK%p:%" PRIx64 ", %s:%d E", (_lock), atomic_load_64(_lock), __FILE__, __LINE__); \ - assert(atomic_load_64((_lock)) >= 0); \ + QW_LOCK_DEBUG("QW WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32((_lock)) >= 0); \ } \ } while (0) diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 79adfaebb3..0a0d319025 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -367,33 +367,33 @@ extern SSchedulerMgmt schMgmt; #define SCH_LOCK(type, _lock) do { \ if (SCH_READ == (type)) { \ - assert(atomic_load_64(_lock) >= 0); \ - SCH_LOCK_DEBUG("SCH RLOCK%p:%" PRIx64 ", %s:%d B", (_lock), atomic_load_64(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32(_lock) >= 0); \ + SCH_LOCK_DEBUG("SCH RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosRLockLatch(_lock); \ - SCH_LOCK_DEBUG("SCH RLOCK%p:%" PRIx64 ", %s:%d E", (_lock), atomic_load_64(_lock), __FILE__, __LINE__); \ - assert(atomic_load_64(_lock) > 0); \ + SCH_LOCK_DEBUG("SCH RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32(_lock) > 0); \ } else { \ - assert(atomic_load_64(_lock) >= 0); \ - SCH_LOCK_DEBUG("SCH WLOCK%p:%" PRIx64 ", %s:%d B", (_lock), atomic_load_64(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32(_lock) >= 0); \ + SCH_LOCK_DEBUG("SCH WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosWLockLatch(_lock); \ - SCH_LOCK_DEBUG("SCH WLOCK%p:%" PRIx64 ", %s:%d E", (_lock), atomic_load_64(_lock), __FILE__, __LINE__); \ - assert(atomic_load_64(_lock) & TD_RWLATCH_WRITE_FLAG_COPY); \ + SCH_LOCK_DEBUG("SCH WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32(_lock) == TD_RWLATCH_WRITE_FLAG_COPY); \ } \ } while (0) #define SCH_UNLOCK(type, _lock) do { \ if (SCH_READ == (type)) { \ - assert(atomic_load_64((_lock)) > 0); \ - SCH_LOCK_DEBUG("SCH RULOCK%p:%" PRIx64 ", %s:%d B", (_lock), atomic_load_64(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32((_lock)) > 0); \ + SCH_LOCK_DEBUG("SCH RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosRUnLockLatch(_lock); \ - SCH_LOCK_DEBUG("SCH RULOCK%p:%" PRIx64 ", %s:%d E", (_lock), atomic_load_64(_lock), __FILE__, __LINE__); \ - assert(atomic_load_64((_lock)) >= 0); \ + SCH_LOCK_DEBUG("SCH RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32((_lock)) >= 0); \ } else { \ - assert(atomic_load_64((_lock)) & TD_RWLATCH_WRITE_FLAG_COPY); \ - SCH_LOCK_DEBUG("SCH WULOCK%p:%" PRIx64 ", %s:%d B", (_lock), atomic_load_64(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32((_lock)) & TD_RWLATCH_WRITE_FLAG_COPY); \ + SCH_LOCK_DEBUG("SCH WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosWUnLockLatch(_lock); \ - SCH_LOCK_DEBUG("SCH WULOCK%p:%" PRIx64 ", %s:%d E", (_lock), atomic_load_64(_lock), __FILE__, __LINE__); \ - assert(atomic_load_64((_lock)) >= 0); \ + SCH_LOCK_DEBUG("SCH WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32((_lock)) >= 0); \ } \ } while (0) @@ -472,6 +472,7 @@ int32_t schLaunchLevelTasks(SSchJob *pJob, SSchLevel *level); int32_t schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTask); int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel *pLevel, int32_t levelNum); int32_t schSwitchTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask); +void schDirectPostJobRes(SSchedulerReq* pReq, int32_t errCode); extern SSchDebug gSCHDebug; diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 394095785d..e482814ee7 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -763,6 +763,17 @@ int32_t schExecJob(SSchJob *pJob, SSchedulerReq *pReq) { return TSDB_CODE_SUCCESS; } +void schDirectPostJobRes(SSchedulerReq* pReq, int32_t errCode) { + if (NULL == pReq || pReq->syncReq) { + return; + } + + if (pReq->execFp) { + (*pReq->execFp)(NULL, pReq->cbParam, errCode); + } else if (pReq->fetchFp) { + (*pReq->fetchFp)(NULL, pReq->cbParam, errCode); + } +} void schProcessOnOpEnd(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq* pReq, int32_t errCode) { int32_t op = 0; @@ -801,17 +812,13 @@ void schProcessOnOpEnd(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq* pReq, int int32_t schProcessOnOpBegin(SSchJob* pJob, SCH_OP_TYPE type, SSchedulerReq* pReq) { int32_t code = 0; - int8_t status = 0; - - if (schJobNeedToStop(pJob, &status)) { - SCH_JOB_ELOG("abort op %s cause of job need to stop, status:%s", schGetOpStr(type), jobTaskStatusStr(status)); - SCH_ERR_RET(TSDB_CODE_SCH_IGNORE_ERROR); - } + int8_t status = SCH_GET_JOB_STATUS(pJob); switch (type) { case SCH_OP_EXEC: if (SCH_OP_NULL != atomic_val_compare_exchange_32(&pJob->opStatus.op, SCH_OP_NULL, type)) { SCH_JOB_ELOG("job already in %s operation", schGetOpStr(pJob->opStatus.op)); + schDirectPostJobRes(pReq, TSDB_CODE_TSC_APP_ERROR); SCH_ERR_RET(TSDB_CODE_TSC_APP_ERROR); } @@ -822,11 +829,16 @@ int32_t schProcessOnOpBegin(SSchJob* pJob, SCH_OP_TYPE type, SSchedulerReq* pReq case SCH_OP_FETCH: if (SCH_OP_NULL != atomic_val_compare_exchange_32(&pJob->opStatus.op, SCH_OP_NULL, type)) { SCH_JOB_ELOG("job already in %s operation", schGetOpStr(pJob->opStatus.op)); + schDirectPostJobRes(pReq, TSDB_CODE_TSC_APP_ERROR); SCH_ERR_RET(TSDB_CODE_TSC_APP_ERROR); } SCH_JOB_DLOG("job start %s operation", schGetOpStr(pJob->opStatus.op)); - + + pJob->userRes.fetchRes = pReq->pFetchRes; + pJob->userRes.fetchFp = pReq->fetchFp; + pJob->userRes.cbParam = pReq->cbParam; + pJob->opStatus.syncReq = pReq->syncReq; if (!SCH_JOB_NEED_FETCH(pJob)) { @@ -839,10 +851,6 @@ int32_t schProcessOnOpBegin(SSchJob* pJob, SCH_OP_TYPE type, SSchedulerReq* pReq SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } - pJob->userRes.fetchRes = pReq->pFetchRes; - pJob->userRes.fetchFp = pReq->fetchFp; - pJob->userRes.cbParam = pReq->cbParam; - break; case SCH_OP_GET_STATUS: if (pJob->status < JOB_TASK_STATUS_INIT || pJob->levelNum <= 0 || NULL == pJob->levels) { @@ -855,6 +863,11 @@ int32_t schProcessOnOpBegin(SSchJob* pJob, SCH_OP_TYPE type, SSchedulerReq* pReq SCH_ERR_RET(TSDB_CODE_TSC_APP_ERROR); } + if (schJobNeedToStop(pJob, &status)) { + SCH_JOB_ELOG("abort op %s cause of job need to stop, status:%s", schGetOpStr(type), jobTaskStatusStr(status)); + SCH_ERR_RET(TSDB_CODE_SCH_IGNORE_ERROR); + } + return TSDB_CODE_SUCCESS; } diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 2257ba8328..1b893739bd 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -35,7 +35,7 @@ int32_t schValidateRspMsgType(SSchJob *pJob, SSchTask *pTask, int32_t msgType) { SCH_TASK_ELOG("rsp msg type mis-match, last sent msgType:%s, rspType:%s", TMSG_INFO(lastMsgType), TMSG_INFO(msgType)); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); - } + } if (taskStatus != JOB_TASK_STATUS_PART_SUCC) { SCH_TASK_ELOG("rsp msg conflicted with task status, status:%s, rspType:%s", jobTaskStatusStr(taskStatus), TMSG_INFO(msgType)); @@ -75,7 +75,7 @@ int32_t schValidateRspMsgType(SSchJob *pJob, SSchTask *pTask, int32_t msgType) { // Note: no more task error processing, handled in function internal int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; - char *msg = pMsg->pData; + char *msg = pMsg->pData; int32_t msgSize = pMsg->len; int32_t msgType = pMsg->msgType; @@ -253,15 +253,15 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa rsp->sversion = ntohl(rsp->sversion); rsp->tversion = ntohl(rsp->tversion); rsp->affectedRows = be64toh(rsp->affectedRows); - + SCH_ERR_JRET(rsp->code); SCH_ERR_JRET(schSaveJobQueryRes(pJob, rsp)); atomic_add_fetch_32(&pJob->resNumOfRows, rsp->affectedRows); - taosMemoryFreeClear(msg); - + taosMemoryFreeClear(msg); + SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); break; @@ -375,7 +375,8 @@ int32_t schHandleCallback(void *param, SDataBuf *pMsg, int32_t rspCode) { SSchTask *pTask = NULL; SSchJob *pJob = NULL; - qDebug("begin to handle rsp msg, type:%s, handle:%p, code:%s", TMSG_INFO(pMsg->msgType), pMsg->handle, tstrerror(rspCode)); + qDebug("begin to handle rsp msg, type:%s, handle:%p, code:%s", TMSG_INFO(pMsg->msgType), pMsg->handle, + tstrerror(rspCode)); SCH_ERR_RET(schProcessOnCbBegin(&pJob, &pTask, pParam->queryId, pParam->refId, pParam->taskId)); @@ -387,7 +388,8 @@ int32_t schHandleCallback(void *param, SDataBuf *pMsg, int32_t rspCode) { taosMemoryFreeClear(pMsg->pData); taosMemoryFreeClear(param); - qDebug("end to handle rsp msg, type:%s, handle:%p, code:%s", TMSG_INFO(pMsg->msgType), pMsg->handle, tstrerror(rspCode)); + qDebug("end to handle rsp msg, type:%s, handle:%p, code:%s", TMSG_INFO(pMsg->msgType), pMsg->handle, + tstrerror(rspCode)); SCH_RET(code); } @@ -424,7 +426,7 @@ int32_t schHandleCommitCallback(void *param, SDataBuf *pMsg, int32_t code) { } int32_t schHandleHbCallback(void *param, SDataBuf *pMsg, int32_t code) { - SSchedulerHbRsp rsp = {0}; + SSchedulerHbRsp rsp = {0}; SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param; if (code) { @@ -453,8 +455,8 @@ _return: SCH_RET(code); } - -int32_t schMakeCallbackParam(SSchJob *pJob, SSchTask *pTask, int32_t msgType, bool isHb, SSchTrans *trans, void **pParam) { +int32_t schMakeCallbackParam(SSchJob *pJob, SSchTask *pTask, int32_t msgType, bool isHb, SSchTrans *trans, + void **pParam) { if (!isHb) { SSchTaskCallbackParam *param = taosMemoryCalloc(1, sizeof(SSchTaskCallbackParam)); if (NULL == param) { @@ -940,7 +942,8 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, if (NULL == addr) { addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); isCandidateAddr = true; - SCH_TASK_DLOG("target candidateIdx %d", pTask->candidateIdx); + SCH_TASK_DLOG("target candidateIdx %d, epInUse %d/%d", pTask->candidateIdx, addr->epSet.inUse, + addr->epSet.numOfEps); } switch (msgType) { diff --git a/source/libs/scheduler/src/schStatus.c b/source/libs/scheduler/src/schStatus.c index 091b1359e0..a4fa4f2839 100644 --- a/source/libs/scheduler/src/schStatus.c +++ b/source/libs/scheduler/src/schStatus.c @@ -77,6 +77,7 @@ int32_t schHandleOpEndEvent(SSchJob* pJob, SCH_OP_TYPE type, SSchedulerReq* pReq int32_t code = errCode; if (NULL == pJob) { + schDirectPostJobRes(pReq, errCode); SCH_RET(code); } diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index a6621d279d..6bb8db593c 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -21,11 +21,9 @@ #include "tref.h" #include "trpc.h" - - void schFreeTask(SSchJob *pJob, SSchTask *pTask) { schDeregisterTaskHb(pJob, pTask); - + if (pTask->candidateAddrs) { taosArrayDestroy(pTask->candidateAddrs); } @@ -45,22 +43,21 @@ void schFreeTask(SSchJob *pJob, SSchTask *pTask) { } } - int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel *pLevel, int32_t levelNum) { int32_t code = 0; - + pTask->plan = pPlan; pTask->level = pLevel; pTask->execId = -1; pTask->maxExecTimes = SCH_TASK_MAX_EXEC_TIMES(pLevel->level, levelNum); pTask->timeoutUsec = SCH_DEFAULT_TASK_TIMEOUT_USEC; pTask->taskId = schGenTaskId(); - pTask->execNodes = taosHashInit(SCH_MAX_CANDIDATE_EP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + pTask->execNodes = + taosHashInit(SCH_MAX_CANDIDATE_EP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); pTask->profile.execTime = taosMemoryCalloc(pTask->maxExecTimes, sizeof(int64_t)); if (NULL == pTask->execNodes || NULL == pTask->profile.execTime) { SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - taosInitReentrantRWLatch(&pTask->lock); SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_INIT); @@ -110,8 +107,8 @@ int32_t schDropTaskExecNode(SSchJob *pJob, SSchTask *pTask, void *handle, int32_ } else { SCH_TASK_DLOG("execId %d removed from execNodeList", execId); } - - if (execId != pTask->execId) { // ignore it + + if (execId != pTask->execId) { // ignore it SCH_TASK_DLOG("execId %d is not current execId %d", execId, pTask->execId); SCH_ERR_RET(TSDB_CODE_SCH_IGNORE_ERROR); } @@ -149,13 +146,13 @@ int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode) if (TSDB_CODE_SCH_IGNORE_ERROR == errCode) { return TSDB_CODE_SCH_IGNORE_ERROR; } - + int8_t status = 0; if (schJobNeedToStop(pJob, &status)) { SCH_TASK_DLOG("no more task failure processing cause of job status %s", jobTaskStatusStr(status)); SCH_ERR_RET(TSDB_CODE_SCH_IGNORE_ERROR); } - + if (SCH_GET_TASK_STATUS(pTask) != JOB_TASK_STATUS_EXEC) { SCH_TASK_ELOG("task already not in EXEC status, status:%s", SCH_GET_TASK_STATUS_STR(pTask)); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); @@ -204,8 +201,6 @@ int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode) SCH_RET(errCode); } - - // Note: no more task error processing, handled in function internal int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { bool moved = false; @@ -264,16 +259,17 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { SSchTask *parent = *(SSchTask **)taosArrayGet(pTask->parents, i); int32_t readyNum = atomic_add_fetch_32(&parent->childReady, 1); - SCH_LOCK_TASK(parent); - SDownstreamSourceNode source = {.type = QUERY_NODE_DOWNSTREAM_SOURCE, - .taskId = pTask->taskId, - .schedId = schMgmt.sId, - .execId = pTask->execId, - .addr = pTask->succeedAddr, - .fetchMsgType = SCH_FETCH_TYPE(pTask), - }; + SCH_LOCK(SCH_WRITE, &parent->planLock); + SDownstreamSourceNode source = { + .type = QUERY_NODE_DOWNSTREAM_SOURCE, + .taskId = pTask->taskId, + .schedId = schMgmt.sId, + .execId = pTask->execId, + .addr = pTask->succeedAddr, + .fetchMsgType = SCH_FETCH_TYPE(pTask), + }; qSetSubplanExecutionNode(parent->plan, pTask->plan->id.groupId, &source); - SCH_UNLOCK_TASK(parent); + SCH_UNLOCK(SCH_WRITE, &parent->planLock); if (SCH_TASK_READY_FOR_LAUNCH(readyNum, parent)) { SCH_TASK_DLOG("all %d children task done, start to launch parent task 0x%" PRIx64, readyNum, parent->taskId); @@ -291,29 +287,29 @@ int32_t schRescheduleTask(SSchJob *pJob, SSchTask *pTask) { return TSDB_CODE_SUCCESS; } - if (SCH_TASK_TIMEOUT(pTask) && JOB_TASK_STATUS_EXEC == pTask->status && - pJob->fetchTask != pTask && taosArrayGetSize(pTask->candidateAddrs) > 1) { + if (SCH_TASK_TIMEOUT(pTask) && JOB_TASK_STATUS_EXEC == pTask->status && pJob->fetchTask != pTask && + taosArrayGetSize(pTask->candidateAddrs) > 1) { SCH_TASK_DLOG("task execId %d will be rescheduled now", pTask->execId); schDropTaskOnExecNode(pJob, pTask); taosHashClear(pTask->execNodes); - + SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, TSDB_CODE_SCH_TIMEOUT_ERROR)); } return TSDB_CODE_SUCCESS; } -int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf* pData, int32_t rspCode) { +int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32_t rspCode) { int32_t code = 0; - + if ((pTask->execId + 1) >= pTask->maxExecTimes) { SCH_TASK_DLOG("task no more retry since reach max try times, execId:%d", pTask->execId); - schSwitchJobStatus(pJob, JOB_TASK_STATUS_FAIL, (void*)&rspCode); + schSwitchJobStatus(pJob, JOB_TASK_STATUS_FAIL, (void *)&rspCode); return TSDB_CODE_SUCCESS; } SCH_TASK_DLOG("task will be redirected now, status:%s", SCH_GET_TASK_STATUS_STR(pTask)); - + schDropTaskOnExecNode(pJob, pTask); taosHashClear(pTask->execNodes); SCH_ERR_JRET(schRemoveTaskFromExecList(pJob, pTask)); @@ -328,25 +324,24 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf* pData, int32 if (pData) { SCH_ERR_JRET(schUpdateTaskCandidateAddr(pJob, pTask, pData->pEpSet)); } - + if (SCH_TASK_NEED_FLOW_CTRL(pJob, pTask)) { if (JOB_TASK_STATUS_EXEC == SCH_GET_TASK_STATUS(pTask)) { SCH_ERR_JRET(schLaunchTasksInFlowCtrlList(pJob, pTask)); } - } + } SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_INIT); - + SCH_ERR_JRET(schLaunchTask(pJob, pTask)); return TSDB_CODE_SUCCESS; } - // merge plan - + pTask->childReady = 0; - + qClearSubplanExecutionNode(pTask->plan); // Note: current error task and upper level merge task @@ -355,10 +350,10 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf* pData, int32 } SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_INIT); - + int32_t childrenNum = taosArrayGetSize(pTask->children); for (int32_t i = 0; i < childrenNum; ++i) { - SSchTask* pChild = taosArrayGetP(pTask->children, i); + SSchTask *pChild = taosArrayGetP(pTask->children, i); SCH_LOCK_TASK(pChild); schDoTaskRedirect(pJob, pChild, NULL, rspCode); SCH_UNLOCK_TASK(pChild); @@ -371,7 +366,7 @@ _return: SCH_RET(schProcessOnTaskFailure(pJob, pTask, code)); } -int32_t schHandleRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf* pData, int32_t rspCode) { +int32_t schHandleRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32_t rspCode) { int32_t code = 0; if (SCH_IS_DATA_BIND_TASK(pTask)) { @@ -537,7 +532,7 @@ int32_t schHandleTaskRetry(SSchJob *pJob, SSchTask *pTask) { SCH_ERR_RET(schRemoveTaskFromExecList(pJob, pTask)); SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_INIT); - + if (SCH_TASK_NEED_FLOW_CTRL(pJob, pTask)) { SCH_ERR_RET(schLaunchTasksInFlowCtrlList(pJob, pTask)); } @@ -545,7 +540,8 @@ int32_t schHandleTaskRetry(SSchJob *pJob, SSchTask *pTask) { schDeregisterTaskHb(pJob, pTask); if (SCH_IS_DATA_BIND_TASK(pTask)) { - SCH_SWITCH_EPSET(&pTask->plan->execNode); + SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + SCH_SWITCH_EPSET(addr); } else { SCH_ERR_RET(schSwitchTaskCandidateAddr(pJob, pTask)); } @@ -558,20 +554,21 @@ int32_t schHandleTaskRetry(SSchJob *pJob, SSchTask *pTask) { int32_t schSetAddrsFromNodeList(SSchJob *pJob, SSchTask *pTask) { int32_t addNum = 0; int32_t nodeNum = 0; - + if (pJob->nodeList) { nodeNum = taosArrayGetSize(pJob->nodeList); for (int32_t i = 0; i < nodeNum && addNum < SCH_MAX_CANDIDATE_EP_NUM; ++i) { SQueryNodeLoad *nload = taosArrayGet(pJob->nodeList, i); SQueryNodeAddr *naddr = &nload->addr; - + if (NULL == taosArrayPush(pTask->candidateAddrs, naddr)) { SCH_TASK_ELOG("taosArrayPush execNode to candidate addrs failed, addNum:%d, errno:%d", addNum, errno); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - SCH_TASK_DLOG("set %dth candidate addr, id %d, fqdn:%s, port:%d", i, naddr->nodeId, SCH_GET_CUR_EP(naddr)->fqdn, SCH_GET_CUR_EP(naddr)->port); + SCH_TASK_DLOG("set %dth candidate addr, id %d, fqdn:%s, port:%d", i, naddr->nodeId, SCH_GET_CUR_EP(naddr)->fqdn, + SCH_GET_CUR_EP(naddr)->port); ++addNum; } @@ -585,7 +582,6 @@ int32_t schSetAddrsFromNodeList(SSchJob *pJob, SSchTask *pTask) { return TSDB_CODE_SUCCESS; } - int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { if (NULL != pTask->candidateAddrs) { return TSDB_CODE_SUCCESS; @@ -628,16 +624,17 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { return TSDB_CODE_SUCCESS; } -int32_t schUpdateTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask, SEpSet* pEpSet) { +int32_t schUpdateTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet) { if (NULL == pTask->candidateAddrs || 1 != taosArrayGetSize(pTask->candidateAddrs)) { - SCH_TASK_ELOG("not able to update cndidate addr, addr num %d", (int32_t)(pTask->candidateAddrs ? taosArrayGetSize(pTask->candidateAddrs): 0)); + SCH_TASK_ELOG("not able to update cndidate addr, addr num %d", + (int32_t)(pTask->candidateAddrs ? taosArrayGetSize(pTask->candidateAddrs) : 0)); SCH_ERR_RET(TSDB_CODE_APP_ERROR); } - SQueryNodeAddr* pAddr = taosArrayGet(pTask->candidateAddrs, 0); + SQueryNodeAddr *pAddr = taosArrayGet(pTask->candidateAddrs, 0); - SEp* pOld = &pAddr->epSet.eps[pAddr->epSet.inUse]; - SEp* pNew = &pEpSet->eps[pEpSet->inUse]; + SEp *pOld = &pAddr->epSet.eps[pAddr->epSet.inUse]; + SEp *pNew = &pEpSet->eps[pEpSet->inUse]; SCH_TASK_DLOG("update task ep from %s:%d to %s:%d", pOld->fqdn, pOld->port, pNew->fqdn, pNew->port); @@ -647,7 +644,7 @@ int32_t schUpdateTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask, SEpSet* pEpSe } int32_t schSwitchTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask) { - int32_t candidateNum = taosArrayGetSize(pTask->candidateAddrs); + int32_t candidateNum = taosArrayGetSize(pTask->candidateAddrs); if (++pTask->candidateIdx >= candidateNum) { pTask->candidateIdx = 0; } @@ -655,8 +652,6 @@ int32_t schSwitchTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask) { return TSDB_CODE_SUCCESS; } - - int32_t schRemoveTaskFromExecList(SSchJob *pJob, SSchTask *pTask) { int32_t code = taosHashRemove(pJob->execTasks, &pTask->taskId, sizeof(pTask->taskId)); if (code) { @@ -692,33 +687,32 @@ void schDropTaskOnExecNode(SSchJob *pJob, SSchTask *pTask) { SCH_TASK_DLOG("task has been dropped on %d exec nodes", size); } - - -int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId* pEpId, SArray* pStatusList) { - int32_t taskNum = (int32_t)taosArrayGetSize(pStatusList); +int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId *pEpId, SArray *pStatusList) { + int32_t taskNum = (int32_t)taosArrayGetSize(pStatusList); SSchTask *pTask = NULL; - SSchJob *pJob = NULL; + SSchJob *pJob = NULL; - qDebug("%d task status in hb rsp from nodeId:%d, fqdn:%s, port:%d", taskNum, pEpId->nodeId, pEpId->ep.fqdn, pEpId->ep.port); + qDebug("%d task status in hb rsp from nodeId:%d, fqdn:%s, port:%d", taskNum, pEpId->nodeId, pEpId->ep.fqdn, + pEpId->ep.port); for (int32_t i = 0; i < taskNum; ++i) { STaskStatus *pStatus = taosArrayGet(pStatusList, i); - int32_t code = 0; - - qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d task status in server: %s", - pStatus->queryId, pStatus->taskId, pStatus->execId, jobTaskStatusStr(pStatus->status)); + int32_t code = 0; + + qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d task status in server: %s", pStatus->queryId, pStatus->taskId, + pStatus->execId, jobTaskStatusStr(pStatus->status)); if (schProcessOnCbBegin(&pJob, &pTask, pStatus->queryId, pStatus->refId, pStatus->taskId)) { continue; } if (pStatus->execId != pTask->execId) { - //TODO + // TODO SCH_TASK_DLOG("execId %d mis-match current execId %d", pStatus->execId, pTask->execId); schProcessOnCbEnd(pJob, pTask, 0); continue; } - + if (pStatus->status == JOB_TASK_STATUS_FAIL) { // RECORD AND HANDLE ERROR!!!! schProcessOnCbEnd(pJob, pTask, 0); @@ -832,7 +826,6 @@ void schDropTaskInHashList(SSchJob *pJob, SHashObj *list) { } } - // Note: no more error processing, handled in function internal int32_t schLaunchFetchTask(SSchJob *pJob) { int32_t code = 0; @@ -851,5 +844,3 @@ _return: SCH_RET(schProcessOnTaskFailure(pJob, pJob->fetchTask, code)); } - - diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 9742f93824..e361c8021c 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -251,6 +251,9 @@ void syncStartStandBy(int64_t rid); bool syncNodeCanChange(SSyncNode* pSyncNode); bool syncNodeCheckNewConfig(SSyncNode* pSyncNode, const SSyncCfg* pNewCfg); +int32_t syncNodeLeaderTransfer(SSyncNode* pSyncNode); +int32_t syncNodeLeaderTransferTo(SSyncNode* pSyncNode, SNodeInfo newLeader); + // for debug -------------- void syncNodePrint(SSyncNode* pObj); void syncNodePrint2(char* s, SSyncNode* pObj); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 50e2588e19..e1c3d4bb33 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -316,6 +316,40 @@ int32_t syncLeaderTransferTo(int64_t rid, SNodeInfo newLeader) { return ret; } +int32_t syncNodeLeaderTransfer(SSyncNode* pSyncNode) { + if (pSyncNode->peersNum == 0) { + sError("only one replica, cannot leader transfer"); + terrno = TSDB_CODE_SYN_ONE_REPLICA; + return -1; + } + + SNodeInfo newLeader = (pSyncNode->peersNodeInfo)[0]; + int32_t ret = syncNodeLeaderTransferTo(pSyncNode, newLeader); + return ret; +} + +int32_t syncNodeLeaderTransferTo(SSyncNode* pSyncNode, SNodeInfo newLeader) { + int32_t ret = 0; + + if (pSyncNode->replicaNum == 1) { + sError("only one replica, cannot leader transfer"); + terrno = TSDB_CODE_SYN_ONE_REPLICA; + return -1; + } + + SyncLeaderTransfer* pMsg = syncLeaderTransferBuild(pSyncNode->vgId); + pMsg->newLeaderId.addr = syncUtilAddr2U64(newLeader.nodeFqdn, newLeader.nodePort); + pMsg->newLeaderId.vgId = pSyncNode->vgId; + pMsg->newNodeInfo = newLeader; + ASSERT(pMsg != NULL); + SRpcMsg rpcMsg = {0}; + syncLeaderTransfer2RpcMsg(pMsg, &rpcMsg); + syncLeaderTransferDestroy(pMsg); + + ret = syncNodePropose(pSyncNode, &rpcMsg, false); + return ret; +} + bool syncCanLeaderTransfer(int64_t rid) { SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { @@ -1113,6 +1147,8 @@ void syncNodeStartStandBy(SSyncNode* pSyncNode) { void syncNodeClose(SSyncNode* pSyncNode) { syncNodeEventLog(pSyncNode, "sync close"); + // leader transfer + int32_t ret; ASSERT(pSyncNode != NULL); @@ -1527,8 +1563,8 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) { char logBuf[256 + 256]; if (pSyncNode != NULL && pSyncNode->pRaftCfg != NULL && pSyncNode->pRaftStore != NULL) { snprintf(logBuf, sizeof(logBuf), - "vgId:%d, sync %s %s, term:%" PRIu64 ", commit:%" PRId64 ", beginlog:%" PRId64 ", lastlog:%" PRId64 - ", lastsnapshot:%" PRId64 + "vgId:%d, sync %s %s, term:%" PRIu64 ", commit:%" PRId64 ", first:%" PRId64 ", last:%" PRId64 + ", snapshot:%" PRId64 ", standby:%d, " "strategy:%d, batch:%d, " "replica-num:%d, " @@ -1548,8 +1584,8 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) { char* s = (char*)taosMemoryMalloc(len); if (pSyncNode != NULL && pSyncNode->pRaftCfg != NULL && pSyncNode->pRaftStore != NULL) { snprintf(s, len, - "vgId:%d, sync %s %s, term:%" PRIu64 ", commit:%" PRId64 ", beginlog:%" PRId64 ", lastlog:%" PRId64 - ", lastsnapshot:%" PRId64 + "vgId:%d, sync %s %s, term:%" PRIu64 ", commit:%" PRId64 ", first:%" PRId64 ", last:%" PRId64 + ", snapshot:%" PRId64 ", standby:%d, " "strategy:%d, batch:%d, " "replica-num:%d, " @@ -1594,8 +1630,8 @@ void syncNodeErrorLog(const SSyncNode* pSyncNode, char* str) { char logBuf[256 + 256]; if (pSyncNode != NULL && pSyncNode->pRaftCfg != NULL && pSyncNode->pRaftStore != NULL) { snprintf(logBuf, sizeof(logBuf), - "vgId:%d, sync %s %s, term:%" PRIu64 ", commit:%" PRId64 ", beginlog:%" PRId64 ", lastlog:%" PRId64 - ", lastsnapshot:%" PRId64 + "vgId:%d, sync %s %s, term:%" PRIu64 ", commit:%" PRId64 ", first:%" PRId64 ", last:%" PRId64 + ", snapshot:%" PRId64 ", standby:%d, " "replica-num:%d, " "lconfig:%" PRId64 ", changing:%d, restore:%d, %s", @@ -1613,8 +1649,8 @@ void syncNodeErrorLog(const SSyncNode* pSyncNode, char* str) { char* s = (char*)taosMemoryMalloc(len); if (pSyncNode != NULL && pSyncNode->pRaftCfg != NULL && pSyncNode->pRaftStore != NULL) { snprintf(s, len, - "vgId:%d, sync %s %s, term:%" PRIu64 ", commit:%" PRId64 ", beginlog:%" PRId64 ", lastlog:%" PRId64 - ", lastsnapshot:%" PRId64 + "vgId:%d, sync %s %s, term:%" PRIu64 ", commit:%" PRId64 ", first:%" PRId64 ", last:%" PRId64 + ", snapshot:%" PRId64 ", standby:%d, " "replica-num:%d, " "lconfig:%" PRId64 ", changing:%d, restore:%d, %s", @@ -1644,8 +1680,8 @@ char* syncNode2SimpleStr(const SSyncNode* pSyncNode) { SyncIndex logBeginIndex = pSyncNode->pLogStore->syncLogBeginIndex(pSyncNode->pLogStore); snprintf(s, len, - "vgId:%d, sync %s, term:%" PRIu64 ", commit:%" PRId64 ", beginlog:%" PRId64 ", lastlog:%" PRId64 - ", lastsnapshot:%" PRId64 + "vgId:%d, sync %s, term:%" PRIu64 ", commit:%" PRId64 ", first:%" PRId64 ", last:%" PRId64 + ", snapshot:%" PRId64 ", standby:%d, " "replica-num:%d, " "lconfig:%" PRId64 ", changing:%d, restore:%d", diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 918a94aa25..edc01c9a05 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -201,6 +201,43 @@ static SyncTerm raftLogLastTerm(struct SSyncLogStore* pLogStore) { return SYNC_TERM_INVALID; } +static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { + SSyncLogStoreData* pData = pLogStore->data; + SWal* pWal = pData->pWal; + + SyncIndex index = 0; + SWalSyncInfo syncMeta; + syncMeta.isWeek = pEntry->isWeak; + syncMeta.seqNum = pEntry->seqNum; + syncMeta.term = pEntry->term; + index = walAppendLog(pWal, pEntry->originalRpcType, syncMeta, pEntry->data, pEntry->dataLen); + if (index < 0) { + int32_t err = terrno; + const char* errStr = tstrerror(err); + int32_t sysErr = errno; + const char* sysErrStr = strerror(errno); + + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "wal write error, index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s", + pEntry->index, err, err, errStr, sysErr, sysErrStr); + syncNodeErrorLog(pData->pSyncNode, logBuf); + + ASSERT(0); + return -1; + } + pEntry->index = index; + + do { + char eventLog[128]; + snprintf(eventLog, sizeof(eventLog), "write index:%" PRId64 ", type:%s,%d, type2:%s,%d", pEntry->index, + TMSG_INFO(pEntry->msgType), pEntry->msgType, TMSG_INFO(pEntry->originalRpcType), pEntry->originalRpcType); + syncNodeEventLog(pData->pSyncNode, eventLog); + } while (0); + + return 0; +} + +#if 0 static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; @@ -213,7 +250,7 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr } int code = 0; - SSyncLogMeta syncMeta; + SWalSyncInfo syncMeta; syncMeta.isWeek = pEntry->isWeak; syncMeta.seqNum = pEntry->seqNum; syncMeta.term = pEntry->term; @@ -243,6 +280,7 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr return code; } +#endif // entry found, return 0 // entry not found, return -1, terrno = TSDB_CODE_WAL_LOG_NOT_EXIST @@ -361,6 +399,8 @@ static int32_t raftLogGetLastEntry(SSyncLogStore* pLogStore, SSyncRaftEntry** pp //------------------------------- // log[0 .. n] + +#if 0 int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; @@ -369,7 +409,7 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { ASSERT(pEntry->index == lastIndex + 1); int code = 0; - SSyncLogMeta syncMeta; + SWalSyncInfo syncMeta; syncMeta.isWeek = pEntry->isWeak; syncMeta.seqNum = pEntry->seqNum; syncMeta.term = pEntry->term; @@ -397,6 +437,44 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { return code; } +#endif + +int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { + SSyncLogStoreData* pData = pLogStore->data; + SWal* pWal = pData->pWal; + + SyncIndex index = 0; + SWalSyncInfo syncMeta; + syncMeta.isWeek = pEntry->isWeak; + syncMeta.seqNum = pEntry->seqNum; + syncMeta.term = pEntry->term; + + index = walAppendLog(pWal, pEntry->originalRpcType, syncMeta, pEntry->data, pEntry->dataLen); + if (index < 0) { + int32_t err = terrno; + const char* errStr = tstrerror(err); + int32_t sysErr = errno; + const char* sysErrStr = strerror(errno); + + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "wal write error, index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s", + pEntry->index, err, err, errStr, sysErr, sysErrStr); + syncNodeErrorLog(pData->pSyncNode, logBuf); + + ASSERT(0); + return -1; + } + pEntry->index = index; + + do { + char eventLog[128]; + snprintf(eventLog, sizeof(eventLog), "write2 index:%" PRId64 ", type:%s,%d, type2:%s,%d", pEntry->index, + TMSG_INFO(pEntry->msgType), pEntry->msgType, TMSG_INFO(pEntry->originalRpcType), pEntry->originalRpcType); + syncNodeEventLog(pData->pSyncNode, eventLog); + } while (0); + + return 0; +} SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { SSyncLogStoreData* pData = pLogStore->data; diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index e73ddedd73..5bb6349d9b 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -311,6 +311,17 @@ void transCtxMerge(STransCtx* dst, STransCtx* src); void* transCtxDumpVal(STransCtx* ctx, int32_t key); void* transCtxDumpBrokenlinkVal(STransCtx* ctx, int32_t* msgType); +// request list +typedef struct STransReq { + queue q; + void* data; +} STransReq; + +void transReqQueueInit(queue* q); +void* transReqQueuePushReq(queue* q); +void* transReqQueueRemove(void* arg); +void transReqQueueClear(queue* q); + // queue sending msgs typedef struct { SArray* q; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 0600b1130d..5d087d5769 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -19,7 +19,7 @@ typedef struct SCliConn { T_REF_DECLARE() uv_connect_t connReq; uv_stream_t* stream; - uv_write_t writeReq; + queue wreqQueue; void* hostThrd; @@ -573,8 +573,7 @@ static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { return; } if (nread < 0) { - tWarn("%s conn %p read error:%s, ref:%d", CONN_GET_INST_LABEL(conn), conn, uv_err_name(nread), - T_REF_VAL_GET(conn)); + tWarn("%s conn %p read error:%s, ref:%d", CONN_GET_INST_LABEL(conn), conn, uv_err_name(nread), T_REF_VAL_GET(conn)); conn->broken = true; cliHandleExcept(conn); } @@ -587,9 +586,10 @@ static SCliConn* cliCreateConn(SCliThrd* pThrd) { uv_tcp_init(pThrd->loop, (uv_tcp_t*)(conn->stream)); conn->stream->data = conn; - conn->writeReq.data = conn; conn->connReq.data = conn; + transReqQueueInit(&conn->wreqQueue); + transQueueInit(&conn->cliMsgs, NULL); QUEUE_INIT(&conn->conn); conn->hostThrd = pThrd; @@ -628,6 +628,8 @@ static void cliDestroy(uv_handle_t* handle) { transCtxCleanup(&conn->ctx); transQueueDestroy(&conn->cliMsgs); tTrace("%s conn %p destroy successfully", CONN_GET_INST_LABEL(conn), conn); + transReqQueueClear(&conn->wreqQueue); + transDestroyBuffer(&conn->readBuf); taosMemoryFree(conn); } @@ -650,7 +652,8 @@ static bool cliHandleNoResp(SCliConn* conn) { return res; } static void cliSendCb(uv_write_t* req, int status) { - SCliConn* pConn = req->data; + SCliConn* pConn = transReqQueueRemove(req); + if (pConn == NULL) return; if (status == 0) { tTrace("%s conn %p data already was written out", CONN_GET_INST_LABEL(pConn), pConn); @@ -708,8 +711,8 @@ void cliSend(SCliConn* pConn) { CONN_SET_PERSIST_BY_APP(pConn); } - pConn->writeReq.data = pConn; - uv_write(&pConn->writeReq, (uv_stream_t*)pConn->stream, &wb, 1, cliSendCb); + uv_write_t* req = transReqQueuePushReq(&pConn->wreqQueue); + uv_write(req, (uv_stream_t*)pConn->stream, &wb, 1, cliSendCb); return; _RETURN: return; diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 812123441c..84af8da513 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -293,6 +293,48 @@ void* transCtxDumpBrokenlinkVal(STransCtx* ctx, int32_t* msgType) { return ret; } +void transReqQueueInit(queue* q) { + // init req queue + QUEUE_INIT(q); +} +void* transReqQueuePushReq(queue* q) { + uv_write_t* req = taosMemoryCalloc(1, sizeof(uv_write_t)); + STransReq* wreq = taosMemoryCalloc(1, sizeof(STransReq)); + wreq->data = req; + req->data = wreq; + QUEUE_PUSH(q, &wreq->q); + return req; +} +void* transReqQueueRemove(void* arg) { + void* ret = NULL; + uv_write_t* req = arg; + STransReq* wreq = req && req->data ? req->data : NULL; + + assert(wreq->data == req); + if (wreq == NULL || wreq->data == NULL) { + taosMemoryFree(wreq->data); + taosMemoryFree(wreq); + return req; + } + + QUEUE_REMOVE(&wreq->q); + + ret = req && req->handle ? req->handle->data : NULL; + taosMemoryFree(wreq->data); + taosMemoryFree(wreq); + + return ret; +} +void transReqQueueClear(queue* q) { + while (!QUEUE_IS_EMPTY(q)) { + queue* h = QUEUE_HEAD(q); + QUEUE_REMOVE(h); + STransReq* req = QUEUE_DATA(h, STransReq, q); + taosMemoryFree(req->data); + taosMemoryFree(req); + } +} + void transQueueInit(STransQueue* queue, void (*freeFunc)(const void* arg)) { queue->q = taosArrayInit(2, sizeof(void*)); queue->freeFunc = (void (*)(const void*))freeFunc; diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index a239f90c29..68e12a1963 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -29,7 +29,7 @@ typedef struct { typedef struct SSvrConn { T_REF_DECLARE() uv_tcp_t* pTcp; - uv_write_t pWriter; + queue wreqQueue; uv_timer_t pTimer; queue queue; @@ -265,8 +265,8 @@ static void uvHandleReq(SSvrConn* pConn) { transMsg.info.refId = pConn->refId; transMsg.info.traceId = pHead->traceId; - tGTrace("%s handle %p conn:%p translated to app, refId:%" PRIu64, transLabel(pTransInst), transMsg.info.handle, - pConn, pConn->refId); + tGTrace("%s handle %p conn:%p translated to app, refId:%" PRIu64, transLabel(pTransInst), transMsg.info.handle, pConn, + pConn->refId); assert(transMsg.info.handle != NULL); if (pHead->noResp == 1) { @@ -316,7 +316,7 @@ void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { memset(&conn->regArg, 0, sizeof(conn->regArg)); } } - transUnrefSrvHandle(conn); + destroyConn(conn, true); } } void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { @@ -331,7 +331,9 @@ void uvOnTimeoutCb(uv_timer_t* handle) { } void uvOnSendCb(uv_write_t* req, int status) { - SSvrConn* conn = req->data; + SSvrConn* conn = transReqQueueRemove(req); + if (conn == NULL) return; + if (status == 0) { tTrace("conn %p data already was written on stream", conn); if (!transQueueEmpty(&conn->srvMsgs)) { @@ -390,7 +392,6 @@ static void uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) { pHead->traceId = pMsg->info.traceId; pHead->hasEpSet = pMsg->info.hasEpSet; - if (pConn->status == ConnNormal) { pHead->msgType = (0 == pMsg->msgType ? pConn->inType + 1 : pMsg->msgType); } else { @@ -433,12 +434,12 @@ static void uvStartSendRespInternal(SSvrMsg* smsg) { uvPrepareSendData(smsg, &wb); transRefSrvHandle(pConn); - uv_write(&pConn->pWriter, (uv_stream_t*)pConn->pTcp, &wb, 1, uvOnSendCb); + uv_write_t* req = transReqQueuePushReq(&pConn->wreqQueue); + uv_write(req, (uv_stream_t*)pConn->pTcp, &wb, 1, uvOnSendCb); } static void uvStartSendResp(SSvrMsg* smsg) { // impl SSvrConn* pConn = smsg->pConn; - if (pConn->broken == true) { // persist by transFreeMsg(smsg->msg.pCont); @@ -635,8 +636,6 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { uv_tcp_init(pThrd->loop, pConn->pTcp); pConn->pTcp->data = pConn; - pConn->pWriter.data = pConn; - transSetConnOption((uv_tcp_t*)pConn->pTcp); if (uv_accept(q, (uv_stream_t*)(pConn->pTcp)) == 0) { @@ -744,6 +743,8 @@ static SSvrConn* createConn(void* hThrd) { SWorkThrd* pThrd = hThrd; SSvrConn* pConn = (SSvrConn*)taosMemoryCalloc(1, sizeof(SSvrConn)); + + transReqQueueInit(&pConn->wreqQueue); QUEUE_INIT(&pConn->queue); QUEUE_PUSH(&pThrd->conn, &pConn->queue); @@ -778,9 +779,6 @@ static void destroyConn(SSvrConn* conn, bool clear) { tTrace("conn %p to be destroyed", conn); uv_close((uv_handle_t*)conn->pTcp, uvDestroyConn); } - //} else { - // uvDestroyConn((uv_handle_t*)conn->pTcp); - //} } } static void destroyConnRegArg(SSvrConn* conn) { @@ -819,6 +817,8 @@ static void uvDestroyConn(uv_handle_t* handle) { SSvrMsg* msg = transQueueGet(&conn->srvMsgs, i); destroySmsg(msg); } + + transReqQueueClear(&conn->wreqQueue); transQueueDestroy(&conn->srvMsgs); QUEUE_REMOVE(&conn->queue); diff --git a/source/libs/wal/inc/walInt.h b/source/libs/wal/inc/walInt.h index 2767780ff3..20667fc918 100644 --- a/source/libs/wal/inc/walInt.h +++ b/source/libs/wal/inc/walInt.h @@ -146,12 +146,12 @@ int walMetaDeserialize(SWal* pWal, const char* bytes); // seek section int walChangeWrite(SWal* pWal, int64_t ver); -int walSetWrite(SWal* pWal); +int walInitWriteFile(SWal* pWal); // seek section end int64_t walGetSeq(); int walSeekWriteVer(SWal* pWal, int64_t ver); -int walRoll(SWal* pWal); +int32_t walRollImpl(SWal* pWal); #ifdef __cplusplus } diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index ecb480223f..991b50f7c0 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -51,10 +51,10 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal) { char fnameStr[WAL_FILE_LEN]; walBuildLogName(pWal, pLastFileInfo->firstVer, fnameStr); - int64_t file_size = 0; - taosStatFile(fnameStr, &file_size, NULL); - int readSize = TMIN(WAL_MAX_SIZE + 2, file_size); - pLastFileInfo->fileSize = file_size; + int64_t fileSize = 0; + taosStatFile(fnameStr, &fileSize, NULL); + int readSize = TMIN(WAL_MAX_SIZE + 2, fileSize); + pLastFileInfo->fileSize = fileSize; TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ); if (pFile == NULL) { @@ -145,6 +145,26 @@ int walCheckAndRepairMeta(SWal* pWal) { int metaFileNum = taosArrayGetSize(pWal->fileInfoSet); int actualFileNum = taosArrayGetSize(pLogInfoArray); +#if 0 + for (int32_t fileNo = actualFileNum - 1; fileNo >= 0; fileNo--) { + SWalFileInfo* pFileInfo = taosArrayGet(pLogInfoArray, fileNo); + char fnameStr[WAL_FILE_LEN]; + walBuildLogName(pWal, pFileInfo->firstVer, fnameStr); + int64_t fileSize = 0; + taosStatFile(fnameStr, &fileSize, NULL); + if (fileSize == 0) { + taosRemoveFile(fnameStr); + walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr); + taosRemoveFile(fnameStr); + taosArrayPop(pLogInfoArray); + } else { + break; + } + } + + actualFileNum = taosArrayGetSize(pLogInfoArray); +#endif + if (metaFileNum > actualFileNum) { taosArrayPopFrontBatch(pWal->fileInfoSet, metaFileNum - actualFileNum); } else if (metaFileNum < actualFileNum) { @@ -164,6 +184,7 @@ int walCheckAndRepairMeta(SWal* pWal) { walBuildLogName(pWal, pLastFileInfo->firstVer, fnameStr); int64_t fileSize = 0; taosStatFile(fnameStr, &fileSize, NULL); + /*ASSERT(fileSize != 0);*/ if (metaFileNum != actualFileNum || pLastFileInfo->fileSize != fileSize) { pLastFileInfo->fileSize = fileSize; @@ -380,9 +401,9 @@ int walLoadMeta(SWal* pWal) { char fnameStr[WAL_FILE_LEN]; walBuildMetaName(pWal, metaVer, fnameStr); // read metafile - int64_t file_size = 0; - taosStatFile(fnameStr, &file_size, NULL); - int size = (int)file_size; + int64_t fileSize = 0; + taosStatFile(fnameStr, &fileSize, NULL); + int size = (int)fileSize; char* buf = taosMemoryMalloc(size + 5); if (buf == NULL) { terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; diff --git a/source/libs/wal/src/walSeek.c b/source/libs/wal/src/walSeek.c index b99206fe98..78d45c84e2 100644 --- a/source/libs/wal/src/walSeek.c +++ b/source/libs/wal/src/walSeek.c @@ -48,7 +48,7 @@ static int64_t walSeekWritePos(SWal* pWal, int64_t ver) { return 0; } -int walSetWrite(SWal* pWal) { +int walInitWriteFile(SWal* pWal) { TdFilePtr pIdxTFile, pLogTFile; SWalFileInfo* pRet = taosArrayGetLast(pWal->fileInfoSet); ASSERT(pRet != NULL); @@ -70,6 +70,7 @@ int walSetWrite(SWal* pWal) { // switch file pWal->pWriteIdxTFile = pIdxTFile; pWal->pWriteLogTFile = pLogTFile; + pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1; return 0; } diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 374aae5a7e..26dc3cdffb 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -207,12 +207,35 @@ int32_t walRollback(SWal *pWal, int64_t ver) { return 0; } +static FORCE_INLINE int32_t walCheckAndRoll(SWal *pWal) { + if (taosArrayGetSize(pWal->fileInfoSet) == 0) { + /*pWal->vers.firstVer = index;*/ + if (walRollImpl(pWal) < 0) { + return -1; + } + } else { + int64_t passed = walGetSeq() - pWal->lastRollSeq; + if (pWal->cfg.rollPeriod != -1 && pWal->cfg.rollPeriod != 0 && passed > pWal->cfg.rollPeriod) { + if (walRollImpl(pWal) < 0) { + return -1; + } + } else if (pWal->cfg.segSize != -1 && pWal->cfg.segSize != 0 && walGetLastFileSize(pWal) > pWal->cfg.segSize) { + if (walRollImpl(pWal) < 0) { + return -1; + } + } + } + return 0; +} + int32_t walBeginSnapshot(SWal *pWal, int64_t ver) { pWal->vers.verInSnapshotting = ver; // check file rolling if (pWal->cfg.retentionPeriod == 0) { taosThreadMutexLock(&pWal->mutex); - walRoll(pWal); + if (walGetLastFileSize(pWal) != 0) { + walRollImpl(pWal); + } taosThreadMutexUnlock(&pWal->mutex); } @@ -282,7 +305,7 @@ END: return code; } -int walRoll(SWal *pWal) { +int32_t walRollImpl(SWal *pWal) { int32_t code = 0; if (pWal->pWriteIdxTFile != NULL) { code = taosCloseFile(&pWal->pWriteIdxTFile); @@ -330,11 +353,13 @@ int walRoll(SWal *pWal) { pWal->lastRollSeq = walGetSeq(); + walSaveMeta(pWal); + END: return code; } -static int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { +static int32_t walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { SWalIdxEntry entry = {.ver = ver, .offset = offset}; int64_t idxOffset = taosLSeekFile(pWal->pWriteIdxTFile, 0, SEEK_END); wDebug("vgId:%d, write index, index:%" PRId64 ", offset:%" PRId64 ", at %" PRId64, pWal->cfg.vgId, ver, offset, @@ -348,61 +373,14 @@ static int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { return 0; } -int32_t walWriteWithSyncInfo(SWal *pWal, int64_t index, tmsg_t msgType, SSyncLogMeta syncMeta, const void *body, - int32_t bodyLen) { - int32_t code = 0; - - // no wal - if (pWal->cfg.level == TAOS_WAL_NOLOG) return 0; - - if (bodyLen > TSDB_MAX_WAL_SIZE) { - terrno = TSDB_CODE_WAL_SIZE_LIMIT; - return -1; - } - taosThreadMutexLock(&pWal->mutex); - - if (index == pWal->vers.lastVer + 1) { - if (taosArrayGetSize(pWal->fileInfoSet) == 0) { - pWal->vers.firstVer = index; - if (walRoll(pWal) < 0) { - taosThreadMutexUnlock(&pWal->mutex); - return -1; - } - } else { - int64_t passed = walGetSeq() - pWal->lastRollSeq; - if (pWal->cfg.rollPeriod != -1 && pWal->cfg.rollPeriod != 0 && passed > pWal->cfg.rollPeriod) { - if (walRoll(pWal) < 0) { - taosThreadMutexUnlock(&pWal->mutex); - return -1; - } - } else if (pWal->cfg.segSize != -1 && pWal->cfg.segSize != 0 && walGetLastFileSize(pWal) > pWal->cfg.segSize) { - if (walRoll(pWal) < 0) { - taosThreadMutexUnlock(&pWal->mutex); - return -1; - } - } - } - } else { - // reject skip log or rewrite log - // must truncate explicitly first - terrno = TSDB_CODE_WAL_INVALID_VER; - taosThreadMutexUnlock(&pWal->mutex); - return -1; - } - - /*if (!tfValid(pWal->pWriteLogTFile)) return -1;*/ - - ASSERT(pWal->writeCur >= 0); - - if (pWal->pWriteIdxTFile == NULL || pWal->pWriteLogTFile == NULL) { - walSetWrite(pWal); - taosLSeekFile(pWal->pWriteLogTFile, 0, SEEK_END); - taosLSeekFile(pWal->pWriteIdxTFile, 0, SEEK_END); - } - - pWal->writeHead.head.version = index; +// TODO gurantee atomicity by truncate failed writing +static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgType, SWalSyncInfo syncMeta, + const void *body, int32_t bodyLen) { + int64_t code = 0; int64_t offset = walGetCurFileOffset(pWal); + + pWal->writeHead.head.version = index; pWal->writeHead.head.bodyLen = bodyLen; pWal->writeHead.head.msgType = msgType; @@ -417,7 +395,8 @@ int32_t walWriteWithSyncInfo(SWal *pWal, int64_t index, tmsg_t msgType, SSyncLog terrno = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); - return -1; + code = -1; + goto END; } if (taosWriteFile(pWal->pWriteLogTFile, (char *)body, bodyLen) != bodyLen) { @@ -425,13 +404,14 @@ int32_t walWriteWithSyncInfo(SWal *pWal, int64_t index, tmsg_t msgType, SSyncLog terrno = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); - return -1; + code = -1; + goto END; } code = walWriteIndex(pWal, index, offset); - if (code != 0) { - // TODO - return -1; + if (code < 0) { + // TODO ftruncate + goto END; } // set status @@ -444,13 +424,88 @@ int32_t walWriteWithSyncInfo(SWal *pWal, int64_t index, tmsg_t msgType, SSyncLog walGetCurFileInfo(pWal)->lastVer = index; walGetCurFileInfo(pWal)->fileSize += sizeof(SWalCkHead) + bodyLen; - taosThreadMutexUnlock(&pWal->mutex); - return 0; +END: + return -1; +} + +int64_t walAppendLog(SWal *pWal, tmsg_t msgType, SWalSyncInfo syncMeta, const void *body, int32_t bodyLen) { + if (bodyLen > TSDB_MAX_WAL_SIZE) { + terrno = TSDB_CODE_WAL_SIZE_LIMIT; + return -1; + } + + taosThreadMutexLock(&pWal->mutex); + + int64_t index = pWal->vers.lastVer + 1; + + if (walCheckAndRoll(pWal) < 0) { + taosThreadMutexUnlock(&pWal->mutex); + return -1; + } + + if (pWal->pWriteIdxTFile == NULL || pWal->pWriteIdxTFile == NULL || pWal->writeCur < 0) { + if (walInitWriteFile(pWal) < 0) { + taosThreadMutexUnlock(&pWal->mutex); + return -1; + } + } + + ASSERT(pWal->pWriteIdxTFile != NULL && pWal->pWriteLogTFile != NULL && pWal->writeCur >= 0); + + if (walWriteImpl(pWal, index, msgType, syncMeta, body, bodyLen) < 0) { + taosThreadMutexUnlock(&pWal->mutex); + return -1; + } + + taosThreadMutexUnlock(&pWal->mutex); + return index; +} + +int32_t walWriteWithSyncInfo(SWal *pWal, int64_t index, tmsg_t msgType, SWalSyncInfo syncMeta, const void *body, + int32_t bodyLen) { + int32_t code = 0; + + if (bodyLen > TSDB_MAX_WAL_SIZE) { + terrno = TSDB_CODE_WAL_SIZE_LIMIT; + return -1; + } + taosThreadMutexLock(&pWal->mutex); + + // concurrency control: + // if logs are write with assigned index, + // smaller index must be write before larger one + if (index != pWal->vers.lastVer + 1) { + terrno = TSDB_CODE_WAL_INVALID_VER; + taosThreadMutexUnlock(&pWal->mutex); + return -1; + } + + if (walCheckAndRoll(pWal) < 0) { + taosThreadMutexUnlock(&pWal->mutex); + return -1; + } + + if (pWal->pWriteIdxTFile == NULL || pWal->pWriteIdxTFile == NULL || pWal->writeCur < 0) { + if (walInitWriteFile(pWal) < 0) { + taosThreadMutexUnlock(&pWal->mutex); + return -1; + } + } + + ASSERT(pWal->pWriteIdxTFile != NULL && pWal->pWriteLogTFile != NULL && pWal->writeCur >= 0); + + if (walWriteImpl(pWal, index, msgType, syncMeta, body, bodyLen) < 0) { + taosThreadMutexUnlock(&pWal->mutex); + return -1; + } + + taosThreadMutexUnlock(&pWal->mutex); + return code; } int32_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, int32_t bodyLen) { - SSyncLogMeta syncMeta = { + SWalSyncInfo syncMeta = { .isWeek = -1, .seqNum = UINT64_MAX, .term = UINT64_MAX, diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 243a234abe..b755a35815 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -106,8 +106,8 @@ int32_t taosMkDir(const char *dirname) { int32_t taosMulMkDir(const char *dirname) { if (dirname == NULL) return -1; - char temp[1024]; - char * pos = temp; + char temp[1024]; + char *pos = temp; int32_t code = 0; #ifdef WINDOWS taosRealPath(dirname, temp, sizeof(temp)); @@ -127,11 +127,11 @@ int32_t taosMulMkDir(const char *dirname) { for (; *pos != '\0'; pos++) { if (*pos == TD_DIRSEP[0]) { *pos = '\0'; - #ifdef WINDOWS +#ifdef WINDOWS code = _mkdir(temp, 0755); - #else +#else code = mkdir(temp, 0755); - #endif +#endif if (code < 0 && errno != EEXIST) { return code; } @@ -140,11 +140,11 @@ int32_t taosMulMkDir(const char *dirname) { } if (*(pos - 1) != TD_DIRSEP[0]) { - #ifdef WINDOWS +#ifdef WINDOWS code = _mkdir(temp, 0755); - #else +#else code = mkdir(temp, 0755); - #endif +#endif if (code < 0 && errno != EEXIST) { return code; } @@ -267,7 +267,7 @@ char *taosDirName(char *name) { } else { name[0] = 0; } - return name; + return name; #else return dirname(name); #endif @@ -334,9 +334,9 @@ bool taosDirEntryIsDir(TdDirEntryPtr pDirEntry) { } char *taosGetDirEntryName(TdDirEntryPtr pDirEntry) { - if (pDirEntry == NULL) { - return NULL; - } + /*if (pDirEntry == NULL) {*/ + /*return NULL;*/ + /*}*/ #ifdef WINDOWS return pDirEntry->findFileData.cFileName; #else diff --git a/source/util/src/tlockfree.c b/source/util/src/tlockfree.c index 3cab16ee83..6f7b6f6901 100644 --- a/source/util/src/tlockfree.c +++ b/source/util/src/tlockfree.c @@ -17,10 +17,8 @@ #include "tlockfree.h" #define TD_RWLATCH_WRITE_FLAG 0x40000000 -#define TD_RWLATCH_REENTRANT_FLAG 0x4000000000000000 void taosInitRWLatch(SRWLatch *pLatch) { *pLatch = 0; } -void taosInitReentrantRWLatch(SRWLatch *pLatch) { *pLatch = TD_RWLATCH_REENTRANT_FLAG; } void taosWLockLatch(SRWLatch *pLatch) { SRWLatch oLatch, nLatch; @@ -28,14 +26,8 @@ void taosWLockLatch(SRWLatch *pLatch) { // Set write flag while (1) { - oLatch = atomic_load_64(pLatch); + oLatch = atomic_load_32(pLatch); if (oLatch & TD_RWLATCH_WRITE_FLAG) { - if (oLatch & TD_RWLATCH_REENTRANT_FLAG) { - nLatch = (((oLatch >> 32) + 1) << 32) | (oLatch & 0xFFFFFFFF); - if (atomic_val_compare_exchange_64(pLatch, oLatch, nLatch) == oLatch) break; - - continue; - } nLoops++; if (nLoops > 1000) { sched_yield(); @@ -45,14 +37,14 @@ void taosWLockLatch(SRWLatch *pLatch) { } nLatch = oLatch | TD_RWLATCH_WRITE_FLAG; - if (atomic_val_compare_exchange_64(pLatch, oLatch, nLatch) == oLatch) break; + if (atomic_val_compare_exchange_32(pLatch, oLatch, nLatch) == oLatch) break; } // wait for all reads end nLoops = 0; while (1) { - oLatch = atomic_load_64(pLatch); - if (0 == (oLatch & 0xFFFFFFF)) break; + oLatch = atomic_load_32(pLatch); + if (oLatch == TD_RWLATCH_WRITE_FLAG) break; nLoops++; if (nLoops > 1000) { sched_yield(); @@ -64,47 +56,27 @@ void taosWLockLatch(SRWLatch *pLatch) { // no reentrant int32_t taosWTryLockLatch(SRWLatch *pLatch) { SRWLatch oLatch, nLatch; - oLatch = atomic_load_64(pLatch); - if (oLatch << 2) { + oLatch = atomic_load_32(pLatch); + if (oLatch) { return -1; } nLatch = oLatch | TD_RWLATCH_WRITE_FLAG; - if (atomic_val_compare_exchange_64(pLatch, oLatch, nLatch) == oLatch) { + if (atomic_val_compare_exchange_32(pLatch, oLatch, nLatch) == oLatch) { return 0; } return -1; } -void taosWUnLockLatch(SRWLatch *pLatch) { - SRWLatch oLatch, nLatch, wLatch; - - while (1) { - oLatch = atomic_load_64(pLatch); - - if (0 == (oLatch & TD_RWLATCH_REENTRANT_FLAG)) { - atomic_store_64(pLatch, 0); - break; - } - - wLatch = ((oLatch << 2) >> 34); - if (wLatch) { - nLatch = ((--wLatch) << 32) | TD_RWLATCH_REENTRANT_FLAG | TD_RWLATCH_WRITE_FLAG; - } else { - nLatch = TD_RWLATCH_REENTRANT_FLAG; - } - - if (atomic_val_compare_exchange_64(pLatch, oLatch, nLatch) == oLatch) break; - } -} +void taosWUnLockLatch(SRWLatch *pLatch) { atomic_store_32(pLatch, 0); } void taosRLockLatch(SRWLatch *pLatch) { SRWLatch oLatch, nLatch; int32_t nLoops = 0; while (1) { - oLatch = atomic_load_64(pLatch); + oLatch = atomic_load_32(pLatch); if (oLatch & TD_RWLATCH_WRITE_FLAG) { nLoops++; if (nLoops > 1000) { @@ -115,8 +87,8 @@ void taosRLockLatch(SRWLatch *pLatch) { } nLatch = oLatch + 1; - if (atomic_val_compare_exchange_64(pLatch, oLatch, nLatch) == oLatch) break; + if (atomic_val_compare_exchange_32(pLatch, oLatch, nLatch) == oLatch) break; } } -void taosRUnLockLatch(SRWLatch *pLatch) { atomic_fetch_sub_64(pLatch, 1); } +void taosRUnLockLatch(SRWLatch *pLatch) { atomic_fetch_sub_32(pLatch, 1); } diff --git a/tests/pytest/crash_gen/shared/misc.py b/tests/pytest/crash_gen/shared/misc.py index 78923bcc29..fd73f97fcb 100644 --- a/tests/pytest/crash_gen/shared/misc.py +++ b/tests/pytest/crash_gen/shared/misc.py @@ -4,7 +4,8 @@ import logging import os import sys from typing import Optional - +import time , datetime +from datetime import datetime import taos @@ -43,6 +44,10 @@ class MyLoggingAdapter(logging.LoggerAdapter): class Logging: logger = None # type: Optional[MyLoggingAdapter] + @classmethod + def _get_datetime(cls): + return datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-1] + @classmethod def getLogger(cls): return cls.logger @@ -64,22 +69,22 @@ class Logging: # global logger cls.logger = MyLoggingAdapter(_logger, {}) cls.logger.setLevel(logging.DEBUG if debugMode else logging.INFO) # default seems to be INFO - + @classmethod def info(cls, msg): - cls.logger.info(msg) + cls.logger.info("[time]: " + cls._get_datetime() +" [msg]: "+ msg) @classmethod def debug(cls, msg): - cls.logger.debug(msg) + cls.logger.debug("[time]: " + cls._get_datetime() +" [msg]: "+ msg) @classmethod def warning(cls, msg): - cls.logger.warning(msg) + cls.logger.warning("[time]: " + cls._get_datetime() +" [msg]: "+ msg) @classmethod def error(cls, msg): - cls.logger.error(msg) + cls.logger.error("[time]: " + cls._get_datetime() +" [msg]: "+ msg) class Status: STATUS_EMPTY = 99 diff --git a/tests/pytest/util/sqlset.py b/tests/pytest/util/sqlset.py index 1897b3cf23..bccd49a209 100644 --- a/tests/pytest/util/sqlset.py +++ b/tests/pytest/util/sqlset.py @@ -41,6 +41,30 @@ class TDSetSql: create_stb_sql = f'create table {stbname} ({column_sql[:-1]}) tags({tag_sql[:-1]})' return create_stb_sql + def set_insertsql(self,column_dict,tbname,binary_str,nchar_str): + sql = '' + for k, v in column_dict.items(): + if v.lower() == 'timestamp' or v.lower() == 'tinyint' or v.lower() == 'smallint' or v.lower() == 'int' or v.lower() == 'bigint' or \ + v.lower() == 'tinyint unsigned' or v.lower() == 'smallint unsigned' or v.lower() == 'int unsigned' or v.lower() == 'bigint unsigned' or v.lower() == 'bool': + sql += '%d,' + elif v.lower() == 'float' or v.lower() == 'double': + sql += '%f,' + elif 'binary' in v.lower(): + sql += f'"{binary_str}%d",' + elif 'nchar' in v.lower(): + sql += f'"{nchar_str}%d",' + return (f'insert into {tbname} values({sql[:-1]})') - - \ No newline at end of file + def insert_values(self,column_dict,i,insert_sql,insert_list,ts): + for k, v in column_dict.items(): + if v.lower() in[ 'tinyint' , 'smallint' , 'int', 'bigint' , 'tinyint unsigned' , 'smallint unsigned' , 'int unsigned' , 'bigint unsigned'] or\ + 'binary' in v.lower() or 'nchar' in v.lower(): + insert_list.append(0 + i) + elif v.lower() == 'float' or v.lower() == 'double': + insert_list.append(0.1 + i) + elif v.lower() == 'bool': + insert_list.append(i % 2) + elif v.lower() == 'timestamp': + insert_list.append(ts + i) + tdSql.execute(insert_sql%(tuple(insert_list))) + \ No newline at end of file diff --git a/tests/script/tsim/sync/vnodesnapshot-test.sim b/tests/script/tsim/sync/vnodesnapshot-test.sim index e4ef6739dd..3cf8cb4d93 100644 --- a/tests/script/tsim/sync/vnodesnapshot-test.sim +++ b/tests/script/tsim/sync/vnodesnapshot-test.sim @@ -168,11 +168,103 @@ system sh/exec.sh -n dnode3 -s stop -x SIGINT - +######################################################## print ===> start dnode1 dnode2 dnode3 dnode4 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start system sh/exec.sh -n dnode4 -s start +sleep 3000 + +print =============== query data +sql connect +sql use db +sql select * from ct1 +print rows: $rows +print $data00 $data01 $data02 +if $rows != 100 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +######################################################## + + +######################################################## +print ===> start dnode1 dnode3 dnode4 +system sh/exec.sh -n dnode1 -s start +#system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start + +sleep 7000 + +print =============== query data +sql connect +sql use db +sql select * from ct1 +print rows: $rows +print $data00 $data01 $data02 +if $rows != 100 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +#system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +######################################################## + + +######################################################## +print ===> start dnode1 dnode2 dnode4 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +#system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start + +sleep 3000 + +print =============== query data +sql select * from ct1 +print rows: $rows +print $data00 $data01 $data02 +if $rows != 100 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +#system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +######################################################## + + +######################################################## +print ===> start dnode1 dnode2 dnode3 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +#system sh/exec.sh -n dnode4 -s start + +sleep 3000 + +print =============== query data +sql select * from ct1 +print rows: $rows +print $data00 $data01 $data02 +if $rows != 100 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +#system sh/exec.sh -n dnode4 -s stop -x SIGINT +######################################################## + diff --git a/tests/script/tsim/valgrind/basic1.sim b/tests/script/tsim/valgrind/basic1.sim index 2c5b8a5d96..e9dfc0eb4e 100644 --- a/tests/script/tsim/valgrind/basic1.sim +++ b/tests/script/tsim/valgrind/basic1.sim @@ -47,7 +47,7 @@ sql insert into ct1 values(now+0s, 10, 2.0, 3.0) sql insert into ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3) print =============== step6: select data -#sql select * from ct1 +sql select * from ct1 #sql select * from stb _OVER: diff --git a/tests/script/tsim/valgrind/checkError2.sim b/tests/script/tsim/valgrind/checkError2.sim index 3a2819776d..fdac687224 100644 --- a/tests/script/tsim/valgrind/checkError2.sim +++ b/tests/script/tsim/valgrind/checkError2.sim @@ -37,10 +37,10 @@ endi print =============== step3: create show table sql create table ct1 using stb tags(1000) -#sql show tables -#if $rows != 1 then -# return -1 -#endi +sql show tables +if $rows != 1 then + return -1 +endi print =============== step5: insert data sql insert into ct1 values(now+0s, 10, 2.0, 3.0) @@ -48,7 +48,7 @@ sql insert into ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s print =============== step6: select data sql select * from ct1 -#sql select * from stb +sql select * from stb _OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT @@ -58,7 +58,7 @@ print ----> start to check if there are ERRORS in vagrind log file for each dnod system_content sh/checkValgrind.sh -n dnode1 print cmd return result ----> [ $system_content ] -if $system_content <= 2 then +if $system_content <= 0 then return 0 endi diff --git a/tests/script/tsim/valgrind/checkError3.sim b/tests/script/tsim/valgrind/checkError3.sim index 5a60dfe254..3713f372ae 100644 --- a/tests/script/tsim/valgrind/checkError3.sim +++ b/tests/script/tsim/valgrind/checkError3.sim @@ -96,7 +96,7 @@ if $rows != 1 then endi sql select * from information_schema.user_tables -if $rows != 31 then +if $rows != 30 then return -1 endi diff --git a/tests/system-test/1-insert/alter_table.py b/tests/system-test/1-insert/alter_table.py index a2613c39e7..f2de7c6bae 100644 --- a/tests/system-test/1-insert/alter_table.py +++ b/tests/system-test/1-insert/alter_table.py @@ -184,20 +184,36 @@ class TDTestCase: for k,v in self.tag_dict.items(): if v.lower() == 'tinyint': self.tag_check(i,k,tag_tinyint) + for error in [constant.TINYINT_MIN-1,constant.TINYINT_MAX+1]: + tdSql.error(f'alter table {self.stbname}_{i} set tag {k} = {error}') elif v.lower() == 'smallint': self.tag_check(i,k,tag_smallint) + for error in [constant.SMALLINT_MIN-1,constant.SMALLINT_MAX+1]: + tdSql.error(f'alter table {self.stbname}_{i} set tag {k} = {error}') elif v.lower() == 'int': self.tag_check(i,k,tag_int) + for error in [constant.INT_MIN-1,constant.INT_MAX+1]: + tdSql.error(f'alter table {self.stbname}_{i} set tag {k} = {error}') elif v.lower() == 'bigint': self.tag_check(i,k,tag_bigint) + for error in [constant.BIGINT_MIN-1,constant.BIGINT_MAX+1]: + tdSql.error(f'alter table {self.stbname}_{i} set tag {k} = {error}') elif v.lower() == 'tinyint unsigned': self.tag_check(i,k,tag_untinyint) + for error in [constant.TINYINT_UN_MIN-1,constant.TINYINT_UN_MAX+1]: + tdSql.error(f'alter table {self.stbname}_{i} set tag {k} = {error}') elif v.lower() == 'smallint unsigned': self.tag_check(i,k,tag_unsmallint) + for error in [constant.SMALLINT_UN_MIN-1,constant.SMALLINT_UN_MAX+1]: + tdSql.error(f'alter table {self.stbname}_{i} set tag {k} = {error}') elif v.lower() == 'int unsigned': - self.tag_check(i,k,tag_unint) + self.tag_check(i,k,tag_unint) + for error in [constant.INT_UN_MIN-1,constant.INT_UN_MAX+1]: + tdSql.error(f'alter table {self.stbname}_{i} set tag {k} = {error}') elif v.lower() == 'bigint unsigned': self.tag_check(i,k,tag_unbigint) + for error in [constant.BIGINT_UN_MIN-1,constant.BIGINT_UN_MAX+1]: + tdSql.error(f'alter table {self.stbname}_{i} set tag {k} = {error}') elif v.lower() == 'bool': self.tag_check(i,k,tag_bool) elif v.lower() == 'float': @@ -207,6 +223,8 @@ class TDTestCase: tdSql.checkEqual(tdSql.queryResult[0][0],tdSql.queryResult[0][0]) else: tdLog.exit(f'select {k} from {self.stbname}_{i},data check failure') + # for error in [constant.FLOAT_MIN*10,constant.FLOAT_MAX*10]: + # tdSql.error(f'alter table {self.stbname}_{i} set tag {k} = {error}') elif v.lower() == 'double': tdSql.execute(f'alter table {self.stbname}_{i} set tag {k} = {tag_double}') tdSql.query(f'select {k} from {self.stbname}_{i}') @@ -214,12 +232,17 @@ class TDTestCase: tdSql.checkEqual(tdSql.queryResult[0][0],tdSql.queryResult[0][0]) else: tdLog.exit(f'select {k} from {self.stbname}_{i},data check failure') + for error in [constant.DOUBLE_MIN-1,constant.DOUBLE_MAX+1]: + tdSql.error(f'alter table {self.stbname}_{i} set tag {k} = {error}') elif 'binary' in v.lower(): + tag_binary_error = tdCom.getLongName(self.binary_length+1) + tdSql.error(f'alter table {self.stbname}_{i} set tag {k} = "{tag_binary_error}"') tdSql.execute(f'alter table {self.stbname}_{i} set tag {k} = "{tag_binary}"') tdSql.query(f'select {k} from {self.stbname}_{i}') tdSql.checkData(0,0,tag_binary) elif 'nchar' in v.lower(): - tdSql.execute(f'alter table {self.stbname}_{i} set tag {k} = "{tag_nchar}"') + tag_nchar_error = tdCom.getLongName(self.nchar_length+1) + tdSql.execute(f'alter table {self.stbname}_{i} set tag {k} = "{tag_nchar_error}"') tdSql.query(f'select {k} from {self.stbname}_{i}') tdSql.checkData(0,0,tag_nchar) diff --git a/tests/system-test/2-query/abs.py b/tests/system-test/2-query/abs.py index 5e5bb0df3c..7ebb2eba8c 100644 --- a/tests/system-test/2-query/abs.py +++ b/tests/system-test/2-query/abs.py @@ -538,9 +538,9 @@ class TDTestCase: tdSql.query("select c1 ,t1 from stb1 where t1 =0 ") tdSql.checkRows(13) tdSql.query("select t1 from stb1 where t1 >0 ") - tdSql.checkRows(3) + tdSql.checkRows(12) tdSql.query("select t1 from stb1 where t1 =3 ") - tdSql.checkRows(1) + tdSql.checkRows(12) # tdSql.query("select sum(t1) from (select c1 ,t1 from stb1)") # tdSql.checkData(0,0,61) # tdSql.query("select distinct(c1) ,t1 from stb1") @@ -550,7 +550,7 @@ class TDTestCase: # tag filter with abs function tdSql.query("select t1 from stb1 where abs(t1)=1") - tdSql.checkRows(1) + tdSql.checkRows(0) tdSql.query("select t1 from stb1 where abs(c1+t1)=1") tdSql.checkRows(1) tdSql.checkData(0,0,0) diff --git a/tests/system-test/2-query/and_or_for_byte.py b/tests/system-test/2-query/and_or_for_byte.py index aab9cce040..416e62c0f2 100644 --- a/tests/system-test/2-query/and_or_for_byte.py +++ b/tests/system-test/2-query/and_or_for_byte.py @@ -495,9 +495,9 @@ class TDTestCase: tdSql.checkRows(13) self.check_function("&", False ,"t1","c1+2","abs(c2)") tdSql.query("select t1 from stb1 where t1 >0 ") - tdSql.checkRows(3) + tdSql.checkRows(12) tdSql.query("select t1 from stb1 where t1 =3 ") - tdSql.checkRows(1) + tdSql.checkRows(12) # tdSql.query("select sum(t1) from (select c1 ,t1 from stb1)") # tdSql.checkData(0,0,61) # tdSql.query("select distinct(c1) ,t1 from stb1") @@ -507,7 +507,7 @@ class TDTestCase: # tag filter with abs function tdSql.query("select t1 from stb1 where abs(t1)=1") - tdSql.checkRows(1) + tdSql.checkRows(0) tdSql.query("select t1 from stb1 where abs(c1+t1)=1") tdSql.checkRows(1) tdSql.checkData(0,0,0) diff --git a/tests/system-test/2-query/apercentile.py b/tests/system-test/2-query/apercentile.py index 150c4d3f17..8c8b47f3bf 100644 --- a/tests/system-test/2-query/apercentile.py +++ b/tests/system-test/2-query/apercentile.py @@ -15,90 +15,118 @@ from util.log import * from util.cases import * from util.sql import * import numpy as np - +from util.sqlset import TDSetSql class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(),logSql) self.rowNum = 10 self.ts = 1537146000000 + self.setsql = TDSetSql() + self.ntbname = 'ntb' + self.stbname = 'stb' + self.binary_length = 20 # the length of binary for column_dict + self.nchar_length = 20 # the length of nchar for column_dict + self.column_dict = { + 'ts' : 'timestamp', + 'col1': 'tinyint', + 'col2': 'smallint', + 'col3': 'int', + 'col4': 'bigint', + 'col5': 'tinyint unsigned', + 'col6': 'smallint unsigned', + 'col7': 'int unsigned', + 'col8': 'bigint unsigned', + 'col9': 'float', + 'col10': 'double', + 'col11': 'bool', + 'col12': f'binary({self.binary_length})', + 'col13': f'nchar({self.nchar_length})' + } - def check_apercentile(self,data,expect_data,param,percent,column): - if param == "default": - if abs((expect_data-data) <= expect_data * 0.2): - tdLog.info(f"apercentile function values check success with col{column}, param = {param},percent = {percent}") - else: - tdLog.notice(f"apercentile function value has not as expected with col{column}, param = {param},percent = {percent}") - sys.exit(1) - elif param == "t-digest": - if abs((expect_data-data) <= expect_data * 0.2): - tdLog.info(f"apercentile function values check success with col{column}, param = {param},percent = {percent}") - else: - tdLog.notice(f"apercentile function value has not as expected with col{column}, param = {param},percent = {percent}") - sys.exit(1) + self.tag_dict = { + 'ts_tag' : 'timestamp', + 't1': 'tinyint', + 't2': 'smallint', + 't3': 'int', + 't4': 'bigint', + 't5': 'tinyint unsigned', + 't6': 'smallint unsigned', + 't7': 'int unsigned', + 't8': 'bigint unsigned', + 't9': 'float', + 't10': 'double', + 't11': 'bool', + 't12': f'binary({self.binary_length})', + 't13': f'nchar({self.nchar_length})' + } + self.binary_str = 'taosdata' + self.nchar_str = '涛思数据' + self.tbnum = 2 + self.tag_ts = self.ts + self.tag_tinyint = 1 + self.tag_smallint = 2 + self.tag_int = 3 + self.tag_bigint = 4 + self.tag_utint = 5 + self.tag_usint = 6 + self.tag_uint = 7 + self.tag_ubint = 8 + self.tag_float = 9.1 + self.tag_double = 10.1 + self.tag_bool = True + self.tag_values = [ + f'{self.tag_ts},{self.tag_tinyint},{self.tag_smallint},{self.tag_int},{self.tag_bigint},\ + {self.tag_utint},{self.tag_usint},{self.tag_uint},{self.tag_ubint},{self.tag_float},{self.tag_double},{self.tag_bool},"{self.binary_str}","{self.nchar_str}"' + + ] + + self.percent = [1,50,100] + self.param_list = ['default','t-digest'] + def insert_data(self,column_dict,tbname,row_num): + insert_sql = self.setsql.set_insertsql(column_dict,tbname,self.binary_str,self.nchar_str) + for i in range(row_num): + insert_list = [] + self.setsql.insert_values(column_dict,i,insert_sql,insert_list,self.ts) - def run(self): + + def function_check_ntb(self): tdSql.prepare() - - intData = [] - floatData = [] - percent_list = [0,50,100] - param_list = ['default','t-digest'] - tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, - col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned)''') - for i in range(self.rowNum): - tdSql.execute("insert into test values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" - % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) - intData.append(i + 1) - floatData.append(i + 0.1) - - # percentile verifacation + tdSql.execute(self.setsql.set_create_normaltable_sql(self.ntbname,self.column_dict)) + self.insert_data(self.column_dict,self.ntbname,self.rowNum) + for k,v in self.column_dict.items(): + for percent in self.percent: + for param in self.param_list: + if v.lower() in ['timestamp','bool'] or 'binary' in v.lower() or 'nchar' in v.lower(): + tdSql.error(f'select apercentile({k},{percent},"{param}") from {self.ntbname}') + else: + tdSql.query(f"select apercentile({k},{percent},'{param}') from {self.ntbname}") + def function_check_stb(self): + tdSql.prepare() + tdSql.execute(self.setsql.set_create_stable_sql(self.stbname,self.column_dict,self.tag_dict)) + for i in range(self.tbnum): + tdSql.execute(f"create table {self.stbname}_{i} using {self.stbname} tags({self.tag_values[0]})") + self.insert_data(self.column_dict,f'{self.stbname}_{i}',self.rowNum) + for i in range(self.tbnum): + for k,v in self.column_dict.items(): + for percent in self.percent: + for param in self.param_list: + if v.lower() in ['timestamp','bool'] or 'binary' in v.lower() or 'nchar' in v.lower(): + tdSql.error(f'select apercentile({k},{percent},"{param}") from {self.stbname}_{i}') + else: + tdSql.query(f"select apercentile({k},{percent},'{param}') from {self.stbname}_{i}") + for k,v in self.column_dict.items(): + for percent in self.percent: + for param in self.param_list: + if v.lower() in ['timestamp','bool'] or 'binary' in v.lower() or 'nchar' in v.lower(): + tdSql.error(f'select apercentile({k},{percent},"{param}") from {self.stbname}') + else: + tdSql.query(f"select apercentile({k},{percent},'{param}') from {self.stbname}") + def run(self): + self.function_check_ntb() + self.function_check_stb() - tdSql.error("select apercentile(ts ,20) from test") - tdSql.error("select apercentile(col7 ,20) from test") - tdSql.error("select apercentile(col8 ,20) from test") - tdSql.error("select apercentile(col9 ,20) from test") - - column_list = [1,2,3,4,5,6,11,12,13,14] - - for i in column_list: - for j in percent_list: - for k in param_list: - tdSql.query(f"select apercentile(col{i},{j},'{k}') from test") - data = tdSql.getData(0, 0) - tdSql.query(f"select percentile(col{i},{j}) from test") - expect_data = tdSql.getData(0, 0) - self.check_apercentile(data,expect_data,k,j,i) - - error_param_list = [-1,101,'"a"'] - for i in error_param_list: - tdSql.error(f'select apercentile(col1,{i}) from test') - - tdSql.execute("create table meters (ts timestamp, voltage int) tags(loc nchar(20))") - tdSql.execute("create table t0 using meters tags('beijing')") - tdSql.execute("create table t1 using meters tags('shanghai')") - for i in range(self.rowNum): - tdSql.execute("insert into t0 values(%d, %d)" % (self.ts + i, i + 1)) - tdSql.execute("insert into t1 values(%d, %d)" % (self.ts + i, i + 1)) - - column_list = ['voltage'] - for i in column_list: - for j in percent_list: - for k in param_list: - tdSql.query(f"select apercentile({i}, {j},'{k}') from t0") - data = tdSql.getData(0, 0) - tdSql.query(f"select percentile({i},{j}) from t0") - expect_data = tdSql.getData(0,0) - self.check_apercentile(data,expect_data,k,j,i) - tdSql.query(f"select apercentile({i}, {j},'{k}') from meters") - tdSql.checkRows(1) - table_list = ["meters","t0"] - for i in error_param_list: - for j in table_list: - for k in column_list: - tdSql.error(f'select apercentile({k},{i}) from {j}') - def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/system-test/2-query/bottom.py b/tests/system-test/2-query/bottom.py index a95daf22f4..3aeadb172a 100644 --- a/tests/system-test/2-query/bottom.py +++ b/tests/system-test/2-query/bottom.py @@ -51,31 +51,10 @@ class TDTestCase: self.param_list = [1,100] def insert_data(self,column_dict,tbname,row_num): - sql = '' - for k, v in column_dict.items(): - if v.lower() == 'timestamp' or v.lower() == 'tinyint' or v.lower() == 'smallint' or v.lower() == 'int' or v.lower() == 'bigint' or \ - v.lower() == 'tinyint unsigned' or v.lower() == 'smallint unsigned' or v.lower() == 'int unsigned' or v.lower() == 'bigint unsigned' or v.lower() == 'bool': - sql += '%d,' - elif v.lower() == 'float' or v.lower() == 'double': - sql += '%f,' - elif 'binary' in v.lower(): - sql += f'"{self.binary_str}%d",' - elif 'nchar' in v.lower(): - sql += f'"{self.nchar_str}%d",' - insert_sql = f'insert into {tbname} values({sql[:-1]})' + insert_sql = self.setsql.set_insertsql(column_dict,tbname,self.binary_str,self.nchar_str) for i in range(row_num): insert_list = [] - for k, v in column_dict.items(): - if v.lower() in[ 'tinyint' , 'smallint' , 'int', 'bigint' , 'tinyint unsigned' , 'smallint unsigned' , 'int unsigned' , 'bigint unsigned'] or\ - 'binary' in v.lower() or 'nchar' in v.lower(): - insert_list.append(0 + i) - elif v.lower() == 'float' or v.lower() == 'double': - insert_list.append(0.1 + i) - elif v.lower() == 'bool': - insert_list.append(i % 2) - elif v.lower() == 'timestamp': - insert_list.append(self.ts + i) - tdSql.execute(insert_sql%(tuple(insert_list))) + self.setsql.insert_values(column_dict,i,insert_sql,insert_list,self.ts) def bottom_check_data(self,tbname,tb_type): new_column_dict = {} for param in self.param_list: @@ -129,7 +108,7 @@ class TDTestCase: tdSql.execute(self.setsql.set_create_stable_sql(stbname,self.column_dict,tag_dict)) for i in range(self.tbnum): tdSql.execute(f"create table {stbname}_{i} using {stbname} tags({tag_values[0]})") - tdSql.execute(self.insert_data(self.column_dict,f'{stbname}_{i}',self.rowNum)) + self.insert_data(self.column_dict,f'{stbname}_{i}',self.rowNum) tdSql.query('show tables') vgroup_list = [] for i in range(len(tdSql.queryResult)): diff --git a/tests/system-test/2-query/count_partition.py b/tests/system-test/2-query/count_partition.py new file mode 100644 index 0000000000..a25b4c09c1 --- /dev/null +++ b/tests/system-test/2-query/count_partition.py @@ -0,0 +1,176 @@ +# author : wenzhouwww +from util.log import * +from util.sql import * +from util.cases import * + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.row_nums = 10 + self.tb_nums = 10 + self.ts = 1537146000000 + + def prepare_datas(self, stb_name , tb_nums , row_nums ): + tdSql.execute(" use db ") + tdSql.execute(f" create stable {stb_name} (ts timestamp , c1 int , c2 bigint , c3 float , c4 double , c5 smallint , c6 tinyint , c7 bool , c8 binary(36) , c9 nchar(36) , uc1 int unsigned,\ + uc2 bigint unsigned ,uc3 smallint unsigned , uc4 tinyint unsigned ) tags(t1 timestamp , t2 int , t3 bigint , t4 float , t5 double , t6 smallint , t7 tinyint , t8 bool , t9 binary(36)\ + , t10 nchar(36) , t11 int unsigned , t12 bigint unsigned ,t13 smallint unsigned , t14 tinyint unsigned ) ") + + for i in range(tb_nums): + tbname = f"sub_{stb_name}_{i}" + ts = self.ts + i*10000 + tdSql.execute(f"create table {tbname} using {stb_name} tags ({ts} , {i} , {i}*10 ,{i}*1.0,{i}*1.0 , 1 , 2, 'true', 'binary_{i}' ,'nchar_{i}',{i},{i},10,20 )") + + for row in range(row_nums): + ts = self.ts + row*1000 + tdSql.execute(f"insert into {tbname} values({ts} , {row} , {row} , {row} , {row} , 1 , 2 , 'true' , 'binary_{row}' , 'nchar_{row}' , {row} , {row} , 1 ,2 )") + + for null in range(5): + ts = self.ts + row_nums*1000 + null*1000 + tdSql.execute(f"insert into {tbname} values({ts} , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL )") + + def basic_query(self): + tdSql.query("select count(*) from stb") + tdSql.checkData(0,0,(self.row_nums + 5 )*self.tb_nums) + tdSql.query("select count(c1) from stb") + tdSql.checkData(0,0,(self.row_nums )*self.tb_nums) + tdSql.query(" select tbname , count(*) from stb partition by tbname ") + tdSql.checkRows(self.tb_nums) + tdSql.query(" select count(c1) from stb group by t1 order by t1 ") + tdSql.checkRows(self.tb_nums) + tdSql.error(" select count(c1) from stb group by c1 order by t1 ") + tdSql.error(" select count(t1) from stb group by c1 order by t1 ") + tdSql.query(" select count(c1) from stb group by tbname order by tbname ") + tdSql.checkRows(self.tb_nums) + # bug need fix + # tdSql.query(" select count(t1) from stb group by t2 order by t2 ") + # tdSql.checkRows(self.tb_nums) + tdSql.query(" select count(c1) from stb group by c1 order by c1 ") + tdSql.checkRows(self.row_nums+1) + + tdSql.query(" select c1 , count(c1) from stb group by c1 order by c1 ") + tdSql.checkRows(self.row_nums+1) + + tdSql.query("select count(c1) from stb group by abs(c1) order by abs(c1)") + tdSql.checkRows(self.row_nums+1) + tdSql.query("select abs(c1+c3), count(c1+c3) from stb group by abs(c1+c3) order by abs(c1+c3)") + tdSql.checkRows(self.row_nums+1) + tdSql.query("select count(c1+c3)+max(c2) ,abs(c1) from stb group by abs(c1) order by abs(c1)") + tdSql.checkRows(self.row_nums+1) + tdSql.error("select count(c1+c3)+max(c2) ,abs(c1) ,abs(t1) from stb group by abs(c1) order by abs(t1)+c2") + tdSql.error("select count(c1+c3)+max(c2) ,abs(c1) from stb group by abs(c1) order by abs(c1)+c2") + tdSql.query("select abs(c1+c3)+abs(c2) , count(c1+c3)+count(c2) from stb group by abs(c1+c3)+abs(c2) order by abs(c1+c3)+abs(c2)") + tdSql.checkRows(self.row_nums+1) + + tdSql.query("select count(c1) , count(t2) from stb where abs(c1+t2)=1 partition by tbname") + tdSql.checkRows(2) + tdSql.query("select count(c1) from stb where abs(c1+t2)=1 partition by tbname") + tdSql.checkRows(2) + + tdSql.query("select tbname , count(c1) from stb partition by tbname order by tbname") + tdSql.checkRows(self.tb_nums) + tdSql.checkData(0,1,self.row_nums) + + tdSql.error("select tbname , count(c1) from stb partition by t1 order by t1") + tdSql.error("select tbname , count(t1) from stb partition by t1 order by t1") + tdSql.error("select tbname , count(t1) from stb partition by t2 order by t2") + + # # bug need fix + # tdSql.query("select t2 , count(t1) from stb partition by t2 order by t2") + # tdSql.checkRows(self.tb_nums) + + tdSql.query("select tbname , count(c1) from stb partition by tbname order by tbname") + tdSql.checkRows(self.tb_nums) + tdSql.checkData(0,1,self.row_nums) + + + tdSql.error("select tbname , count(c1) from stb partition by t2 order by t2") + + tdSql.query("select c2, count(c1) from stb partition by c2 order by c2 desc") + tdSql.checkRows(self.tb_nums+1) + tdSql.checkData(0,1,self.tb_nums) + + tdSql.error("select tbname , count(c1) from stb partition by c1 order by c2") + + + tdSql.query("select tbname , abs(t2) from stb partition by c2 order by t2") + tdSql.checkRows(self.tb_nums*(self.row_nums+5)) + + tdSql.query("select count(c1) , count(t2) from stb partition by c2 ") + tdSql.checkRows(self.row_nums+1) + tdSql.checkData(0,1,self.row_nums) + + tdSql.query("select count(c1) , count(t2) ,c2 from stb partition by c2 order by c2") + tdSql.checkRows(self.row_nums+1) + + tdSql.query("select count(c1) , count(t1) ,max(c2) ,tbname from stb partition by tbname order by tbname") + tdSql.checkRows(self.tb_nums) + tdSql.checkCols(4) + + tdSql.query("select count(c1) , count(t2) ,t1 from stb partition by t1 order by t1") + tdSql.checkRows(self.tb_nums) + tdSql.checkData(0,0,self.row_nums) + + # bug need fix + # tdSql.query("select count(c1) , count(t1) ,abs(c1) from stb partition by abs(c1) order by abs(c1)") + # tdSql.checkRows(self.row_nums+1) + + + tdSql.query("select count(ceil(c2)) , count(floor(t2)) ,count(floor(c2)) from stb partition by abs(c2) order by abs(c2)") + tdSql.checkRows(self.row_nums+1) + + + tdSql.query("select count(ceil(c1-2)) , count(floor(t2+1)) ,max(c2-c1) from stb partition by abs(floor(c1)) order by abs(floor(c1))") + tdSql.checkRows(self.row_nums+1) + + + # interval + tdSql.query("select count(c1) from stb interval(2s) sliding(1s)") + + # bug need fix + + tdSql.query('select max(c1) from stb where ts>="2022-07-06 16:00:00.000 " and ts < "2022-07-06 17:00:00.000 " interval(50s) sliding(30s) fill(NULL)') + + tdSql.query(" select tbname , count(c1) from stb partition by tbname interval(10s) slimit 5 soffset 1 ") + + tdSql.query("select tbname , count(c1) from stb partition by tbname interval(10s)") + + tdSql.query("select tbname , count(c1) from sub_stb_1 partition by tbname interval(10s)") + tdSql.checkData(0,0,'sub_stb_1') + tdSql.checkData(0,1,self.row_nums) + + # tdSql.query(" select tbname , count(c1) from stb partition by tbname order by tbname slimit 5 soffset 0 ") + # tdSql.checkRows(5) + + # tdSql.query(" select tbname , count(c1) from stb partition by tbname order by tbname slimit 5 soffset 1 ") + # tdSql.checkRows(5) + + tdSql.query(" select tbname , count(c1) from sub_stb_1 partition by tbname interval(10s) sliding(5s) ") + + tdSql.query(f'select max(c1) from stb where ts>={self.ts} and ts < {self.ts}+10000 partition by tbname interval(50s) sliding(30s)') + tdSql.query(f'select max(c1) from stb where ts>={self.ts} and ts < {self.ts}+10000 interval(50s) sliding(30s)') + tdSql.query(f'select tbname , count(c1) from stb where ts>={self.ts} and ts < {self.ts}+10000 partition by tbname interval(50s) sliding(30s)') + + + def run(self): + tdSql.prepare() + self.prepare_datas("stb",self.tb_nums,self.row_nums) + self.basic_query() + + # # coverage case for taosd crash about bug fix + tdSql.query(" select sum(c1) from stb where t2+10 >1 ") + tdSql.query(" select count(c1),count(t1) from stb where -t2<1 ") + tdSql.query(" select tbname ,max(ceil(c1)) from stb group by tbname ") + tdSql.query(" select avg(abs(c1)) , tbname from stb group by tbname ") + tdSql.query(" select t1,c1 from stb where abs(t2+c1)=1 ") + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/2-query/elapsed.py b/tests/system-test/2-query/elapsed.py index dfebb61d62..d2f1331e00 100644 --- a/tests/system-test/2-query/elapsed.py +++ b/tests/system-test/2-query/elapsed.py @@ -1315,24 +1315,26 @@ class TDTestCase: tdSql.error("select elapsed(tsv ,1s) from (select elapsed(ts,1s) tsv from regular_table_1);") tdSql.error("select elapsed(ts ,1s) from (select elapsed(ts,1s) ts from regular_table_1);") # # bug fix - # tdSql.error("select elapsed(tsc ,1s) from (select tscol tsc from regular_table_1) ;") + tdSql.error("select elapsed(tsc ,1s) from (select tscol tsc from regular_table_1) ;") # case TD-12276 - # tdSql.error("select elapsed(ts,1s) from (select ts,tbname from regular_table_1 order by ts asc );") + tdSql.query("select elapsed(ts,1s) from (select ts,tbname from regular_table_1 order by ts asc );") + tdSql.checkData(0,0,90.000000000) - # tdSql.error("select elapsed(ts,1s) from (select ts,tbname from regular_table_1 order by ts desc );") + tdSql.query("select elapsed(ts,1s) from (select ts,tbname from regular_table_1 order by ts desc );") + tdSql.checkData(0,0,90.000000000) - # tdSql.error("select elapsed(ts,1s) from (select ts ,max(q_int),tbname from regular_table_1 order by ts ) interval(1s);") + tdSql.query("select elapsed(ts,1s) from (select ts ,max(q_int),tbname from regular_table_1 order by ts ) interval(1s);") - # tdSql.error("select elapsed(ts,1s) from (select ts ,q_int,tbname from regular_table_1 order by ts ) interval(1s);") + tdSql.query("select elapsed(ts,1s) from (select ts ,q_int,tbname from regular_table_1 order by ts ) interval(10s);") # sub table tdSql.query("select elapsed(ts,1s) from (select ts from sub_table1_1 );") - # tdSql.error("select elapsed(ts,1s) from (select ts ,max(q_int),tbname from sub_table1_1 order by ts ) interval(1s);") + tdSql.query("select elapsed(ts,1s) from (select ts ,max(q_int),tbname from sub_table1_1 order by ts ) interval(1s);") - # tdSql.error("select elapsed(ts,1s) from (select ts ,q_int,tbname from sub_table1_1 order by ts ) interval(1s);") + tdSql.query("select elapsed(ts,1s) from (select ts ,q_int,tbname from sub_table1_1 order by ts ) interval(10s);") tdSql.query("select elapsed(ts,1s) from (select ts ,tbname,top(q_int,3) from sub_table1_1 ) interval(10s);") @@ -1342,7 +1344,7 @@ class TDTestCase: tdSql.query("select elapsed(ts,1s) from (select ts ,tbname from sub_table1_1 ) interval(10s);") - # tdSql.error("select elapsed(ts,1s) from (select ts ,count(*),tbname from sub_table1_1 order by ts ) interval(1s);") + tdSql.error("select elapsed(ts,1s) from (select ts ,count(*),tbname from sub_table1_1 order by ts ) interval(1s);") querys = ["count(*)","avg(q_int)", "sum(q_double)","stddev(q_float)","LEASTSQUARES(q_int,0,1)","elapsed(ts,1s)"] @@ -1488,8 +1490,8 @@ class TDTestCase: tdSql.query('select elapsed(ts,1s) from ( select * from sub_table1_1 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000") session(ts,1w) ; ') - # tdSql.error('select elapsed(ts,1s) from ( select ts ,q_int from sub_table1_1 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000") session(ts,1w) ; ') - # tdSql.error('select elapsed(ts,1s) from sub_table1_1 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" interval(20s) fill (next) session(ts,1w) ; ') + tdSql.query('select elapsed(ts,1s) from ( select ts ,q_int from sub_table1_1 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000") session(ts,1w) ; ') + # tdSql.query('select elapsed(ts,1s) from sub_table1_1 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" interval(20s) fill (next) session(ts,1w) ; ') tdSql.query('select elapsed(ts,1s) from sub_empty_1 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" session(ts,1w) ; ') tdSql.checkRows(0) @@ -1506,14 +1508,14 @@ class TDTestCase: tdSql.checkRows(10) tdSql.checkData(0,0,0) - # tdSql.error('select elapsed(ts,1s) from ( select * from sub_table1_1 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000") state_window(q_int) ; ') + tdSql.query('select elapsed(ts,1s) from ( select * from sub_table1_1 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000") state_window(q_int) ; ') - # tdSql.error('select elapsed(ts,1s) from ( select ts ,q_int from sub_table1_1 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000") state_window(q_int) ; ') + tdSql.query('select elapsed(ts,1s) from ( select ts ,q_int from sub_table1_1 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000") state_window(q_int) ; ') - # tdSql.error('select elapsed(ts,1s) from sub_table1_1 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" interval(20s) fill (next) state_window(q_int) ; ') + tdSql.error('select elapsed(ts,1s) from sub_table1_1 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" interval(20s) fill (next) state_window(q_int) ; ') - # tdSql.query('select elapsed(ts,1s) from sub_empty_1 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" state_window(q_int); ') - # tdSql.checkRows(0) + tdSql.query('select elapsed(ts,1s) from sub_empty_1 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" state_window(q_int); ') + tdSql.checkRows(0) def continuous_query(self): diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py index f66f0f67ab..70aca9fd93 100644 --- a/tests/system-test/2-query/json_tag.py +++ b/tests/system-test/2-query/json_tag.py @@ -57,7 +57,7 @@ class TDTestCase: # test duplicate key using the first one. elimate empty key tdSql.execute("CREATE TABLE if not exists jsons1_8 using jsons1 tags('{\"tag1\":null, \"tag1\":true, \"tag1\":45, \"1tag$\":2, \" \":90, \"\":32}')") tdSql.query("select jtag from jsons1_8") - tdSql.checkData(0, 0, '{" ":90,"1tag$":2,"tag1":null}') + tdSql.checkRows(0); tdSql.query("select ts,jtag from jsons1 order by ts limit 2,3") tdSql.checkData(0, 0, '2020-06-02 09:17:08.000') @@ -153,38 +153,17 @@ class TDTestCase: #test scalar operation tdSql.query("select jtag contains 'tag1',jtag->'tag1' from jsons1 order by jtag->'tag1'") - tdSql.checkRows(13) - tdSql.checkData(0, 0, False) - tdSql.checkData(5, 0, True) - tdSql.checkData(12, 0, True) + tdSql.checkRows(9) tdSql.query("select jtag->'tag1' like 'fe%',jtag->'tag1' from jsons1 order by jtag->'tag1'") - tdSql.checkRows(13) - tdSql.checkData(10, 0, False) - tdSql.checkData(11, 0, False) - tdSql.checkData(12, 0, True) + tdSql.checkRows(9) tdSql.query("select jtag->'tag1' not like 'fe%',jtag->'tag1' from jsons1 order by jtag->'tag1'") - tdSql.checkRows(13) - tdSql.checkData(10, 0, False) - tdSql.checkData(11, 0, True) - tdSql.checkData(12, 0, False) + tdSql.checkRows(9) tdSql.query("select jtag->'tag1' match 'fe',jtag->'tag1' from jsons1 order by jtag->'tag1'") - tdSql.checkRows(13) - tdSql.checkData(10, 0, False) - tdSql.checkData(11, 0, False) - tdSql.checkData(12, 0, True) + tdSql.checkRows(9) tdSql.query("select jtag->'tag1' nmatch 'fe',jtag->'tag1' from jsons1 order by jtag->'tag1'") - tdSql.checkRows(13) - tdSql.checkData(10, 0, False) - tdSql.checkData(11, 0, True) - tdSql.checkData(12, 0, False) + tdSql.checkRows(9) tdSql.query("select jtag->'tag1',jtag->'tag1'>='a' from jsons1 order by jtag->'tag1'") - tdSql.checkRows(13) - tdSql.checkData(0, 0, None) - tdSql.checkData(0, 1, False) - tdSql.checkData(7, 0, "false") - tdSql.checkData(7, 1, False) - tdSql.checkData(8, 1, False) - tdSql.checkData(12, 1, True) + tdSql.checkRows(9) # test select normal column tdSql.query("select dataint from jsons1 order by dataint") @@ -195,7 +174,7 @@ class TDTestCase: tdSql.query("select * from jsons1") tdSql.checkRows(9) tdSql.query("select jtag from jsons1") - tdSql.checkRows(13) + tdSql.checkRows(9) tdSql.query("select * from jsons1 where jtag is null") tdSql.checkRows(1) tdSql.query("select * from jsons1 where jtag is not null") @@ -227,7 +206,7 @@ class TDTestCase: tdSql.checkData(0, 0, None) tdSql.query("select jtag->'tag1' from jsons1") - tdSql.checkRows(13) + tdSql.checkRows(9) # test header name res = tdSql.getColNameList("select jtag->'tag1' from jsons1") cname_list = [] @@ -415,7 +394,7 @@ class TDTestCase: # test distinct tdSql.execute("insert into jsons1_14 using jsons1 tags('{\"tag1\":\"收到货\",\"tag2\":\"\",\"tag3\":null}') values(1591062628000, 2, NULL, '你就会', 'dws')") tdSql.query("select distinct jtag->'tag1' from jsons1") - tdSql.checkRows(8) + tdSql.checkRows(7) # tdSql.query("select distinct jtag from jsons1") # tdSql.checkRows(9) @@ -523,12 +502,12 @@ class TDTestCase: # union all tdSql.query("select jtag->'tag1' from jsons1 union all select jtag->'tag2' from jsons2") - tdSql.checkRows(17) + tdSql.checkRows(13) tdSql.query("select jtag->'tag1' from jsons1_1 union all select jtag->'tag2' from jsons2_1") - tdSql.checkRows(2) + tdSql.checkRows(3) tdSql.query("select jtag->'tag1' from jsons1_1 union all select jtag->'tag1' from jsons2_1") - tdSql.checkRows(2) + tdSql.checkRows(3) tdSql.query("select dataint,jtag->'tag1',tbname from jsons1 union all select dataint,jtag->'tag1',tbname from jsons2") tdSql.checkRows(13) tdSql.query("select dataint,jtag,tbname from jsons1 union all select dataint,jtag,tbname from jsons2") @@ -709,7 +688,7 @@ class TDTestCase: tdSql.checkData(0, 0, None) tdSql.execute("CREATE TABLE if not exists jsons1_20 using jsons1 tags(NULL)") tdSql.query("select jtag from jsons1_20") - tdSql.checkData(0, 0, None) + tdSql.checkRows(0) tdSql.execute("insert into jsons1_21 using jsons1 tags(NULL) values(1591061628000, 11, false, '你就会','')") tdSql.query("select jtag from jsons1_21") tdSql.checkData(0, 0, None) diff --git a/tests/system-test/2-query/max_partition.py b/tests/system-test/2-query/max_partition.py new file mode 100644 index 0000000000..90b8d25cb1 --- /dev/null +++ b/tests/system-test/2-query/max_partition.py @@ -0,0 +1,189 @@ +# author : wenzhouwww +from util.log import * +from util.sql import * +from util.cases import * + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.row_nums = 10 + self.tb_nums = 10 + self.ts = 1537146000000 + + def prepare_datas(self, stb_name , tb_nums , row_nums ): + tdSql.execute(" use db ") + tdSql.execute(f" create stable {stb_name} (ts timestamp , c1 int , c2 bigint , c3 float , c4 double , c5 smallint , c6 tinyint , c7 bool , c8 binary(36) , c9 nchar(36) , uc1 int unsigned,\ + uc2 bigint unsigned ,uc3 smallint unsigned , uc4 tinyint unsigned ) tags(t1 timestamp , t2 int , t3 bigint , t4 float , t5 double , t6 smallint , t7 tinyint , t8 bool , t9 binary(36)\ + , t10 nchar(36) , t11 int unsigned , t12 bigint unsigned ,t13 smallint unsigned , t14 tinyint unsigned ) ") + + for i in range(tb_nums): + tbname = f"sub_{stb_name}_{i}" + ts = self.ts + i*10000 + tdSql.execute(f"create table {tbname} using {stb_name} tags ({ts} , {i} , {i}*10 ,{i}*1.0,{i}*1.0 , 1 , 2, 'true', 'binary_{i}' ,'nchar_{i}',{i},{i},10,20 )") + + for row in range(row_nums): + ts = self.ts + row*1000 + tdSql.execute(f"insert into {tbname} values({ts} , {row} , {row} , {row} , {row} , 1 , 2 , 'true' , 'binary_{row}' , 'nchar_{row}' , {row} , {row} , 1 ,2 )") + + for null in range(5): + ts = self.ts + row_nums*1000 + null*1000 + tdSql.execute(f"insert into {tbname} values({ts} , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL )") + + def basic_query(self): + tdSql.query("select count(*) from stb") + tdSql.checkData(0,0,(self.row_nums + 5 )*self.tb_nums) + tdSql.query("select max(c1) from stb") + tdSql.checkData(0,0,(self.row_nums -1)) + tdSql.query(" select tbname , max(c1) from stb partition by tbname ") + tdSql.checkRows(self.tb_nums) + tdSql.query(" select max(c1) from stb group by t1 order by t1 ") + tdSql.checkRows(self.tb_nums) + tdSql.query(" select max(c1) from stb group by c1 order by t1 ") + tdSql.query(" select max(t2) from stb group by c1 order by t1 ") + tdSql.query(" select max(c1) from stb group by tbname order by tbname ") + tdSql.checkRows(self.tb_nums) + # bug need fix + # tdSql.query(" select max(t1) from stb group by t2 order by t2 ") + # tdSql.checkRows(self.tb_nums) + tdSql.query(" select max(c1) from stb group by c1 order by c1 ") + tdSql.checkRows(self.row_nums+1) + + tdSql.query(" select c1 , max(c1) from stb group by c1 order by c1 ") + tdSql.checkRows(self.row_nums+1) + + # support selective functions + tdSql.query(" select c1 ,c2 ,c3 , max(c1) ,c4 ,c5 ,t11 from stb group by c1 order by c1 desc ") + tdSql.checkRows(self.row_nums+1) + + tdSql.query(" select c1, tbname , max(c1) ,c4 ,c5 ,t11 from stb group by c1 order by c1 desc ") + tdSql.checkRows(self.row_nums+1) + + # bug need fix + # tdSql.query(" select tbname , max(c1) from sub_stb_1 where c1 is null group by c1 order by c1 desc ") + # tdSql.checkRows(1) + # tdSql.checkData(0,0,"sub_stb_1") + + tdSql.query("select max(c1) ,c2 ,t2,tbname from stb group by abs(c1) order by abs(c1)") + tdSql.checkRows(self.row_nums+1) + tdSql.query("select abs(c1+c3), count(c1+c3) ,max(c1+t2) from stb group by abs(c1+c3) order by abs(c1+c3)") + tdSql.checkRows(self.row_nums+1) + tdSql.query("select max(c1+c3)+min(c2) ,abs(c1) from stb group by abs(c1) order by abs(c1)") + tdSql.checkRows(self.row_nums+1) + tdSql.error("select count(c1+c3)+max(c2) ,abs(c1) ,abs(t1) from stb group by abs(c1) order by abs(t1)+c2") + tdSql.error("select count(c1+c3)+max(c2) ,abs(c1) from stb group by abs(c1) order by abs(c1)+c2") + tdSql.query("select abs(c1+c3)+abs(c2) , count(c1+c3)+max(c2) from stb group by abs(c1+c3)+abs(c2) order by abs(c1+c3)+abs(c2)") + tdSql.checkRows(self.row_nums+1) + + tdSql.query(" select max(c1) , max(t2) from stb where abs(c1+t2)=1 partition by tbname ") + tdSql.checkRows(2) + tdSql.query(" select max(c1) from stb where abs(c1+t2)=1 partition by tbname ") + tdSql.checkRows(2) + + tdSql.query(" select tbname , max(c1) from stb partition by tbname order by tbname ") + tdSql.checkRows(self.tb_nums) + tdSql.checkData(0,1,self.row_nums-1) + + tdSql.query("select tbname , max(c2) from stb partition by t1 order by t1") + tdSql.query("select tbname , max(t2) from stb partition by t1 order by t1") + tdSql.query("select tbname , max(t2) from stb partition by t2 order by t2") + + # # bug need fix + # tdSql.query("select t2 , max(t2) from stb partition by t2 order by t2") + # tdSql.checkRows(self.tb_nums) + + tdSql.query("select tbname , max(c1) from stb partition by tbname order by tbname") + tdSql.checkRows(self.tb_nums) + tdSql.checkData(0,1,self.row_nums-1) + + + tdSql.query("select tbname , max(c1) from stb partition by t2 order by t2") + + tdSql.query("select c2, max(c1) from stb partition by c2 order by c2 desc") + tdSql.checkRows(self.tb_nums+1) + tdSql.checkData(0,1,self.row_nums-1) + + tdSql.query("select tbname , max(c1) from stb partition by c1 order by c2") + + + tdSql.query("select tbname , abs(t2) from stb partition by c2 order by t2") + tdSql.checkRows(self.tb_nums*(self.row_nums+5)) + + tdSql.query("select max(c1) , count(t2) from stb partition by c2 ") + tdSql.checkRows(self.row_nums+1) + tdSql.checkData(0,1,self.row_nums) + + tdSql.query("select count(c1) , max(t2) ,c2 from stb partition by c2 order by c2") + tdSql.checkRows(self.row_nums+1) + + tdSql.query("select count(c1) , count(t1) ,max(c2) ,tbname from stb partition by tbname order by tbname") + tdSql.checkRows(self.tb_nums) + tdSql.checkCols(4) + + tdSql.query("select count(c1) , max(t2) ,t1 from stb partition by t1 order by t1") + tdSql.checkRows(self.tb_nums) + tdSql.checkData(0,0,self.row_nums) + + # bug need fix + # tdSql.query("select count(c1) , max(t1) ,abs(c1) from stb partition by abs(c1) order by abs(c1)") + # tdSql.checkRows(self.row_nums+1) + + + tdSql.query("select max(ceil(c2)) , max(floor(t2)) ,max(floor(c2)) from stb partition by abs(c2) order by abs(c2)") + tdSql.checkRows(self.row_nums+1) + + + tdSql.query("select max(ceil(c1-2)) , max(floor(t2+1)) ,max(c2-c1) from stb partition by abs(floor(c1)) order by abs(floor(c1))") + tdSql.checkRows(self.row_nums+1) + + + # interval + tdSql.query("select max(c1) from stb interval(2s) sliding(1s)") + + # bug need fix + + tdSql.query('select max(c1) from stb where ts>="2022-07-06 16:00:00.000 " and ts < "2022-07-06 17:00:00.000 " interval(50s) sliding(30s) fill(NULL)') + + tdSql.query(" select tbname , count(c1) from stb partition by tbname interval(10s) slimit 5 soffset 1 ") + + tdSql.query("select tbname , max(c1) from stb partition by tbname interval(10s)") + tdSql.checkRows(self.row_nums*2) + + tdSql.query("select tbname , count(c1) from sub_stb_1 partition by tbname interval(10s)") + tdSql.checkData(0,0,'sub_stb_1') + tdSql.checkData(0,1,self.row_nums) + + # bug need fix + # tdSql.query(" select tbname , max(c1) from stb partition by tbname order by tbname slimit 5 soffset 0 ") + # tdSql.checkRows(5) + + # tdSql.query(" select tbname , max(c1) from stb partition by tbname order by tbname slimit 5 soffset 1 ") + # tdSql.checkRows(5) + + tdSql.query(" select tbname , max(c1) from sub_stb_1 partition by tbname interval(10s) sliding(5s) ") + + tdSql.query(f'select max(c1) from stb where ts>={self.ts} and ts < {self.ts}+1000 interval(50s) sliding(30s)') + tdSql.query(f'select tbname , max(c1) from stb where ts>={self.ts} and ts < {self.ts}+1000 interval(50s) sliding(30s)') + + + def run(self): + tdSql.prepare() + self.prepare_datas("stb",self.tb_nums,self.row_nums) + self.basic_query() + + # # coverage case for taosd crash about bug fix + tdSql.query(" select sum(c1) from stb where t2+10 >1 ") + tdSql.query(" select count(c1),count(t1) from stb where -t2<1 ") + tdSql.query(" select tbname ,max(ceil(c1)) from stb group by tbname ") + tdSql.query(" select avg(abs(c1)) , tbname from stb group by tbname ") + tdSql.query(" select t1,c1 from stb where abs(t2+c1)=1 ") + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/2-query/percentile.py b/tests/system-test/2-query/percentile.py index c2584fd394..21bb8763dc 100644 --- a/tests/system-test/2-query/percentile.py +++ b/tests/system-test/2-query/percentile.py @@ -11,12 +11,14 @@ # -*- coding: utf-8 -*- -from platform import java_ver + from util.log import * from util.cases import * from util.sql import * import numpy as np +from util.sqlset import TDSetSql + class TDTestCase: def init(self, conn, logSql): @@ -25,55 +27,141 @@ class TDTestCase: self.rowNum = 10 self.ts = 1537146000000 + self.setsql = TDSetSql() + self.ntbname = 'ntb' + self.stbname = 'stb' + self.binary_length = 20 # the length of binary for column_dict + self.nchar_length = 20 # the length of nchar for column_dict + self.column_dict = { + 'ts' : 'timestamp', + 'col1': 'tinyint', + 'col2': 'smallint', + 'col3': 'int', + 'col4': 'bigint', + 'col5': 'tinyint unsigned', + 'col6': 'smallint unsigned', + 'col7': 'int unsigned', + 'col8': 'bigint unsigned', + 'col9': 'float', + 'col10': 'double', + 'col11': 'bool', + 'col12': f'binary({self.binary_length})', + 'col13': f'nchar({self.nchar_length})' + } - def run(self): - tdSql.prepare() - + self.tag_dict = { + 'ts_tag' : 'timestamp', + 't1': 'tinyint', + 't2': 'smallint', + 't3': 'int', + 't4': 'bigint', + 't5': 'tinyint unsigned', + 't6': 'smallint unsigned', + 't7': 'int unsigned', + 't8': 'bigint unsigned', + 't9': 'float', + 't10': 'double', + 't11': 'bool', + 't12': f'binary({self.binary_length})', + 't13': f'nchar({self.nchar_length})' + } + self.binary_str = 'taosdata' + self.nchar_str = '涛思数据' + self.tbnum = 2 + self.tag_ts = self.ts + self.tag_tinyint = 1 + self.tag_smallint = 2 + self.tag_int = 3 + self.tag_bigint = 4 + self.tag_utint = 5 + self.tag_usint = 6 + self.tag_uint = 7 + self.tag_ubint = 8 + self.tag_float = 9.1 + self.tag_double = 10.1 + self.tag_bool = True + self.tag_values = [ + f'{self.tag_ts},{self.tag_tinyint},{self.tag_smallint},{self.tag_int},{self.tag_bigint},\ + {self.tag_utint},{self.tag_usint},{self.tag_uint},{self.tag_ubint},{self.tag_float},{self.tag_double},{self.tag_bool},"{self.binary_str}","{self.nchar_str}"' + + ] + + self.param = [1,50,100] + + def insert_data(self,column_dict,tbname,row_num): intData = [] floatData = [] - - tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, - col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned)''') - for i in range(self.rowNum): - tdSql.execute("insert into test values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" - % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) - intData.append(i + 1) - floatData.append(i + 0.1) - - # percentile verifacation - tdSql.error("select percentile(ts ,20) from test") - tdSql.error("select percentile(col7 ,20) from test") - tdSql.error("select percentile(col8 ,20) from test") - tdSql.error("select percentile(col9 ,20) from test") - column_list = [1,2,3,4,11,12,13,14] - percent_list = [0,50,100] - for i in column_list: - for j in percent_list: - tdSql.query(f"select percentile(col{i}, {j}) from test") - tdSql.checkData(0, 0, np.percentile(intData, j)) - - for i in [5,6]: - for j in percent_list: - tdSql.query(f"select percentile(col{i}, {j}) from test") - tdSql.checkData(0, 0, np.percentile(floatData, j)) - - tdSql.execute("create table meters (ts timestamp, voltage int) tags(loc nchar(20))") - tdSql.execute("create table t0 using meters tags('beijing')") - tdSql.execute("create table t1 using meters tags('shanghai')") - for i in range(self.rowNum): - tdSql.execute("insert into t0 values(%d, %d)" % (self.ts + i, i + 1)) - tdSql.execute("insert into t1 values(%d, %d)" % (self.ts + i, i + 1)) - - # tdSql.error("select percentile(voltage, 20) from meters") - - - - tdSql.execute("create table st(ts timestamp, k int)") - tdSql.execute("insert into st values(now, -100)(now+1a,-99)") - tdSql.query("select apercentile(k, 20) from st") - tdSql.checkData(0, 0, -100.00) - - + insert_sql = self.setsql.set_insertsql(column_dict,tbname,self.binary_str,self.nchar_str) + for i in range(row_num): + insert_list = [] + self.setsql.insert_values(column_dict,i,insert_sql,insert_list,self.ts) + intData.append(i) + floatData.append(i + 0.1) + return intData,floatData + def check_tags(self,tags,param,num,value): + tdSql.query(f'select percentile({tags}, {param}) from {self.stbname}_{num}') + print(tdSql.queryResult) + tdSql.checkEqual(tdSql.queryResult[0][0], value) + def function_check_ntb(self): + tdSql.prepare() + tdSql.execute(self.setsql.set_create_normaltable_sql(self.ntbname,self.column_dict)) + intData,floatData = self.insert_data(self.column_dict,self.ntbname,self.rowNum) + for k,v in self.column_dict.items(): + for param in self.param: + if v.lower() in ['timestamp','bool'] or 'binary' in v.lower() or 'nchar' in v.lower(): + tdSql.error(f'select percentile({k},{param}) from {self.ntbname}') + elif v.lower() in ['tinyint','smallint','int','bigint','tinyint unsigned','smallint unsigned','int unsigned','bigint unsigned']: + tdSql.query(f'select percentile({k}, {param}) from {self.ntbname}') + tdSql.checkData(0, 0, np.percentile(intData, param)) + else: + tdSql.query(f'select percentile({k}, {param}) from {self.ntbname}') + tdSql.checkData(0, 0, np.percentile(floatData, param)) + def function_check_ctb(self): + tdSql.prepare() + tdSql.execute(self.setsql.set_create_stable_sql(self.stbname,self.column_dict,self.tag_dict)) + for i in range(self.tbnum): + tdSql.execute(f"create table {self.stbname}_{i} using {self.stbname} tags({self.tag_values[0]})") + intData,floatData = self.insert_data(self.column_dict,f'{self.stbname}_{i}',self.rowNum) + for i in range(self.tbnum): + for k,v in self.column_dict.items(): + for param in self.param: + if v.lower() in ['timestamp','bool'] or 'binary' in v.lower() or 'nchar' in v.lower(): + tdSql.error(f'select percentile({k},{param}) from {self.stbname}_{i}') + elif v.lower() in ['tinyint','smallint','int','bigint','tinyint unsigned','smallint unsigned','int unsigned','bigint unsigned']: + tdSql.query(f'select percentile({k}, {param}) from {self.stbname}_{i}') + tdSql.checkData(0, 0, np.percentile(intData, param)) + else: + tdSql.query(f'select percentile({k}, {param}) from {self.stbname}_{i}') + tdSql.checkData(0, 0, np.percentile(floatData, param)) + + #!bug TD-17119 + # for k,v in self.tag_dict.items(): + # for param in self.param: + # if v.lower() in ['timestamp','bool'] or 'binary' in v.lower() or 'nchar' in v.lower(): + # tdSql.error(f'select percentile({k},{param}) from {self.stbname}_{i}') + # elif v.lower() == 'tinyint': + # self.check_tags(k,param,i,self.tag_tinyint) + # elif v.lower() == 'smallint': + # self.check_tags(k,param,i,self.tag_smallint) + # elif v.lower() == 'int': + # self.check_tags(k,param,i,self.tag_int) + # elif v.lower() == 'bigint': + # self.check_tags(k,param,i,self.tag_bigint) + # elif v.lower() == 'tinyint unsigned': + # self.check_tags(k,param,i,self.tag_utint) + # elif v.lower() == 'smallint unsigned': + # self.check_tags(k,param,i,self.tag_usint) + # elif v.lower() == 'int unsigned': + # self.check_tags(k,param,i,self.tag_uint) + # elif v.lower() == 'bigint unsigned': + # self.check_tags(k,param,i,self.tag_ubint) + # elif v.lower() == 'float': + # self.check_tags(k,param,i,self.tag_float) + # elif v.lower() == 'double': + # self.check_tags(k,param,i,self.tag_double) + def run(self): + self.function_check_ntb() + self.function_check_ctb() def stop(self): tdSql.close() diff --git a/tests/system-test/2-query/top.py b/tests/system-test/2-query/top.py index 86e201ea9a..6120f81e6d 100644 --- a/tests/system-test/2-query/top.py +++ b/tests/system-test/2-query/top.py @@ -49,32 +49,10 @@ class TDTestCase: self.param_list = [1,100] def insert_data(self,column_dict,tbname,row_num): - sql = '' - for k, v in column_dict.items(): - if v.lower() == 'timestamp' or v.lower() == 'tinyint' or v.lower() == 'smallint' or v.lower() == 'int' or v.lower() == 'bigint' or \ - v.lower() == 'tinyint unsigned' or v.lower() == 'smallint unsigned' or v.lower() == 'int unsigned' or v.lower() == 'bigint unsigned' or v.lower() == 'bool': - sql += '%d,' - elif v.lower() == 'float' or v.lower() == 'double': - sql += '%f,' - elif 'binary' in v.lower(): - sql += f'"{self.binary_str}%d",' - elif 'nchar' in v.lower(): - sql += f'"{self.nchar_str}%d",' - insert_sql = f'insert into {tbname} values({sql[:-1]})' + insert_sql = self.setsql.set_insertsql(column_dict,tbname,self.binary_str,self.nchar_str) for i in range(row_num): insert_list = [] - for k, v in column_dict.items(): - if v.lower() in[ 'tinyint' , 'smallint' , 'int', 'bigint' , 'tinyint unsigned' , 'smallint unsigned' , 'int unsigned' , 'bigint unsigned'] or\ - 'binary' in v.lower() or 'nchar' in v.lower(): - insert_list.append(1 + i) - elif v.lower() == 'float' or v.lower() == 'double': - insert_list.append(0.1 + i) - elif v.lower() == 'bool': - insert_list.append(i % 2) - elif v.lower() == 'timestamp': - insert_list.append(self.ts + i) - tdSql.execute(insert_sql%(tuple(insert_list))) - pass + self.setsql.insert_values(column_dict,i,insert_sql,insert_list,self.ts) def top_check_data(self,tbname,tb_type): new_column_dict = {} for param in self.param_list: @@ -86,7 +64,7 @@ class TDTestCase: tdSql.checkRows(self.rowNum) values_list = [] for i in range(self.rowNum): - tp = (self.rowNum-i,) + tp = (self.rowNum-i-1,) values_list.insert(0,tp) tdSql.checkEqual(tdSql.queryResult,values_list) elif tb_type == 'stable': @@ -96,7 +74,7 @@ class TDTestCase: tdSql.checkRows(param) values_list = [] for i in range(param): - tp = (self.rowNum-i,) + tp = (self.rowNum-i-1,) values_list.insert(0,tp) tdSql.checkEqual(tdSql.queryResult,values_list) elif tb_type == 'stable': @@ -132,7 +110,7 @@ class TDTestCase: for i in range(self.tbnum): tdSql.execute(f"create table {stbname}_{i} using {stbname} tags({tag_values[0]})") - tdSql.execute(self.insert_data(self.column_dict,f'{stbname}_{i}',self.rowNum)) + self.insert_data(self.column_dict,f'{stbname}_{i}',self.rowNum) tdSql.query('show tables') vgroup_list = [] for i in range(len(tdSql.queryResult)): diff --git a/tests/system-test/7-tmq/tmqAutoCreateTbl.py b/tests/system-test/7-tmq/tmqAutoCreateTbl.py new file mode 100644 index 0000000000..6a9f10ebbf --- /dev/null +++ b/tests/system-test/7-tmq/tmqAutoCreateTbl.py @@ -0,0 +1,333 @@ + +import taos +import sys +import time +import socket +import os +import threading +from enum import Enum + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def __init__(self): + self.vgroups = 2 + self.ctbNum = 100 + self.rowsPerTbl = 10000 + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 3, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 500, + 'rowsPerTbl': 1000, + 'batchNum': 500, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 3, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("restart taosd to ensure that the data falls into the disk") + # tdDnodes.stop(1) + # tdDnodes.start(1) + tdSql.query("flush database %s"%(paraDict['dbName'])) + return + + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 1000, + 'rowsPerTbl': 1000, + 'batchNum': 400, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 5, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 1} + + # paraDict['vgroups'] = self.vgroups + # paraDict['ctbNum'] = self.ctbNum + # paraDict['rowsPerTbl'] = self.rowsPerTbl + + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("create topics from stb1") + topicFromStb1 = 'topic_stb1' + queryString = "select ts, c1, c2 from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicFromStb1, queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + topicList = topicFromStb1 + ifcheckdata = 0 + ifManualCommit = 0 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:true,\ + auto.commit.interval.ms:500,\ + auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + + # time.sleep(3) + tmqCom.getStartCommitNotifyFromTmqsim() + tdLog.info("================= restart dnode ===========================") + tdDnodes.stop(1) + tdDnodes.start(1) + time.sleep(5) + + tdLog.info("insert process end, and start to check consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + tdSql.query(queryString) + totalRowsInserted = tdSql.getRows() + + if totalConsumeRows != totalRowsInserted: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, totalRowsInserted)) + tdLog.exit("tmq consume rows error!") + + tdSql.query("drop topic %s"%topicFromStb1) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def tmqCase2(self): + tdLog.printNoPrefix("======== test case 2: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 1000, + 'rowsPerTbl': 1000, + 'batchNum': 1000, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 5, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 1} + + # paraDict['vgroups'] = self.vgroups + # paraDict['ctbNum'] = self.ctbNum + # paraDict['rowsPerTbl'] = self.rowsPerTbl + + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("create topics from stb1") + topicFromStb1 = 'topic_stb1' + queryString = "select ts, c1, c2 from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicFromStb1, queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 + topicList = topicFromStb1 + ifcheckdata = 0 + ifManualCommit = 0 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:true,\ + auto.commit.interval.ms:1000,\ + auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + + tdLog.info("create some new child table and insert data ") + tmqCom.insert_data_with_autoCreateTbl(tdSql,paraDict["dbName"],paraDict["stbName"],"ctb",paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"]) + + tmqCom.getStartCommitNotifyFromTmqsim() + tdLog.info("================= restart dnode ===========================") + tdDnodes.stop(1) + tdDnodes.start(1) + time.sleep(5) + + tdLog.info("insert process end, and start to check consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + tdSql.query(queryString) + totalRowsInserted = tdSql.getRows() + + if totalConsumeRows != totalRowsInserted: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, totalRowsInserted)) + tdLog.exit("tmq consume rows error!") + + tdSql.query("drop topic %s"%topicFromStb1) + + tdLog.printNoPrefix("======== test case 2 end ...... ") + + # 自动建表完成数据插入，启动消费 + def tmqCase3(self): + tdLog.printNoPrefix("======== test case 3: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 1000, + 'rowsPerTbl': 1000, + 'batchNum': 400, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 5, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 1} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + tdLog.info("insert data by auto create ctb") + tmqCom.insert_data_with_autoCreateTbl(tdSql,paraDict["dbName"],paraDict["stbName"],"ctb",paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"]) + + tdLog.info("create topics from stb1") + topicFromStb1 = 'topic_stb1' + queryString = "select ts, c1, c2 from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicFromStb1, queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + topicList = topicFromStb1 + ifcheckdata = 0 + ifManualCommit = 0 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:true,\ + auto.commit.interval.ms:1000,\ + auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + + # tdLog.info("================= restart dnode ===========================") + # tdDnodes.stop(1) + # tdDnodes.start(1) + # time.sleep(2) + + tdLog.info("insert process end, and start to check consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + tdSql.query(queryString) + totalRowsInserted = tdSql.getRows() + + if totalConsumeRows != totalRowsInserted: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, totalRowsInserted)) + tdLog.exit("tmq consume rows error!") + + tdSql.query("drop topic %s"%topicFromStb1) + + tdLog.printNoPrefix("======== test case 3 end ...... ") + + + def run(self): + tdSql.prepare() + + # self.tmqCase1() + # self.tmqCase2() + self.tmqCase3() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmqCommon.py b/tests/system-test/7-tmq/tmqCommon.py index 7f9d36bd26..2030563a9a 100644 --- a/tests/system-test/7-tmq/tmqCommon.py +++ b/tests/system-test/7-tmq/tmqCommon.py @@ -361,19 +361,25 @@ class TMQCom: if startTs == 0: t = time.time() startTs = int(round(t * 1000)) - + #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) rowsBatched = 0 for i in range(ctbNum): - sql += " %s.%s_%d using %s.%s tags (%d) values "%(dbName,ctbPrefix,i+ctbStartIdx,dbName,stbName,i) + tagBinaryValue = 'beijing' + if (i % 2 == 0): + tagBinaryValue = 'shanghai' + elif (i % 3 == 0): + tagBinaryValue = 'changsha' + + sql += " %s.%s_%d using %s.%s tags (%d, %d, %d, '%s', '%s') values "%(dbName,ctbPrefix,i+ctbStartIdx,dbName,stbName,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx,tagBinaryValue,tagBinaryValue) for j in range(rowsPerTbl): - sql += "(%d, %d, 'tmqrow_%d') "%(startTs + j, j, j) + sql += "(%d, %d, %d, %d, 'binary_%d', 'nchar_%d', now) "%(startTs+j, j,j, j,i+ctbStartIdx,rowsBatched) rowsBatched += 1 if ((rowsBatched == batchNum) or (j == rowsPerTbl - 1)): tsql.execute(sql) rowsBatched = 0 if j < rowsPerTbl - 1: - sql = "insert into %s.%s_%d using %s.%s tags (%d) values " %(dbName,ctbPrefix,i+ctbStartIdx,dbName,stbName,i) + sql = "insert into %s.%s_%d using %s.%s tags (%d, %d, %d, '%s', '%s') values " %(dbName,ctbPrefix,i+ctbStartIdx,dbName,stbName,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx,tagBinaryValue,tagBinaryValue) else: sql = "insert into " #end sql diff --git a/tests/system-test/99-TDcase/TD-17255.py b/tests/system-test/99-TDcase/TD-17255.py new file mode 100644 index 0000000000..9eb8d531f7 --- /dev/null +++ b/tests/system-test/99-TDcase/TD-17255.py @@ -0,0 +1,333 @@ + +import taos +import sys +import time +import socket +import os +import threading +from enum import Enum + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def __init__(self): + self.vgroups = 2 + self.ctbNum = 100 + self.rowsPerTbl = 10000 + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 3, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 500, + 'rowsPerTbl': 1000, + 'batchNum': 500, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 3, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("restart taosd to ensure that the data falls into the disk") + # tdDnodes.stop(1) + # tdDnodes.start(1) + tdSql.query("flush database %s"%(paraDict['dbName'])) + return + + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 1000, + 'rowsPerTbl': 1000, + 'batchNum': 400, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 5, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 1} + + # paraDict['vgroups'] = self.vgroups + # paraDict['ctbNum'] = self.ctbNum + # paraDict['rowsPerTbl'] = self.rowsPerTbl + + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("create topics from stb1") + topicFromStb1 = 'topic_stb1' + queryString = "select ts, c1, c2 from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicFromStb1, queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + topicList = topicFromStb1 + ifcheckdata = 0 + ifManualCommit = 0 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:true,\ + auto.commit.interval.ms:500,\ + auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + + # time.sleep(3) + tmqCom.getStartCommitNotifyFromTmqsim() + tdLog.info("================= restart dnode ===========================") + tdDnodes.stop(1) + tdDnodes.start(1) + time.sleep(5) + + tdLog.info("insert process end, and start to check consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + tdSql.query(queryString) + totalRowsInserted = tdSql.getRows() + + if totalConsumeRows != totalRowsInserted: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, totalRowsInserted)) + tdLog.exit("tmq consume rows error!") + + tdSql.query("drop topic %s"%topicFromStb1) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def tmqCase2(self): + tdLog.printNoPrefix("======== test case 2: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 1000, + 'rowsPerTbl': 1000, + 'batchNum': 1000, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 5, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 1} + + # paraDict['vgroups'] = self.vgroups + # paraDict['ctbNum'] = self.ctbNum + # paraDict['rowsPerTbl'] = self.rowsPerTbl + + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("create topics from stb1") + topicFromStb1 = 'topic_stb1' + queryString = "select ts, c1, c2 from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicFromStb1, queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 + topicList = topicFromStb1 + ifcheckdata = 0 + ifManualCommit = 0 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:true,\ + auto.commit.interval.ms:1000,\ + auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + + tdLog.info("create some new child table and insert data ") + tmqCom.insert_data_with_autoCreateTbl(tdSql,paraDict["dbName"],paraDict["stbName"],"ctb",paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"]) + + tmqCom.getStartCommitNotifyFromTmqsim() + tdLog.info("================= restart dnode ===========================") + tdDnodes.stop(1) + tdDnodes.start(1) + time.sleep(5) + + tdLog.info("insert process end, and start to check consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + tdSql.query(queryString) + totalRowsInserted = tdSql.getRows() + + if totalConsumeRows != totalRowsInserted: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, totalRowsInserted)) + tdLog.exit("tmq consume rows error!") + + tdSql.query("drop topic %s"%topicFromStb1) + + tdLog.printNoPrefix("======== test case 2 end ...... ") + + # 自动建表完成数据插入，启动消费 + def tmqCase3(self): + tdLog.printNoPrefix("======== test case 3: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 1000, + 'rowsPerTbl': 1000, + 'batchNum': 400, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 5, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 1} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + tdLog.info("insert data by auto create ctb") + tmqCom.insert_data_with_autoCreateTbl(tdSql,paraDict["dbName"],paraDict["stbName"],"ctb",paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"]) + + tdLog.info("create topics from stb1") + topicFromStb1 = 'topic_stb1' + queryString = "select ts, c1, c2 from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicFromStb1, queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + topicList = topicFromStb1 + ifcheckdata = 0 + ifManualCommit = 0 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:true,\ + auto.commit.interval.ms:1000,\ + auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + + # tdLog.info("================= restart dnode ===========================") + # tdDnodes.stop(1) + # tdDnodes.start(1) + # time.sleep(2) + + tdLog.info("insert process end, and start to check consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + tdSql.query(queryString) + totalRowsInserted = tdSql.getRows() + + if totalConsumeRows != totalRowsInserted: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, totalRowsInserted)) + tdLog.exit("tmq consume rows error!") + + tdSql.query("drop topic %s"%topicFromStb1) + + tdLog.printNoPrefix("======== test case 3 end ...... ") + + + def run(self): + tdSql.prepare() + + self.tmqCase1() + # self.tmqCase2() + self.tmqCase3() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 3b0dd76a30..2d1ca64914 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -118,9 +118,11 @@ python3 ./test.py -f 2-query/distribute_agg_stddev.py python3 ./test.py -f 2-query/twa.py python3 ./test.py -f 2-query/irate.py python3 ./test.py -f 2-query/and_or_for_byte.py - +python3 ./test.py -f 2-query/count_partition.py python3 ./test.py -f 2-query/function_null.py -#python3 ./test.py -f 2-query/queryQnode.py +python3 ./test.py -f 2-query/queryQnode.py +python3 ./test.py -f 2-query/max_partition.py + python3 ./test.py -f 6-cluster/5dnode1mnode.py #BUG python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 -M 3 @@ -263,6 +265,8 @@ python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 2 python3 ./test.py -f 2-query/twa.py -Q 2 python3 ./test.py -f 2-query/irate.py -Q 2 python3 ./test.py -f 2-query/function_null.py -Q 2 +python3 ./test.py -f 2-query/count_partition.py -Q 2 +python3 ./test.py -f 2-query/max_partition.py -Q 2 #------------querPolicy 3----------- @@ -348,3 +352,5 @@ python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 3 python3 ./test.py -f 2-query/twa.py -Q 3 python3 ./test.py -f 2-query/irate.py -Q 3 python3 ./test.py -f 2-query/function_null.py -Q 3 +python3 ./test.py -f 2-query/count_partition.py -Q 3 +python3 ./test.py -f 2-query/max_partition.py -Q 3 \ No newline at end of file diff --git a/tests/test/c/sdbDump.c b/tests/test/c/sdbDump.c index 42f1bb4062..4d0f582dc6 100644 --- a/tests/test/c/sdbDump.c +++ b/tests/test/c/sdbDump.c @@ -72,6 +72,7 @@ void dumpDb(SSdb *pSdb, SJson *json) { tjsonAddIntegerToObject(item, "buffer", pObj->cfg.buffer); tjsonAddIntegerToObject(item, "pageSize", pObj->cfg.pageSize); tjsonAddIntegerToObject(item, "pages", pObj->cfg.pages); + tjsonAddIntegerToObject(item, "cacheLastSize", pObj->cfg.cacheLastSize); tjsonAddIntegerToObject(item, "daysPerFile", pObj->cfg.daysPerFile); tjsonAddIntegerToObject(item, "daysToKeep0", pObj->cfg.daysToKeep0); tjsonAddIntegerToObject(item, "daysToKeep1", pObj->cfg.daysToKeep1); @@ -84,7 +85,7 @@ void dumpDb(SSdb *pSdb, SJson *json) { tjsonAddIntegerToObject(item, "compression", pObj->cfg.compression); tjsonAddIntegerToObject(item, "replications", pObj->cfg.replications); tjsonAddIntegerToObject(item, "strict", pObj->cfg.strict); - tjsonAddIntegerToObject(item, "cacheLastRow", pObj->cfg.cacheLastRow); + tjsonAddIntegerToObject(item, "cacheLast", pObj->cfg.cacheLast); tjsonAddIntegerToObject(item, "hashMethod", pObj->cfg.hashMethod); tjsonAddIntegerToObject(item, "numOfRetensions", pObj->cfg.numOfRetensions); tjsonAddIntegerToObject(item, "schemaless", pObj->cfg.schemaless); diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index 85fe456670..b4f86d52b5 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -52,6 +52,7 @@ typedef struct { // char autoOffsetRest[16]; // none, earliest, latest TdFilePtr pConsumeRowsFile; + TdFilePtr pConsumeMetaFile; int32_t ifCheckData; int64_t expectMsgCnt; @@ -445,7 +446,7 @@ static void dumpToFileForCheck(TdFilePtr pFile, TAOS_ROW row, TAOS_FIELD* fields taosFprintfFile(pFile, "\n"); } -static int32_t msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIndex) { +static int32_t data_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIndex) { char buf[1024]; int32_t totalRows = 0; @@ -496,6 +497,52 @@ static int32_t msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIndex) return totalRows; } + +static int32_t meta_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIndex) { + char buf[1024]; + int32_t totalRows = 0; + + // printf("topic: %s\n", tmq_get_topic_name(msg)); + int32_t vgroupId = tmq_get_vgroup_id(msg); + const char* dbName = tmq_get_db_name(msg); + + taosFprintfFile(g_fp, "consumerId: %d, msg index:%" PRId64 "\n", pInfo->consumerId, msgIndex); + taosFprintfFile(g_fp, "dbName: %s, topic: %s, vgroupId: %d\n", dbName != NULL ? dbName : "invalid table", + tmq_get_topic_name(msg), vgroupId); + + { + tmq_raw_data *raw = tmq_get_raw_meta(msg); + + if(raw){ + TAOS_RES* pRes = taos_query(pInfo->taos, "use metadb"); + if (taos_errno(pRes) != 0) { + pError("error when use metadb, reason:%s\n", taos_errstr(pRes)); + taosFprintfFile(g_fp, "error when use metadb, reason:%s\n", taos_errstr(pRes)); + taosCloseFile(&g_fp); + taos_free_result(pRes); + exit(-1); + } + taos_free_result(pRes); + taosFprintfFile(g_fp, "raw:%p\n", raw); + + int32_t ret = taos_write_raw_meta(pInfo->taos, raw); + taosMemoryFree(raw); + } + + char* result = tmq_get_json_meta(msg); + if(result){ + //printf("meta result: %s\n", result); + taosFprintfFile(pInfo->pConsumeMetaFile, "%s\n", result); + taosMemoryFree(result); + } + } + + totalRows++; + + return totalRows; +} + + int queryDB(TAOS* taos, char* command) { TAOS_RES* pRes = taos_query(taos, command); int code = taos_errno(pRes); @@ -526,7 +573,7 @@ int32_t notifyMainScript(SThreadInfo* pInfo, int32_t cmdId) { static int32_t g_once_commit_flag = 0; static void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) { - pError("tmq_commit_cb_print() commit %d\n", code); + taosFprintfFile(g_fp, "tmq_commit_cb_print() commit %d\n", code); if (0 == g_once_commit_flag) { g_once_commit_flag = 1; @@ -630,8 +677,12 @@ void loop_consume(SThreadInfo* pInfo) { // getCurrentTimeString(tmpString)); sprintf(filename, "%s/../log/consumerid_%d.txt", configDir, pInfo->consumerId); pInfo->pConsumeRowsFile = taosOpenFile(filename, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM); - if (pInfo->pConsumeRowsFile == NULL) { - taosFprintfFile(g_fp, "%s create file fail for save rows content\n", getCurrentTimeString(tmpString)); + + sprintf(filename, "%s/../log/meta_consumerid_%d.txt", configDir, pInfo->consumerId); + pInfo->pConsumeMetaFile = taosOpenFile(filename, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM); + + if (pInfo->pConsumeRowsFile == NULL || pInfo->pConsumeMetaFile == NULL) { + taosFprintfFile(g_fp, "%s create file fail for save rows or save meta\n", getCurrentTimeString(tmpString)); return; } } @@ -645,7 +696,11 @@ void loop_consume(SThreadInfo* pInfo) { TAOS_RES* tmqMsg = tmq_consumer_poll(pInfo->tmq, consumeDelay); if (tmqMsg) { if (0 != g_stConfInfo.showMsgFlag) { - totalRows += msg_process(tmqMsg, pInfo, totalMsgs); + tmq_res_t msgType = tmq_get_res_type(tmqMsg); + if (msgType == TMQ_RES_TABLE_META) { + totalRows += meta_msg_process(tmqMsg, pInfo, totalMsgs); + } else if (msgType == TMQ_RES_DATA) + totalRows += data_msg_process(tmqMsg, pInfo, totalMsgs); } taos_free_result(tmqMsg); diff --git a/tools/taosws-rs b/tools/taosws-rs index 7a94ffab45..6dccac192a 160000 --- a/tools/taosws-rs +++ b/tools/taosws-rs @@ -1 +1 @@ -Subproject commit 7a94ffab45f08e16f09b3f430fe75d717054adb6 +Subproject commit 6dccac192a2ae7dd78718ab926201aab5419327a