Merge branch '3.0' into enh/TD-21161-3.0
This commit is contained in:
kailixu
commit
e976fcb91d
|
|
@ -10,6 +10,8 @@ if (NOT DEFINED TD_SOURCE_DIR)
|
|||
set( TD_SOURCE_DIR ${PROJECT_SOURCE_DIR} )
|
||||
endif()
|
||||
|
||||
SET(TD_COMMUNITY_DIR ${PROJECT_SOURCE_DIR})
|
||||
|
||||
set(TD_SUPPORT_DIR "${TD_SOURCE_DIR}/cmake")
|
||||
set(TD_CONTRIB_DIR "${TD_SOURCE_DIR}/contrib")
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
IF (DEFINED VERNUMBER)
|
||||
SET(TD_VER_NUMBER ${VERNUMBER})
|
||||
ELSE ()
|
||||
SET(TD_VER_NUMBER "3.0.3.1")
|
||||
SET(TD_VER_NUMBER "3.0.3.2")
|
||||
ENDIF ()
|
||||
|
||||
IF (DEFINED VERCOMPATIBLE)
|
||||
|
|
@ -16,7 +16,7 @@ find_program(HAVE_GIT NAMES git)
|
|||
IF (DEFINED GITINFO)
|
||||
SET(TD_VER_GIT ${GITINFO})
|
||||
ELSEIF (HAVE_GIT)
|
||||
execute_process(COMMAND git log -1 --format=%H WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} OUTPUT_VARIABLE GIT_COMMITID)
|
||||
execute_process(COMMAND git log -1 --format=%H WORKING_DIRECTORY ${TD_COMMUNITY_DIR} OUTPUT_VARIABLE GIT_COMMITID)
|
||||
#message(STATUS "git log result:${GIT_COMMITID}")
|
||||
IF (GIT_COMMITID)
|
||||
string (REGEX REPLACE "[\n\t\r]" "" GIT_COMMITID ${GIT_COMMITID})
|
||||
|
|
@ -30,6 +30,23 @@ ELSE ()
|
|||
SET(TD_VER_GIT "no git commit id")
|
||||
ENDIF ()
|
||||
|
||||
IF (DEFINED GITINFOI)
|
||||
SET(TD_VER_GIT_INTERNAL ${GITINFOI})
|
||||
ELSEIF (HAVE_GIT)
|
||||
execute_process(COMMAND git log -1 --format=%H WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} OUTPUT_VARIABLE GIT_COMMITID)
|
||||
message(STATUS "git log result:${GIT_COMMITID}")
|
||||
IF (GIT_COMMITID)
|
||||
string (REGEX REPLACE "[\n\t\r]" "" GIT_COMMITID ${GIT_COMMITID})
|
||||
SET(TD_VER_GIT_INTERNAL ${GIT_COMMITID})
|
||||
ELSE ()
|
||||
message(STATUS "not a git repository")
|
||||
SET(TD_VER_GIT "no git commit id")
|
||||
ENDIF ()
|
||||
ELSE ()
|
||||
message(STATUS "no git cmd")
|
||||
SET(TD_VER_GIT_INTERNAL "no git commit id")
|
||||
ENDIF ()
|
||||
|
||||
IF (DEFINED VERDATE)
|
||||
SET(TD_VER_DATE ${VERDATE})
|
||||
ELSE ()
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
# taos-tools
|
||||
ExternalProject_Add(taos-tools
|
||||
GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
|
||||
GIT_TAG d194dc9
|
||||
GIT_TAG 63635fc
|
||||
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
|
||||
BINARY_DIR ""
|
||||
#BUILD_IN_SOURCE TRUE
|
||||
|
|
|
|||
|
|
@ -368,7 +368,6 @@ conf := &tmq.ConfigMap{
|
|||
"td.connect.port": "6030",
|
||||
"client.id": "test_tmq_c",
|
||||
"enable.auto.commit": "false",
|
||||
"enable.heartbeat.background": "true",
|
||||
"experimental.snapshot.enable": "true",
|
||||
"msg.with.table.name": "true",
|
||||
}
|
||||
|
|
|
|||
|
|
@ -35,8 +35,8 @@ database_option: {
|
|||
| TABLE_SUFFIX value
|
||||
| TSDB_PAGESIZE value
|
||||
| WAL_RETENTION_PERIOD value
|
||||
| WAL_ROLL_PERIOD value
|
||||
| WAL_RETENTION_SIZE value
|
||||
| WAL_ROLL_PERIOD value
|
||||
| WAL_SEGMENT_SIZE value
|
||||
}
|
||||
```
|
||||
|
|
@ -75,11 +75,10 @@ database_option: {
|
|||
- TABLE_PREFIX：The prefix length in the table name that is ignored when distributing table to vnode based on table name.
|
||||
- TABLE_SUFFIX：The suffix length in the table name that is ignored when distributing table to vnode based on table name.
|
||||
- TSDB_PAGESIZE: The page size of the data storage engine in a vnode. The unit is KB. The default is 4 KB. The range is 1 to 16384, that is, 1 KB to 16 MB.
|
||||
- WAL_RETENTION_PERIOD: specifies the maximum time of which WAL files are to be kept after consumption. This parameter is used for data subscription. Enter a time in seconds. The default value 0. A value of 0 indicates that WAL files are not required to keep after consumption. -1: the time of WAL files to keep has no upper limit.
|
||||
- WAL_RETENTION_SIZE: specifies the maximum total size of which WAL files are to be kept after consumption. This parameter is used for data subscription. Enter a size in KB. The default value is 0. A value of 0 indicates that WAL files are not required to keep after consumption. -1: the total size of WAL files to keep has no upper limit.
|
||||
- WAL_RETENTION_PERIOD: specifies the maximum time of which WAL files are to be kept for consumption. This parameter is used for data subscription. Enter a time in seconds. The default value 0. A value of 0 indicates that WAL files are not required to keep for consumption. Alter it with a proper value at first to create topics.
|
||||
- WAL_RETENTION_SIZE: specifies the maximum total size of which WAL files are to be kept for consumption. This parameter is used for data subscription. Enter a size in KB. The default value is 0. A value of 0 indicates that the total size of WAL files to keep for consumption has no upper limit.
|
||||
- WAL_ROLL_PERIOD: specifies the time after which WAL files are rotated. After this period elapses, a new WAL file is created. The default value is 0. A value of 0 indicates that a new WAL file is created only after TSDB data in memory are flushed to disk.
|
||||
- WAL_SEGMENT_SIZE: specifies the maximum size of a WAL file. After the current WAL file reaches this size, a new WAL file is created. The default value is 0. A value of 0 indicates that a new WAL file is created only after TSDB data in memory are flushed to disk.
|
||||
|
||||
### Example Statement
|
||||
|
||||
```sql
|
||||
|
|
@ -123,6 +122,8 @@ alter_database_option: {
|
|||
| WAL_LEVEL value
|
||||
| WAL_FSYNC_PERIOD value
|
||||
| KEEP value
|
||||
| WAL_RETENTION_PERIOD value
|
||||
| WAL_RETENTION_SIZE value
|
||||
}
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -13,12 +13,11 @@ create_definition:
|
|||
col_name column_definition
|
||||
|
||||
column_definition:
|
||||
type_name [COMMENT 'string_value']
|
||||
type_name
|
||||
```
|
||||
|
||||
**More explanations**
|
||||
- Each supertable can have a maximum of 4096 columns, including tags. The minimum number of columns is 3: a timestamp column used as the key, one tag column, and one data column.
|
||||
- When you create a supertable, you can add comments to columns and tags.
|
||||
- The TAGS keyword defines the tag columns for the supertable. The following restrictions apply to tag columns:
|
||||
- A tag column can use the TIMESTAMP data type, but the values in the column must be fixed numbers. Timestamps including formulae, such as "now + 10s", cannot be stored in a tag column.
|
||||
- The name of a tag column cannot be the same as the name of any other column.
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ The following data types can be used in the schema for standard tables.
|
|||
| - | :------- | :-------- | :------- |
|
||||
| 1 | ALTER ACCOUNT | Deprecated| This Enterprise Edition-only statement has been removed. It returns the error "This statement is no longer supported."
|
||||
| 2 | ALTER ALL DNODES | Added | Modifies the configuration of all dnodes.
|
||||
| 3 | ALTER DATABASE | Modified | Deprecated<ul><li>QUORUM: Specified the required number of confirmations. STRICT is now used to specify strong or weak consistency. The STRICT parameter cannot be modified. </li><li>BLOCKS: Specified the memory blocks used by each vnode. BUFFER is now used to specify the size of the write cache pool for each vnode. </li><li>UPDATE: Specified whether update operations were supported. All databases now support updating data in certain columns. </li><li>CACHELAST: Specified how to cache the newest row of data. CACHEMODEL now replaces CACHELAST. </li><li>COMP: Cannot be modified. <br/>Added</li><li>CACHEMODEL: Specifies whether to cache the latest subtable data. </li><li>CACHESIZE: Specifies the size of the cache for the newest subtable data. </li><li>WAL_FSYNC_PERIOD: Replaces the FSYNC parameter. </li><li>WAL_LEVEL: Replaces the WAL parameter. <br/>Modified</li><li>REPLICA: Cannot be modified. </li><li>KEEP: Now supports units. </li></ul>
|
||||
| 3 | ALTER DATABASE | Modified | Deprecated<ul><li>QUORUM: Specified the required number of confirmations. STRICT is now used to specify strong or weak consistency. The STRICT parameter cannot be modified. </li><li>BLOCKS: Specified the memory blocks used by each vnode. BUFFER is now used to specify the size of the write cache pool for each vnode. </li><li>UPDATE: Specified whether update operations were supported. All databases now support updating data in certain columns. </li><li>CACHELAST: Specified how to cache the newest row of data. CACHEMODEL now replaces CACHELAST. </li><li>COMP: Cannot be modified. <br/>Added</li><li>CACHEMODEL: Specifies whether to cache the latest subtable data. </li><li>CACHESIZE: Specifies the size of the cache for the newest subtable data. </li><li>WAL_FSYNC_PERIOD: Replaces the FSYNC parameter. </li><li>WAL_LEVEL: Replaces the WAL parameter. </li><li>WAL_RETENTION_PERIOD: specifies the time after which WAL files are deleted. This parameter is used for data subscription. </li><li>WAL_RETENTION_SIZE: specifies the size at which WAL files are deleted. This parameter is used for data subscription. <br/>Modified</li><li>REPLICA: Cannot be modified. </li><li>KEEP: Now supports units. </li></ul>
|
||||
| 4 | ALTER STABLE | Modified | Deprecated<ul><li>CHANGE TAG: Modified the name of a tag. Replaced by RENAME TAG. <br/>Added</li><li>RENAME TAG: Replaces CHANGE TAG. </li><li>COMMENT: Specifies comments for a supertable. </li></ul>
|
||||
| 5 | ALTER TABLE | Modified | Deprecated<ul><li>CHANGE TAG: Modified the name of a tag. Replaced by RENAME TAG. <br/>Added</li><li>RENAME TAG: Replaces CHANGE TAG. </li><li>COMMENT: Specifies comments for a standard table. </li><li>TTL: Specifies the time-to-live for a standard table. </li></ul>
|
||||
| 6 | ALTER USER | Modified | Deprecated<ul><li>PRIVILEGE: Specified user permissions. Replaced by GRANT and REVOKE. <br/>Added</li><li>ENABLE: Enables or disables a user. </li><li>SYSINFO: Specifies whether a user can query system information. </li></ul>
|
||||
|
|
|
|||
|
|
@ -120,7 +120,7 @@ _taosSql_ implements Go's `database/sql/driver` interface via cgo. You can use t
|
|||
|
||||
Use `taosSql` as `driverName` and use a correct [DSN](#DSN) as `dataSourceName`, DSN supports the following parameters.
|
||||
|
||||
* configPath specifies the `taos.cfg` directory
|
||||
* cfg specifies the `taos.cfg` directory
|
||||
|
||||
For example:
|
||||
|
||||
|
|
|
|||
|
|
@ -10,6 +10,14 @@ For TDengine 2.x installation packages by version, please visit [here](https://w
|
|||
|
||||
import Release from "/components/ReleaseV3";
|
||||
|
||||
## 3.0.3.2
|
||||
|
||||
<Release type="tdengine" version="3.0.3.2" />
|
||||
|
||||
## 3.0.3.1
|
||||
|
||||
<Release type="tdengine" version="3.0.3.1" />
|
||||
|
||||
## 3.0.3.1
|
||||
|
||||
<Release type="tdengine" version="3.0.3.1" />
|
||||
|
|
|
|||
|
|
@ -10,6 +10,10 @@ For other historical version installers, please visit [here](https://www.taosdat
|
|||
|
||||
import Release from "/components/ReleaseV3";
|
||||
|
||||
## 2.4.11
|
||||
|
||||
<Release type="tools" version="2.4.11" />
|
||||
|
||||
## 2.4.10
|
||||
|
||||
<Release type="tools" version="2.4.10" />
|
||||
|
|
|
|||
|
|
@ -35,7 +35,6 @@ func main() {
|
|||
"td.connect.port": "6030",
|
||||
"client.id": "test_tmq_client",
|
||||
"enable.auto.commit": "false",
|
||||
"enable.heartbeat.background": "true",
|
||||
"experimental.snapshot.enable": "true",
|
||||
"msg.with.table.name": "true",
|
||||
})
|
||||
|
|
|
|||
|
|
@ -367,7 +367,6 @@ conf := &tmq.ConfigMap{
|
|||
"td.connect.port": "6030",
|
||||
"client.id": "test_tmq_c",
|
||||
"enable.auto.commit": "false",
|
||||
"enable.heartbeat.background": "true",
|
||||
"experimental.snapshot.enable": "true",
|
||||
"msg.with.table.name": "true",
|
||||
}
|
||||
|
|
@ -419,7 +418,6 @@ consumer = Consumer({"group.id": "local", "td.connect.ip": "127.0.0.1"})
|
|||
| `auto.commit.interval.ms` | string | 以毫秒为单位的自动提交时间间隔 | 默认值：5000 ms |
|
||||
| `auto.offset.reset` | string | 消费组订阅的初始位置 | 可选：`earliest`(default), `latest`, `none` |
|
||||
| `experimental.snapshot.enable` | string | 是否允许从 TSDB 消费数据 | 合法值：`true`, `false` |
|
||||
| `enable.heartbeat.background` | string | 启用后台心跳，启用后即使长时间不 poll 消息也不会造成离线 | 合法值：`true`, `false` |
|
||||
|
||||
</TabItem>
|
||||
|
||||
|
|
|
|||
|
|
@ -122,7 +122,7 @@ _taosSql_ 通过 cgo 实现了 Go 的 `database/sql/driver` 接口。只需要
|
|||
|
||||
使用 `taosSql` 作为 `driverName` 并且使用一个正确的 [DSN](#DSN) 作为 `dataSourceName`，DSN 支持的参数：
|
||||
|
||||
* configPath 指定 taos.cfg 目录
|
||||
* cfg 指定 taos.cfg 目录
|
||||
|
||||
示例：
|
||||
|
||||
|
|
|
|||
|
|
@ -35,8 +35,8 @@ database_option: {
|
|||
| TABLE_SUFFIX value
|
||||
| TSDB_PAGESIZE value
|
||||
| WAL_RETENTION_PERIOD value
|
||||
| WAL_ROLL_PERIOD value
|
||||
| WAL_RETENTION_SIZE value
|
||||
| WAL_ROLL_PERIOD value
|
||||
| WAL_SEGMENT_SIZE value
|
||||
}
|
||||
```
|
||||
|
|
@ -75,11 +75,10 @@ database_option: {
|
|||
- TABLE_PREFIX：内部存储引擎根据表名分配存储该表数据的 VNODE 时要忽略的前缀的长度。
|
||||
- TABLE_SUFFIX：内部存储引擎根据表名分配存储该表数据的 VNODE 时要忽略的后缀的长度。
|
||||
- TSDB_PAGESIZE：一个 VNODE 中时序数据存储引擎的页大小，单位为 KB，默认为 4 KB。范围为 1 到 16384，即 1 KB到 16 MB。
|
||||
- WAL_RETENTION_PERIOD：数据订阅已消费WAL日志，WAL文件的最大额外保留的时长策略。单位为 s。默认为 0，表示无需额外保留。-1, 表示额外保留，时间无上限。
|
||||
- WAL_RETENTION_SIZE：数据订阅已消费WAL日志，WAL文件的最大额外保留的累计大小策略。单位为 KB。默认为 0，表示无需额外保留。-1, 表示额外保留，累计大小无上限。
|
||||
- WAL_RETENTION_PERIOD: 为了数据订阅消费，需要WAL日志文件额外保留的最大时长策略。WAL日志清理，不受订阅客户端消费状态影响。单位为 s。默认为 0，表示无需为订阅保留。新建订阅，应先设置恰当的时长策略。
|
||||
- WAL_RETENTION_SIZE：为了数据订阅消费，需要WAL日志文件额外保留的最大累计大小策略。单位为 KB。默认为 0，表示累计大小无上限。
|
||||
- WAL_ROLL_PERIOD：wal 文件切换时长，单位为 s。当WAL文件创建并写入后，经过该时间，会自动创建一个新的WAL文件。默认为 0，即仅在TSDB落盘时创建新文件。
|
||||
- WAL_SEGMENT_SIZE：wal 单个文件大小，单位为 KB。当前写入文件大小超过上限后会自动创建一个新的WAL文件。默认为 0，即仅在TSDB落盘时创建新文件。
|
||||
|
||||
### 创建数据库示例
|
||||
|
||||
```sql
|
||||
|
|
|
|||
|
|
@ -13,12 +13,11 @@ create_definition:
|
|||
col_name column_definition
|
||||
|
||||
column_definition:
|
||||
type_name [COMMENT 'string_value']
|
||||
type_name
|
||||
```
|
||||
|
||||
**使用说明**
|
||||
- 超级表中列的最大个数为 4096，需要注意，这里的 4096 是包含 TAG 列在内的，最小个数为 3，包含一个时间戳主键、一个 TAG 列和一个数据列。
|
||||
- 建表时可以给列或标签附加注释。
|
||||
- TAGS语法指定超级表的标签列，标签列需要遵循以下约定：
|
||||
- TAGS 中的 TIMESTAMP 列写入数据时需要提供给定值，而暂不支持四则运算，例如 NOW + 10s 这类表达式。
|
||||
- TAGS 列名不能与其他列名相同。
|
||||
|
|
|
|||
|
|
@ -269,7 +269,7 @@ description: TDengine 保留关键字的详细列表
|
|||
- SPLIT
|
||||
- STABLE
|
||||
- STABLES
|
||||
- STAR
|
||||
- START
|
||||
- STATE
|
||||
- STATE_WINDOW
|
||||
- STATEMENT
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ description: "TDengine 3.0 版本的语法变更说明"
|
|||
| - | :------- | :-------- | :------- |
|
||||
| 1 | ALTER ACCOUNT | 废除 | 2.x中为企业版功能，3.0不再支持。语法暂时保留了，执行报“This statement is no longer supported”错误。
|
||||
| 2 | ALTER ALL DNODES | 新增 | 修改所有DNODE的参数。
|
||||
| 3 | ALTER DATABASE | 调整 | 废除<ul><li>QUORUM：写入需要的副本确认数。3.0版本使用STRICT来指定强一致还是弱一致。3.0.0版本STRICT暂不支持修改。</li><li>BLOCKS：VNODE使用的内存块数。3.0版本使用BUFFER来表示VNODE写入内存池的大小。</li><li>UPDATE：更新操作的支持模式。3.0版本所有数据库都支持部分列更新。</li><li>CACHELAST：缓存最新一行数据的模式。3.0版本用CACHEMODEL代替。</li><li>COMP：3.0版本暂不支持修改。<br/>新增</li><li>CACHEMODEL：表示是否在内存中缓存子表的最近数据。</li><li>CACHESIZE：表示缓存子表最近数据的内存大小。</li><li>WAL_FSYNC_PERIOD：代替原FSYNC参数。</li><li>WAL_LEVEL：代替原WAL参数。<br/>调整</li><li>REPLICA：3.0.0版本暂不支持修改。</li><li>KEEP：3.0版本新增支持带单位的设置方式。</li></ul>
|
||||
| 3 | ALTER DATABASE | 调整 | 废除<ul><li>QUORUM：写入需要的副本确认数。3.0版本使用STRICT来指定强一致还是弱一致。3.0.0版本STRICT暂不支持修改。</li><li>BLOCKS：VNODE使用的内存块数。3.0版本使用BUFFER来表示VNODE写入内存池的大小。</li><li>UPDATE：更新操作的支持模式。3.0版本所有数据库都支持部分列更新。</li><li>CACHELAST：缓存最新一行数据的模式。3.0版本用CACHEMODEL代替。</li><li>COMP：3.0版本暂不支持修改。<br/>新增<li>CACHEMODEL：表示是否在内存中缓存子表的最近数据。</li><li>CACHESIZE：表示缓存子表最近数据的内存大小。</li><li>WAL_FSYNC_PERIOD：代替原FSYNC参数。</li><li>WAL_LEVEL：代替原WAL参数。</li><li>WAL_RETENTION_PERIOD：3.0.4.0版本新增，wal文件的额外保留策略，用于数据订阅。</li><li>WAL_RETENTION_SIZE：3.0.4.0版本新增，wal文件的额外保留策略，用于数据订阅。<br/>调整</li><li>REPLICA：3.0.0版本暂不支持修改。</li><li>KEEP：3.0版本新增支持带单位的设置方式。</li></ul>
|
||||
| 4 | ALTER STABLE | 调整 | 废除<ul><li>CHANGE TAG：修改标签列的名称。3.0版本使用RENAME TAG代替。<br/>新增</li><li>RENAME TAG：代替原CHANGE TAG子句。</li><li>COMMENT：修改超级表的注释。</li></ul>
|
||||
| 5 | ALTER TABLE | 调整 | 废除<ul><li>CHANGE TAG：修改标签列的名称。3.0版本使用RENAME TAG代替。<br/>新增</li><li>RENAME TAG：代替原CHANGE TAG子句。</li><li>COMMENT：修改表的注释。</li><li>TTL：修改表的生命周期。</li></ul>
|
||||
| 6 | ALTER USER | 调整 | 废除<ul><li>PRIVILEGE：修改用户权限。3.0版本使用GRANT和REVOKE来授予和回收权限。<br/>新增</li><li>ENABLE：启用或停用此用户。</li><li>SYSINFO：修改用户是否可查看系统信息。</li></ul>
|
||||
|
|
|
|||
|
|
@ -10,6 +10,10 @@ TDengine 2.x 各版本安装包请访问[这里](https://www.taosdata.com/all-do
|
|||
|
||||
import Release from "/components/ReleaseV3";
|
||||
|
||||
## 3.0.3.2
|
||||
|
||||
<Release type="tdengine" version="3.0.3.2" />
|
||||
|
||||
## 3.0.3.1
|
||||
|
||||
<Release type="tdengine" version="3.0.3.1" />
|
||||
|
|
|
|||
|
|
@ -10,6 +10,10 @@ taosTools 各版本安装包下载链接如下：
|
|||
|
||||
import Release from "/components/ReleaseV3";
|
||||
|
||||
## 2.4.11
|
||||
|
||||
<Release type="tools" version="2.4.11" />
|
||||
|
||||
## 2.4.10
|
||||
|
||||
<Release type="tools" version="2.4.10" />
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@
|
|||
<description>Demo project for TDengine</description>
|
||||
|
||||
<properties>
|
||||
<spring.version>5.3.20</spring.version>
|
||||
<spring.version>5.3.26</spring.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
|
|
|
|||
|
|
@ -185,7 +185,7 @@ typedef struct SBlockID {
|
|||
typedef struct SDataBlockInfo {
|
||||
STimeWindow window;
|
||||
int32_t rowSize;
|
||||
int32_t rows; // todo hide this attribute
|
||||
int64_t rows; // todo hide this attribute
|
||||
uint32_t capacity;
|
||||
SBlockID id;
|
||||
int16_t hasVarCol;
|
||||
|
|
|
|||
|
|
@ -178,7 +178,7 @@ int32_t getJsonValueLen(const char* data);
|
|||
|
||||
int32_t colDataSetVal(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull);
|
||||
int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull);
|
||||
int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, uint32_t numOfRows);
|
||||
int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, uint32_t numOfRows, bool trimValue);
|
||||
int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int32_t* capacity,
|
||||
const SColumnInfoData* pSource, int32_t numOfRow2);
|
||||
int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows,
|
||||
|
|
|
|||
|
|
@ -1054,6 +1054,7 @@ typedef struct {
|
|||
int64_t signature;
|
||||
char* pComment;
|
||||
char* pCode;
|
||||
int8_t orReplace;
|
||||
} SCreateFuncReq;
|
||||
|
||||
int32_t tSerializeSCreateFuncReq(void* buf, int32_t bufLen, SCreateFuncReq* pReq);
|
||||
|
|
@ -1669,7 +1670,7 @@ int32_t tSerializeSBalanceVgroupLeaderReq(void* buf, int32_t bufLen, SBalanceVgr
|
|||
int32_t tDeserializeSBalanceVgroupLeaderReq(void* buf, int32_t bufLen, SBalanceVgroupLeaderReq* pReq);
|
||||
|
||||
typedef struct {
|
||||
int32_t vgId;
|
||||
int32_t vgId;
|
||||
} SForceBecomeFollowerReq;
|
||||
|
||||
int32_t tSerializeSForceBecomeFollowerReq(void* buf, int32_t bufLen, SForceBecomeFollowerReq* pReq);
|
||||
|
|
@ -3206,9 +3207,9 @@ typedef struct {
|
|||
SArray* blockTbName;
|
||||
SArray* blockSchema;
|
||||
// the following attributes are extended from SMqDataRsp
|
||||
int32_t createTableNum;
|
||||
SArray* createTableLen;
|
||||
SArray* createTableReq;
|
||||
int32_t createTableNum;
|
||||
SArray* createTableLen;
|
||||
SArray* createTableReq;
|
||||
} STaosxRsp;
|
||||
|
||||
int32_t tEncodeSTaosxRsp(SEncoder* pEncoder, const STaosxRsp* pRsp);
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ typedef enum {
|
|||
WRITE_QUEUE,
|
||||
APPLY_QUEUE,
|
||||
SYNC_QUEUE,
|
||||
SYNC_CTRL_QUEUE,
|
||||
SYNC_RD_QUEUE,
|
||||
STREAM_QUEUE,
|
||||
QUEUE_MAX,
|
||||
} EQueueType;
|
||||
|
|
|
|||
|
|
@ -260,7 +260,7 @@ enum {
|
|||
|
||||
TD_NEW_MSG_SEG(TDMT_SYNC_MSG)
|
||||
TD_DEF_MSG_TYPE(TDMT_SYNC_TIMEOUT, "sync-timer", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_SYNC_PING, "sync-ping", NULL, NULL) // no longer used
|
||||
TD_DEF_MSG_TYPE(TDMT_SYNC_TIMEOUT_ELECTION, "sync-elect", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_SYNC_PING_REPLY, "sync-ping-reply", NULL, NULL) // no longer used
|
||||
TD_DEF_MSG_TYPE(TDMT_SYNC_CLIENT_REQUEST, "sync-client-request", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_SYNC_CLIENT_REQUEST_BATCH, "sync-client-request-batch", NULL, NULL)
|
||||
|
|
|
|||
|
|
@ -64,7 +64,7 @@ static FORCE_INLINE int64_t taosGetTimestampToday(int32_t precision) {
|
|||
: 1000000000;
|
||||
time_t t = taosTime(NULL);
|
||||
struct tm tm;
|
||||
taosLocalTime(&t, &tm);
|
||||
taosLocalTime(&t, &tm, NULL);
|
||||
tm.tm_hour = 0;
|
||||
tm.tm_min = 0;
|
||||
tm.tm_sec = 0;
|
||||
|
|
|
|||
|
|
@ -208,131 +208,131 @@
|
|||
#define TK_AGGREGATE 190
|
||||
#define TK_BUFSIZE 191
|
||||
#define TK_LANGUAGE 192
|
||||
#define TK_STREAM 193
|
||||
#define TK_INTO 194
|
||||
#define TK_TRIGGER 195
|
||||
#define TK_AT_ONCE 196
|
||||
#define TK_WINDOW_CLOSE 197
|
||||
#define TK_IGNORE 198
|
||||
#define TK_EXPIRED 199
|
||||
#define TK_FILL_HISTORY 200
|
||||
#define TK_UPDATE 201
|
||||
#define TK_SUBTABLE 202
|
||||
#define TK_KILL 203
|
||||
#define TK_CONNECTION 204
|
||||
#define TK_TRANSACTION 205
|
||||
#define TK_BALANCE 206
|
||||
#define TK_VGROUP 207
|
||||
#define TK_LEADER 208
|
||||
#define TK_MERGE 209
|
||||
#define TK_REDISTRIBUTE 210
|
||||
#define TK_SPLIT 211
|
||||
#define TK_DELETE 212
|
||||
#define TK_INSERT 213
|
||||
#define TK_NULL 214
|
||||
#define TK_NK_QUESTION 215
|
||||
#define TK_NK_ARROW 216
|
||||
#define TK_ROWTS 217
|
||||
#define TK_QSTART 218
|
||||
#define TK_QEND 219
|
||||
#define TK_QDURATION 220
|
||||
#define TK_WSTART 221
|
||||
#define TK_WEND 222
|
||||
#define TK_WDURATION 223
|
||||
#define TK_IROWTS 224
|
||||
#define TK_ISFILLED 225
|
||||
#define TK_CAST 226
|
||||
#define TK_NOW 227
|
||||
#define TK_TODAY 228
|
||||
#define TK_TIMEZONE 229
|
||||
#define TK_CLIENT_VERSION 230
|
||||
#define TK_SERVER_VERSION 231
|
||||
#define TK_SERVER_STATUS 232
|
||||
#define TK_CURRENT_USER 233
|
||||
#define TK_CASE 234
|
||||
#define TK_WHEN 235
|
||||
#define TK_THEN 236
|
||||
#define TK_ELSE 237
|
||||
#define TK_BETWEEN 238
|
||||
#define TK_IS 239
|
||||
#define TK_NK_LT 240
|
||||
#define TK_NK_GT 241
|
||||
#define TK_NK_LE 242
|
||||
#define TK_NK_GE 243
|
||||
#define TK_NK_NE 244
|
||||
#define TK_MATCH 245
|
||||
#define TK_NMATCH 246
|
||||
#define TK_CONTAINS 247
|
||||
#define TK_IN 248
|
||||
#define TK_JOIN 249
|
||||
#define TK_INNER 250
|
||||
#define TK_SELECT 251
|
||||
#define TK_DISTINCT 252
|
||||
#define TK_WHERE 253
|
||||
#define TK_PARTITION 254
|
||||
#define TK_BY 255
|
||||
#define TK_SESSION 256
|
||||
#define TK_STATE_WINDOW 257
|
||||
#define TK_EVENT_WINDOW 258
|
||||
#define TK_SLIDING 259
|
||||
#define TK_FILL 260
|
||||
#define TK_VALUE 261
|
||||
#define TK_VALUE_F 262
|
||||
#define TK_NONE 263
|
||||
#define TK_PREV 264
|
||||
#define TK_NULL_F 265
|
||||
#define TK_LINEAR 266
|
||||
#define TK_NEXT 267
|
||||
#define TK_HAVING 268
|
||||
#define TK_RANGE 269
|
||||
#define TK_EVERY 270
|
||||
#define TK_ORDER 271
|
||||
#define TK_SLIMIT 272
|
||||
#define TK_SOFFSET 273
|
||||
#define TK_LIMIT 274
|
||||
#define TK_OFFSET 275
|
||||
#define TK_ASC 276
|
||||
#define TK_NULLS 277
|
||||
#define TK_ABORT 278
|
||||
#define TK_AFTER 279
|
||||
#define TK_ATTACH 280
|
||||
#define TK_BEFORE 281
|
||||
#define TK_BEGIN 282
|
||||
#define TK_BITAND 283
|
||||
#define TK_BITNOT 284
|
||||
#define TK_BITOR 285
|
||||
#define TK_BLOCKS 286
|
||||
#define TK_CHANGE 287
|
||||
#define TK_COMMA 288
|
||||
#define TK_CONCAT 289
|
||||
#define TK_CONFLICT 290
|
||||
#define TK_COPY 291
|
||||
#define TK_DEFERRED 292
|
||||
#define TK_DELIMITERS 293
|
||||
#define TK_DETACH 294
|
||||
#define TK_DIVIDE 295
|
||||
#define TK_DOT 296
|
||||
#define TK_EACH 297
|
||||
#define TK_FAIL 298
|
||||
#define TK_FILE 299
|
||||
#define TK_FOR 300
|
||||
#define TK_GLOB 301
|
||||
#define TK_ID 302
|
||||
#define TK_IMMEDIATE 303
|
||||
#define TK_IMPORT 304
|
||||
#define TK_INITIALLY 305
|
||||
#define TK_INSTEAD 306
|
||||
#define TK_ISNULL 307
|
||||
#define TK_KEY 308
|
||||
#define TK_MODULES 309
|
||||
#define TK_NK_BITNOT 310
|
||||
#define TK_NK_SEMI 311
|
||||
#define TK_NOTNULL 312
|
||||
#define TK_OF 313
|
||||
#define TK_PLUS 314
|
||||
#define TK_PRIVILEGE 315
|
||||
#define TK_RAISE 316
|
||||
#define TK_REPLACE 317
|
||||
#define TK_REPLACE 193
|
||||
#define TK_STREAM 194
|
||||
#define TK_INTO 195
|
||||
#define TK_TRIGGER 196
|
||||
#define TK_AT_ONCE 197
|
||||
#define TK_WINDOW_CLOSE 198
|
||||
#define TK_IGNORE 199
|
||||
#define TK_EXPIRED 200
|
||||
#define TK_FILL_HISTORY 201
|
||||
#define TK_UPDATE 202
|
||||
#define TK_SUBTABLE 203
|
||||
#define TK_KILL 204
|
||||
#define TK_CONNECTION 205
|
||||
#define TK_TRANSACTION 206
|
||||
#define TK_BALANCE 207
|
||||
#define TK_VGROUP 208
|
||||
#define TK_LEADER 209
|
||||
#define TK_MERGE 210
|
||||
#define TK_REDISTRIBUTE 211
|
||||
#define TK_SPLIT 212
|
||||
#define TK_DELETE 213
|
||||
#define TK_INSERT 214
|
||||
#define TK_NULL 215
|
||||
#define TK_NK_QUESTION 216
|
||||
#define TK_NK_ARROW 217
|
||||
#define TK_ROWTS 218
|
||||
#define TK_QSTART 219
|
||||
#define TK_QEND 220
|
||||
#define TK_QDURATION 221
|
||||
#define TK_WSTART 222
|
||||
#define TK_WEND 223
|
||||
#define TK_WDURATION 224
|
||||
#define TK_IROWTS 225
|
||||
#define TK_ISFILLED 226
|
||||
#define TK_CAST 227
|
||||
#define TK_NOW 228
|
||||
#define TK_TODAY 229
|
||||
#define TK_TIMEZONE 230
|
||||
#define TK_CLIENT_VERSION 231
|
||||
#define TK_SERVER_VERSION 232
|
||||
#define TK_SERVER_STATUS 233
|
||||
#define TK_CURRENT_USER 234
|
||||
#define TK_CASE 235
|
||||
#define TK_WHEN 236
|
||||
#define TK_THEN 237
|
||||
#define TK_ELSE 238
|
||||
#define TK_BETWEEN 239
|
||||
#define TK_IS 240
|
||||
#define TK_NK_LT 241
|
||||
#define TK_NK_GT 242
|
||||
#define TK_NK_LE 243
|
||||
#define TK_NK_GE 244
|
||||
#define TK_NK_NE 245
|
||||
#define TK_MATCH 246
|
||||
#define TK_NMATCH 247
|
||||
#define TK_CONTAINS 248
|
||||
#define TK_IN 249
|
||||
#define TK_JOIN 250
|
||||
#define TK_INNER 251
|
||||
#define TK_SELECT 252
|
||||
#define TK_DISTINCT 253
|
||||
#define TK_WHERE 254
|
||||
#define TK_PARTITION 255
|
||||
#define TK_BY 256
|
||||
#define TK_SESSION 257
|
||||
#define TK_STATE_WINDOW 258
|
||||
#define TK_EVENT_WINDOW 259
|
||||
#define TK_SLIDING 260
|
||||
#define TK_FILL 261
|
||||
#define TK_VALUE 262
|
||||
#define TK_VALUE_F 263
|
||||
#define TK_NONE 264
|
||||
#define TK_PREV 265
|
||||
#define TK_NULL_F 266
|
||||
#define TK_LINEAR 267
|
||||
#define TK_NEXT 268
|
||||
#define TK_HAVING 269
|
||||
#define TK_RANGE 270
|
||||
#define TK_EVERY 271
|
||||
#define TK_ORDER 272
|
||||
#define TK_SLIMIT 273
|
||||
#define TK_SOFFSET 274
|
||||
#define TK_LIMIT 275
|
||||
#define TK_OFFSET 276
|
||||
#define TK_ASC 277
|
||||
#define TK_NULLS 278
|
||||
#define TK_ABORT 279
|
||||
#define TK_AFTER 280
|
||||
#define TK_ATTACH 281
|
||||
#define TK_BEFORE 282
|
||||
#define TK_BEGIN 283
|
||||
#define TK_BITAND 284
|
||||
#define TK_BITNOT 285
|
||||
#define TK_BITOR 286
|
||||
#define TK_BLOCKS 287
|
||||
#define TK_CHANGE 288
|
||||
#define TK_COMMA 289
|
||||
#define TK_CONCAT 290
|
||||
#define TK_CONFLICT 291
|
||||
#define TK_COPY 292
|
||||
#define TK_DEFERRED 293
|
||||
#define TK_DELIMITERS 294
|
||||
#define TK_DETACH 295
|
||||
#define TK_DIVIDE 296
|
||||
#define TK_DOT 297
|
||||
#define TK_EACH 298
|
||||
#define TK_FAIL 299
|
||||
#define TK_FILE 300
|
||||
#define TK_FOR 301
|
||||
#define TK_GLOB 302
|
||||
#define TK_ID 303
|
||||
#define TK_IMMEDIATE 304
|
||||
#define TK_IMPORT 305
|
||||
#define TK_INITIALLY 306
|
||||
#define TK_INSTEAD 307
|
||||
#define TK_ISNULL 308
|
||||
#define TK_KEY 309
|
||||
#define TK_MODULES 310
|
||||
#define TK_NK_BITNOT 311
|
||||
#define TK_NK_SEMI 312
|
||||
#define TK_NOTNULL 313
|
||||
#define TK_OF 314
|
||||
#define TK_PLUS 315
|
||||
#define TK_PRIVILEGE 316
|
||||
#define TK_RAISE 317
|
||||
#define TK_RESTRICT 318
|
||||
#define TK_ROW 319
|
||||
#define TK_SEMI 320
|
||||
|
|
|
|||
|
|
@ -78,7 +78,8 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers, int32_t v
|
|||
* @param SReadHandle
|
||||
* @return
|
||||
*/
|
||||
qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int32_t vgId, int32_t* numOfCols, SSchemaWrapper** pSchema);
|
||||
qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int32_t vgId, int32_t* numOfCols,
|
||||
uint64_t id);
|
||||
|
||||
/**
|
||||
* set the task Id, usually used by message queue process
|
||||
|
|
@ -89,6 +90,7 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int3
|
|||
void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId);
|
||||
|
||||
int32_t qSetStreamOpOpen(qTaskInfo_t tinfo);
|
||||
|
||||
/**
|
||||
* Set multiple input data blocks for the stream scan.
|
||||
* @param tinfo
|
||||
|
|
@ -149,7 +151,6 @@ int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* table
|
|||
* @param handle
|
||||
* @return
|
||||
*/
|
||||
|
||||
int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bool* hasMore, SLocalFetch* pLocal);
|
||||
|
||||
int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pBlock, uint64_t* useconds);
|
||||
|
|
@ -162,6 +163,7 @@ void qCleanExecTaskBlockBuf(qTaskInfo_t tinfo);
|
|||
* @return
|
||||
*/
|
||||
int32_t qAsyncKillTask(qTaskInfo_t tinfo, int32_t rspCode);
|
||||
int32_t qKillTask(qTaskInfo_t tinfo, int32_t rspCode);
|
||||
|
||||
bool qTaskIsExecuting(qTaskInfo_t qinfo);
|
||||
|
||||
|
|
@ -171,14 +173,6 @@ bool qTaskIsExecuting(qTaskInfo_t qinfo);
|
|||
*/
|
||||
void qDestroyTask(qTaskInfo_t tinfo);
|
||||
|
||||
/**
|
||||
* Extract the qualified table id list, and than pass them to the TSDB driver to load the required table data blocks.
|
||||
*
|
||||
* @param iter the table iterator to traverse all tables belongs to a super table, or an invert index
|
||||
* @return
|
||||
*/
|
||||
int32_t qGetQualifiedTableIdList(void* pTableList, const char* tagCond, int32_t tagCondLen, SArray* pTableIdList);
|
||||
|
||||
void qProcessRspMsg(void* parent, struct SRpcMsg* pMsg, struct SEpSet* pEpSet);
|
||||
|
||||
int32_t qGetExplainExecInfo(qTaskInfo_t tinfo, SArray* pExecInfoList /*,int32_t* resNum, SExplainExecInfo** pRes*/);
|
||||
|
|
|
|||
|
|
@ -112,7 +112,7 @@ typedef struct SResultDataInfo {
|
|||
typedef struct SInputColumnInfoData {
|
||||
int32_t totalRows; // total rows in current columnar data
|
||||
int32_t startRowIndex; // handle started row index
|
||||
int32_t numOfRows; // the number of rows needs to be handled
|
||||
int64_t numOfRows; // the number of rows needs to be handled
|
||||
int32_t numOfInputCols; // PTS is not included
|
||||
bool colDataSMAIsSet; // if agg is set or not
|
||||
SColumnInfoData *pPTS; // primary timestamp column
|
||||
|
|
|
|||
|
|
@ -241,6 +241,7 @@ int32_t fmGetUdafExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet);
|
|||
int32_t fmSetInvertFunc(int32_t funcId, SFuncExecFuncs* pFpSet);
|
||||
int32_t fmSetNormalFunc(int32_t funcId, SFuncExecFuncs* pFpSet);
|
||||
bool fmIsInvertible(int32_t funcId);
|
||||
char* fmGetFuncName(int32_t funcId);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
|
|
|||
|
|
@ -438,6 +438,7 @@ typedef struct SDropStreamStmt {
|
|||
|
||||
typedef struct SCreateFunctionStmt {
|
||||
ENodeType type;
|
||||
bool orReplace;
|
||||
bool ignoreExists;
|
||||
char funcName[TSDB_FUNC_NAME_LEN];
|
||||
bool isAgg;
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ typedef enum {
|
|||
JOB_TASK_STATUS_INIT,
|
||||
JOB_TASK_STATUS_EXEC,
|
||||
JOB_TASK_STATUS_PART_SUCC,
|
||||
JOB_TASK_STATUS_FETCH,
|
||||
JOB_TASK_STATUS_SUCC,
|
||||
JOB_TASK_STATUS_FAIL,
|
||||
JOB_TASK_STATUS_DROP,
|
||||
|
|
|
|||
|
|
@ -23,20 +23,19 @@ extern "C" {
|
|||
#ifndef _STREAM_STATE_H_
|
||||
#define _STREAM_STATE_H_
|
||||
|
||||
typedef struct SStreamTask SStreamTask;
|
||||
|
||||
typedef bool (*state_key_cmpr_fn)(void* pKey1, void* pKey2);
|
||||
|
||||
typedef struct STdbState {
|
||||
SStreamTask* pOwner;
|
||||
TDB* db;
|
||||
TTB* pStateDb;
|
||||
TTB* pFuncStateDb;
|
||||
TTB* pFillStateDb; // todo refactor
|
||||
TTB* pSessionStateDb;
|
||||
TTB* pParNameDb;
|
||||
TTB* pParTagDb;
|
||||
TXN* txn;
|
||||
struct SStreamTask* pOwner;
|
||||
|
||||
TDB* db;
|
||||
TTB* pStateDb;
|
||||
TTB* pFuncStateDb;
|
||||
TTB* pFillStateDb; // todo refactor
|
||||
TTB* pSessionStateDb;
|
||||
TTB* pParNameDb;
|
||||
TTB* pParTagDb;
|
||||
TXN* txn;
|
||||
} STdbState;
|
||||
|
||||
// incremental state storage
|
||||
|
|
@ -45,7 +44,7 @@ typedef struct {
|
|||
int32_t number;
|
||||
} SStreamState;
|
||||
|
||||
SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int32_t szPage, int32_t pages);
|
||||
SStreamState* streamStateOpen(char* path, struct SStreamTask* pTask, bool specPath, int32_t szPage, int32_t pages);
|
||||
void streamStateClose(SStreamState* pState);
|
||||
int32_t streamStateBegin(SStreamState* pState);
|
||||
int32_t streamStateCommit(SStreamState* pState);
|
||||
|
|
|
|||
|
|
@ -295,7 +295,7 @@ typedef struct {
|
|||
SEpSet epSet;
|
||||
} SStreamChildEpInfo;
|
||||
|
||||
typedef struct SStreamTask {
|
||||
struct SStreamTask {
|
||||
int64_t streamId;
|
||||
int32_t taskId;
|
||||
int32_t totalLevel;
|
||||
|
|
@ -362,8 +362,7 @@ typedef struct SStreamTask {
|
|||
|
||||
int64_t checkpointingId;
|
||||
int32_t checkpointAlignCnt;
|
||||
|
||||
} SStreamTask;
|
||||
};
|
||||
|
||||
int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo);
|
||||
int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo);
|
||||
|
|
@ -587,7 +586,7 @@ int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, S
|
|||
int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, int32_t code);
|
||||
|
||||
int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, SRpcMsg* pMsg);
|
||||
int32_t streamProcessRetrieveRsp(SStreamTask* pTask, SStreamRetrieveRsp* pRsp);
|
||||
// int32_t streamProcessRetrieveRsp(SStreamTask* pTask, SStreamRetrieveRsp* pRsp);
|
||||
|
||||
int32_t streamTryExec(SStreamTask* pTask);
|
||||
int32_t streamSchedExec(SStreamTask* pTask);
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@ extern "C" {
|
|||
#define SYNC_HEARTBEAT_REPLY_SLOW_MS 1500
|
||||
#define SYNC_SNAP_RESEND_MS 1000 * 60
|
||||
|
||||
#define SYNC_VND_COMMIT_MIN_MS 1000
|
||||
#define SYNC_VND_COMMIT_MIN_MS 3000
|
||||
|
||||
#define SYNC_MAX_BATCH_SIZE 1
|
||||
#define SYNC_INDEX_BEGIN 0
|
||||
|
|
|
|||
|
|
@ -91,7 +91,7 @@ static FORCE_INLINE int64_t taosGetMonoTimestampMs() {
|
|||
}
|
||||
|
||||
char *taosStrpTime(const char *buf, const char *fmt, struct tm *tm);
|
||||
struct tm *taosLocalTime(const time_t *timep, struct tm *result);
|
||||
struct tm *taosLocalTime(const time_t *timep, struct tm *result, char *buf);
|
||||
struct tm *taosLocalTimeNolock(struct tm *result, const time_t *timep, int dst);
|
||||
time_t taosTime(time_t *t);
|
||||
time_t taosMktime(struct tm *timep);
|
||||
|
|
|
|||
|
|
@ -69,14 +69,6 @@ void* taosArrayAddBatch(SArray* pArray, const void* pData, int32_t nEles);
|
|||
*/
|
||||
void taosArrayRemoveDuplicate(SArray* pArray, __compar_fn_t comparFn, void (*fp)(void*));
|
||||
|
||||
/**
|
||||
*
|
||||
* @param pArray
|
||||
* @param comparFn
|
||||
* @param fp
|
||||
*/
|
||||
void taosArrayRemoveDuplicateP(SArray* pArray, __compar_fn_t comparFn, void (*fp)(void*));
|
||||
|
||||
/**
|
||||
* add all element from the source array list into the destination
|
||||
* @param pArray
|
||||
|
|
@ -216,6 +208,8 @@ void taosArrayDestroyP(SArray* pArray, FDelete fp);
|
|||
|
||||
void taosArrayDestroyEx(SArray* pArray, FDelete fp);
|
||||
|
||||
void taosArraySwap(SArray* a, SArray* b);
|
||||
|
||||
/**
|
||||
* sort the array
|
||||
* @param pArray
|
||||
|
|
@ -252,14 +246,6 @@ void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* par
|
|||
int32_t taosEncodeArray(void** buf, const SArray* pArray, FEncode encode);
|
||||
void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t dataSz);
|
||||
|
||||
/**
|
||||
* swap array
|
||||
* @param a
|
||||
* @param b
|
||||
* @return
|
||||
*/
|
||||
void taosArraySwap(SArray* a, SArray* b);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -48,6 +48,13 @@ SSHashObj *tSimpleHashInit(size_t capacity, _hash_fn_t fn);
|
|||
*/
|
||||
int32_t tSimpleHashGetSize(const SSHashObj *pHashObj);
|
||||
|
||||
/**
|
||||
* set the free function pointer
|
||||
* @param pHashObj
|
||||
* @param freeFp
|
||||
*/
|
||||
void tSimpleHashSetFreeFp(SSHashObj* pHashObj, _hash_free_fn_t freeFp);
|
||||
|
||||
int32_t tSimpleHashPrint(const SSHashObj *pHashObj);
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ extern "C" {
|
|||
extern char version[];
|
||||
extern char compatible_version[];
|
||||
extern char gitinfo[];
|
||||
extern char gitinfoOfInternal[];
|
||||
extern char buildinfo[];
|
||||
|
||||
#ifdef __cplusplus
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
[Unit]
|
||||
Description=TDengine server service
|
||||
Description=server service
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
|
|
|
|||
|
|
@ -90,45 +90,33 @@ cp %{_compiledir}/../include/libs/function/taosudf.h %{buildroot}%{homepat
|
|||
cp -r %{_compiledir}/../examples/* %{buildroot}%{homepath}/examples
|
||||
|
||||
if [ -f %{_compiledir}/build/bin/jemalloc-config ]; then
|
||||
mkdir -p %{buildroot}%{userlocalpath}/bin
|
||||
mkdir -p %{buildroot}%{userlocalpath}/lib
|
||||
mkdir -p %{buildroot}%{userlocalpath}/lib/pkgconfig
|
||||
mkdir -p %{buildroot}%{userlocalpath}/include
|
||||
mkdir -p %{buildroot}%{userlocalpath}/include/jemalloc
|
||||
mkdir -p %{buildroot}%{userlocalpath}/share
|
||||
mkdir -p %{buildroot}%{userlocalpath}/share/doc
|
||||
mkdir -p %{buildroot}%{userlocalpath}/share/doc/jemalloc
|
||||
mkdir -p %{buildroot}%{userlocalpath}/share/man
|
||||
mkdir -p %{buildroot}%{userlocalpath}/share/man/man3
|
||||
mkdir -p %{buildroot}%{homepath}/jemalloc/ ||:
|
||||
mkdir -p %{buildroot}%{homepath}/jemalloc/include/jemalloc/ ||:
|
||||
mkdir -p %{buildroot}%{homepath}/jemalloc/lib/ ||:
|
||||
mkdir -p %{buildroot}%{homepath}/jemalloc/lib/pkgconfig ||:
|
||||
|
||||
cp %{_compiledir}/build/bin/jemalloc-config %{buildroot}%{userlocalpath}/bin/
|
||||
cp %{_compiledir}/build/bin/jemalloc-config %{buildroot}%{homepath}/jemalloc/bin
|
||||
if [ -f %{_compiledir}/build/bin/jemalloc.sh ]; then
|
||||
cp %{_compiledir}/build/bin/jemalloc.sh %{buildroot}%{userlocalpath}/bin/
|
||||
cp %{_compiledir}/build/bin/jemalloc.sh %{buildroot}%{homepath}/jemalloc/bin
|
||||
fi
|
||||
if [ -f %{_compiledir}/build/bin/jeprof ]; then
|
||||
cp %{_compiledir}/build/bin/jeprof %{buildroot}%{userlocalpath}/bin/
|
||||
cp %{_compiledir}/build/bin/jeprof %{buildroot}%{homepath}/jemalloc/bin
|
||||
fi
|
||||
if [ -f %{_compiledir}/build/include/jemalloc/jemalloc.h ]; then
|
||||
cp %{_compiledir}/build/include/jemalloc/jemalloc.h %{buildroot}%{userlocalpath}/include/jemalloc/
|
||||
cp %{_compiledir}/build/include/jemalloc/jemalloc.h %{buildroot}%{homepath}/jemalloc/include/jemalloc/
|
||||
fi
|
||||
if [ -f %{_compiledir}/build/lib/libjemalloc.so.2 ]; then
|
||||
cp %{_compiledir}/build/lib/libjemalloc.so.2 %{buildroot}%{userlocalpath}/lib/
|
||||
ln -sf libjemalloc.so.2 %{buildroot}%{userlocalpath}/lib/libjemalloc.so
|
||||
cp %{_compiledir}/build/lib/libjemalloc.so.2 %{buildroot}%{homepath}/jemalloc/lib
|
||||
ln -sf libjemalloc.so.2 %{buildroot}%{homepath}/jemalloc/lib/libjemalloc.so
|
||||
fi
|
||||
if [ -f %{_compiledir}/build/lib/libjemalloc.a ]; then
|
||||
cp %{_compiledir}/build/lib/libjemalloc.a %{buildroot}%{userlocalpath}/lib/
|
||||
cp %{_compiledir}/build/lib/libjemalloc.a %{buildroot}%{homepath}/jemalloc/lib
|
||||
fi
|
||||
if [ -f %{_compiledir}/build/lib/libjemalloc_pic.a ]; then
|
||||
cp %{_compiledir}/build/lib/libjemalloc_pic.a %{buildroot}%{userlocalpath}/lib/
|
||||
cp %{_compiledir}/build/lib/libjemalloc_pic.a %{buildroot}%{homepath}/jemalloc/lib
|
||||
fi
|
||||
if [ -f %{_compiledir}/build/lib/pkgconfig/jemalloc.pc ]; then
|
||||
cp %{_compiledir}/build/lib/pkgconfig/jemalloc.pc %{buildroot}%{userlocalpath}/lib/pkgconfig/
|
||||
fi
|
||||
if [ -f %{_compiledir}/build/share/doc/jemalloc/jemalloc.html ]; then
|
||||
cp %{_compiledir}/build/share/doc/jemalloc/jemalloc.html %{buildroot}%{userlocalpath}/share/doc/jemalloc/
|
||||
fi
|
||||
if [ -f %{_compiledir}/build/share/man/man3/jemalloc.3 ]; then
|
||||
cp %{_compiledir}/build/share/man/man3/jemalloc.3 %{buildroot}%{userlocalpath}/share/man/man3/
|
||||
cp %{_compiledir}/build/lib/pkgconfig/jemalloc.pc %{buildroot}%{homepath}/jemalloc/lib/pkgconfig
|
||||
fi
|
||||
fi
|
||||
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
# is required to use systemd to manage services at boot
|
||||
|
||||
set -e
|
||||
#set -x
|
||||
# set -x
|
||||
|
||||
verMode=edge
|
||||
pagMode=full
|
||||
|
|
@ -34,21 +34,25 @@ benchmarkName="taosBenchmark"
|
|||
dumpName="taosdump"
|
||||
demoName="taosdemo"
|
||||
xname="taosx"
|
||||
explorerName="${clientName}-explorer"
|
||||
|
||||
clientName2="taos"
|
||||
serverName2="taosd"
|
||||
serverName2="${clientName2}d"
|
||||
configFile2="${clientName2}.cfg"
|
||||
productName2="TDengine"
|
||||
emailName2="taosdata.com"
|
||||
xname2="${clientName2}x"
|
||||
adapterName2="${clientName2}adapter"
|
||||
|
||||
explorerName="${clientName2}-explorer"
|
||||
benchmarkName2="${clientName2}Benchmark"
|
||||
demoName2="${clientName2}demo"
|
||||
dumpName2="${clientName2}dump"
|
||||
uninstallScript2="rm${clientName2}"
|
||||
|
||||
historyFile="${clientName2}_history"
|
||||
logDir="/var/log/${clientName2}"
|
||||
configDir="/etc/${clientName2}"
|
||||
installDir="/usr/local/${clientName}"
|
||||
installDir="/usr/local/${clientName2}"
|
||||
|
||||
data_dir=${dataDir}
|
||||
log_dir=${logDir}
|
||||
|
|
@ -206,15 +210,15 @@ function install_main_path() {
|
|||
|
||||
function install_bin() {
|
||||
# Remove links
|
||||
${csudo}rm -f ${bin_link_dir}/${clientName} || :
|
||||
${csudo}rm -f ${bin_link_dir}/${serverName} || :
|
||||
${csudo}rm -f ${bin_link_dir}/${clientName2} || :
|
||||
${csudo}rm -f ${bin_link_dir}/${serverName2} || :
|
||||
${csudo}rm -f ${bin_link_dir}/${udfdName} || :
|
||||
${csudo}rm -f ${bin_link_dir}/${adapterName} || :
|
||||
${csudo}rm -f ${bin_link_dir}/${uninstallScript} || :
|
||||
${csudo}rm -f ${bin_link_dir}/${demoName} || :
|
||||
${csudo}rm -f ${bin_link_dir}/${benchmarkName} || :
|
||||
${csudo}rm -f ${bin_link_dir}/${dumpName} || :
|
||||
${csudo}rm -f ${bin_link_dir}/${xname} || :
|
||||
${csudo}rm -f ${bin_link_dir}/${uninstallScript2} || :
|
||||
${csudo}rm -f ${bin_link_dir}/${demoName2} || :
|
||||
${csudo}rm -f ${bin_link_dir}/${benchmarkName2} || :
|
||||
${csudo}rm -f ${bin_link_dir}/${dumpName2} || :
|
||||
${csudo}rm -f ${bin_link_dir}/${xname2} || :
|
||||
${csudo}rm -f ${bin_link_dir}/${explorerName} || :
|
||||
${csudo}rm -f ${bin_link_dir}/set_core || :
|
||||
${csudo}rm -f ${bin_link_dir}/TDinsight.sh || :
|
||||
|
|
@ -222,24 +226,23 @@ function install_bin() {
|
|||
${csudo}cp -r ${script_dir}/bin/* ${install_main_dir}/bin && ${csudo}chmod 0555 ${install_main_dir}/bin/*
|
||||
|
||||
#Make link
|
||||
[ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} || :
|
||||
[ -x ${install_main_dir}/bin/${serverName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} || :
|
||||
[ -x ${install_main_dir}/bin/${clientName2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${clientName2} ${bin_link_dir}/${clientName2} || :
|
||||
[ -x ${install_main_dir}/bin/${serverName2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${serverName2} ${bin_link_dir}/${serverName2} || :
|
||||
[ -x ${install_main_dir}/bin/${udfdName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${udfdName} ${bin_link_dir}/${udfdName} || :
|
||||
[ -x ${install_main_dir}/bin/${adapterName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${adapterName} ${bin_link_dir}/${adapterName} || :
|
||||
[ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${demoName} || :
|
||||
[ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${benchmarkName} || :
|
||||
[ -x ${install_main_dir}/bin/${dumpName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${dumpName} || :
|
||||
[ -x ${install_main_dir}/bin/${xname} ] && ${csudo}ln -sf ${install_main_dir}/bin/${xname} ${bin_link_dir}/${xname} || :
|
||||
[ -x ${install_main_dir}/bin/${adapterName2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${adapterName2} ${bin_link_dir}/${adapterName2} || :
|
||||
[ -x ${install_main_dir}/bin/${benchmarkName2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${benchmarkName2} ${bin_link_dir}/${demoName2} || :
|
||||
[ -x ${install_main_dir}/bin/${benchmarkName2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${benchmarkName2} ${bin_link_dir}/${benchmarkName2} || :
|
||||
[ -x ${install_main_dir}/bin/${dumpName2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${dumpName2} ${bin_link_dir}/${dumpName2} || :
|
||||
[ -x ${install_main_dir}/bin/${xname2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${xname2} ${bin_link_dir}/${xname2} || :
|
||||
[ -x ${install_main_dir}/bin/${explorerName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${explorerName} ${bin_link_dir}/${explorerName} || :
|
||||
[ -x ${install_main_dir}/bin/TDinsight.sh ] && ${csudo}ln -sf ${install_main_dir}/bin/TDinsight.sh ${bin_link_dir}/TDinsight.sh || :
|
||||
[ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -sf ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || :
|
||||
[ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo}ln -sf ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || :
|
||||
if [ "$clientName2" == "${clientName}" ]; then
|
||||
[ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || :
|
||||
fi
|
||||
[ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || :
|
||||
|
||||
if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then
|
||||
[ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName2} || :
|
||||
[ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${benchmarkName2} || :
|
||||
[ -x ${install_main_dir}/bin/${dumpName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${dumpName2} || :
|
||||
[ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -sf ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript2} || :
|
||||
[ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript2} || :
|
||||
fi
|
||||
}
|
||||
|
||||
|
|
@ -399,7 +402,7 @@ function set_hostname() {
|
|||
${csudo}sed -i -r "s/#*\s*(HOSTNAME=\s*).*/\1$newHostname/" /etc/sysconfig/network || :
|
||||
fi
|
||||
|
||||
${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/${configFile}
|
||||
${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/${configFile2}
|
||||
serverFqdn=$newHostname
|
||||
|
||||
if [[ -e /etc/hosts ]]; then
|
||||
|
|
@ -433,7 +436,7 @@ function set_ipAsFqdn() {
|
|||
echo -e -n "${GREEN}Unable to get local ip, use 127.0.0.1${NC}"
|
||||
localFqdn="127.0.0.1"
|
||||
# Write the local FQDN to configuration file
|
||||
${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/${configFile}
|
||||
${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/${configFile2}
|
||||
serverFqdn=$localFqdn
|
||||
echo
|
||||
return
|
||||
|
|
@ -455,7 +458,7 @@ function set_ipAsFqdn() {
|
|||
read -p "Please choose an IP from local IP list:" localFqdn
|
||||
else
|
||||
# Write the local FQDN to configuration file
|
||||
${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/${configFile}
|
||||
${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/${configFile2}
|
||||
serverFqdn=$localFqdn
|
||||
break
|
||||
fi
|
||||
|
|
@ -519,15 +522,15 @@ function install_adapter_config() {
|
|||
|
||||
function install_config() {
|
||||
|
||||
if [ ! -f "${cfg_install_dir}/${configFile}" ]; then
|
||||
if [ ! -f "${cfg_install_dir}/${configFile2}" ]; then
|
||||
${csudo}mkdir -p ${cfg_install_dir}
|
||||
[ -f ${script_dir}/cfg/${configFile} ] && ${csudo}cp ${script_dir}/cfg/${configFile} ${cfg_install_dir}
|
||||
[ -f ${script_dir}/cfg/${configFile2} ] && ${csudo}cp ${script_dir}/cfg/${configFile2} ${cfg_install_dir}
|
||||
${csudo}chmod 644 ${cfg_install_dir}/*
|
||||
else
|
||||
${csudo}cp -f ${script_dir}/cfg/${configFile} ${cfg_install_dir}/${configFile}.new
|
||||
${csudo}cp -f ${script_dir}/cfg/${configFile2} ${cfg_install_dir}/${configFile2}.new
|
||||
fi
|
||||
|
||||
${csudo}ln -sf ${cfg_install_dir}/${configFile} ${install_main_dir}/cfg
|
||||
${csudo}ln -sf ${cfg_install_dir}/${configFile2} ${install_main_dir}/cfg
|
||||
|
||||
[ ! -z $1 ] && return 0 || : # only install client
|
||||
|
||||
|
|
@ -548,7 +551,7 @@ function install_config() {
|
|||
read firstEp
|
||||
while true; do
|
||||
if [ ! -z "$firstEp" ]; then
|
||||
${csudo}sed -i -r "s/#*\s*(firstEp\s*).*/\1$firstEp/" ${cfg_install_dir}/${configFile}
|
||||
${csudo}sed -i -r "s/#*\s*(firstEp\s*).*/\1$firstEp/" ${cfg_install_dir}/${configFile2}
|
||||
break
|
||||
else
|
||||
break
|
||||
|
|
@ -600,8 +603,8 @@ function install_web() {
|
|||
|
||||
|
||||
function clean_service_on_sysvinit() {
|
||||
if ps aux | grep -v grep | grep ${serverName} &>/dev/null; then
|
||||
${csudo}service ${serverName} stop || :
|
||||
if ps aux | grep -v grep | grep ${serverName2} &>/dev/null; then
|
||||
${csudo}service ${serverName2} stop || :
|
||||
fi
|
||||
|
||||
if ps aux | grep -v grep | grep tarbitrator &>/dev/null; then
|
||||
|
|
@ -609,30 +612,30 @@ function clean_service_on_sysvinit() {
|
|||
fi
|
||||
|
||||
if ((${initd_mod} == 1)); then
|
||||
if [ -e ${service_config_dir}/${serverName} ]; then
|
||||
${csudo}chkconfig --del ${serverName} || :
|
||||
if [ -e ${service_config_dir}/${serverName2} ]; then
|
||||
${csudo}chkconfig --del ${serverName2} || :
|
||||
fi
|
||||
|
||||
if [ -e ${service_config_dir}/tarbitratord ]; then
|
||||
${csudo}chkconfig --del tarbitratord || :
|
||||
fi
|
||||
elif ((${initd_mod} == 2)); then
|
||||
if [ -e ${service_config_dir}/${serverName} ]; then
|
||||
${csudo}insserv -r ${serverName} || :
|
||||
if [ -e ${service_config_dir}/${serverName2} ]; then
|
||||
${csudo}insserv -r ${serverName2} || :
|
||||
fi
|
||||
if [ -e ${service_config_dir}/tarbitratord ]; then
|
||||
${csudo}insserv -r tarbitratord || :
|
||||
fi
|
||||
elif ((${initd_mod} == 3)); then
|
||||
if [ -e ${service_config_dir}/${serverName} ]; then
|
||||
${csudo}update-rc.d -f ${serverName} remove || :
|
||||
if [ -e ${service_config_dir}/${serverName2} ]; then
|
||||
${csudo}update-rc.d -f ${serverName2} remove || :
|
||||
fi
|
||||
if [ -e ${service_config_dir}/tarbitratord ]; then
|
||||
${csudo}update-rc.d -f tarbitratord remove || :
|
||||
fi
|
||||
fi
|
||||
|
||||
${csudo}rm -f ${service_config_dir}/${serverName} || :
|
||||
${csudo}rm -f ${service_config_dir}/${serverName2} || :
|
||||
${csudo}rm -f ${service_config_dir}/tarbitratord || :
|
||||
|
||||
if $(which init &>/dev/null); then
|
||||
|
|
@ -653,24 +656,24 @@ function install_service_on_sysvinit() {
|
|||
fi
|
||||
|
||||
if ((${initd_mod} == 1)); then
|
||||
${csudo}chkconfig --add ${serverName} || :
|
||||
${csudo}chkconfig --level 2345 ${serverName} on || :
|
||||
${csudo}chkconfig --add ${serverName2} || :
|
||||
${csudo}chkconfig --level 2345 ${serverName2} on || :
|
||||
elif ((${initd_mod} == 2)); then
|
||||
${csudo}insserv ${serverName} || :
|
||||
${csudo}insserv -d ${serverName} || :
|
||||
${csudo}insserv ${serverName2} || :
|
||||
${csudo}insserv -d ${serverName2} || :
|
||||
elif ((${initd_mod} == 3)); then
|
||||
${csudo}update-rc.d ${serverName} defaults || :
|
||||
${csudo}update-rc.d ${serverName2} defaults || :
|
||||
fi
|
||||
}
|
||||
|
||||
function clean_service_on_systemd() {
|
||||
taosd_service_config="${service_config_dir}/${serverName}.service"
|
||||
if systemctl is-active --quiet ${serverName}; then
|
||||
service_config="${service_config_dir}/${serverName2}.service"
|
||||
if systemctl is-active --quiet ${serverName2}; then
|
||||
echo "${productName} is running, stopping it..."
|
||||
${csudo}systemctl stop ${serverName} &>/dev/null || echo &>/dev/null
|
||||
${csudo}systemctl stop ${serverName2} &>/dev/null || echo &>/dev/null
|
||||
fi
|
||||
${csudo}systemctl disable ${serverName} &>/dev/null || echo &>/dev/null
|
||||
${csudo}rm -f ${taosd_service_config}
|
||||
${csudo}systemctl disable ${serverName2} &>/dev/null || echo &>/dev/null
|
||||
${csudo}rm -f ${service_config}
|
||||
|
||||
tarbitratord_service_config="${service_config_dir}/tarbitratord.service"
|
||||
if systemctl is-active --quiet tarbitratord; then
|
||||
|
|
@ -687,19 +690,19 @@ function clean_service_on_systemd() {
|
|||
function install_service_on_systemd() {
|
||||
clean_service_on_systemd
|
||||
|
||||
[ -f ${script_dir}/cfg/${serverName}.service ] &&
|
||||
${csudo}cp ${script_dir}/cfg/${serverName}.service \
|
||||
[ -f ${script_dir}/cfg/${serverName2}.service ] &&
|
||||
${csudo}cp ${script_dir}/cfg/${serverName2}.service \
|
||||
${service_config_dir}/ || :
|
||||
|
||||
# if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then
|
||||
# [ -f ${script_dir}/cfg/${serverName}.service ] &&
|
||||
# ${csudo}cp ${script_dir}/cfg/${serverName}.service \
|
||||
# [ -f ${script_dir}/cfg/${serverName2}.service ] &&
|
||||
# ${csudo}cp ${script_dir}/cfg/${serverName2}.service \
|
||||
# ${service_config_dir}/${serverName2}.service || :
|
||||
# fi
|
||||
|
||||
${csudo}systemctl daemon-reload
|
||||
|
||||
${csudo}systemctl enable ${serverName}
|
||||
${csudo}systemctl enable ${serverName2}
|
||||
|
||||
${csudo}systemctl daemon-reload
|
||||
}
|
||||
|
|
@ -719,7 +722,7 @@ function install_service() {
|
|||
elif ((${service_mod} == 1)); then
|
||||
install_service_on_sysvinit
|
||||
else
|
||||
kill_process ${serverName}
|
||||
kill_process ${serverName2}
|
||||
fi
|
||||
}
|
||||
|
||||
|
|
@ -756,10 +759,10 @@ function is_version_compatible() {
|
|||
if [ -f ${script_dir}/driver/vercomp.txt ]; then
|
||||
min_compatible_version=$(cat ${script_dir}/driver/vercomp.txt)
|
||||
else
|
||||
min_compatible_version=$(${script_dir}/bin/${serverName} -V | head -1 | cut -d ' ' -f 5)
|
||||
min_compatible_version=$(${script_dir}/bin/${serverName2} -V | head -1 | cut -d ' ' -f 5)
|
||||
fi
|
||||
|
||||
exist_version=$(${installDir}/bin/${serverName} -V | head -1 | cut -d ' ' -f 3)
|
||||
exist_version=$(${installDir}/bin/${serverName2} -V | head -1 | cut -d ' ' -f 3)
|
||||
vercomp $exist_version "3.0.0.0"
|
||||
case $? in
|
||||
2)
|
||||
|
|
@ -829,13 +832,13 @@ function updateProduct() {
|
|||
|
||||
echo -e "${GREEN}Start to update ${productName2}...${NC}"
|
||||
# Stop the service if running
|
||||
if ps aux | grep -v grep | grep ${serverName} &>/dev/null; then
|
||||
if ps aux | grep -v grep | grep ${serverName2} &>/dev/null; then
|
||||
if ((${service_mod} == 0)); then
|
||||
${csudo}systemctl stop ${serverName} || :
|
||||
${csudo}systemctl stop ${serverName2} || :
|
||||
elif ((${service_mod} == 1)); then
|
||||
${csudo}service ${serverName} stop || :
|
||||
${csudo}service ${serverName2} stop || :
|
||||
else
|
||||
kill_process ${serverName}
|
||||
kill_process ${serverName2}
|
||||
fi
|
||||
sleep 1
|
||||
fi
|
||||
|
|
@ -862,21 +865,21 @@ function updateProduct() {
|
|||
openresty_work=false
|
||||
|
||||
echo
|
||||
echo -e "${GREEN_DARK}To configure ${productName2} ${NC}: edit ${cfg_install_dir}/${configFile}"
|
||||
[ -f ${configDir}/taosadapter.toml ] && [ -f ${installDir}/bin/taosadapter ] && \
|
||||
echo -e "${GREEN_DARK}To configure ${clientName2} Adapter ${NC}: edit ${configDir}/taosadapter.toml"
|
||||
echo -e "${GREEN_DARK}To configure ${productName2} ${NC}: edit ${cfg_install_dir}/${configFile2}"
|
||||
[ -f ${configDir}/${clientName2}adapter.toml ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \
|
||||
echo -e "${GREEN_DARK}To configure ${clientName2} Adapter ${NC}: edit ${configDir}/${clientName2}adapter.toml"
|
||||
if ((${service_mod} == 0)); then
|
||||
echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}systemctl start ${serverName}${NC}"
|
||||
[ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \
|
||||
echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}systemctl start taosadapter ${NC}"
|
||||
echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}systemctl start ${serverName2}${NC}"
|
||||
[ -f ${service_config_dir}/${clientName2}adapter.service ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \
|
||||
echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}systemctl start ${clientName2}adapter ${NC}"
|
||||
elif ((${service_mod} == 1)); then
|
||||
echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}service ${serverName} start${NC}"
|
||||
[ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \
|
||||
echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}service taosadapter start${NC}"
|
||||
echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}service ${serverName2} start${NC}"
|
||||
[ -f ${service_config_dir}/${clientName2}adapter.service ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \
|
||||
echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}service ${clientName2}adapter start${NC}"
|
||||
else
|
||||
echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ./${serverName}${NC}"
|
||||
[ -f ${installDir}/bin/taosadapter ] && \
|
||||
echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: taosadapter &${NC}"
|
||||
echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ./${serverName2}${NC}"
|
||||
[ -f ${installDir}/bin/${clientName2}adapter ] && \
|
||||
echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${clientName2}adapter &${NC}"
|
||||
fi
|
||||
|
||||
if [ ${openresty_work} = 'true' ]; then
|
||||
|
|
@ -887,7 +890,7 @@ function updateProduct() {
|
|||
|
||||
if ((${prompt_force} == 1)); then
|
||||
echo ""
|
||||
echo -e "${RED}Please run '${serverName} --force-keep-file' at first time for the exist ${productName2} $exist_version!${NC}"
|
||||
echo -e "${RED}Please run '${serverName2} --force-keep-file' at first time for the exist ${productName2} $exist_version!${NC}"
|
||||
fi
|
||||
echo
|
||||
echo -e "\033[44;32;1m${productName2} is updated successfully!${NC}"
|
||||
|
|
@ -899,7 +902,7 @@ function updateProduct() {
|
|||
echo -e "\033[44;32;1m${productName2} client is updated successfully!${NC}"
|
||||
fi
|
||||
|
||||
rm -rf $(tar -tf ${tarName} | grep -v "^\./$")
|
||||
rm -rf $(tar -tf ${tarName} | grep -Ev "^\./$|^\/")
|
||||
}
|
||||
|
||||
function installProduct() {
|
||||
|
|
@ -944,21 +947,21 @@ function installProduct() {
|
|||
|
||||
# Ask if to start the service
|
||||
echo
|
||||
echo -e "${GREEN_DARK}To configure ${productName2} ${NC}: edit ${cfg_install_dir}/${configFile}"
|
||||
[ -f ${configDir}/taosadapter.toml ] && [ -f ${installDir}/bin/taosadapter ] && \
|
||||
echo -e "${GREEN_DARK}To configure ${clientName2} Adapter ${NC}: edit ${configDir}/taosadapter.toml"
|
||||
echo -e "${GREEN_DARK}To configure ${productName2} ${NC}: edit ${cfg_install_dir}/${configFile2}"
|
||||
[ -f ${configDir}/${clientName2}adapter.toml ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \
|
||||
echo -e "${GREEN_DARK}To configure ${clientName2} Adapter ${NC}: edit ${configDir}/${clientName2}adapter.toml"
|
||||
if ((${service_mod} == 0)); then
|
||||
echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}systemctl start ${serverName}${NC}"
|
||||
[ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \
|
||||
echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}systemctl start taosadapter ${NC}"
|
||||
echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}systemctl start ${serverName2}${NC}"
|
||||
[ -f ${service_config_dir}/${clientName2}adapter.service ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \
|
||||
echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}systemctl start ${clientName2}adapter ${NC}"
|
||||
elif ((${service_mod} == 1)); then
|
||||
echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}service ${serverName} start${NC}"
|
||||
[ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \
|
||||
echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}service taosadapter start${NC}"
|
||||
echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}service ${serverName2} start${NC}"
|
||||
[ -f ${service_config_dir}/${clientName2}adapter.service ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \
|
||||
echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}service ${clientName2}adapter start${NC}"
|
||||
else
|
||||
echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${serverName}${NC}"
|
||||
[ -f ${installDir}/bin/taosadapter ] && \
|
||||
echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: taosadapter &${NC}"
|
||||
echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${serverName2}${NC}"
|
||||
[ -f ${installDir}/bin/${clientName2}adapter ] && \
|
||||
echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${clientName2}adapter &${NC}"
|
||||
fi
|
||||
|
||||
if [ ! -z "$firstEp" ]; then
|
||||
|
|
@ -991,7 +994,7 @@ function installProduct() {
|
|||
fi
|
||||
|
||||
touch ~/.${historyFile}
|
||||
rm -rf $(tar -tf ${tarName} | grep -v "^\./$")
|
||||
rm -rf $(tar -tf ${tarName} | grep -Ev "^\./$|^\/")
|
||||
}
|
||||
|
||||
## ==============================Main program starts from here============================
|
||||
|
|
@ -1002,7 +1005,7 @@ if [ "$verType" == "server" ]; then
|
|||
echo -e "\033[44;31;5mThe default data directory ${data_dir} contains old data of ${productName2} 2.x, please clear it before installing!\033[0m"
|
||||
else
|
||||
# Install server and client
|
||||
if [ -x ${bin_dir}/${serverName} ]; then
|
||||
if [ -x ${bin_dir}/${serverName2} ]; then
|
||||
update_flag=1
|
||||
updateProduct
|
||||
else
|
||||
|
|
@ -1012,7 +1015,7 @@ if [ "$verType" == "server" ]; then
|
|||
elif [ "$verType" == "client" ]; then
|
||||
interactiveFqdn=no
|
||||
# Only install client
|
||||
if [ -x ${bin_dir}/${clientName} ]; then
|
||||
if [ -x ${bin_dir}/${clientName2} ]; then
|
||||
update_flag=1
|
||||
updateProduct client
|
||||
else
|
||||
|
|
|
|||
|
|
@ -95,7 +95,7 @@ function install_main_path() {
|
|||
${csudo}mkdir -p ${install_main_dir}/cfg
|
||||
${csudo}mkdir -p ${install_main_dir}/bin
|
||||
${csudo}mkdir -p ${install_main_dir}/driver
|
||||
if [ $productName == "TDengine" ]; then
|
||||
if [ "$productName2" == "TDengine" ]; then
|
||||
${csudo}mkdir -p ${install_main_dir}/examples
|
||||
fi
|
||||
${csudo}mkdir -p ${install_main_dir}/include
|
||||
|
|
@ -118,18 +118,19 @@ function install_bin() {
|
|||
#Make link
|
||||
[ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} || :
|
||||
if [ "$osType" != "Darwin" ]; then
|
||||
[ -x ${install_main_dir}/bin/taosdemo ] && ${csudo}ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || :
|
||||
[ -x ${install_main_dir}/bin/${demoName2} ] && ${csudo}ln -s ${install_main_dir}/bin/${demoName2} ${bin_link_dir}/${demoName2} || :
|
||||
fi
|
||||
[ -x ${install_main_dir}/bin/remove_client.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove_client.sh ${bin_link_dir}/${uninstallScript} || :
|
||||
[ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || :
|
||||
|
||||
if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then
|
||||
#Make link
|
||||
[ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName2} || :
|
||||
[ -x ${install_main_dir}/bin/${clientName2} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName2} ${bin_link_dir}/${clientName2} || :
|
||||
if [ "$osType" != "Darwin" ]; then
|
||||
[ -x ${install_main_dir}/bin/taosdemo ] && ${csudo}ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/${demoName2} || :
|
||||
[ -x ${install_main_dir}/bin/${demoName2} ] && ${csudo}ln -s ${install_main_dir}/bin/${demoName2} ${bin_link_dir}/${demoName2} || :
|
||||
[ -x ${install_main_dir}/bin/${benchmarkName2} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName2} ${bin_link_dir}/${benchmarkName2} || :
|
||||
fi
|
||||
[ -x ${install_main_dir}/bin/remove_client.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove_client.sh ${bin_link_dir}/${uninstallScript2} || :
|
||||
[ -x ${install_main_dir}/bin/remove_client.sh ] && ${csudo}ln -sf ${install_main_dir}/bin/remove_client.sh ${bin_link_dir}/${uninstallScript2} || :
|
||||
fi
|
||||
}
|
||||
|
||||
|
|
@ -305,7 +306,7 @@ function update_TDengine() {
|
|||
echo
|
||||
echo -e "\033[44;32;1m${productName2} client is updated successfully!${NC}"
|
||||
|
||||
rm -rf $(tar -tf ${tarName})
|
||||
rm -rf $(tar -tf ${tarName} | grep -Ev "^\./$|^\/")
|
||||
}
|
||||
|
||||
function install_TDengine() {
|
||||
|
|
@ -332,7 +333,7 @@ function install_TDengine() {
|
|||
echo
|
||||
echo -e "\033[44;32;1m${productName2} client is installed successfully!${NC}"
|
||||
|
||||
rm -rf $(tar -tf ${tarName})
|
||||
rm -rf $(tar -tf ${tarName} | grep -Ev "^\./$|^\/")
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
#
|
||||
# Generate tar.gz package for linux client in all os system
|
||||
set -e
|
||||
# set -x
|
||||
set -x
|
||||
|
||||
curr_dir=$(pwd)
|
||||
compile_dir=$1
|
||||
|
|
@ -23,9 +23,12 @@ clientName2="${12}"
|
|||
|
||||
productName="TDengine"
|
||||
clientName="taos"
|
||||
benchmarkName="taosBenchmark"
|
||||
configFile="taos.cfg"
|
||||
tarName="package.tar.gz"
|
||||
|
||||
benchmarkName2="${clientName2}Benchmark"
|
||||
|
||||
if [ "$osType" != "Darwin" ]; then
|
||||
script_dir="$(dirname $(readlink -f $0))"
|
||||
top_dir="$(readlink -f ${script_dir}/../..)"
|
||||
|
|
@ -53,11 +56,12 @@ fi
|
|||
|
||||
# Directories and files.
|
||||
|
||||
if [ "$verMode" == "cluster" ]; then
|
||||
sed -i 's/verMode=edge/verMode=cluster/g' ${script_dir}/remove_client.sh
|
||||
sed -i "s/clientName2=\"taos\"/clientName2=\"${clientName2}\"/g" ${script_dir}/remove_client.sh
|
||||
sed -i "s/productName2=\"TDengine\"/productName2=\"${productName2}\"/g" ${script_dir}/remove_client.sh
|
||||
fi
|
||||
#if [ "$verMode" == "cluster" ]; then
|
||||
# sed -i 's/verMode=edge/verMode=cluster/g' ${script_dir}/remove_client.sh
|
||||
# sed -i "s/clientName2=\"taos\"/clientName2=\"${clientName2}\"/g" ${script_dir}/remove_client.sh
|
||||
# sed -i "s/configFile2=\"taos\"/configFile2=\"${clientName2}\"/g" ${script_dir}/remove_client.sh
|
||||
# sed -i "s/productName2=\"TDengine\"/productName2=\"${productName2}\"/g" ${script_dir}/remove_client.sh
|
||||
#fi
|
||||
|
||||
if [ "$osType" != "Darwin" ]; then
|
||||
if [ "$pagMode" == "lite" ]; then
|
||||
|
|
@ -66,6 +70,7 @@ if [ "$osType" != "Darwin" ]; then
|
|||
${script_dir}/remove_client.sh"
|
||||
else
|
||||
bin_files="${build_dir}/bin/${clientName} \
|
||||
${build_dir}/bin/${benchmarkName} \
|
||||
${script_dir}/remove_client.sh \
|
||||
${script_dir}/set_core.sh \
|
||||
${script_dir}/get_client.sh"
|
||||
|
|
@ -153,6 +158,7 @@ if [ "$verMode" == "cluster" ]; then
|
|||
sed -i 's/verMode=edge/verMode=cluster/g' install_client_temp.sh
|
||||
sed -i "s/serverName2=\"taosd\"/serverName2=\"${serverName2}\"/g" install_client_temp.sh
|
||||
sed -i "s/clientName2=\"taos\"/clientName2=\"${clientName2}\"/g" install_client_temp.sh
|
||||
sed -i "s/configFile2=\"taos.cfg\"/configFile2=\"${clientName2}.cfg\"/g" install_client_temp.sh
|
||||
sed -i "s/productName2=\"TDengine\"/productName2=\"${productName2}\"/g" install_client_temp.sh
|
||||
sed -i "s/emailName2=\"taosdata.com\"/emailName2=\"${cusEmail2}\"/g" install_client_temp.sh
|
||||
|
||||
|
|
|
|||
|
|
@ -51,6 +51,8 @@ fi
|
|||
|
||||
if [ -d ${top_dir}/tools/taos-tools/packaging/deb ]; then
|
||||
cd ${top_dir}/tools/taos-tools/packaging/deb
|
||||
|
||||
taostools_ver=$(git for-each-ref --sort=taggerdate --format '%(tag)' refs/tags|grep -v taos | tail -1)
|
||||
[ -z "$taos_tools_ver" ] && taos_tools_ver="0.1.0"
|
||||
|
||||
taostools_ver=$(git for-each-ref --sort=taggerdate --format '%(tag)' refs/tags|grep -v taos | tail -1)
|
||||
|
|
@ -96,7 +98,7 @@ else
|
|||
${taostools_bin_files} \
|
||||
${taosx_bin} \
|
||||
${explorer_bin_files} \
|
||||
${build_dir}/bin/taosadapter \
|
||||
${build_dir}/bin/${clientName}adapter \
|
||||
${build_dir}/bin/udfd \
|
||||
${script_dir}/remove.sh \
|
||||
${script_dir}/set_core.sh \
|
||||
|
|
@ -135,12 +137,12 @@ mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc
|
|||
|
||||
mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/${configFile} ${install_dir}/cfg/${configFile}
|
||||
|
||||
if [ -f "${compile_dir}/test/cfg/taosadapter.toml" ]; then
|
||||
cp ${compile_dir}/test/cfg/taosadapter.toml ${install_dir}/cfg || :
|
||||
if [ -f "${compile_dir}/test/cfg/${clientName}adapter.toml" ]; then
|
||||
cp ${compile_dir}/test/cfg/${clientName}adapter.toml ${install_dir}/cfg || :
|
||||
fi
|
||||
|
||||
if [ -f "${compile_dir}/test/cfg/taosadapter.service" ]; then
|
||||
cp ${compile_dir}/test/cfg/taosadapter.service ${install_dir}/cfg || :
|
||||
if [ -f "${compile_dir}/test/cfg/${clientName}adapter.service" ]; then
|
||||
cp ${compile_dir}/test/cfg/${clientName}adapter.service ${install_dir}/cfg || :
|
||||
fi
|
||||
|
||||
if [ -f "${cfg_dir}/${serverName}.service" ]; then
|
||||
|
|
@ -152,16 +154,16 @@ mkdir -p ${install_dir}/init.d && cp ${init_file_deb} ${install_dir}/init.d/${se
|
|||
mkdir -p ${install_dir}/init.d && cp ${init_file_rpm} ${install_dir}/init.d/${serverName}.rpm
|
||||
|
||||
if [ $adapterName != "taosadapter" ]; then
|
||||
mv ${install_dir}/cfg/taosadapter.toml ${install_dir}/cfg/$adapterName.toml
|
||||
mv ${install_dir}/cfg/${clientName2}adapter.toml ${install_dir}/cfg/$adapterName.toml
|
||||
sed -i "s/path = \"\/var\/log\/taos\"/path = \"\/var\/log\/${productName}\"/g" ${install_dir}/cfg/$adapterName.toml
|
||||
sed -i "s/password = \"taosdata\"/password = \"${defaultPasswd}\"/g" ${install_dir}/cfg/$adapterName.toml
|
||||
|
||||
mv ${install_dir}/cfg/taosadapter.service ${install_dir}/cfg/$adapterName.service
|
||||
mv ${install_dir}/cfg/${clientName2}adapter.service ${install_dir}/cfg/$adapterName.service
|
||||
sed -i "s/TDengine/${productName}/g" ${install_dir}/cfg/$adapterName.service
|
||||
sed -i "s/taosAdapter/${adapterName}/g" ${install_dir}/cfg/$adapterName.service
|
||||
sed -i "s/taosadapter/${adapterName}/g" ${install_dir}/cfg/$adapterName.service
|
||||
|
||||
mv ${install_dir}/bin/taosadapter ${install_dir}/bin/${adapterName}
|
||||
mv ${install_dir}/bin/${clientName2}adapter ${install_dir}/bin/${adapterName}
|
||||
mv ${install_dir}/bin/taosd-dump-cfg.gdb ${install_dir}/bin/${serverName}-dump-cfg.gdb
|
||||
fi
|
||||
|
||||
|
|
@ -233,9 +235,9 @@ if [ "$verMode" == "cluster" ]; then
|
|||
sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/bin/remove.sh >>remove_temp.sh
|
||||
sed -i "s/serverName2=\"taosd\"/serverName2=\"${serverName2}\"/g" remove_temp.sh
|
||||
sed -i "s/clientName2=\"taos\"/clientName2=\"${clientName2}\"/g" remove_temp.sh
|
||||
sed -i "s/configFile2=\"taos.cfg\"/configFile2=\"${clientName2}.cfg\"/g" remove_temp.sh
|
||||
sed -i "s/productName2=\"TDengine\"/productName2=\"${productName2}\"/g" remove_temp.sh
|
||||
cusDomain=`echo "${cusEmail2}" | sed 's/^[^@]*@//'`
|
||||
echo "domain is ${cusDomain}"
|
||||
sed -i "s/emailName2=\"taosdata.com\"/emailName2=\"${cusDomain}\"/g" remove_temp.sh
|
||||
mv remove_temp.sh ${install_dir}/bin/remove.sh
|
||||
fi
|
||||
|
|
@ -264,8 +266,10 @@ if [ "$verMode" == "cluster" ]; then
|
|||
sed -i 's/verMode=edge/verMode=cluster/g' install_temp.sh
|
||||
sed -i "s/serverName2=\"taosd\"/serverName2=\"${serverName2}\"/g" install_temp.sh
|
||||
sed -i "s/clientName2=\"taos\"/clientName2=\"${clientName2}\"/g" install_temp.sh
|
||||
sed -i "s/configFile2=\"taos.cfg\"/configFile2=\"${clientName2}.cfg\"/g" install_temp.sh
|
||||
sed -i "s/productName2=\"TDengine\"/productName2=\"${productName2}\"/g" install_temp.sh
|
||||
sed -i "s/emailName2=\"taosdata.com\"/emailName2=\"${cusEmail2}\"/g" install_temp.sh
|
||||
cusDomain=`echo "${cusEmail2}" | sed 's/^[^@]*@//'`
|
||||
sed -i "s/emailName2=\"taosdata.com\"/emailName2=\"${cusDomain}\"/g" install_temp.sh
|
||||
mv install_temp.sh ${install_dir}/install.sh
|
||||
fi
|
||||
if [ "$verMode" == "cloud" ]; then
|
||||
|
|
|
|||
|
|
@ -145,6 +145,59 @@ function install_include() {
|
|||
log_print "install include success"
|
||||
}
|
||||
|
||||
function install_jemalloc() {
|
||||
jemalloc_dir=${script_dir}/../jemalloc
|
||||
|
||||
if [ -d ${jemalloc_dir} ]; then
|
||||
${csudo}/usr/bin/install -c -d /usr/local/bin
|
||||
|
||||
if [ -f ${jemalloc_dir}/bin/jemalloc-config ]; then
|
||||
${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc-config /usr/local/bin
|
||||
fi
|
||||
if [ -f ${jemalloc_dir}/bin/jemalloc.sh ]; then
|
||||
${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc.sh /usr/local/bin
|
||||
fi
|
||||
if [ -f ${jemalloc_dir}/bin/jeprof ]; then
|
||||
${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jeprof /usr/local/bin
|
||||
fi
|
||||
if [ -f ${jemalloc_dir}/include/jemalloc/jemalloc.h ]; then
|
||||
${csudo}/usr/bin/install -c -d /usr/local/include/jemalloc
|
||||
${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/include/jemalloc/jemalloc.h /usr/local/include/jemalloc
|
||||
fi
|
||||
if [ -f ${jemalloc_dir}/lib/libjemalloc.so.2 ]; then
|
||||
${csudo}/usr/bin/install -c -d /usr/local/lib
|
||||
${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.so.2 /usr/local/lib
|
||||
${csudo}ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so
|
||||
${csudo}/usr/bin/install -c -d /usr/local/lib
|
||||
if [ -f ${jemalloc_dir}/lib/libjemalloc.a ]; then
|
||||
${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.a /usr/local/lib
|
||||
fi
|
||||
if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then
|
||||
${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc_pic.a /usr/local/lib
|
||||
fi
|
||||
if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then
|
||||
${csudo}/usr/bin/install -c -d /usr/local/lib/pkgconfig
|
||||
${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/lib/pkgconfig/jemalloc.pc /usr/local/lib/pkgconfig
|
||||
fi
|
||||
fi
|
||||
if [ -f ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html ]; then
|
||||
${csudo}/usr/bin/install -c -d /usr/local/share/doc/jemalloc
|
||||
${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html /usr/local/share/doc/jemalloc
|
||||
fi
|
||||
if [ -f ${jemalloc_dir}/share/man/man3/jemalloc.3 ]; then
|
||||
${csudo}/usr/bin/install -c -d /usr/local/share/man/man3
|
||||
${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3
|
||||
fi
|
||||
|
||||
if [ -d /etc/ld.so.conf.d ]; then
|
||||
echo "/usr/local/lib" | ${csudo}tee /etc/ld.so.conf.d/jemalloc.conf >/dev/null || echo -e "failed to write /etc/ld.so.conf.d/jemalloc.conf"
|
||||
${csudo}ldconfig
|
||||
else
|
||||
echo "/etc/ld.so.conf.d not found!"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
function install_lib() {
|
||||
log_print "start install lib from ${lib_dir} to ${lib_link_dir}"
|
||||
${csudo}rm -f ${lib_link_dir}/libtaos* || :
|
||||
|
|
@ -667,6 +720,7 @@ function install_TDengine() {
|
|||
# Install include, lib, binary and service
|
||||
install_include &&
|
||||
install_lib &&
|
||||
install_jemalloc
|
||||
install_bin
|
||||
|
||||
if [[ "$?" != 0 ]];then
|
||||
|
|
|
|||
|
|
@ -40,11 +40,16 @@ serverName2="taosd"
|
|||
clientName2="taos"
|
||||
productName2="TDengine"
|
||||
|
||||
adapterName2="${clientName2}adapter"
|
||||
demoName2="${clientName2}demo"
|
||||
benchmarkName2="${clientName2}Benchmark"
|
||||
dumpName2="${clientName2}dump"
|
||||
keeperName2="${clientName2}keeper"
|
||||
xName2="${clientName2}x"
|
||||
explorerName2="${clientName2}-explorer"
|
||||
uninstallScript2="rm${clientName2}"
|
||||
|
||||
installDir="/usr/local/${clientName}"
|
||||
installDir="/usr/local/${clientName2}"
|
||||
|
||||
#install main path
|
||||
install_main_dir=${installDir}
|
||||
|
|
@ -55,8 +60,8 @@ local_bin_link_dir="/usr/local/bin"
|
|||
|
||||
|
||||
service_config_dir="/etc/systemd/system"
|
||||
taos_service_name=${serverName}
|
||||
taosadapter_service_name="taosadapter"
|
||||
taos_service_name=${serverName2}
|
||||
taosadapter_service_name="${clientName2}adapter"
|
||||
tarbitrator_service_name="tarbitratord"
|
||||
csudo=""
|
||||
if command -v sudo >/dev/null; then
|
||||
|
|
@ -84,14 +89,14 @@ else
|
|||
fi
|
||||
|
||||
function kill_taosadapter() {
|
||||
pid=$(ps -ef | grep "taosadapter" | grep -v "grep" | awk '{print $2}')
|
||||
pid=$(ps -ef | grep "${adapterName2}" | grep -v "grep" | awk '{print $2}')
|
||||
if [ -n "$pid" ]; then
|
||||
${csudo}kill -9 $pid || :
|
||||
fi
|
||||
}
|
||||
|
||||
function kill_taosd() {
|
||||
pid=$(ps -ef | grep ${serverName} | grep -v "grep" | awk '{print $2}')
|
||||
pid=$(ps -ef | grep ${serverName2} | grep -v "grep" | awk '{print $2}')
|
||||
if [ -n "$pid" ]; then
|
||||
${csudo}kill -9 $pid || :
|
||||
fi
|
||||
|
|
@ -109,17 +114,17 @@ function clean_bin() {
|
|||
${csudo}rm -f ${bin_link_dir}/${clientName} || :
|
||||
${csudo}rm -f ${bin_link_dir}/${serverName} || :
|
||||
${csudo}rm -f ${bin_link_dir}/udfd || :
|
||||
${csudo}rm -f ${bin_link_dir}/taosadapter || :
|
||||
${csudo}rm -f ${bin_link_dir}/taosBenchmark || :
|
||||
${csudo}rm -f ${bin_link_dir}/taosdemo || :
|
||||
${csudo}rm -f ${bin_link_dir}/taosdump || :
|
||||
${csudo}rm -f ${bin_link_dir}/${uninstallScript} || :
|
||||
${csudo}rm -f ${bin_link_dir}/${adapterName2} || :
|
||||
${csudo}rm -f ${bin_link_dir}/${benchmarkName2} || :
|
||||
${csudo}rm -f ${bin_link_dir}/${demoName2} || :
|
||||
${csudo}rm -f ${bin_link_dir}/${dumpName2} || :
|
||||
${csudo}rm -f ${bin_link_dir}/${uninstallScript} || :
|
||||
${csudo}rm -f ${bin_link_dir}/tarbitrator || :
|
||||
${csudo}rm -f ${bin_link_dir}/set_core || :
|
||||
${csudo}rm -f ${bin_link_dir}/TDinsight.sh || :
|
||||
${csudo}rm -f ${bin_link_dir}/taoskeeper || :
|
||||
${csudo}rm -f ${bin_link_dir}/taosx || :
|
||||
${csudo}rm -f ${bin_link_dir}/taos-explorer || :
|
||||
${csudo}rm -f ${bin_link_dir}/${keeperName2} || :
|
||||
${csudo}rm -f ${bin_link_dir}/${xName2} || :
|
||||
${csudo}rm -f ${bin_link_dir}/${explorerName2} || :
|
||||
|
||||
if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then
|
||||
${csudo}rm -f ${bin_link_dir}/${clientName2} || :
|
||||
|
|
@ -130,8 +135,8 @@ function clean_bin() {
|
|||
}
|
||||
|
||||
function clean_local_bin() {
|
||||
${csudo}rm -f ${local_bin_link_dir}/taosBenchmark || :
|
||||
${csudo}rm -f ${local_bin_link_dir}/taosdemo || :
|
||||
${csudo}rm -f ${local_bin_link_dir}/${benchmarkName2} || :
|
||||
${csudo}rm -f ${local_bin_link_dir}/${demoName2} || :
|
||||
}
|
||||
|
||||
function clean_lib() {
|
||||
|
|
@ -173,7 +178,7 @@ function clean_service_on_systemd() {
|
|||
${csudo}systemctl disable ${taos_service_name} &>/dev/null || echo &>/dev/null
|
||||
${csudo}rm -f ${taosd_service_config}
|
||||
|
||||
taosadapter_service_config="${service_config_dir}/taosadapter.service"
|
||||
taosadapter_service_config="${service_config_dir}/${clientName2}adapter.service"
|
||||
if systemctl is-active --quiet ${taosadapter_service_name}; then
|
||||
echo "${productName2} ${clientName2}Adapter is running, stopping it..."
|
||||
${csudo}systemctl stop ${taosadapter_service_name} &>/dev/null || echo &>/dev/null
|
||||
|
|
@ -235,8 +240,8 @@ function clean_service_on_sysvinit() {
|
|||
function clean_service_on_launchctl() {
|
||||
${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || :
|
||||
${csudo}rm /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || :
|
||||
${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosadapter.plist > /dev/null 2>&1 || :
|
||||
${csudo}rm /Library/LaunchDaemons/com.taosdata.taosadapter.plist > /dev/null 2>&1 || :
|
||||
${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.${clientName2}adapter.plist > /dev/null 2>&1 || :
|
||||
${csudo}rm /Library/LaunchDaemons/com.taosdata.${clientName2}adapter.plist > /dev/null 2>&1 || :
|
||||
}
|
||||
|
||||
function clean_service() {
|
||||
|
|
|
|||
|
|
@ -15,11 +15,12 @@ uninstallScript="rmtaos"
|
|||
clientName2="taos"
|
||||
productName2="TDengine"
|
||||
|
||||
benchmarkName2="${clientName}Benchmark"
|
||||
dumpName2="${clientName}dump"
|
||||
uninstallScript2="rm${clientName}"
|
||||
benchmarkName2="${clientName2}Benchmark"
|
||||
demoName2="${clientName2}demo"
|
||||
dumpName2="${clientName2}dump"
|
||||
uninstallScript2="rm${clientName2}"
|
||||
|
||||
installDir="/usr/local/${clientName}"
|
||||
installDir="/usr/local/${clientName2}"
|
||||
|
||||
#install main path
|
||||
install_main_dir=${installDir}
|
||||
|
|
@ -44,14 +45,17 @@ function kill_client() {
|
|||
|
||||
function clean_bin() {
|
||||
# Remove link
|
||||
${csudo}rm -f ${bin_link_dir}/${clientName} || :
|
||||
${csudo}rm -f ${bin_link_dir}/taosdemo || :
|
||||
${csudo}rm -f ${bin_link_dir}/taosdump || :
|
||||
${csudo}rm -f ${bin_link_dir}/${clientName2} || :
|
||||
${csudo}rm -f ${bin_link_dir}/${demoName2} || :
|
||||
${csudo}rm -f ${bin_link_dir}/${benchmarkName2} || :
|
||||
${csudo}rm -f ${bin_link_dir}/${dumpName2} || :
|
||||
${csudo}rm -f ${bin_link_dir}/${uninstallScript} || :
|
||||
${csudo}rm -f ${bin_link_dir}/set_core || :
|
||||
|
||||
if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then
|
||||
${csudo}rm -f ${bin_link_dir}/${clientName2} || :
|
||||
${csudo}rm -f ${bin_link_dir}/${demoName2} || :
|
||||
${csudo}rm -f ${bin_link_dir}/${benchmarkName2} || :
|
||||
${csudo}rm -f ${bin_link_dir}/${dumpName2} || :
|
||||
${csudo}rm -f ${bin_link_dir}/${uninstallScript2} || :
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -76,7 +76,7 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) {
|
|||
goto End;
|
||||
}
|
||||
|
||||
if ((code = taosCheckVersionCompatibleFromStr(version, connectRsp.sVer, 2)) != 0) {
|
||||
if ((code = taosCheckVersionCompatibleFromStr(version, connectRsp.sVer, 3)) != 0) {
|
||||
setErrno(pRequest, code);
|
||||
tsem_post(&pRequest->body.rspSem);
|
||||
goto End;
|
||||
|
|
@ -506,7 +506,7 @@ int32_t processShowVariablesRsp(void* param, SDataBuf* pMsg, int32_t code) {
|
|||
code = setQueryResultFromRsp(&pRequest->body.resInfo, pRes, false, true);
|
||||
}
|
||||
|
||||
if(code != 0){
|
||||
if (code != 0) {
|
||||
taosMemoryFree(pRes);
|
||||
}
|
||||
tFreeSShowVariablesRsp(&rsp);
|
||||
|
|
|
|||
|
|
@ -25,9 +25,7 @@
|
|||
#include "tref.h"
|
||||
#include "ttimer.h"
|
||||
|
||||
static tb_uid_t processSuid(tb_uid_t suid, char* db){
|
||||
return suid + MurmurHash3_32(db, strlen(db));
|
||||
}
|
||||
static tb_uid_t processSuid(tb_uid_t suid, char* db) { return suid + MurmurHash3_32(db, strlen(db)); }
|
||||
|
||||
static char* buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id,
|
||||
int8_t t) {
|
||||
|
|
@ -363,7 +361,8 @@ static char* processCreateTable(SMqMetaRsp* metaRsp) {
|
|||
if (pCreateReq->type == TSDB_CHILD_TABLE) {
|
||||
string = buildCreateCTableJson(req.pReqs, req.nReqs);
|
||||
} else if (pCreateReq->type == TSDB_NORMAL_TABLE) {
|
||||
string = buildCreateTableJson(&pCreateReq->ntb.schemaRow, NULL, pCreateReq->name, pCreateReq->uid, TSDB_NORMAL_TABLE);
|
||||
string =
|
||||
buildCreateTableJson(&pCreateReq->ntb.schemaRow, NULL, pCreateReq->name, pCreateReq->uid, TSDB_NORMAL_TABLE);
|
||||
}
|
||||
uDebug("processCreateTable :%s", string);
|
||||
}
|
||||
|
|
@ -709,7 +708,7 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) {
|
|||
pReq.source = TD_REQ_FROM_TAOX;
|
||||
pReq.igExists = true;
|
||||
|
||||
uDebug("taosCreateStb name:%s suid:%"PRId64" processSuid:%"PRId64, req.name, req.suid, pReq.suid);
|
||||
uDebug("taosCreateStb name:%s suid:%" PRId64 " processSuid:%" PRId64, req.name, req.suid, pReq.suid);
|
||||
STscObj* pTscObj = pRequest->pTscObj;
|
||||
SName tableName;
|
||||
tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName), pReq.name);
|
||||
|
|
@ -776,20 +775,20 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) {
|
|||
goto end;
|
||||
}
|
||||
|
||||
SCatalog* pCatalog = NULL;
|
||||
SCatalog* pCatalog = NULL;
|
||||
code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
goto end;
|
||||
}
|
||||
SRequestConnInfo conn = {.pTrans = pRequest->pTscObj->pAppInfo->pTransporter,
|
||||
.requestId = pRequest->requestId,
|
||||
.requestObjRefId = pRequest->self,
|
||||
.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp)};
|
||||
SName pName = {0};
|
||||
.requestId = pRequest->requestId,
|
||||
.requestObjRefId = pRequest->self,
|
||||
.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp)};
|
||||
SName pName = {0};
|
||||
toName(pRequest->pTscObj->acctId, pRequest->pDb, req.name, &pName);
|
||||
STableMeta *pTableMeta = NULL;
|
||||
STableMeta* pTableMeta = NULL;
|
||||
code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta);
|
||||
if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST){
|
||||
if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) {
|
||||
code = TSDB_CODE_SUCCESS;
|
||||
taosMemoryFreeClear(pTableMeta);
|
||||
goto end;
|
||||
|
|
@ -804,9 +803,9 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) {
|
|||
// build drop stable
|
||||
pReq.igNotExists = true;
|
||||
pReq.source = TD_REQ_FROM_TAOX;
|
||||
// pReq.suid = processSuid(req.suid, pRequest->pDb);
|
||||
// pReq.suid = processSuid(req.suid, pRequest->pDb);
|
||||
|
||||
uDebug("taosDropStb name:%s suid:%"PRId64" new suid:%"PRId64, req.name, req.suid, pReq.suid);
|
||||
uDebug("taosDropStb name:%s suid:%" PRId64 " new suid:%" PRId64, req.name, req.suid, pReq.suid);
|
||||
STscObj* pTscObj = pRequest->pTscObj;
|
||||
SName tableName = {0};
|
||||
tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName), pReq.name);
|
||||
|
|
@ -831,7 +830,7 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) {
|
|||
launchQueryImpl(pRequest, &pQuery, true, NULL);
|
||||
|
||||
if (pRequest->code == TSDB_CODE_SUCCESS) {
|
||||
// SCatalog* pCatalog = NULL;
|
||||
// SCatalog* pCatalog = NULL;
|
||||
catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog);
|
||||
catalogRemoveTableMeta(pCatalog, &tableName);
|
||||
}
|
||||
|
|
@ -924,8 +923,8 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) {
|
|||
if (pCreateReq->type == TSDB_CHILD_TABLE) {
|
||||
STableMeta* pTableMeta = NULL;
|
||||
SName sName = {0};
|
||||
tb_uid_t oldSuid = pCreateReq->ctb.suid;
|
||||
// pCreateReq->ctb.suid = processSuid(pCreateReq->ctb.suid, pRequest->pDb);
|
||||
tb_uid_t oldSuid = pCreateReq->ctb.suid;
|
||||
// pCreateReq->ctb.suid = processSuid(pCreateReq->ctb.suid, pRequest->pDb);
|
||||
toName(pTscObj->acctId, pRequest->pDb, pCreateReq->ctb.stbName, &sName);
|
||||
code = catalogGetTableMeta(pCatalog, &conn, &sName, &pTableMeta);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
|
|
@ -933,7 +932,8 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) {
|
|||
goto end;
|
||||
}
|
||||
pCreateReq->ctb.suid = pTableMeta->uid;
|
||||
uDebug("taosCreateTable name:%s sname:%s suid:%"PRId64" new suid:%"PRId64, pCreateReq->name, pCreateReq->ctb.stbName, oldSuid, pCreateReq->ctb.suid);
|
||||
uDebug("taosCreateTable name:%s sname:%s suid:%" PRId64 " new suid:%" PRId64, pCreateReq->name,
|
||||
pCreateReq->ctb.stbName, oldSuid, pCreateReq->ctb.suid);
|
||||
|
||||
for (int32_t i = 0; i < taosArrayGetSize(pCreateReq->ctb.tagName); i++) {
|
||||
char* tName = taosArrayGet(pCreateReq->ctb.tagName, i);
|
||||
|
|
@ -1066,7 +1066,7 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) {
|
|||
for (int32_t iReq = 0; iReq < req.nReqs; iReq++) {
|
||||
pDropReq = req.pReqs + iReq;
|
||||
pDropReq->igNotExists = true;
|
||||
// pDropReq->suid = processSuid(pDropReq->suid, pRequest->pDb);
|
||||
// pDropReq->suid = processSuid(pDropReq->suid, pRequest->pDb);
|
||||
|
||||
SVgroupInfo pInfo = {0};
|
||||
SName pName = {0};
|
||||
|
|
@ -1076,9 +1076,9 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) {
|
|||
goto end;
|
||||
}
|
||||
|
||||
STableMeta *pTableMeta = NULL;
|
||||
STableMeta* pTableMeta = NULL;
|
||||
code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta);
|
||||
if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST){
|
||||
if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) {
|
||||
code = TSDB_CODE_SUCCESS;
|
||||
taosMemoryFreeClear(pTableMeta);
|
||||
continue;
|
||||
|
|
@ -1090,7 +1090,7 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) {
|
|||
tb_uid_t oldSuid = pDropReq->suid;
|
||||
pDropReq->suid = pTableMeta->suid;
|
||||
taosMemoryFreeClear(pTableMeta);
|
||||
uDebug("taosDropTable name:%s suid:%"PRId64" new suid:%"PRId64, pDropReq->name, oldSuid, pDropReq->suid);
|
||||
uDebug("taosDropTable name:%s suid:%" PRId64 " new suid:%" PRId64, pDropReq->name, oldSuid, pDropReq->suid);
|
||||
|
||||
taosArrayPush(pRequest->tableList, &pName);
|
||||
SVgroupDropTableBatch* pTableBatch = taosHashGet(pVgroupHashmap, &pInfo.vgId, sizeof(pInfo.vgId));
|
||||
|
|
@ -1106,7 +1106,7 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) {
|
|||
}
|
||||
}
|
||||
|
||||
if (taosHashGetSize(pVgroupHashmap) == 0){
|
||||
if (taosHashGetSize(pVgroupHashmap) == 0) {
|
||||
goto end;
|
||||
}
|
||||
SArray* pBufArray = serializeVgroupsDropTableBatch(pVgroupHashmap);
|
||||
|
|
@ -1373,7 +1373,7 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch
|
|||
uError("WriteRaw:catalogGetTableMeta failed. table name: %s", tbname);
|
||||
goto end;
|
||||
}
|
||||
// uError("td23101 0vgId:%d, vgId:%d, name:%s, uid:%"PRIu64, vgData.vgId, pTableMeta->vgId, tbname, pTableMeta->uid);
|
||||
// uError("td23101 0vgId:%d, vgId:%d, name:%s, uid:%"PRIu64, vgData.vgId, pTableMeta->vgId, tbname, pTableMeta->uid);
|
||||
|
||||
pQuery = smlInitHandle();
|
||||
if (pQuery == NULL) {
|
||||
|
|
@ -1382,7 +1382,8 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch
|
|||
}
|
||||
pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
|
||||
taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData));
|
||||
// uError("td23101 1vgId:%d, numEps:%d, name:%s, uid:%"PRIu64, vgData.vgId, vgData.epSet.numOfEps, tbname, pTableMeta->uid);
|
||||
// uError("td23101 1vgId:%d, numEps:%d, name:%s, uid:%"PRIu64, vgData.vgId, vgData.epSet.numOfEps, tbname,
|
||||
// pTableMeta->uid);
|
||||
|
||||
code = rawBlockBindData(pQuery, pTableMeta, pData, NULL, fields, numFields, false);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
|
|
@ -1601,6 +1602,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) {
|
|||
uError("WriteRaw:rawBlockBindData failed");
|
||||
goto end;
|
||||
}
|
||||
taosMemoryFreeClear(pTableMeta);
|
||||
}
|
||||
|
||||
code = smlBuildOutput(pQuery, pVgHash);
|
||||
|
|
@ -1707,6 +1709,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
|
|||
tDecoderInit(&decoderTmp, *dataTmp, *lenTmp);
|
||||
if (tDecodeSVCreateTbReq(&decoderTmp, &pCreateReq) < 0) {
|
||||
tDecoderClear(&decoderTmp);
|
||||
tDestroySVCreateTbReq(&pCreateReq, TSDB_MSG_FLG_DECODE);
|
||||
uError("WriteRaw: tDecodeSVCreateTbReq error");
|
||||
code = TSDB_CODE_TMQ_INVALID_MSG;
|
||||
goto end;
|
||||
|
|
@ -1715,15 +1718,19 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
|
|||
if (pCreateReq.type != TSDB_CHILD_TABLE) {
|
||||
uError("WriteRaw:pCreateReq.type != TSDB_CHILD_TABLE. table name: %s", tbName);
|
||||
code = TSDB_CODE_TSC_INVALID_VALUE;
|
||||
tDecoderClear(&decoderTmp);
|
||||
tDestroySVCreateTbReq(&pCreateReq, TSDB_MSG_FLG_DECODE);
|
||||
goto end;
|
||||
}
|
||||
if (strcmp(tbName, pCreateReq.name) == 0) {
|
||||
cloneSVreateTbReq(&pCreateReq, &pCreateReqDst);
|
||||
// pCreateReqDst->ctb.suid = processSuid(pCreateReqDst->ctb.suid, pRequest->pDb);
|
||||
// pCreateReqDst->ctb.suid = processSuid(pCreateReqDst->ctb.suid, pRequest->pDb);
|
||||
tDecoderClear(&decoderTmp);
|
||||
tDestroySVCreateTbReq(&pCreateReq, TSDB_MSG_FLG_DECODE);
|
||||
break;
|
||||
}
|
||||
tDecoderClear(&decoderTmp);
|
||||
tDestroySVCreateTbReq(&pCreateReq, TSDB_MSG_FLG_DECODE);
|
||||
}
|
||||
|
||||
SVgroupInfo vg;
|
||||
|
|
@ -1774,6 +1781,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
|
|||
goto end;
|
||||
}
|
||||
pCreateReqDst = NULL;
|
||||
taosMemoryFreeClear(pTableMeta);
|
||||
}
|
||||
|
||||
code = smlBuildOutput(pQuery, pVgHash);
|
||||
|
|
|
|||
|
|
@ -256,20 +256,15 @@ cleanup:
|
|||
kvVal->type = TSDB_DATA_TYPE_FLOAT; \
|
||||
kvVal->f = (float)result;
|
||||
|
||||
#define SET_BIGINT \
|
||||
if (smlDoubleToInt64OverFlow(result)) { \
|
||||
errno = 0; \
|
||||
int64_t tmp = taosStr2Int64(pVal, &endptr, 10); \
|
||||
if (errno == ERANGE) { \
|
||||
smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal); \
|
||||
return false; \
|
||||
} \
|
||||
kvVal->type = TSDB_DATA_TYPE_BIGINT; \
|
||||
kvVal->i = tmp; \
|
||||
return true; \
|
||||
} \
|
||||
kvVal->type = TSDB_DATA_TYPE_BIGINT; \
|
||||
kvVal->i = (int64_t)result;
|
||||
#define SET_BIGINT \
|
||||
errno = 0; \
|
||||
int64_t tmp = taosStr2Int64(pVal, &endptr, 10); \
|
||||
if (errno == ERANGE) { \
|
||||
smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal); \
|
||||
return false; \
|
||||
} \
|
||||
kvVal->type = TSDB_DATA_TYPE_BIGINT; \
|
||||
kvVal->i = tmp;
|
||||
|
||||
#define SET_INT \
|
||||
if (!IS_VALID_INT(result)) { \
|
||||
|
|
@ -287,20 +282,15 @@ cleanup:
|
|||
kvVal->type = TSDB_DATA_TYPE_SMALLINT; \
|
||||
kvVal->i = result;
|
||||
|
||||
#define SET_UBIGINT \
|
||||
if (result >= (double)UINT64_MAX || result < 0) { \
|
||||
errno = 0; \
|
||||
uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10); \
|
||||
if (errno == ERANGE || result < 0) { \
|
||||
smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal); \
|
||||
return false; \
|
||||
} \
|
||||
kvVal->type = TSDB_DATA_TYPE_UBIGINT; \
|
||||
kvVal->u = tmp; \
|
||||
return true; \
|
||||
} \
|
||||
kvVal->type = TSDB_DATA_TYPE_UBIGINT; \
|
||||
kvVal->u = result;
|
||||
#define SET_UBIGINT \
|
||||
errno = 0; \
|
||||
uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10); \
|
||||
if (errno == ERANGE || result < 0) { \
|
||||
smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal); \
|
||||
return false; \
|
||||
} \
|
||||
kvVal->type = TSDB_DATA_TYPE_UBIGINT; \
|
||||
kvVal->u = tmp;
|
||||
|
||||
#define SET_UINT \
|
||||
if (!IS_VALID_UINT(result)) { \
|
||||
|
|
@ -637,8 +627,8 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO
|
|||
for (int j = 0; j < taosArrayGetSize(cols); ++j) {
|
||||
SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j);
|
||||
ESchemaAction action = SCHEMA_ACTION_NULL;
|
||||
int code = smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, &action, info);
|
||||
if(code != 0){
|
||||
int code = smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, &action, info);
|
||||
if (code != 0) {
|
||||
return code;
|
||||
}
|
||||
if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_ADD_TAG) {
|
||||
|
|
@ -649,11 +639,11 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO
|
|||
taosArrayPush(results, &field);
|
||||
} else if (action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) {
|
||||
uint16_t *index = (uint16_t *)taosHashGet(schemaHash, kv->key, kv->keyLen);
|
||||
if(index == NULL){
|
||||
if (index == NULL) {
|
||||
uError("smlBuildFieldsList get error, key:%s", kv->key);
|
||||
return TSDB_CODE_SML_INVALID_DATA;
|
||||
}
|
||||
uint16_t newIndex = *index;
|
||||
uint16_t newIndex = *index;
|
||||
if (isTag) newIndex -= numOfCols;
|
||||
SField *field = (SField *)taosArrayGet(results, newIndex);
|
||||
field->bytes = getBytes(kv->type, kv->length);
|
||||
|
|
@ -750,7 +740,8 @@ end:
|
|||
}
|
||||
|
||||
static int32_t smlModifyDBSchemas(SSmlHandle *info) {
|
||||
uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas start, format:%d, needModifySchema:%d", info->id, info->dataFormat, info->needModifySchema);
|
||||
uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas start, format:%d, needModifySchema:%d", info->id, info->dataFormat,
|
||||
info->needModifySchema);
|
||||
if (info->dataFormat && !info->needModifySchema) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
|
@ -820,7 +811,8 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
|
|||
goto end;
|
||||
}
|
||||
if (action != SCHEMA_ACTION_NULL) {
|
||||
uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas change table tag, table:%s, action:%d", info->id, pName.tname, action);
|
||||
uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas change table tag, table:%s, action:%d", info->id, pName.tname,
|
||||
action);
|
||||
SArray *pColumns =
|
||||
taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField));
|
||||
SArray *pTags =
|
||||
|
|
@ -838,7 +830,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
|
|||
}
|
||||
}
|
||||
code = smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->tags, pTags,
|
||||
pTableMeta->tableInfo.numOfColumns, true);
|
||||
pTableMeta->tableInfo.numOfColumns, true);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
uError("SML:0x%" PRIx64 " smlBuildFieldsList tag2 failed. %s", info->id, pName.tname);
|
||||
goto end;
|
||||
|
|
@ -872,7 +864,8 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
|
|||
goto end;
|
||||
}
|
||||
if (action != SCHEMA_ACTION_NULL) {
|
||||
uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas change table col, table:%s, action:%d", info->id, pName.tname, action);
|
||||
uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas change table col, table:%s, action:%d", info->id, pName.tname,
|
||||
action);
|
||||
SArray *pColumns =
|
||||
taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField));
|
||||
SArray *pTags =
|
||||
|
|
@ -891,7 +884,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
|
|||
}
|
||||
|
||||
code = smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->cols, pColumns,
|
||||
pTableMeta->tableInfo.numOfColumns, false);
|
||||
pTableMeta->tableInfo.numOfColumns, false);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
uError("SML:0x%" PRIx64 " smlBuildFieldsList col2 failed. %s", info->id, pName.tname);
|
||||
goto end;
|
||||
|
|
@ -939,10 +932,11 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
|
|||
}
|
||||
|
||||
sTableData->tableMeta = pTableMeta;
|
||||
uDebug("SML:0x%" PRIx64 "modify schema uid:%" PRIu64 ", sversion:%d, tversion:%d", info->id, pTableMeta->uid, pTableMeta->sversion, pTableMeta->tversion)
|
||||
tmp = (SSmlSTableMeta **)taosHashIterate(info->superTables, tmp);
|
||||
uDebug("SML:0x%" PRIx64 "modify schema uid:%" PRIu64 ", sversion:%d, tversion:%d", info->id, pTableMeta->uid,
|
||||
pTableMeta->sversion, pTableMeta->tversion) tmp = (SSmlSTableMeta **)taosHashIterate(info->superTables, tmp);
|
||||
}
|
||||
uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas end success, format:%d, needModifySchema:%d", info->id, info->dataFormat, info->needModifySchema);
|
||||
uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas end success, format:%d, needModifySchema:%d", info->id, info->dataFormat,
|
||||
info->needModifySchema);
|
||||
|
||||
return 0;
|
||||
|
||||
|
|
@ -950,7 +944,8 @@ end:
|
|||
taosHashCleanup(hashTmp);
|
||||
taosMemoryFreeClear(pTableMeta);
|
||||
catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1);
|
||||
uError("SML:0x%" PRIx64 " smlModifyDBSchemas end failed:%d:%s, format:%d, needModifySchema:%d", info->id, code, tstrerror(code), info->dataFormat, info->needModifySchema);
|
||||
uError("SML:0x%" PRIx64 " smlModifyDBSchemas end failed:%d:%s, format:%d, needModifySchema:%d", info->id, code,
|
||||
tstrerror(code), info->dataFormat, info->needModifySchema);
|
||||
|
||||
return code;
|
||||
}
|
||||
|
|
@ -1128,7 +1123,7 @@ SSmlHandle *smlBuildSmlInfo(TAOS *taos) {
|
|||
}
|
||||
if (taos != NULL) {
|
||||
info->taos = acquireTscObj(*(int64_t *)taos);
|
||||
if(info->taos == NULL){
|
||||
if (info->taos == NULL) {
|
||||
goto cleanup;
|
||||
}
|
||||
code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog);
|
||||
|
|
@ -1171,7 +1166,9 @@ static int32_t smlPushCols(SArray *colsArray, SArray *cols) {
|
|||
for (size_t i = 0; i < taosArrayGetSize(cols); i++) {
|
||||
SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i);
|
||||
taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES);
|
||||
if(terrno == TSDB_CODE_DUP_KEY){return terrno;}
|
||||
if (terrno == TSDB_CODE_DUP_KEY) {
|
||||
return terrno;
|
||||
}
|
||||
}
|
||||
|
||||
taosArrayPush(colsArray, &kvHash);
|
||||
|
|
@ -1179,23 +1176,25 @@ static int32_t smlPushCols(SArray *colsArray, SArray *cols) {
|
|||
}
|
||||
|
||||
static int32_t smlParseLineBottom(SSmlHandle *info) {
|
||||
uDebug("SML:0x%" PRIx64 " smlParseLineBottom start, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum);
|
||||
uDebug("SML:0x%" PRIx64 " smlParseLineBottom start, format:%d, linenum:%d", info->id, info->dataFormat,
|
||||
info->lineNum);
|
||||
if (info->dataFormat) return TSDB_CODE_SUCCESS;
|
||||
|
||||
for (int32_t i = 0; i < info->lineNum; i++) {
|
||||
SSmlLineInfo *elements = info->lines + i;
|
||||
SSmlTableInfo *tinfo = NULL;
|
||||
if (info->protocol == TSDB_SML_LINE_PROTOCOL) {
|
||||
SSmlTableInfo** tmp = (SSmlTableInfo **)taosHashGet(info->childTables, elements->measure, elements->measureTagsLen);
|
||||
if(tmp) tinfo = *tmp;
|
||||
SSmlTableInfo **tmp =
|
||||
(SSmlTableInfo **)taosHashGet(info->childTables, elements->measure, elements->measureTagsLen);
|
||||
if (tmp) tinfo = *tmp;
|
||||
} else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) {
|
||||
SSmlTableInfo** tmp = (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag,
|
||||
elements->measureLen + elements->tagsLen);
|
||||
if(tmp) tinfo = *tmp;
|
||||
SSmlTableInfo **tmp = (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag,
|
||||
elements->measureLen + elements->tagsLen);
|
||||
if (tmp) tinfo = *tmp;
|
||||
} else {
|
||||
SSmlTableInfo** tmp = (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag,
|
||||
elements->measureLen + elements->tagsLen);
|
||||
if(tmp) tinfo = *tmp;
|
||||
SSmlTableInfo **tmp = (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag,
|
||||
elements->measureLen + elements->tagsLen);
|
||||
if (tmp) tinfo = *tmp;
|
||||
}
|
||||
|
||||
if (tinfo == NULL) {
|
||||
|
|
@ -1222,7 +1221,8 @@ static int32_t smlParseLineBottom(SSmlHandle *info) {
|
|||
SSmlSTableMeta **tableMeta =
|
||||
(SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen);
|
||||
if (tableMeta) { // update meta
|
||||
uDebug("SML:0x%" PRIx64 " smlParseLineBottom update meta, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum);
|
||||
uDebug("SML:0x%" PRIx64 " smlParseLineBottom update meta, format:%d, linenum:%d", info->id, info->dataFormat,
|
||||
info->lineNum);
|
||||
ret = smlUpdateMeta((*tableMeta)->colHash, (*tableMeta)->cols, elements->colArray, false, &info->msgBuf);
|
||||
if (ret == TSDB_CODE_SUCCESS) {
|
||||
ret = smlUpdateMeta((*tableMeta)->tagHash, (*tableMeta)->tags, tinfo->tags, true, &info->msgBuf);
|
||||
|
|
@ -1237,10 +1237,13 @@ static int32_t smlParseLineBottom(SSmlHandle *info) {
|
|||
// uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id);
|
||||
// return ret;
|
||||
// }
|
||||
uDebug("SML:0x%" PRIx64 " smlParseLineBottom add meta, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum);
|
||||
uDebug("SML:0x%" PRIx64 " smlParseLineBottom add meta, format:%d, linenum:%d", info->id, info->dataFormat,
|
||||
info->lineNum);
|
||||
SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat);
|
||||
smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags);
|
||||
if(terrno == TSDB_CODE_DUP_KEY){return terrno;}
|
||||
if (terrno == TSDB_CODE_DUP_KEY) {
|
||||
return terrno;
|
||||
}
|
||||
smlInsertMeta(meta->colHash, meta->cols, elements->colArray);
|
||||
taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES);
|
||||
}
|
||||
|
|
@ -1254,11 +1257,12 @@ static int32_t smlInsertData(SSmlHandle *info) {
|
|||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
uDebug("SML:0x%" PRIx64 " smlInsertData start, format:%d", info->id, info->dataFormat);
|
||||
|
||||
if(info->pRequest->dbList == NULL){
|
||||
if (info->pRequest->dbList == NULL) {
|
||||
info->pRequest->dbList = taosArrayInit(1, TSDB_DB_FNAME_LEN);
|
||||
}
|
||||
void* data = taosArrayReserve(info->pRequest->dbList, 1);
|
||||
memcpy(data, info->pRequest->pDb, TSDB_DB_FNAME_LEN > strlen(info->pRequest->pDb) ? strlen(info->pRequest->pDb) : TSDB_DB_FNAME_LEN);
|
||||
void *data = taosArrayReserve(info->pRequest->dbList, 1);
|
||||
memcpy(data, info->pRequest->pDb,
|
||||
TSDB_DB_FNAME_LEN > strlen(info->pRequest->pDb) ? strlen(info->pRequest->pDb) : TSDB_DB_FNAME_LEN);
|
||||
|
||||
SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL);
|
||||
while (oneTable) {
|
||||
|
|
@ -1268,7 +1272,7 @@ static int32_t smlInsertData(SSmlHandle *info) {
|
|||
tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname));
|
||||
memcpy(pName.tname, tableData->childTableName, strlen(tableData->childTableName));
|
||||
|
||||
if(info->pRequest->tableList == NULL){
|
||||
if (info->pRequest->tableList == NULL) {
|
||||
info->pRequest->tableList = taosArrayInit(1, sizeof(SName));
|
||||
}
|
||||
taosArrayPush(info->pRequest->tableList, &pName);
|
||||
|
|
@ -1297,7 +1301,8 @@ static int32_t smlInsertData(SSmlHandle *info) {
|
|||
// use tablemeta of stable to save vgid and uid of child table
|
||||
(*pMeta)->tableMeta->vgId = vg.vgId;
|
||||
(*pMeta)->tableMeta->uid = tableData->uid; // one table merge data block together according uid
|
||||
uDebug("SML:0x%" PRIx64 " smlInsertData table:%s, uid:%" PRIu64 ", format:%d", info->id, pName.tname, tableData->uid, info->dataFormat);
|
||||
uDebug("SML:0x%" PRIx64 " smlInsertData table:%s, uid:%" PRIu64 ", format:%d", info->id, pName.tname,
|
||||
tableData->uid, info->dataFormat);
|
||||
|
||||
code = smlBindData(info->pQuery, info->dataFormat, tableData->tags, (*pMeta)->cols, tableData->cols,
|
||||
(*pMeta)->tableMeta, tableData->childTableName, tableData->sTableName, tableData->sTableNameLen,
|
||||
|
|
@ -1320,7 +1325,8 @@ static int32_t smlInsertData(SSmlHandle *info) {
|
|||
atomic_add_fetch_64((int64_t *)&pActivity->numOfInsertsReq, 1);
|
||||
|
||||
launchQueryImpl(info->pRequest, info->pQuery, true, NULL);
|
||||
uDebug("SML:0x%" PRIx64 " smlInsertData end, format:%d, code:%d,%s", info->id, info->dataFormat, info->pRequest->code, tstrerror(info->pRequest->code));
|
||||
uDebug("SML:0x%" PRIx64 " smlInsertData end, format:%d, code:%d,%s", info->id, info->dataFormat, info->pRequest->code,
|
||||
tstrerror(info->pRequest->code));
|
||||
|
||||
return info->pRequest->code;
|
||||
}
|
||||
|
|
@ -1331,11 +1337,11 @@ static void smlPrintStatisticInfo(SSmlHandle *info) {
|
|||
" smlInsertLines result, code:%d, msg:%s, lineNum:%d,stable num:%d,ctable num:%d,create stable num:%d,alter stable tag num:%d,alter stable col num:%d \
|
||||
parse cost:%" PRId64 ",schema cost:%" PRId64 ",bind cost:%" PRId64 ",rpc cost:%" PRId64 ",total cost:%" PRId64
|
||||
"",
|
||||
info->id, info->cost.code, tstrerror(info->cost.code), info->cost.lineNum, info->cost.numOfSTables, info->cost.numOfCTables,
|
||||
info->cost.numOfCreateSTables, info->cost.numOfAlterTagSTables, info->cost.numOfAlterColSTables,
|
||||
info->cost.schemaTime - info->cost.parseTime, info->cost.insertBindTime - info->cost.schemaTime,
|
||||
info->cost.insertRpcTime - info->cost.insertBindTime, info->cost.endTime - info->cost.insertRpcTime,
|
||||
info->cost.endTime - info->cost.parseTime);
|
||||
info->id, info->cost.code, tstrerror(info->cost.code), info->cost.lineNum, info->cost.numOfSTables,
|
||||
info->cost.numOfCTables, info->cost.numOfCreateSTables, info->cost.numOfAlterTagSTables,
|
||||
info->cost.numOfAlterColSTables, info->cost.schemaTime - info->cost.parseTime,
|
||||
info->cost.insertBindTime - info->cost.schemaTime, info->cost.insertRpcTime - info->cost.insertBindTime,
|
||||
info->cost.endTime - info->cost.insertRpcTime, info->cost.endTime - info->cost.parseTime);
|
||||
}
|
||||
|
||||
int32_t smlClearForRerun(SSmlHandle *info) {
|
||||
|
|
@ -1412,14 +1418,15 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char
|
|||
}
|
||||
}
|
||||
|
||||
char cTmp = 0; // for print tmp if is raw
|
||||
if(info->isRawLine){
|
||||
char cTmp = 0; // for print tmp if is raw
|
||||
if (info->isRawLine) {
|
||||
cTmp = tmp[len - 1];
|
||||
tmp[len - 1] = '\0';
|
||||
}
|
||||
|
||||
uDebug("SML:0x%" PRIx64 " smlParseLine israw:%d, numLines:%d, protocol:%d, len:%d, sql:%s", info->id, info->isRawLine, numLines, info->protocol, len, tmp);
|
||||
if(info->isRawLine){
|
||||
uDebug("SML:0x%" PRIx64 " smlParseLine israw:%d, numLines:%d, protocol:%d, len:%d, sql:%s", info->id,
|
||||
info->isRawLine, numLines, info->protocol, len, tmp);
|
||||
if (info->isRawLine) {
|
||||
tmp[len - 1] = cTmp;
|
||||
}
|
||||
|
||||
|
|
@ -1515,9 +1522,9 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine,
|
|||
return NULL;
|
||||
}
|
||||
SRequestObj *request = NULL;
|
||||
SSmlHandle *info = NULL;
|
||||
int cnt = 0;
|
||||
while(1){
|
||||
SSmlHandle *info = NULL;
|
||||
int cnt = 0;
|
||||
while (1) {
|
||||
request = (SRequestObj *)createRequest(*(int64_t *)taos, TSDB_SQL_INSERT, reqid);
|
||||
if (request == NULL) {
|
||||
uError("SML:taos_schemaless_insert error request is null");
|
||||
|
|
@ -1571,15 +1578,16 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine,
|
|||
request->code = code;
|
||||
info->cost.endTime = taosGetTimestampUs();
|
||||
info->cost.code = code;
|
||||
if(code == TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER || code == TSDB_CODE_SDB_OBJ_CREATING
|
||||
|| code == TSDB_CODE_PAR_VALUE_TOO_LONG || code == TSDB_CODE_MND_TRANS_CONFLICT){
|
||||
if(cnt++ >= 10){
|
||||
uInfo("SML:%"PRIx64" retry:%d/10 end code:%d, msg:%s", info->id, cnt, code, tstrerror(code));
|
||||
if (code == TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER || code == TSDB_CODE_SDB_OBJ_CREATING ||
|
||||
code == TSDB_CODE_PAR_VALUE_TOO_LONG || code == TSDB_CODE_MND_TRANS_CONFLICT) {
|
||||
if (cnt++ >= 10) {
|
||||
uInfo("SML:%" PRIx64 " retry:%d/10 end code:%d, msg:%s", info->id, cnt, code, tstrerror(code));
|
||||
break;
|
||||
}
|
||||
taosMsleep(100);
|
||||
refreshMeta(request->pTscObj, request);
|
||||
uInfo("SML:%"PRIx64" retry:%d/10,ver is old retry or object is creating code:%d, msg:%s", info->id, cnt, code, tstrerror(code));
|
||||
uInfo("SML:%" PRIx64 " retry:%d/10,ver is old retry or object is creating code:%d, msg:%s", info->id, cnt, code,
|
||||
tstrerror(code));
|
||||
smlDestroyInfo(info);
|
||||
info = NULL;
|
||||
taos_free_result(request);
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -52,15 +52,15 @@ void printResult(TAOS_RES* pRes) {
|
|||
int32_t n = 0;
|
||||
char str[512] = {0};
|
||||
while ((pRow = taos_fetch_row(pRes)) != NULL) {
|
||||
// int32_t* length = taos_fetch_lengths(pRes);
|
||||
// for(int32_t i = 0; i < numOfFields; ++i) {
|
||||
// printf("(%d):%d " , i, length[i]);
|
||||
// }
|
||||
// printf("\n");
|
||||
//
|
||||
// int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
|
||||
// printf("%s\n", str);
|
||||
// memset(str, 0, sizeof(str));
|
||||
// int32_t* length = taos_fetch_lengths(pRes);
|
||||
// for(int32_t i = 0; i < numOfFields; ++i) {
|
||||
// printf("(%d):%d " , i, length[i]);
|
||||
// }
|
||||
// printf("\n");
|
||||
//
|
||||
// int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
|
||||
// printf("%s\n", str);
|
||||
// memset(str, 0, sizeof(str));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -112,7 +112,7 @@ void createNewTable(TAOS* pConn, int32_t index) {
|
|||
}
|
||||
taos_free_result(pRes);
|
||||
|
||||
for(int32_t i = 0; i < 100; i += 20) {
|
||||
for (int32_t i = 0; i < 100; i += 20) {
|
||||
char sql[1024] = {0};
|
||||
sprintf(sql,
|
||||
"insert into tu%d values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)"
|
||||
|
|
@ -131,7 +131,7 @@ void createNewTable(TAOS* pConn, int32_t index) {
|
|||
}
|
||||
}
|
||||
|
||||
void *queryThread(void *arg) {
|
||||
void* queryThread(void* arg) {
|
||||
TAOS* pConn = taos_connect("192.168.0.209", "root", "taosdata", NULL, 0);
|
||||
if (pConn == NULL) {
|
||||
printf("failed to connect to db, reason:%s", taos_errstr(pConn));
|
||||
|
|
@ -141,9 +141,10 @@ void *queryThread(void *arg) {
|
|||
int64_t el = 0;
|
||||
|
||||
for (int32_t i = 0; i < 5000000; ++i) {
|
||||
int64_t st = taosGetTimestampUs();
|
||||
int64_t st = taosGetTimestampUs();
|
||||
TAOS_RES* pRes = taos_query(pConn,
|
||||
"SELECT _wstart as ts,max(usage_user) FROM benchmarkcpu.host_49 WHERE ts >= 1451618560000 AND ts < 1451622160000 INTERVAL(1m) ;");
|
||||
"SELECT _wstart as ts,max(usage_user) FROM benchmarkcpu.host_49 WHERE ts >= "
|
||||
"1451618560000 AND ts < 1451622160000 INTERVAL(1m) ;");
|
||||
if (taos_errno(pRes) != 0) {
|
||||
printf("failed, reason:%s\n", taos_errstr(pRes));
|
||||
} else {
|
||||
|
|
@ -153,7 +154,7 @@ void *queryThread(void *arg) {
|
|||
taos_free_result(pRes);
|
||||
el += (taosGetTimestampUs() - st);
|
||||
if (i % 1000 == 0 && i != 0) {
|
||||
printf("total:%d, avg time:%.2fms\n", i, el/(double)(i*1000));
|
||||
printf("total:%d, avg time:%.2fms\n", i, el / (double)(i * 1000));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -163,9 +164,7 @@ void *queryThread(void *arg) {
|
|||
|
||||
static int32_t numOfThreads = 1;
|
||||
|
||||
void tmq_commit_cb_print(tmq_t *pTmq, int32_t code, void *param) {
|
||||
printf("success, code:%d\n", code);
|
||||
}
|
||||
void tmq_commit_cb_print(tmq_t* pTmq, int32_t code, void* param) { printf("success, code:%d\n", code); }
|
||||
|
||||
void* doConsumeData(void* param) {
|
||||
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
|
||||
|
|
@ -205,7 +204,7 @@ void* doConsumeData(void* param) {
|
|||
while (1) {
|
||||
TAOS_RES* pRes = tmq_consumer_poll(tmq, timeout);
|
||||
if (pRes) {
|
||||
char buf[1024];
|
||||
char buf[1024];
|
||||
|
||||
const char* topicName = tmq_get_topic_name(pRes);
|
||||
const char* dbName = tmq_get_db_name(pRes);
|
||||
|
|
@ -226,7 +225,7 @@ void* doConsumeData(void* param) {
|
|||
precision = taos_result_precision(pRes);
|
||||
taos_print_row(buf, row, fields, numOfFields);
|
||||
totalRows += 1;
|
||||
// printf("precision: %d, row content: %s\n", precision, buf);
|
||||
// printf("precision: %d, row content: %s\n", precision, buf);
|
||||
}
|
||||
|
||||
taos_free_result(pRes);
|
||||
|
|
@ -491,17 +490,17 @@ TEST(clientCase, create_stable_Test) {
|
|||
ASSERT_EQ(numOfFields, 0);
|
||||
taos_free_result(pRes);
|
||||
|
||||
// pRes = taos_query(pConn, "create stable if not exists abc1.`123_$^)` (ts timestamp, `abc` int) tags(a int)");
|
||||
// if (taos_errno(pRes) != 0) {
|
||||
// printf("failed to create super table 123_$^), reason:%s\n", taos_errstr(pRes));
|
||||
// }
|
||||
//
|
||||
// pRes = taos_query(pConn, "use abc1");
|
||||
// taos_free_result(pRes);
|
||||
// pRes = taos_query(pConn, "drop stable `123_$^)`");
|
||||
// if (taos_errno(pRes) != 0) {
|
||||
// printf("failed to drop super table 123_$^), reason:%s\n", taos_errstr(pRes));
|
||||
// }
|
||||
// pRes = taos_query(pConn, "create stable if not exists abc1.`123_$^)` (ts timestamp, `abc` int) tags(a int)");
|
||||
// if (taos_errno(pRes) != 0) {
|
||||
// printf("failed to create super table 123_$^), reason:%s\n", taos_errstr(pRes));
|
||||
// }
|
||||
//
|
||||
// pRes = taos_query(pConn, "use abc1");
|
||||
// taos_free_result(pRes);
|
||||
// pRes = taos_query(pConn, "drop stable `123_$^)`");
|
||||
// if (taos_errno(pRes) != 0) {
|
||||
// printf("failed to drop super table 123_$^), reason:%s\n", taos_errstr(pRes));
|
||||
// }
|
||||
|
||||
taos_close(pConn);
|
||||
}
|
||||
|
|
@ -666,8 +665,7 @@ TEST(clientCase, create_multiple_tables) {
|
|||
|
||||
for (int32_t i = 0; i < 500; i += 2) {
|
||||
char sql[512] = {0};
|
||||
snprintf(sql, tListLen(sql),
|
||||
"create table t_x_%d using st1 tags(2) t_x_%d using st1 tags(5)", i, i + 1);
|
||||
snprintf(sql, tListLen(sql), "create table t_x_%d using st1 tags(2) t_x_%d using st1 tags(5)", i, i + 1);
|
||||
TAOS_RES* pres = taos_query(pConn, sql);
|
||||
if (taos_errno(pres) != 0) {
|
||||
printf("failed to create table %d\n, reason:%s", i, taos_errstr(pres));
|
||||
|
|
@ -701,7 +699,7 @@ TEST(clientCase, show_table_Test) {
|
|||
int32_t numOfFields = taos_num_fields(pRes);
|
||||
|
||||
int32_t count = 0;
|
||||
char str[512] = {0};
|
||||
char str[512] = {0};
|
||||
|
||||
while ((pRow = taos_fetch_row(pRes)) != NULL) {
|
||||
int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
|
||||
|
|
@ -712,30 +710,30 @@ TEST(clientCase, show_table_Test) {
|
|||
taos_close(pConn);
|
||||
}
|
||||
|
||||
//TEST(clientCase, drop_stable_Test) {
|
||||
// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
|
||||
// assert(pConn != nullptr);
|
||||
// TEST(clientCase, drop_stable_Test) {
|
||||
// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
|
||||
// assert(pConn != nullptr);
|
||||
//
|
||||
// TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1");
|
||||
// if (taos_errno(pRes) != 0) {
|
||||
// printf("error in creating db, reason:%s\n", taos_errstr(pRes));
|
||||
// }
|
||||
// taos_free_result(pRes);
|
||||
// TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1");
|
||||
// if (taos_errno(pRes) != 0) {
|
||||
// printf("error in creating db, reason:%s\n", taos_errstr(pRes));
|
||||
// }
|
||||
// taos_free_result(pRes);
|
||||
//
|
||||
// pRes = taos_query(pConn, "use abc1");
|
||||
// if (taos_errno(pRes) != 0) {
|
||||
// printf("error in using db, reason:%s\n", taos_errstr(pRes));
|
||||
// }
|
||||
// taos_free_result(pRes);
|
||||
// pRes = taos_query(pConn, "use abc1");
|
||||
// if (taos_errno(pRes) != 0) {
|
||||
// printf("error in using db, reason:%s\n", taos_errstr(pRes));
|
||||
// }
|
||||
// taos_free_result(pRes);
|
||||
//
|
||||
// pRes = taos_query(pConn, "drop stable st1");
|
||||
// if (taos_errno(pRes) != 0) {
|
||||
// printf("failed to drop stable, reason:%s\n", taos_errstr(pRes));
|
||||
// }
|
||||
// pRes = taos_query(pConn, "drop stable st1");
|
||||
// if (taos_errno(pRes) != 0) {
|
||||
// printf("failed to drop stable, reason:%s\n", taos_errstr(pRes));
|
||||
// }
|
||||
//
|
||||
// taos_free_result(pRes);
|
||||
// taos_close(pConn);
|
||||
//}
|
||||
// taos_free_result(pRes);
|
||||
// taos_close(pConn);
|
||||
// }
|
||||
|
||||
TEST(clientCase, generated_request_id_test) {
|
||||
SHashObj* phash = taosHashInit(10000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK);
|
||||
|
|
@ -744,7 +742,7 @@ TEST(clientCase, generated_request_id_test) {
|
|||
uint64_t v = generateRequestId();
|
||||
void* result = taosHashGet(phash, &v, sizeof(v));
|
||||
if (result != nullptr) {
|
||||
// printf("0x%llx, index:%d\n", v, i);
|
||||
// printf("0x%llx, index:%d\n", v, i);
|
||||
}
|
||||
assert(result == nullptr);
|
||||
taosHashPut(phash, &v, sizeof(v), NULL, 0);
|
||||
|
|
@ -832,7 +830,7 @@ TEST(clientCase, projection_query_tables) {
|
|||
TEST(clientCase, tsbs_perf_test) {
|
||||
TdThread qid[20] = {0};
|
||||
|
||||
for(int32_t i = 0; i < numOfThreads; ++i) {
|
||||
for (int32_t i = 0; i < numOfThreads; ++i) {
|
||||
taosThreadCreate(&qid[i], NULL, queryThread, NULL);
|
||||
}
|
||||
getchar();
|
||||
|
|
@ -858,8 +856,8 @@ TEST(clientCase, projection_query_stables) {
|
|||
|
||||
char str[512] = {0};
|
||||
while ((pRow = taos_fetch_row(pRes)) != NULL) {
|
||||
// int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
|
||||
// printf("%s\n", str);
|
||||
// int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
|
||||
// printf("%s\n", str);
|
||||
}
|
||||
|
||||
taos_free_result(pRes);
|
||||
|
|
@ -898,7 +896,8 @@ use test;
|
|||
create table m1(ts timestamp, k int) tags(a int);
|
||||
create table tm0 using m1 tags(1);
|
||||
create table tm1 using m1 tags(2);
|
||||
insert into tm0 values('2021-1-1 1:1:1.120', 1) ('2021-1-1 1:1:2.9', 2) tm1 values('2021-1-1 1:1:1.120', 11) ('2021-1-1 1:1:2.99', 22);
|
||||
insert into tm0 values('2021-1-1 1:1:1.120', 1) ('2021-1-1 1:1:2.9', 2) tm1 values('2021-1-1 1:1:1.120', 11) ('2021-1-1
|
||||
1:1:2.99', 22);
|
||||
|
||||
*/
|
||||
TEST(clientCase, async_api_test) {
|
||||
|
|
@ -912,7 +911,7 @@ TEST(clientCase, async_api_test) {
|
|||
printf("failed, reason:%s\n", taos_errstr(pRes));
|
||||
}
|
||||
|
||||
int32_t n = 0;
|
||||
int32_t n = 0;
|
||||
TAOS_ROW pRow = NULL;
|
||||
TAOS_FIELD* pFields = taos_fetch_fields(pRes);
|
||||
int32_t numOfFields = taos_num_fields(pRes);
|
||||
|
|
@ -920,8 +919,8 @@ TEST(clientCase, async_api_test) {
|
|||
char str[512] = {0};
|
||||
while ((pRow = taos_fetch_row(pRes)) != NULL) {
|
||||
int32_t* length = taos_fetch_lengths(pRes);
|
||||
for(int32_t i = 0; i < numOfFields; ++i) {
|
||||
printf("(%d):%d " , i, length[i]);
|
||||
for (int32_t i = 0; i < numOfFields; ++i) {
|
||||
printf("(%d):%d ", i, length[i]);
|
||||
}
|
||||
printf("\n");
|
||||
|
||||
|
|
@ -963,8 +962,8 @@ TEST(clientCase, update_test) {
|
|||
|
||||
taos_free_result(pRes);
|
||||
|
||||
char s[256] = {0};
|
||||
for(int32_t i = 0; i < 17000; ++i) {
|
||||
char s[256] = {0};
|
||||
for (int32_t i = 0; i < 17000; ++i) {
|
||||
sprintf(s, "insert into tup values(now+%da, %d)", i, i);
|
||||
pRes = taos_query(pConn, s);
|
||||
taos_free_result(pRes);
|
||||
|
|
@ -998,7 +997,8 @@ TEST(clientCase, sub_db_test) {
|
|||
|
||||
// 创建订阅 topics 列表
|
||||
tmq_list_t* topicList = tmq_list_new();
|
||||
tmq_list_append(topicList, "topic_db1");
|
||||
tmq_list_append(topicList, "topic_t1");
|
||||
tmq_list_append(topicList, "topic_s2");
|
||||
|
||||
// 启动订阅
|
||||
tmq_subscribe(tmq, topicList);
|
||||
|
|
@ -1027,7 +1027,7 @@ TEST(clientCase, sub_db_test) {
|
|||
printf("db: %s\n", dbName);
|
||||
printf("vgroup id: %d\n", vgroupId);
|
||||
|
||||
if (count ++ > 200) {
|
||||
if (count++ > 200) {
|
||||
tmq_unsubscribe(tmq);
|
||||
break;
|
||||
}
|
||||
|
|
@ -1091,7 +1091,7 @@ TEST(clientCase, sub_tb_test) {
|
|||
while (1) {
|
||||
TAOS_RES* pRes = tmq_consumer_poll(tmq, timeout);
|
||||
if (pRes) {
|
||||
char buf[1024];
|
||||
char buf[1024];
|
||||
|
||||
const char* topicName = tmq_get_topic_name(pRes);
|
||||
const char* dbName = tmq_get_db_name(pRes);
|
||||
|
|
@ -1116,9 +1116,9 @@ TEST(clientCase, sub_tb_test) {
|
|||
}
|
||||
|
||||
taos_free_result(pRes);
|
||||
// if ((++count) > 1) {
|
||||
// break;
|
||||
// }
|
||||
// if ((++count) > 1) {
|
||||
// break;
|
||||
// }
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
|
|
@ -1133,11 +1133,11 @@ TEST(clientCase, sub_tb_mt_test) {
|
|||
taos_options(TSDB_OPTION_CONFIGDIR, "~/first/cfg");
|
||||
TdThread qid[20] = {0};
|
||||
|
||||
for(int32_t i = 0; i < 1; ++i) {
|
||||
for (int32_t i = 0; i < 1; ++i) {
|
||||
taosThreadCreate(&qid[i], NULL, doConsumeData, NULL);
|
||||
}
|
||||
|
||||
for(int32_t i = 0; i < 4; ++i) {
|
||||
for (int32_t i = 0; i < 4; ++i) {
|
||||
taosThreadJoin(qid[i], NULL);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -147,9 +147,17 @@ int32_t colDataReserve(SColumnInfoData* pColumnInfoData, size_t newSize) {
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static void doCopyNItems(struct SColumnInfoData* pColumnInfoData, int32_t currentRow, const char* pData,
|
||||
int32_t itemLen, int32_t numOfRows) {
|
||||
ASSERT(pColumnInfoData->info.bytes >= itemLen);
|
||||
static int32_t doCopyNItems(struct SColumnInfoData* pColumnInfoData, int32_t currentRow, const char* pData,
|
||||
int32_t itemLen, int32_t numOfRows, bool trimValue) {
|
||||
if (pColumnInfoData->info.bytes < itemLen) {
|
||||
uWarn("column/tag actual data len %d is bigger than schema len %d, trim it:%d", itemLen, pColumnInfoData->info.bytes, trimValue);
|
||||
if (trimValue) {
|
||||
itemLen = pColumnInfoData->info.bytes;
|
||||
} else {
|
||||
return TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER;
|
||||
}
|
||||
}
|
||||
|
||||
size_t start = 1;
|
||||
|
||||
// the first item
|
||||
|
|
@ -178,10 +186,12 @@ static void doCopyNItems(struct SColumnInfoData* pColumnInfoData, int32_t curren
|
|||
|
||||
pColumnInfoData->varmeta.length += numOfRows * itemLen;
|
||||
}
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData,
|
||||
uint32_t numOfRows) {
|
||||
uint32_t numOfRows, bool trimValue) {
|
||||
int32_t len = pColumnInfoData->info.bytes;
|
||||
if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
|
||||
len = varDataTLen(pData);
|
||||
|
|
@ -193,8 +203,7 @@ int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow,
|
|||
}
|
||||
}
|
||||
|
||||
doCopyNItems(pColumnInfoData, currentRow, pData, len, numOfRows);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
return doCopyNItems(pColumnInfoData, currentRow, pData, len, numOfRows, trimValue);
|
||||
}
|
||||
|
||||
static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, const SColumnInfoData* pSource,
|
||||
|
|
@ -973,7 +982,7 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) {
|
|||
taosSort(pColInfoData->pData, pDataBlock->info.rows, pColInfoData->info.bytes, fn);
|
||||
|
||||
int64_t p1 = taosGetTimestampUs();
|
||||
uDebug("blockDataSort easy cost:%" PRId64 ", rows:%d\n", p1 - p0, pDataBlock->info.rows);
|
||||
uDebug("blockDataSort easy cost:%" PRId64 ", rows:%" PRId64 "\n", p1 - p0, pDataBlock->info.rows);
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
} else { // var data type
|
||||
|
|
@ -1026,6 +1035,7 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) {
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
#if 0
|
||||
typedef struct SHelper {
|
||||
int32_t index;
|
||||
union {
|
||||
|
|
@ -1074,59 +1084,20 @@ SHelper* createTupleIndex_rv(int32_t numOfRows, SArray* pOrderInfo, SSDataBlock*
|
|||
int32_t dataBlockCompar_rv(const void* p1, const void* p2, const void* param) {
|
||||
const SSDataBlockSortHelper* pHelper = (const SSDataBlockSortHelper*)param;
|
||||
|
||||
// SSDataBlock* pDataBlock = pHelper->pDataBlock;
|
||||
|
||||
SHelper* left = (SHelper*)p1;
|
||||
SHelper* right = (SHelper*)p2;
|
||||
|
||||
SArray* pInfo = pHelper->orderInfo;
|
||||
|
||||
int32_t offset = 0;
|
||||
// for(int32_t i = 0; i < pInfo->size; ++i) {
|
||||
// SBlockOrderInfo* pOrder = TARRAY_GET_ELEM(pInfo, 0);
|
||||
// SColumnInfoData* pColInfoData = pOrder->pColData;//TARRAY_GET_ELEM(pDataBlock->pDataBlock, pOrder->colIndex);
|
||||
|
||||
// if (pColInfoData->hasNull) {
|
||||
// bool leftNull = colDataIsNull(pColInfoData, pDataBlock->info.rows, left, pDataBlock->pBlockAgg);
|
||||
// bool rightNull = colDataIsNull(pColInfoData, pDataBlock->info.rows, right, pDataBlock->pBlockAgg);
|
||||
// if (leftNull && rightNull) {
|
||||
// continue; // continue to next slot
|
||||
// }
|
||||
//
|
||||
// if (rightNull) {
|
||||
// return pHelper->nullFirst? 1:-1;
|
||||
// }
|
||||
//
|
||||
// if (leftNull) {
|
||||
// return pHelper->nullFirst? -1:1;
|
||||
// }
|
||||
// }
|
||||
|
||||
// void* left1 = colDataGetData(pColInfoData, left);
|
||||
// void* right1 = colDataGetData(pColInfoData, right);
|
||||
|
||||
// switch(pColInfoData->info.type) {
|
||||
// case TSDB_DATA_TYPE_INT: {
|
||||
int32_t leftx = *(int32_t*)left->pData; //*(int32_t*)(left->pData + offset);
|
||||
int32_t rightx = *(int32_t*)right->pData; //*(int32_t*)(right->pData + offset);
|
||||
|
||||
// offset += pColInfoData->info.bytes;
|
||||
if (leftx == rightx) {
|
||||
// break;
|
||||
return 0;
|
||||
} else {
|
||||
// if (pOrder->order == TSDB_ORDER_ASC) {
|
||||
return (leftx < rightx) ? -1 : 1;
|
||||
// } else {
|
||||
// return (leftx < rightx)? 1:-1;
|
||||
// }
|
||||
}
|
||||
// }
|
||||
// default:
|
||||
// assert(0);
|
||||
// }
|
||||
// }
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -1170,6 +1141,7 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF
|
|||
// destroyTupleIndex(index);
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
void blockDataCleanup(SSDataBlock* pDataBlock) {
|
||||
blockDataEmpty(pDataBlock);
|
||||
|
|
@ -1180,7 +1152,7 @@ void blockDataCleanup(SSDataBlock* pDataBlock) {
|
|||
|
||||
void blockDataEmpty(SSDataBlock* pDataBlock) {
|
||||
SDataBlockInfo* pInfo = &pDataBlock->info;
|
||||
if (pInfo->capacity == 0 || pInfo->rows > pDataBlock->info.capacity) {
|
||||
if (pInfo->capacity == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -1739,14 +1711,14 @@ int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) {
|
|||
int64_t tbUid = pBlock->info.id.uid;
|
||||
int16_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
|
||||
int16_t hasVarCol = pBlock->info.hasVarCol;
|
||||
int32_t rows = pBlock->info.rows;
|
||||
int64_t rows = pBlock->info.rows;
|
||||
int32_t sz = taosArrayGetSize(pBlock->pDataBlock);
|
||||
|
||||
int32_t tlen = 0;
|
||||
tlen += taosEncodeFixedI64(buf, tbUid);
|
||||
tlen += taosEncodeFixedI16(buf, numOfCols);
|
||||
tlen += taosEncodeFixedI16(buf, hasVarCol);
|
||||
tlen += taosEncodeFixedI32(buf, rows);
|
||||
tlen += taosEncodeFixedI64(buf, rows);
|
||||
tlen += taosEncodeFixedI32(buf, sz);
|
||||
for (int32_t i = 0; i < sz; i++) {
|
||||
SColumnInfoData* pColData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i);
|
||||
|
|
@ -1777,7 +1749,7 @@ void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock) {
|
|||
buf = taosDecodeFixedU64(buf, &pBlock->info.id.uid);
|
||||
buf = taosDecodeFixedI16(buf, &numOfCols);
|
||||
buf = taosDecodeFixedI16(buf, &pBlock->info.hasVarCol);
|
||||
buf = taosDecodeFixedI32(buf, &pBlock->info.rows);
|
||||
buf = taosDecodeFixedI64(buf, &pBlock->info.rows);
|
||||
buf = taosDecodeFixedI32(buf, &sz);
|
||||
pBlock->pDataBlock = taosArrayInit(sz, sizeof(SColumnInfoData));
|
||||
for (int32_t i = 0; i < sz; i++) {
|
||||
|
|
@ -1864,7 +1836,9 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) {
|
|||
}
|
||||
}
|
||||
struct tm ptm = {0};
|
||||
taosLocalTime(&tt, &ptm);
|
||||
if (taosLocalTime(&tt, &ptm, buf) == NULL) {
|
||||
return buf;
|
||||
}
|
||||
size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", &ptm);
|
||||
|
||||
if (precision == TSDB_TIME_PRECISION_NANO) {
|
||||
|
|
@ -1878,6 +1852,7 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) {
|
|||
return buf;
|
||||
}
|
||||
|
||||
#if 0
|
||||
void blockDebugShowDataBlock(SSDataBlock* pBlock, const char* flag) {
|
||||
SArray* dataBlocks = taosArrayInit(1, sizeof(SSDataBlock*));
|
||||
taosArrayPush(dataBlocks, &pBlock);
|
||||
|
|
@ -1970,6 +1945,8 @@ void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag) {
|
|||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
// for debug
|
||||
char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) {
|
||||
int32_t size = 2048;
|
||||
|
|
@ -1981,7 +1958,7 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf)
|
|||
int32_t len = 0;
|
||||
len += snprintf(dumpBuf + len, size - len,
|
||||
"===stream===%s|block type %d|child id %d|group id:%" PRIu64 "|uid:%" PRId64
|
||||
"|rows:%d|version:%" PRIu64 "|cal start:%" PRIu64 "|cal end:%" PRIu64 "|tbl:%s\n",
|
||||
"|rows:%" PRId64 "|version:%" PRIu64 "|cal start:%" PRIu64 "|cal end:%" PRIu64 "|tbl:%s\n",
|
||||
flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId, pDataBlock->info.id.groupId,
|
||||
pDataBlock->info.id.uid, pDataBlock->info.rows, pDataBlock->info.version,
|
||||
pDataBlock->info.calWin.skey, pDataBlock->info.calWin.ekey, pDataBlock->info.parTbName);
|
||||
|
|
|
|||
|
|
@ -2439,6 +2439,12 @@ _exit:
|
|||
int32_t tColDataAddValueByDataBlock(SColData *pColData, int8_t type, int32_t bytes, int32_t nRows, char *lengthOrbitmap,
|
||||
char *data) {
|
||||
int32_t code = 0;
|
||||
if(data == NULL){
|
||||
for (int32_t i = 0; i < nRows; ++i) {
|
||||
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0);
|
||||
}
|
||||
goto _exit;
|
||||
}
|
||||
|
||||
if (IS_VAR_DATA_TYPE(type)) { // var-length data type
|
||||
for (int32_t i = 0; i < nRows; ++i) {
|
||||
|
|
@ -2447,6 +2453,11 @@ int32_t tColDataAddValueByDataBlock(SColData *pColData, int8_t type, int32_t byt
|
|||
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0);
|
||||
if (code) goto _exit;
|
||||
} else {
|
||||
if(ASSERT(varDataTLen(data + offset) <= bytes)){
|
||||
uError("var data length invalid, varDataTLen(data + offset):%d <= bytes:%d", (int)varDataTLen(data + offset), bytes);
|
||||
code = TSDB_CODE_INVALID_PARA;
|
||||
goto _exit;
|
||||
}
|
||||
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, (uint8_t *)varDataVal(data + offset),
|
||||
varDataLen(data + offset));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1150,7 +1150,7 @@ int32_t tDeserializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) {
|
|||
if (tDecodeI64(&decoder, &vload.compStorage) < 0) return -1;
|
||||
if (tDecodeI64(&decoder, &vload.pointsWritten) < 0) return -1;
|
||||
if (tDecodeI32(&decoder, &vload.numOfCachedTables) < 0) return -1;
|
||||
if (tDecodeI32(&decoder, (int32_t*)&reserved) < 0) return -1;
|
||||
if (tDecodeI32(&decoder, (int32_t *)&reserved) < 0) return -1;
|
||||
if (tDecodeI64(&decoder, &reserved) < 0) return -1;
|
||||
if (tDecodeI64(&decoder, &reserved) < 0) return -1;
|
||||
if (taosArrayPush(pReq->pVloads, &vload) == NULL) {
|
||||
|
|
@ -1702,6 +1702,8 @@ int32_t tSerializeSCreateFuncReq(void *buf, int32_t bufLen, SCreateFuncReq *pReq
|
|||
if (tEncodeCStr(&encoder, pReq->pComment) < 0) return -1;
|
||||
}
|
||||
|
||||
if (tEncodeI8(&encoder, pReq->orReplace) < 0) return -1;
|
||||
|
||||
tEndEncode(&encoder);
|
||||
|
||||
int32_t tlen = encoder.pos;
|
||||
|
|
@ -1744,6 +1746,12 @@ int32_t tDeserializeSCreateFuncReq(void *buf, int32_t bufLen, SCreateFuncReq *pR
|
|||
if (tDecodeCStrTo(&decoder, pReq->pComment) < 0) return -1;
|
||||
}
|
||||
|
||||
if (!tDecodeIsEnd(&decoder)) {
|
||||
if (tDecodeI8(&decoder, &pReq->orReplace) < 0) return -1;
|
||||
} else {
|
||||
pReq->orReplace = false;
|
||||
}
|
||||
|
||||
tEndDecode(&decoder);
|
||||
|
||||
tDecoderClear(&decoder);
|
||||
|
|
@ -6895,7 +6903,7 @@ int32_t tDecodeSMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) {
|
|||
}
|
||||
|
||||
void tDeleteSMqDataRsp(SMqDataRsp *pRsp) {
|
||||
pRsp->blockDataLen = taosArrayDestroy(pRsp->blockDataLen);;
|
||||
pRsp->blockDataLen = taosArrayDestroy(pRsp->blockDataLen);
|
||||
taosArrayDestroyP(pRsp->blockData, (FDelete)taosMemoryFree);
|
||||
pRsp->blockData = NULL;
|
||||
taosArrayDestroyP(pRsp->blockSchema, (FDelete)tDeleteSSchemaWrapper);
|
||||
|
|
|
|||
|
|
@ -727,7 +727,7 @@ int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision) {
|
|||
|
||||
struct tm tm;
|
||||
time_t tt = (time_t)(t / TSDB_TICK_PER_SECOND(precision));
|
||||
taosLocalTime(&tt, &tm);
|
||||
taosLocalTime(&tt, &tm, NULL);
|
||||
int32_t mon = tm.tm_year * 12 + tm.tm_mon + (int32_t)numOfMonth;
|
||||
tm.tm_year = mon / 12;
|
||||
tm.tm_mon = mon % 12;
|
||||
|
|
@ -750,11 +750,11 @@ int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char
|
|||
|
||||
struct tm tm;
|
||||
time_t t = (time_t)skey;
|
||||
taosLocalTime(&t, &tm);
|
||||
taosLocalTime(&t, &tm, NULL);
|
||||
int32_t smon = tm.tm_year * 12 + tm.tm_mon;
|
||||
|
||||
t = (time_t)ekey;
|
||||
taosLocalTime(&t, &tm);
|
||||
taosLocalTime(&t, &tm, NULL);
|
||||
int32_t emon = tm.tm_year * 12 + tm.tm_mon;
|
||||
|
||||
if (unit == 'y') {
|
||||
|
|
@ -774,7 +774,7 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio
|
|||
start /= (int64_t)(TSDB_TICK_PER_SECOND(precision));
|
||||
struct tm tm;
|
||||
time_t tt = (time_t)start;
|
||||
taosLocalTime(&tt, &tm);
|
||||
taosLocalTime(&tt, &tm, NULL);
|
||||
tm.tm_sec = 0;
|
||||
tm.tm_min = 0;
|
||||
tm.tm_hour = 0;
|
||||
|
|
@ -867,13 +867,17 @@ const char* fmtts(int64_t ts) {
|
|||
|
||||
if (ts > -62135625943 && ts < 32503651200) {
|
||||
time_t t = (time_t)ts;
|
||||
taosLocalTime(&t, &tm);
|
||||
if (taosLocalTime(&t, &tm, buf) == NULL) {
|
||||
return buf;
|
||||
}
|
||||
pos += strftime(buf + pos, sizeof(buf), "s=%Y-%m-%d %H:%M:%S", &tm);
|
||||
}
|
||||
|
||||
if (ts > -62135625943000 && ts < 32503651200000) {
|
||||
time_t t = (time_t)(ts / 1000);
|
||||
taosLocalTime(&t, &tm);
|
||||
if (taosLocalTime(&t, &tm, buf) == NULL) {
|
||||
return buf;
|
||||
}
|
||||
if (pos > 0) {
|
||||
buf[pos++] = ' ';
|
||||
buf[pos++] = '|';
|
||||
|
|
@ -885,7 +889,9 @@ const char* fmtts(int64_t ts) {
|
|||
|
||||
{
|
||||
time_t t = (time_t)(ts / 1000000);
|
||||
taosLocalTime(&t, &tm);
|
||||
if (taosLocalTime(&t, &tm, buf) == NULL) {
|
||||
return buf;
|
||||
}
|
||||
if (pos > 0) {
|
||||
buf[pos++] = ' ';
|
||||
buf[pos++] = '|';
|
||||
|
|
@ -937,7 +943,9 @@ void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precision)
|
|||
ASSERT(false);
|
||||
}
|
||||
|
||||
taosLocalTime(", &ptm);
|
||||
if (taosLocalTime(", &ptm, buf) == NULL) {
|
||||
return;
|
||||
}
|
||||
int32_t length = (int32_t)strftime(ts, 40, "%Y-%m-%dT%H:%M:%S", &ptm);
|
||||
length += snprintf(ts + length, fractionLen, format, mod);
|
||||
length += (int32_t)strftime(ts + length, 40 - length, "%z", &ptm);
|
||||
|
|
|
|||
|
|
@ -18,7 +18,23 @@
|
|||
#include "mnode.h"
|
||||
#include "tconfig.h"
|
||||
#include "tglobal.h"
|
||||
#include "version.h"
|
||||
|
||||
#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL)
|
||||
#include "cus_name.h"
|
||||
#else
|
||||
#ifndef CUS_NAME
|
||||
#define CUS_NAME "TDengine"
|
||||
#endif
|
||||
|
||||
#ifndef CUS_PROMPT
|
||||
#define CUS_PROMPT "taos"
|
||||
#endif
|
||||
|
||||
#ifndef CUS_EMAIL
|
||||
#define CUS_EMAIL "<support@taosdata.com>"
|
||||
#endif
|
||||
#endif
|
||||
// clang-format off
|
||||
#define DM_APOLLO_URL "The apollo string to use when configuring the server, such as: -a 'jsonFile:./tests/cfg.json', cfg.json text can be '{\"fqdn\":\"td1\"}'."
|
||||
#define DM_CFG_DIR "Configuration directory."
|
||||
|
|
@ -83,11 +99,11 @@ void dmLogCrash(int signum, void *sigInfo, void *context) {
|
|||
taosIgnSignal(SIGFPE);
|
||||
taosIgnSignal(SIGSEGV);
|
||||
|
||||
char *pMsg = NULL;
|
||||
char *pMsg = NULL;
|
||||
const char *flags = "UTL FATAL ";
|
||||
ELogLevel level = DEBUG_FATAL;
|
||||
int32_t dflag = 255;
|
||||
int64_t msgLen= -1;
|
||||
int64_t msgLen = -1;
|
||||
|
||||
if (tsEnableCrashReport) {
|
||||
if (taosGenCrashJsonMsg(signum, &pMsg, dmGetClusterId(), global.startTime)) {
|
||||
|
|
@ -203,6 +219,9 @@ static void dmPrintVersion() {
|
|||
#endif
|
||||
printf("%s version: %s compatible_version: %s\n", releaseName, version, compatible_version);
|
||||
printf("gitinfo: %s\n", gitinfo);
|
||||
#ifdef TD_ENTERPRISE
|
||||
printf("gitinfoOfInternal: %s\n", gitinfoOfInternal);
|
||||
#endif
|
||||
printf("buildInfo: %s\n", buildinfo);
|
||||
}
|
||||
|
||||
|
|
@ -228,7 +247,7 @@ static void dmDumpCfg() {
|
|||
}
|
||||
|
||||
static int32_t dmInitLog() {
|
||||
return taosCreateLog("taosdlog", 1, configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0);
|
||||
return taosCreateLog(CUS_PROMPT"dlog", 1, configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0);
|
||||
}
|
||||
|
||||
static void taosCleanupArgs() {
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ typedef struct SMnodeMgmt {
|
|||
SSingleWorker readWorker;
|
||||
SSingleWorker writeWorker;
|
||||
SSingleWorker syncWorker;
|
||||
SSingleWorker syncCtrlWorker;
|
||||
SSingleWorker syncRdWorker;
|
||||
bool stopped;
|
||||
int32_t refCount;
|
||||
TdThreadRwlock lock;
|
||||
|
|
@ -54,7 +54,7 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt);
|
|||
void mmStopWorker(SMnodeMgmt *pMgmt);
|
||||
int32_t mmPutMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg);
|
||||
int32_t mmPutMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg);
|
||||
int32_t mmPutMsgToSyncCtrlQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg);
|
||||
int32_t mmPutMsgToSyncRdQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg);
|
||||
int32_t mmPutMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg);
|
||||
int32_t mmPutMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg);
|
||||
int32_t mmPutMsgToFetchQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg);
|
||||
|
|
|
|||
|
|
@ -189,7 +189,7 @@ SArray *mmGetMsgHandles() {
|
|||
if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_INDEX_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_DISABLE_WRITE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT_ELECTION, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_BATCH, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
|
||||
|
|
@ -199,13 +199,16 @@ SArray *mmGetMsgHandles() {
|
|||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_BATCH, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_SEND, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_RSP, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT, mmPutMsgToSyncCtrlQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, mmPutMsgToSyncCtrlQueue, 1) == NULL) goto _OVER;
|
||||
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_FORCE_FOLLOWER_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_RSP, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT_REPLY, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER;
|
||||
|
||||
code = 0;
|
||||
|
||||
_OVER:
|
||||
|
|
|
|||
|
|
@ -111,8 +111,8 @@ int32_t mmPutMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) {
|
|||
return mmPutMsgToWorker(pMgmt, &pMgmt->syncWorker, pMsg);
|
||||
}
|
||||
|
||||
int32_t mmPutMsgToSyncCtrlQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) {
|
||||
return mmPutMsgToWorker(pMgmt, &pMgmt->syncCtrlWorker, pMsg);
|
||||
int32_t mmPutMsgToSyncRdQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) {
|
||||
return mmPutMsgToWorker(pMgmt, &pMgmt->syncRdWorker, pMsg);
|
||||
}
|
||||
|
||||
int32_t mmPutMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) {
|
||||
|
|
@ -151,8 +151,8 @@ int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
|
|||
case SYNC_QUEUE:
|
||||
pWorker = &pMgmt->syncWorker;
|
||||
break;
|
||||
case SYNC_CTRL_QUEUE:
|
||||
pWorker = &pMgmt->syncCtrlWorker;
|
||||
case SYNC_RD_QUEUE:
|
||||
pWorker = &pMgmt->syncRdWorker;
|
||||
break;
|
||||
default:
|
||||
terrno = TSDB_CODE_INVALID_PARA;
|
||||
|
|
@ -238,12 +238,12 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
|
|||
SSingleWorkerCfg scCfg = {
|
||||
.min = 1,
|
||||
.max = 1,
|
||||
.name = "mnode-sync-ctrl",
|
||||
.name = "mnode-sync-rd",
|
||||
.fp = (FItem)mmProcessSyncMsg,
|
||||
.param = pMgmt,
|
||||
};
|
||||
if (tSingleWorkerInit(&pMgmt->syncCtrlWorker, &scCfg) != 0) {
|
||||
dError("failed to start mnode mnode-sync-ctrl worker since %s", terrstr());
|
||||
if (tSingleWorkerInit(&pMgmt->syncRdWorker, &scCfg) != 0) {
|
||||
dError("failed to start mnode mnode-sync-rd worker since %s", terrstr());
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
|
@ -259,6 +259,6 @@ void mmStopWorker(SMnodeMgmt *pMgmt) {
|
|||
tSingleWorkerCleanup(&pMgmt->readWorker);
|
||||
tSingleWorkerCleanup(&pMgmt->writeWorker);
|
||||
tSingleWorkerCleanup(&pMgmt->syncWorker);
|
||||
tSingleWorkerCleanup(&pMgmt->syncCtrlWorker);
|
||||
tSingleWorkerCleanup(&pMgmt->syncRdWorker);
|
||||
dDebug("mnode workers are closed");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -59,7 +59,7 @@ typedef struct {
|
|||
SVnode *pImpl;
|
||||
SMultiWorker pWriteW;
|
||||
SMultiWorker pSyncW;
|
||||
SMultiWorker pSyncCtrlW;
|
||||
SMultiWorker pSyncRdW;
|
||||
SMultiWorker pApplyW;
|
||||
STaosQueue *pQueryQ;
|
||||
STaosQueue *pStreamQ;
|
||||
|
|
@ -107,7 +107,7 @@ int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc);
|
|||
|
||||
int32_t vmPutMsgToWriteQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
|
||||
int32_t vmPutMsgToSyncQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
|
||||
int32_t vmPutMsgToSyncCtrlQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
|
||||
int32_t vmPutMsgToSyncRdQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
|
||||
int32_t vmPutMsgToQueryQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
|
||||
int32_t vmPutMsgToFetchQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
|
||||
int32_t vmPutMsgToStreamQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
|
||||
|
|
|
|||
|
|
@ -549,7 +549,7 @@ SArray *vmGetMsgHandles() {
|
|||
if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
|
||||
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT_ELECTION, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_BATCH, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
|
||||
|
|
@ -559,14 +559,15 @@ SArray *vmGetMsgHandles() {
|
|||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_BATCH, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_SEND, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_RSP, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
|
||||
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT, vmPutMsgToSyncCtrlQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, vmPutMsgToSyncCtrlQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_FORCE_FOLLOWER, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
|
||||
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_RSP, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT_REPLY, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER;
|
||||
|
||||
code = 0;
|
||||
|
||||
_OVER:
|
||||
|
|
|
|||
|
|
@ -98,9 +98,9 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal)
|
|||
pVnode->pSyncW.queue->threadId);
|
||||
tMultiWorkerCleanup(&pVnode->pSyncW);
|
||||
|
||||
dInfo("vgId:%d, wait for vnode sync ctrl queue:%p is empty, thread:%08" PRId64, pVnode->vgId,
|
||||
pVnode->pSyncCtrlW.queue, pVnode->pSyncCtrlW.queue->threadId);
|
||||
tMultiWorkerCleanup(&pVnode->pSyncCtrlW);
|
||||
dInfo("vgId:%d, wait for vnode sync rd queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncRdW.queue,
|
||||
pVnode->pSyncRdW.queue->threadId);
|
||||
tMultiWorkerCleanup(&pVnode->pSyncRdW);
|
||||
|
||||
dInfo("vgId:%d, wait for vnode apply queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pApplyW.queue,
|
||||
pVnode->pApplyW.queue->threadId);
|
||||
|
|
|
|||
|
|
@ -216,9 +216,9 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp
|
|||
dGTrace("vgId:%d, msg:%p put into vnode-sync queue", pVnode->vgId, pMsg);
|
||||
taosWriteQitem(pVnode->pSyncW.queue, pMsg);
|
||||
break;
|
||||
case SYNC_CTRL_QUEUE:
|
||||
dGTrace("vgId:%d, msg:%p put into vnode-sync-ctrl queue", pVnode->vgId, pMsg);
|
||||
taosWriteQitem(pVnode->pSyncCtrlW.queue, pMsg);
|
||||
case SYNC_RD_QUEUE:
|
||||
dGTrace("vgId:%d, msg:%p put into vnode-sync-rd queue", pVnode->vgId, pMsg);
|
||||
taosWriteQitem(pVnode->pSyncRdW.queue, pMsg);
|
||||
break;
|
||||
case APPLY_QUEUE:
|
||||
dGTrace("vgId:%d, msg:%p put into vnode-apply queue", pVnode->vgId, pMsg);
|
||||
|
|
@ -234,9 +234,7 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp
|
|||
return code;
|
||||
}
|
||||
|
||||
int32_t vmPutMsgToSyncCtrlQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
|
||||
return vmPutMsgToQueue(pMgmt, pMsg, SYNC_CTRL_QUEUE);
|
||||
}
|
||||
int32_t vmPutMsgToSyncRdQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return vmPutMsgToQueue(pMgmt, pMsg, SYNC_RD_QUEUE); }
|
||||
|
||||
int32_t vmPutMsgToSyncQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return vmPutMsgToQueue(pMgmt, pMsg, SYNC_QUEUE); }
|
||||
|
||||
|
|
@ -327,18 +325,18 @@ int32_t vmGetQueueSize(SVnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype) {
|
|||
int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
|
||||
SMultiWorkerCfg wcfg = {.max = 1, .name = "vnode-write", .fp = (FItems)vnodeProposeWriteMsg, .param = pVnode->pImpl};
|
||||
SMultiWorkerCfg scfg = {.max = 1, .name = "vnode-sync", .fp = (FItems)vmProcessSyncQueue, .param = pVnode};
|
||||
SMultiWorkerCfg sccfg = {.max = 1, .name = "vnode-sync-ctrl", .fp = (FItems)vmProcessSyncQueue, .param = pVnode};
|
||||
SMultiWorkerCfg sccfg = {.max = 1, .name = "vnode-sync-rd", .fp = (FItems)vmProcessSyncQueue, .param = pVnode};
|
||||
SMultiWorkerCfg acfg = {.max = 1, .name = "vnode-apply", .fp = (FItems)vnodeApplyWriteMsg, .param = pVnode->pImpl};
|
||||
(void)tMultiWorkerInit(&pVnode->pWriteW, &wcfg);
|
||||
(void)tMultiWorkerInit(&pVnode->pSyncW, &scfg);
|
||||
(void)tMultiWorkerInit(&pVnode->pSyncCtrlW, &sccfg);
|
||||
(void)tMultiWorkerInit(&pVnode->pSyncRdW, &sccfg);
|
||||
(void)tMultiWorkerInit(&pVnode->pApplyW, &acfg);
|
||||
|
||||
pVnode->pQueryQ = tQWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FItem)vmProcessQueryQueue);
|
||||
pVnode->pStreamQ = tAutoQWorkerAllocQueue(&pMgmt->streamPool, pVnode, (FItem)vmProcessStreamQueue);
|
||||
pVnode->pFetchQ = tWWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FItems)vmProcessFetchQueue);
|
||||
|
||||
if (pVnode->pWriteW.queue == NULL || pVnode->pSyncW.queue == NULL || pVnode->pSyncCtrlW.queue == NULL ||
|
||||
if (pVnode->pWriteW.queue == NULL || pVnode->pSyncW.queue == NULL || pVnode->pSyncRdW.queue == NULL ||
|
||||
pVnode->pApplyW.queue == NULL || pVnode->pQueryQ == NULL || pVnode->pStreamQ == NULL || pVnode->pFetchQ == NULL) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return -1;
|
||||
|
|
@ -348,8 +346,8 @@ int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
|
|||
pVnode->pWriteW.queue->threadId);
|
||||
dInfo("vgId:%d, sync-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncW.queue,
|
||||
pVnode->pSyncW.queue->threadId);
|
||||
dInfo("vgId:%d, sync-ctrl-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncCtrlW.queue,
|
||||
pVnode->pSyncCtrlW.queue->threadId);
|
||||
dInfo("vgId:%d, sync-rd-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncRdW.queue,
|
||||
pVnode->pSyncRdW.queue->threadId);
|
||||
dInfo("vgId:%d, apply-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pApplyW.queue,
|
||||
pVnode->pApplyW.queue->threadId);
|
||||
dInfo("vgId:%d, query-queue:%p is alloced", pVnode->vgId, pVnode->pQueryQ);
|
||||
|
|
|
|||
|
|
@ -393,7 +393,7 @@ typedef struct {
|
|||
} SSmaObj;
|
||||
|
||||
typedef struct {
|
||||
char name[TSDB_TABLE_FNAME_LEN];
|
||||
char name[TSDB_INDEX_FNAME_LEN];
|
||||
char stb[TSDB_TABLE_FNAME_LEN];
|
||||
char db[TSDB_DB_FNAME_LEN];
|
||||
char dstTbName[TSDB_TABLE_FNAME_LEN];
|
||||
|
|
|
|||
|
|
@ -81,9 +81,7 @@ bool mndRebTryStart() {
|
|||
return old == 0;
|
||||
}
|
||||
|
||||
void mndRebEnd() {
|
||||
mndRebCntDec();
|
||||
}
|
||||
void mndRebEnd() { mndRebCntDec(); }
|
||||
|
||||
void mndRebCntInc() {
|
||||
int32_t val = atomic_add_fetch_32(&mqRebInExecCnt, 1);
|
||||
|
|
@ -247,6 +245,13 @@ static SMqRebInfo *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) {
|
|||
return pRebInfo;
|
||||
}
|
||||
|
||||
static void freeRebalanceItem(void *param) {
|
||||
SMqRebInfo *pInfo = param;
|
||||
taosArrayDestroy(pInfo->lostConsumers);
|
||||
taosArrayDestroy(pInfo->newConsumers);
|
||||
taosArrayDestroy(pInfo->removedConsumers);
|
||||
}
|
||||
|
||||
static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) {
|
||||
SMnode *pMnode = pMsg->info.node;
|
||||
SSdb *pSdb = pMnode->pSdb;
|
||||
|
|
@ -262,8 +267,21 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) {
|
|||
}
|
||||
|
||||
SMqDoRebalanceMsg *pRebMsg = rpcMallocCont(sizeof(SMqDoRebalanceMsg));
|
||||
if (pRebMsg == NULL) {
|
||||
mError("failed to create the rebalance msg, size:%d, quit mq timer", (int32_t)sizeof(SMqDoRebalanceMsg));
|
||||
mndRebEnd();
|
||||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
|
||||
pRebMsg->rebSubHash = taosHashInit(64, MurmurHash3_32, true, HASH_NO_LOCK);
|
||||
// TODO set cleanfp
|
||||
if (pRebMsg->rebSubHash == NULL) {
|
||||
mError("failed to create rebalance hashmap");
|
||||
rpcFreeCont(pRebMsg);
|
||||
mndRebEnd();
|
||||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
|
||||
taosHashSetFreeFp(pRebMsg->rebSubHash, freeRebalanceItem);
|
||||
|
||||
// iterate all consumers, find all modification
|
||||
while (1) {
|
||||
|
|
@ -356,7 +374,7 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) {
|
|||
} else {
|
||||
taosHashCleanup(pRebMsg->rebSubHash);
|
||||
rpcFreeCont(pRebMsg);
|
||||
mDebug("mq rebalance finished, no modification");
|
||||
mDebug("mq timer finished, no need to re-balance");
|
||||
mndRebEnd();
|
||||
}
|
||||
return 0;
|
||||
|
|
@ -576,7 +594,7 @@ int32_t mndSetConsumerCommitLogs(SMnode *pMnode, STrans *pTrans, SMqConsumerObj
|
|||
return 0;
|
||||
}
|
||||
|
||||
static int32_t validateTopics(const SArray* pTopicList, SMnode* pMnode, const char* pUser) {
|
||||
static int32_t validateTopics(const SArray *pTopicList, SMnode *pMnode, const char *pUser) {
|
||||
int32_t numOfTopics = taosArrayGetSize(pTopicList);
|
||||
|
||||
for (int32_t i = 0; i < numOfTopics; i++) {
|
||||
|
|
@ -597,8 +615,13 @@ static int32_t validateTopics(const SArray* pTopicList, SMnode* pMnode, const ch
|
|||
return 0;
|
||||
}
|
||||
|
||||
static void* topicNameDup(void* p){
|
||||
return taosStrdup((char*) p);
|
||||
static void *topicNameDup(void *p) { return taosStrdup((char *)p); }
|
||||
|
||||
static void freeItem(void *param) {
|
||||
void *pItem = *(void **)param;
|
||||
if (pItem != NULL) {
|
||||
taosMemoryFree(pItem);
|
||||
}
|
||||
}
|
||||
|
||||
int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
|
||||
|
|
@ -616,7 +639,7 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
|
|||
int32_t code = -1;
|
||||
SArray *pTopicList = subscribe.topicNames;
|
||||
taosArraySort(pTopicList, taosArrayCompareString);
|
||||
taosArrayRemoveDuplicateP(pTopicList, taosArrayCompareString, taosMemoryFree);
|
||||
taosArrayRemoveDuplicate(pTopicList, taosArrayCompareString, freeItem);
|
||||
|
||||
int32_t newTopicNum = taosArrayGetSize(pTopicList);
|
||||
|
||||
|
|
@ -633,8 +656,8 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
|
|||
|
||||
pExistedConsumer = mndAcquireConsumer(pMnode, consumerId);
|
||||
if (pExistedConsumer == NULL) {
|
||||
mInfo("receive subscribe request from new consumer:0x%" PRIx64" cgroup:%s, numOfTopics:%d", consumerId,
|
||||
subscribe.cgroup, (int32_t) taosArrayGetSize(pTopicList));
|
||||
mInfo("receive subscribe request from new consumer:0x%" PRIx64 " cgroup:%s, numOfTopics:%d", consumerId,
|
||||
subscribe.cgroup, (int32_t)taosArrayGetSize(pTopicList));
|
||||
|
||||
pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup);
|
||||
tstrncpy(pConsumerNew->clientId, subscribe.clientId, tListLen(pConsumerNew->clientId));
|
||||
|
|
@ -656,8 +679,9 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
|
|||
/*taosRLockLatch(&pExistedConsumer->lock);*/
|
||||
int32_t status = atomic_load_32(&pExistedConsumer->status);
|
||||
|
||||
mInfo("receive subscribe request from existed consumer:0x%" PRIx64 " cgroup:%s, current status:%d(%s), subscribe topic num: %d",
|
||||
consumerId, subscribe.cgroup, status,mndConsumerStatusName(status), newTopicNum);
|
||||
mInfo("receive subscribe request from existed consumer:0x%" PRIx64
|
||||
" cgroup:%s, current status:%d(%s), subscribe topic num: %d",
|
||||
consumerId, subscribe.cgroup, status, mndConsumerStatusName(status), newTopicNum);
|
||||
|
||||
if (status != MQ_CONSUMER_STATUS__READY) {
|
||||
terrno = TSDB_CODE_MND_CONSUMER_NOT_READY;
|
||||
|
|
@ -674,7 +698,7 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
|
|||
taosArrayDestroy(pConsumerNew->assignedTopics);
|
||||
pConsumerNew->assignedTopics = taosArrayDup(pTopicList, topicNameDup);
|
||||
|
||||
int32_t oldTopicNum = (pExistedConsumer->currentTopics)? taosArrayGetSize(pExistedConsumer->currentTopics):0;
|
||||
int32_t oldTopicNum = (pExistedConsumer->currentTopics) ? taosArrayGetSize(pExistedConsumer->currentTopics) : 0;
|
||||
|
||||
int32_t i = 0, j = 0;
|
||||
while (i < oldTopicNum || j < newTopicNum) {
|
||||
|
|
@ -839,12 +863,59 @@ static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer) {
|
|||
}
|
||||
|
||||
static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer) {
|
||||
mDebug("consumer:0x%" PRIx64 " perform delete action, status:(%d)%s", pConsumer->consumerId,
|
||||
pConsumer->status, mndConsumerStatusName(pConsumer->status));
|
||||
mDebug("consumer:0x%" PRIx64 " perform delete action, status:(%d)%s", pConsumer->consumerId, pConsumer->status,
|
||||
mndConsumerStatusName(pConsumer->status));
|
||||
tDeleteSMqConsumerObj(pConsumer);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void updateConsumerStatus(SMqConsumerObj *pConsumer) {
|
||||
int32_t status = pConsumer->status;
|
||||
|
||||
if (taosArrayGetSize(pConsumer->rebNewTopics) == 0 && taosArrayGetSize(pConsumer->rebRemovedTopics) == 0) {
|
||||
if (status == MQ_CONSUMER_STATUS__MODIFY || status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) {
|
||||
pConsumer->status = MQ_CONSUMER_STATUS__READY;
|
||||
} else if (status == MQ_CONSUMER_STATUS__LOST_IN_REB || status == MQ_CONSUMER_STATUS__LOST) {
|
||||
pConsumer->status = MQ_CONSUMER_STATUS__LOST_REBD;
|
||||
}
|
||||
} else {
|
||||
if (status == MQ_CONSUMER_STATUS__MODIFY || status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) {
|
||||
pConsumer->status = MQ_CONSUMER_STATUS__MODIFY;
|
||||
} else if (status == MQ_CONSUMER_STATUS__LOST || status == MQ_CONSUMER_STATUS__LOST_IN_REB) {
|
||||
pConsumer->status = MQ_CONSUMER_STATUS__LOST;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// remove from new topic
|
||||
static void removeFromNewTopicList(SMqConsumerObj *pConsumer, const char *pTopic) {
|
||||
int32_t size = taosArrayGetSize(pConsumer->rebNewTopics);
|
||||
for (int32_t i = 0; i < taosArrayGetSize(pConsumer->rebNewTopics); i++) {
|
||||
char *p = taosArrayGetP(pConsumer->rebNewTopics, i);
|
||||
if (strcmp(pTopic, p) == 0) {
|
||||
taosArrayRemove(pConsumer->rebNewTopics, i);
|
||||
taosMemoryFree(p);
|
||||
|
||||
mDebug("consumer:0x%" PRIx64 " remove new topic:%s in the topic list, remain newTopics:%d", pConsumer->consumerId,
|
||||
pTopic, (int)taosArrayGetSize(pConsumer->rebNewTopics));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// remove from removed topic
|
||||
static void removeFromRemoveTopicList(SMqConsumerObj *pConsumer, const char *pTopic) {
|
||||
int32_t size = taosArrayGetSize(pConsumer->rebRemovedTopics);
|
||||
for (int32_t i = 0; i < size; i++) {
|
||||
char *p = taosArrayGetP(pConsumer->rebRemovedTopics, i);
|
||||
if (strcmp(pTopic, p) == 0) {
|
||||
taosArrayRemove(pConsumer->rebRemovedTopics, i);
|
||||
taosMemoryFree(p);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, SMqConsumerObj *pNewConsumer) {
|
||||
mDebug("consumer:0x%" PRIx64 " perform update action, update type:%d, subscribe-time:%" PRId64 ", uptime:%" PRId64,
|
||||
pOldConsumer->consumerId, pNewConsumer->updateType, pOldConsumer->subscribeTime, pOldConsumer->upTime);
|
||||
|
|
@ -855,6 +926,7 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
|
|||
/*A(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0);*/
|
||||
/*A(taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0);*/
|
||||
|
||||
// this new consumer has identical topics with one existed consumers.
|
||||
if (taosArrayGetSize(pNewConsumer->rebNewTopics) == 0 && taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 0) {
|
||||
pOldConsumer->status = MQ_CONSUMER_STATUS__READY;
|
||||
} else {
|
||||
|
|
@ -871,7 +943,6 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
|
|||
pNewConsumer->assignedTopics = tmp;
|
||||
|
||||
pOldConsumer->subscribeTime = pNewConsumer->upTime;
|
||||
|
||||
pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY;
|
||||
}
|
||||
} else if (pNewConsumer->updateType == CONSUMER_UPDATE__LOST) {
|
||||
|
|
@ -911,71 +982,48 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
|
|||
pOldConsumer->rebalanceTime = pNewConsumer->upTime;
|
||||
|
||||
} else if (pNewConsumer->updateType == CONSUMER_UPDATE__ADD) {
|
||||
/*A(taosArrayGetSize(pNewConsumer->rebNewTopics) == 1);*/
|
||||
/*A(taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 0);*/
|
||||
ASSERT(taosArrayGetSize(pNewConsumer->rebNewTopics) == 1 && taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 0);
|
||||
char *pNewTopic = taosStrdup(taosArrayGetP(pNewConsumer->rebNewTopics, 0));
|
||||
|
||||
char *addedTopic = taosStrdup(taosArrayGetP(pNewConsumer->rebNewTopics, 0));
|
||||
// not exist in current topic
|
||||
|
||||
bool existing = false;
|
||||
#if 1
|
||||
bool existing = false;
|
||||
int32_t numOfExistedTopics = taosArrayGetSize(pOldConsumer->currentTopics);
|
||||
for (int32_t i = 0; i < numOfExistedTopics; i++) {
|
||||
char *topic = taosArrayGetP(pOldConsumer->currentTopics, i);
|
||||
if (strcmp(topic, addedTopic) == 0) {
|
||||
if (strcmp(topic, pNewTopic) == 0) {
|
||||
existing = true;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
// remove from new topic
|
||||
for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->rebNewTopics); i++) {
|
||||
char *topic = taosArrayGetP(pOldConsumer->rebNewTopics, i);
|
||||
if (strcmp(addedTopic, topic) == 0) {
|
||||
taosArrayRemove(pOldConsumer->rebNewTopics, i);
|
||||
taosMemoryFree(topic);
|
||||
break;
|
||||
}
|
||||
}
|
||||
removeFromNewTopicList(pOldConsumer, pNewTopic);
|
||||
|
||||
// add to current topic
|
||||
if (!existing) {
|
||||
taosArrayPush(pOldConsumer->currentTopics, &addedTopic);
|
||||
taosArrayPush(pOldConsumer->currentTopics, &pNewTopic);
|
||||
taosArraySort(pOldConsumer->currentTopics, taosArrayCompareString);
|
||||
} else {
|
||||
taosMemoryFree(addedTopic);
|
||||
taosMemoryFree(pNewTopic);
|
||||
}
|
||||
|
||||
// set status
|
||||
int32_t status = pOldConsumer->status;
|
||||
if (taosArrayGetSize(pOldConsumer->rebNewTopics) == 0 && taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0) {
|
||||
if (status == MQ_CONSUMER_STATUS__MODIFY || status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) {
|
||||
pOldConsumer->status = MQ_CONSUMER_STATUS__READY;
|
||||
} else if (status == MQ_CONSUMER_STATUS__LOST_IN_REB || status == MQ_CONSUMER_STATUS__LOST) {
|
||||
pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_REBD;
|
||||
}
|
||||
} else {
|
||||
if (status == MQ_CONSUMER_STATUS__MODIFY || status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) {
|
||||
pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY_IN_REB;
|
||||
} else if (status == MQ_CONSUMER_STATUS__LOST || status == MQ_CONSUMER_STATUS__LOST_IN_REB) {
|
||||
pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_IN_REB;
|
||||
}
|
||||
}
|
||||
updateConsumerStatus(pOldConsumer);
|
||||
|
||||
// the re-balance is triggered when the new consumer is launched.
|
||||
pOldConsumer->rebalanceTime = pNewConsumer->upTime;
|
||||
|
||||
atomic_add_fetch_32(&pOldConsumer->epoch, 1);
|
||||
mDebug("consumer:0x%" PRIx64 " state (%d)%s -> (%d)%s, new epoch:%d, reb-time:%" PRId64 ", current topics:%d",
|
||||
mDebug("consumer:0x%" PRIx64 " state (%d)%s -> (%d)%s, new epoch:%d, reb-time:%" PRId64
|
||||
", current topics:%d, newTopics:%d, removeTopics:%d",
|
||||
pOldConsumer->consumerId, status, mndConsumerStatusName(status), pOldConsumer->status,
|
||||
mndConsumerStatusName(pOldConsumer->status),
|
||||
pOldConsumer->epoch, pOldConsumer->rebalanceTime, (int)taosArrayGetSize(pOldConsumer->currentTopics));
|
||||
mndConsumerStatusName(pOldConsumer->status), pOldConsumer->epoch, pOldConsumer->rebalanceTime,
|
||||
(int)taosArrayGetSize(pOldConsumer->currentTopics), (int)taosArrayGetSize(pOldConsumer->rebNewTopics),
|
||||
(int)taosArrayGetSize(pOldConsumer->rebRemovedTopics));
|
||||
|
||||
} else if (pNewConsumer->updateType == CONSUMER_UPDATE__REMOVE) {
|
||||
/*A(taosArrayGetSize(pNewConsumer->rebNewTopics) == 0);*/
|
||||
/*A(taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 1);*/
|
||||
char *removedTopic = taosArrayGetP(pNewConsumer->rebRemovedTopics, 0);
|
||||
|
||||
// not exist in new topic
|
||||
#if 0
|
||||
for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->rebNewTopics); i++) {
|
||||
char *topic = taosArrayGetP(pOldConsumer->rebNewTopics, i);
|
||||
|
|
@ -984,14 +1032,7 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
|
|||
#endif
|
||||
|
||||
// remove from removed topic
|
||||
for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->rebRemovedTopics); i++) {
|
||||
char *topic = taosArrayGetP(pOldConsumer->rebRemovedTopics, i);
|
||||
if (strcmp(removedTopic, topic) == 0) {
|
||||
taosArrayRemove(pOldConsumer->rebRemovedTopics, i);
|
||||
taosMemoryFree(topic);
|
||||
break;
|
||||
}
|
||||
}
|
||||
removeFromRemoveTopicList(pOldConsumer, removedTopic);
|
||||
|
||||
// remove from current topic
|
||||
int32_t i = 0;
|
||||
|
|
@ -1004,32 +1045,20 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
|
|||
break;
|
||||
}
|
||||
}
|
||||
// must find the topic
|
||||
/*A(i < sz);*/
|
||||
|
||||
// set status
|
||||
int32_t status = pOldConsumer->status;
|
||||
if (taosArrayGetSize(pOldConsumer->rebNewTopics) == 0 && taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0) {
|
||||
if (status == MQ_CONSUMER_STATUS__MODIFY || status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) {
|
||||
pOldConsumer->status = MQ_CONSUMER_STATUS__READY;
|
||||
} else if (status == MQ_CONSUMER_STATUS__LOST_IN_REB || status == MQ_CONSUMER_STATUS__LOST) {
|
||||
pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_REBD;
|
||||
}
|
||||
} else {
|
||||
if (status == MQ_CONSUMER_STATUS__MODIFY || status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) {
|
||||
pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY_IN_REB;
|
||||
} else if (status == MQ_CONSUMER_STATUS__LOST || status == MQ_CONSUMER_STATUS__LOST_IN_REB) {
|
||||
pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_IN_REB;
|
||||
}
|
||||
}
|
||||
updateConsumerStatus(pOldConsumer);
|
||||
|
||||
pOldConsumer->rebalanceTime = pNewConsumer->upTime;
|
||||
atomic_add_fetch_32(&pOldConsumer->epoch, 1);
|
||||
|
||||
mDebug("consumer:0x%" PRIx64 " state %d(%s) -> %d(%s), new epoch:%d, reb-time:%" PRId64 ", current topics:%d",
|
||||
mDebug("consumer:0x%" PRIx64 " state (%d)%s -> (%d)%s, new epoch:%d, reb-time:%" PRId64
|
||||
", current topics:%d, newTopics:%d, removeTopics:%d",
|
||||
pOldConsumer->consumerId, status, mndConsumerStatusName(status), pOldConsumer->status,
|
||||
mndConsumerStatusName(pOldConsumer->status),
|
||||
pOldConsumer->epoch, pOldConsumer->rebalanceTime, (int)taosArrayGetSize(pOldConsumer->currentTopics));
|
||||
mndConsumerStatusName(pOldConsumer->status), pOldConsumer->epoch, pOldConsumer->rebalanceTime,
|
||||
(int)taosArrayGetSize(pOldConsumer->currentTopics), (int)taosArrayGetSize(pOldConsumer->rebNewTopics),
|
||||
(int)taosArrayGetSize(pOldConsumer->rebRemovedTopics));
|
||||
}
|
||||
|
||||
taosWUnLockLatch(&pOldConsumer->lock);
|
||||
|
|
@ -1105,8 +1134,8 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *
|
|||
colDataSetVal(pColInfo, numOfRows, (const char *)clientId, false);
|
||||
|
||||
// status
|
||||
char status[20 + VARSTR_HEADER_SIZE] = {0};
|
||||
const char* pStatusName = mndConsumerStatusName(pConsumer->status);
|
||||
char status[20 + VARSTR_HEADER_SIZE] = {0};
|
||||
const char *pStatusName = mndConsumerStatusName(pConsumer->status);
|
||||
STR_TO_VARSTR(status, pStatusName);
|
||||
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
|
|
|
|||
|
|
@ -138,7 +138,7 @@ static void *mndBuildDropIdxReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStbOb
|
|||
mInfo("idx: %s start to build drop index req", pIdx->name);
|
||||
|
||||
len = tSerializeSDropIdxReq(NULL, 0, &req);
|
||||
if (ret < 0) {
|
||||
if (len < 0) {
|
||||
goto _err;
|
||||
}
|
||||
|
||||
|
|
@ -672,7 +672,7 @@ _OVER:
|
|||
static int32_t mndAddIndex(SMnode *pMnode, SRpcMsg *pReq, SCreateTagIndexReq *req, SDbObj *pDb, SStbObj *pStb) {
|
||||
int32_t code = -1;
|
||||
SIdxObj idxObj = {0};
|
||||
memcpy(idxObj.name, req->idxName, TSDB_TABLE_FNAME_LEN);
|
||||
memcpy(idxObj.name, req->idxName, TSDB_INDEX_FNAME_LEN);
|
||||
memcpy(idxObj.stb, pStb->name, TSDB_TABLE_FNAME_LEN);
|
||||
memcpy(idxObj.db, pDb->name, TSDB_DB_FNAME_LEN);
|
||||
memcpy(idxObj.colName, req->colName, TSDB_COL_NAME_LEN);
|
||||
|
|
|
|||
|
|
@ -226,7 +226,7 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) {
|
|||
goto _OVER;
|
||||
}
|
||||
|
||||
if ((code = taosCheckVersionCompatibleFromStr(connReq.sVer, version, 2)) != 0) {
|
||||
if ((code = taosCheckVersionCompatibleFromStr(connReq.sVer, version, 3)) != 0) {
|
||||
terrno = code;
|
||||
goto _OVER;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -197,24 +197,20 @@ static SMqRebInfo *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) {
|
|||
return pRebSub;
|
||||
}
|
||||
|
||||
static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebOutputObj *pOutput) {
|
||||
int32_t totalVgNum = pOutput->pSub->vgNum;
|
||||
const char *sub = pOutput->pSub->key;
|
||||
mInfo("sub:%s mq re-balance %d vgroups", sub, pOutput->pSub->vgNum);
|
||||
static void doRemoveExistedConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash, const SMqRebInputObj *pInput) {
|
||||
int32_t numOfRemoved = taosArrayGetSize(pInput->pRebInfo->removedConsumers);
|
||||
const char *pSubKey = pOutput->pSub->key;
|
||||
|
||||
// 1. build temporary hash(vgId -> SMqRebOutputVg) to store modified vg
|
||||
SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
|
||||
|
||||
// 2. check and get actual removed consumers, put their vg into hash
|
||||
int32_t removedNum = taosArrayGetSize(pInput->pRebInfo->removedConsumers);
|
||||
int32_t actualRemoved = 0;
|
||||
for (int32_t i = 0; i < removedNum; i++) {
|
||||
for (int32_t i = 0; i < numOfRemoved; i++) {
|
||||
uint64_t consumerId = *(uint64_t *)taosArrayGet(pInput->pRebInfo->removedConsumers, i);
|
||||
|
||||
SMqConsumerEp *pConsumerEp = taosHashGet(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t));
|
||||
|
||||
// consumer exists till now
|
||||
if (pConsumerEp) {
|
||||
actualRemoved++;
|
||||
|
||||
int32_t consumerVgNum = taosArrayGetSize(pConsumerEp->vgs);
|
||||
for (int32_t j = 0; j < consumerVgNum; j++) {
|
||||
SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, j);
|
||||
|
|
@ -223,52 +219,67 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
|
|||
.newConsumerId = -1,
|
||||
.pVgEp = pVgEp,
|
||||
};
|
||||
|
||||
taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg));
|
||||
mInfo("sub:%s mq re-balance remove vgId:%d from consumer:0x%" PRIx64, sub, pVgEp->vgId, consumerId);
|
||||
mInfo("sub:%s mq re-balance remove vgId:%d from consumer:%" PRIx64, pSubKey, pVgEp->vgId, consumerId);
|
||||
}
|
||||
|
||||
taosArrayDestroy(pConsumerEp->vgs);
|
||||
taosHashRemove(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t));
|
||||
|
||||
// put into removed
|
||||
taosArrayPush(pOutput->removedConsumers, &consumerId);
|
||||
}
|
||||
}
|
||||
|
||||
if (removedNum != actualRemoved) {
|
||||
mError("sub:%s mq re-balance removedNum:%d not matched with actual:%d", sub, removedNum, actualRemoved);
|
||||
if (numOfRemoved != actualRemoved) {
|
||||
mError("sub:%s mq re-balance removedNum:%d not matched with actual:%d", pSubKey, numOfRemoved, actualRemoved);
|
||||
} else {
|
||||
mInfo("sub:%s removed %d consumers", pSubKey, numOfRemoved);
|
||||
}
|
||||
}
|
||||
|
||||
// if previously no consumer, there are vgs not assigned
|
||||
{
|
||||
int32_t consumerVgNum = taosArrayGetSize(pOutput->pSub->unassignedVgs);
|
||||
for (int32_t i = 0; i < consumerVgNum; i++) {
|
||||
SMqVgEp *pVgEp = *(SMqVgEp **)taosArrayPop(pOutput->pSub->unassignedVgs);
|
||||
SMqRebOutputVg rebOutput = {
|
||||
.oldConsumerId = -1,
|
||||
.newConsumerId = -1,
|
||||
.pVgEp = pVgEp,
|
||||
};
|
||||
taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &rebOutput, sizeof(SMqRebOutputVg));
|
||||
mInfo("sub:%s mq re-balance remove vgId:%d from unassigned", sub, pVgEp->vgId);
|
||||
}
|
||||
static void doAddNewConsumers(SMqRebOutputObj *pOutput, const SMqRebInputObj *pInput) {
|
||||
int32_t numOfNewConsumers = taosArrayGetSize(pInput->pRebInfo->newConsumers);
|
||||
const char *pSubKey = pOutput->pSub->key;
|
||||
|
||||
for (int32_t i = 0; i < numOfNewConsumers; i++) {
|
||||
int64_t consumerId = *(int64_t *)taosArrayGet(pInput->pRebInfo->newConsumers, i);
|
||||
|
||||
SMqConsumerEp newConsumerEp;
|
||||
newConsumerEp.consumerId = consumerId;
|
||||
newConsumerEp.vgs = taosArrayInit(0, sizeof(void *));
|
||||
|
||||
taosHashPut(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t), &newConsumerEp, sizeof(SMqConsumerEp));
|
||||
taosArrayPush(pOutput->newConsumers, &consumerId);
|
||||
mInfo("sub:%s mq rebalance add new consumer:%" PRIx64, pSubKey, consumerId);
|
||||
}
|
||||
}
|
||||
|
||||
// 3. calc vg number of each consumer
|
||||
int32_t afterRebConsumerNum = pInput->oldConsumerNum + taosArrayGetSize(pInput->pRebInfo->newConsumers) -
|
||||
taosArrayGetSize(pInput->pRebInfo->removedConsumers);
|
||||
int32_t minVgCnt = 0;
|
||||
int32_t imbConsumerNum = 0;
|
||||
// calc num
|
||||
if (afterRebConsumerNum) {
|
||||
minVgCnt = totalVgNum / afterRebConsumerNum;
|
||||
imbConsumerNum = totalVgNum % afterRebConsumerNum;
|
||||
static void addUnassignedVgroups(SMqRebOutputObj *pOutput, SHashObj *pHash) {
|
||||
const char *pSubKey = pOutput->pSub->key;
|
||||
int32_t numOfVgroups = taosArrayGetSize(pOutput->pSub->unassignedVgs);
|
||||
|
||||
for (int32_t i = 0; i < numOfVgroups; i++) {
|
||||
SMqVgEp *pVgEp = *(SMqVgEp **)taosArrayPop(pOutput->pSub->unassignedVgs);
|
||||
SMqRebOutputVg rebOutput = {
|
||||
.oldConsumerId = -1,
|
||||
.newConsumerId = -1,
|
||||
.pVgEp = pVgEp,
|
||||
};
|
||||
|
||||
taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &rebOutput, sizeof(SMqRebOutputVg));
|
||||
mInfo("sub:%s mq re-balance remove vgId:%d from unassigned", pSubKey, pVgEp->vgId);
|
||||
}
|
||||
}
|
||||
|
||||
mInfo("sub:%s mq re-balance %d consumers: at least %d vgs each, %d consumers has more vgs", sub,
|
||||
afterRebConsumerNum, minVgCnt, imbConsumerNum);
|
||||
static void transferVgroupsForConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash, int32_t minVgCnt,
|
||||
int32_t imbConsumerNum) {
|
||||
const char *pSubKey = pOutput->pSub->key;
|
||||
|
||||
// 4. first scan: remove consumer more than wanted, put to remove hash
|
||||
int32_t imbCnt = 0;
|
||||
void *pIter = NULL;
|
||||
|
||||
while (1) {
|
||||
pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter);
|
||||
if (pIter == NULL) {
|
||||
|
|
@ -276,8 +287,8 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
|
|||
}
|
||||
|
||||
SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter;
|
||||
int32_t consumerVgNum = taosArrayGetSize(pConsumerEp->vgs);
|
||||
|
||||
int32_t consumerVgNum = taosArrayGetSize(pConsumerEp->vgs);
|
||||
// all old consumers still existing are touched
|
||||
// TODO optimize: touch only consumer whose vgs changed
|
||||
taosArrayPush(pOutput->touchedConsumers, &pConsumerEp->consumerId);
|
||||
|
|
@ -296,13 +307,13 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
|
|||
.pVgEp = pVgEp,
|
||||
};
|
||||
taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg));
|
||||
mInfo("sub:%s mq rebalance remove vgId:%d from consumer:0x%" PRIx64 ",(first scan)", sub, pVgEp->vgId,
|
||||
mInfo("sub:%s mq rebalance remove vgId:%d from consumer:0x%" PRIx64 ",(first scan)", pSubKey, pVgEp->vgId,
|
||||
pConsumerEp->consumerId);
|
||||
}
|
||||
imbCnt++;
|
||||
}
|
||||
} else {
|
||||
// pop until equal minVg
|
||||
// all the remain consumers should only have the number of vgroups, which is equalled to the value of minVg
|
||||
while (taosArrayGetSize(pConsumerEp->vgs) > minVgCnt) {
|
||||
SMqVgEp *pVgEp = *(SMqVgEp **)taosArrayPop(pConsumerEp->vgs);
|
||||
SMqRebOutputVg outputVg = {
|
||||
|
|
@ -311,36 +322,67 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
|
|||
.pVgEp = pVgEp,
|
||||
};
|
||||
taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg));
|
||||
mInfo("sub:%s mq rebalance remove vgId:%d from consumer:0x%" PRIx64 ",(first scan)", sub, pVgEp->vgId,
|
||||
mInfo("sub:%s mq rebalance remove vgId:%d from consumer:0x%" PRIx64 ",(first scan)", pSubKey, pVgEp->vgId,
|
||||
pConsumerEp->consumerId);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 5. add new consumer into sub
|
||||
{
|
||||
int32_t consumerNum = taosArrayGetSize(pInput->pRebInfo->newConsumers);
|
||||
for (int32_t i = 0; i < consumerNum; i++) {
|
||||
int64_t consumerId = *(int64_t *)taosArrayGet(pInput->pRebInfo->newConsumers, i);
|
||||
static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebOutputObj *pOutput) {
|
||||
int32_t totalVgNum = pOutput->pSub->vgNum;
|
||||
const char *pSubKey = pOutput->pSub->key;
|
||||
|
||||
SMqConsumerEp newConsumerEp;
|
||||
newConsumerEp.consumerId = consumerId;
|
||||
newConsumerEp.vgs = taosArrayInit(0, sizeof(void *));
|
||||
taosHashPut(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t), &newConsumerEp, sizeof(SMqConsumerEp));
|
||||
taosArrayPush(pOutput->newConsumers, &consumerId);
|
||||
mInfo("sub:%s mq rebalance add new consumer:0x%" PRIx64, sub, consumerId);
|
||||
}
|
||||
int32_t numOfRemoved = taosArrayGetSize(pInput->pRebInfo->removedConsumers);
|
||||
int32_t numOfAdded = taosArrayGetSize(pInput->pRebInfo->newConsumers);
|
||||
mInfo("sub:%s mq re-balance %d vgroups, existed consumers:%d, added:%d, removed:%d", pSubKey, totalVgNum,
|
||||
pInput->oldConsumerNum, numOfAdded, numOfRemoved);
|
||||
|
||||
// 1. build temporary hash(vgId -> SMqRebOutputVg) to store modified vg
|
||||
SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
|
||||
|
||||
// 2. check and get actual removed consumers, put their vg into hash
|
||||
doRemoveExistedConsumers(pOutput, pHash, pInput);
|
||||
|
||||
// 3. if previously no consumer, there are vgs not assigned
|
||||
addUnassignedVgroups(pOutput, pHash);
|
||||
|
||||
// 4. calc vg number of each consumer
|
||||
int32_t numOfFinal = pInput->oldConsumerNum + numOfAdded - numOfRemoved;
|
||||
|
||||
int32_t minVgCnt = 0;
|
||||
int32_t imbConsumerNum = 0;
|
||||
|
||||
// calc num
|
||||
if (numOfFinal) {
|
||||
minVgCnt = totalVgNum / numOfFinal;
|
||||
imbConsumerNum = totalVgNum % numOfFinal;
|
||||
mInfo("sub:%s mq re-balance %d consumers: at least %d vgs each, %d consumers has 1 more vgroups than avg value",
|
||||
pSubKey, numOfFinal, minVgCnt, imbConsumerNum);
|
||||
} else {
|
||||
mInfo("sub:%s no consumer subscribe this topic", pSubKey);
|
||||
}
|
||||
|
||||
// 6. second scan: find consumer do not have enough vg, extract from temporary hash and assign to new consumer.
|
||||
// 5. first scan: remove vgroups from te consumers, who have more vgroups than the threashold value that is
|
||||
// minVgCnt, and then put them into the recycled hash list
|
||||
transferVgroupsForConsumers(pOutput, pHash, minVgCnt, imbConsumerNum);
|
||||
|
||||
// 6. add new consumer into sub
|
||||
doAddNewConsumers(pOutput, pInput);
|
||||
|
||||
// 7. second scan: find consumer do not have enough vgroups, extract from temporary hash and assign to them
|
||||
// All related vg should be put into rebVgs
|
||||
SMqRebOutputVg *pRebVg = NULL;
|
||||
void *pRemovedIter = NULL;
|
||||
pIter = NULL;
|
||||
void *pIter = NULL;
|
||||
|
||||
while (1) {
|
||||
pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter);
|
||||
if (pIter == NULL) break;
|
||||
if (pIter == NULL) {
|
||||
break;
|
||||
}
|
||||
|
||||
SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter;
|
||||
|
||||
// push until equal minVg
|
||||
|
|
@ -348,8 +390,8 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
|
|||
// iter hash and find one vg
|
||||
pRemovedIter = taosHashIterate(pHash, pRemovedIter);
|
||||
if (pRemovedIter == NULL) {
|
||||
mError("sub:%s removed iter is null", sub);
|
||||
continue;
|
||||
mError("sub:%s removed iter is null", pSubKey);
|
||||
break;
|
||||
}
|
||||
|
||||
pRebVg = (SMqRebOutputVg *)pRemovedIter;
|
||||
|
|
@ -409,15 +451,15 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
|
|||
|
||||
taosArrayPush(pOutput->pSub->unassignedVgs, &pRebOutput->pVgEp);
|
||||
taosArrayPush(pOutput->rebVgs, pRebOutput);
|
||||
mInfo("sub:%s mq re-balance unassign vgId:%d (second scan)", sub, pRebOutput->pVgEp->vgId);
|
||||
mInfo("sub:%s mq re-balance unassign vgId:%d (second scan)", pSubKey, pRebOutput->pVgEp->vgId);
|
||||
}
|
||||
}
|
||||
|
||||
// 8. generate logs
|
||||
mInfo("sub:%s mq re-balance calculation completed, re-balanced vg", sub);
|
||||
mInfo("sub:%s mq re-balance calculation completed, re-balanced vg", pSubKey);
|
||||
for (int32_t i = 0; i < taosArrayGetSize(pOutput->rebVgs); i++) {
|
||||
SMqRebOutputVg *pOutputRebVg = taosArrayGet(pOutput->rebVgs, i);
|
||||
mInfo("sub:%s mq re-balance vgId:%d, moved from consumer:0x%" PRIx64 ", to consumer:0x%" PRIx64, sub,
|
||||
mInfo("sub:%s mq re-balance vgId:%d, moved from consumer:0x%" PRIx64 ", to consumer:0x%" PRIx64, pSubKey,
|
||||
pOutputRebVg->pVgEp->vgId, pOutputRebVg->oldConsumerId, pOutputRebVg->newConsumerId);
|
||||
}
|
||||
{
|
||||
|
|
@ -427,10 +469,10 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
|
|||
if (pIter == NULL) break;
|
||||
SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter;
|
||||
int32_t sz = taosArrayGetSize(pConsumerEp->vgs);
|
||||
mInfo("sub:%s mq re-balance final cfg: consumer:0x%" PRIx64 " has %d vg", sub, pConsumerEp->consumerId, sz);
|
||||
mInfo("sub:%s mq re-balance final cfg: consumer:0x%" PRIx64 " has %d vg", pSubKey, pConsumerEp->consumerId, sz);
|
||||
for (int32_t i = 0; i < sz; i++) {
|
||||
SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, i);
|
||||
mInfo("sub:%s mq re-balance final cfg: vg %d to consumer:0x%" PRIx64, sub, pVgEp->vgId,
|
||||
mInfo("sub:%s mq re-balance final cfg: vg %d to consumer:0x%" PRIx64, pSubKey, pVgEp->vgId,
|
||||
pConsumerEp->consumerId);
|
||||
}
|
||||
}
|
||||
|
|
@ -555,17 +597,23 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) {
|
|||
SMnode *pMnode = pMsg->info.node;
|
||||
SMqDoRebalanceMsg *pReq = pMsg->pCont;
|
||||
void *pIter = NULL;
|
||||
bool rebalanceOnce = false; // to ensure only once.
|
||||
|
||||
mInfo("mq re-balance start");
|
||||
mInfo("mq re-balance start, total required re-balanced trans:%d", taosHashGetSize(pReq->rebSubHash));
|
||||
|
||||
// here we only handle one topic rebalance requirement to ensure the atomic execution of this transaction.
|
||||
while (1) {
|
||||
if (rebalanceOnce) {
|
||||
break;
|
||||
}
|
||||
|
||||
pIter = taosHashIterate(pReq->rebSubHash, pIter);
|
||||
if (pIter == NULL) {
|
||||
break;
|
||||
}
|
||||
|
||||
SMqRebInputObj rebInput = {0};
|
||||
|
||||
// todo handle the malloc failure
|
||||
SMqRebInputObj rebInput = {0};
|
||||
SMqRebOutputObj rebOutput = {0};
|
||||
rebOutput.newConsumers = taosArrayInit(0, sizeof(int64_t));
|
||||
rebOutput.removedConsumers = taosArrayInit(0, sizeof(int64_t));
|
||||
|
|
@ -582,9 +630,10 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) {
|
|||
char topic[TSDB_TOPIC_FNAME_LEN];
|
||||
char cgroup[TSDB_CGROUP_LEN];
|
||||
mndSplitSubscribeKey(pRebInfo->key, topic, cgroup, true);
|
||||
|
||||
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic);
|
||||
if (pTopic == NULL) {
|
||||
mError("mq re-balance %s ignored since topic %s not exist", pRebInfo->key, topic);
|
||||
mError("mq re-balance %s ignored since topic %s doesn't exist", pRebInfo->key, topic);
|
||||
continue;
|
||||
}
|
||||
|
||||
|
|
@ -604,11 +653,13 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) {
|
|||
mndReleaseTopic(pMnode, pTopic);
|
||||
|
||||
rebInput.oldConsumerNum = 0;
|
||||
mInfo("topic:%s has no consumers sub yet", topic);
|
||||
} else {
|
||||
taosRLockLatch(&pSub->lock);
|
||||
rebInput.oldConsumerNum = taosHashGetSize(pSub->consumerHash);
|
||||
rebOutput.pSub = tCloneSubscribeObj(pSub);
|
||||
taosRUnLockLatch(&pSub->lock);
|
||||
mInfo("topic:%s has %d consumers sub till now", pRebInfo->key, rebInput.oldConsumerNum);
|
||||
mndReleaseSubscribe(pMnode, pSub);
|
||||
}
|
||||
|
||||
|
|
@ -623,16 +674,14 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) {
|
|||
mError("mq re-balance persist output error, possibly vnode splitted or dropped");
|
||||
}
|
||||
|
||||
taosArrayDestroy(pRebInfo->lostConsumers);
|
||||
taosArrayDestroy(pRebInfo->newConsumers);
|
||||
taosArrayDestroy(pRebInfo->removedConsumers);
|
||||
|
||||
taosArrayDestroy(rebOutput.newConsumers);
|
||||
taosArrayDestroy(rebOutput.touchedConsumers);
|
||||
taosArrayDestroy(rebOutput.removedConsumers);
|
||||
taosArrayDestroy(rebOutput.rebVgs);
|
||||
tDeleteSubscribeObj(rebOutput.pSub);
|
||||
taosMemoryFree(rebOutput.pSub);
|
||||
|
||||
rebalanceOnce = true;
|
||||
}
|
||||
|
||||
// reset flag
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ static int32_t mndSyncEqCtrlMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) {
|
|||
return -1;
|
||||
}
|
||||
|
||||
int32_t code = tmsgPutToQueue(msgcb, SYNC_CTRL_QUEUE, pMsg);
|
||||
int32_t code = tmsgPutToQueue(msgcb, SYNC_RD_QUEUE, pMsg);
|
||||
if (code != 0) {
|
||||
rpcFreeCont(pMsg->pCont);
|
||||
pMsg->pCont = NULL;
|
||||
|
|
|
|||
|
|
@ -1093,10 +1093,13 @@ int32_t mndTransProcessRsp(SRpcMsg *pRsp) {
|
|||
pAction->msgReceived = 1;
|
||||
pAction->errCode = pRsp->code;
|
||||
pTrans->lastErrorNo = pRsp->code;
|
||||
|
||||
mInfo("trans:%d, %s:%d response is received, code:0x%x, accept:0x%x retry:0x%x", transId,
|
||||
mndTransStr(pAction->stage), action, pRsp->code, pAction->acceptableCode, pAction->retryCode);
|
||||
} else {
|
||||
mInfo("trans:%d, invalid action, index:%d, code:0x%x", transId, action, pRsp->code);
|
||||
}
|
||||
|
||||
mInfo("trans:%d, %s:%d response is received, code:0x%x, accept:0x%x retry:0x%x", transId, mndTransStr(pAction->stage),
|
||||
action, pRsp->code, pAction->acceptableCode, pAction->retryCode);
|
||||
mndTransExecute(pMnode, pTrans, true);
|
||||
|
||||
_OVER:
|
||||
|
|
|
|||
|
|
@ -57,7 +57,7 @@ target_sources(
|
|||
|
||||
# tq
|
||||
"src/tq/tq.c"
|
||||
"src/tq/tqExec.c"
|
||||
"src/tq/tqScan.c"
|
||||
"src/tq/tqMeta.c"
|
||||
"src/tq/tqRead.c"
|
||||
"src/tq/tqOffset.c"
|
||||
|
|
|
|||
|
|
@ -178,11 +178,11 @@ typedef struct STsdbReader STsdbReader;
|
|||
|
||||
int32_t tsdbSetTableList(STsdbReader *pReader, const void *pTableList, int32_t num);
|
||||
int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables,
|
||||
SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr);
|
||||
SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr, bool countOnly);
|
||||
|
||||
void tsdbReaderSetId(STsdbReader* pReader, const char* idstr);
|
||||
void tsdbReaderClose(STsdbReader *pReader);
|
||||
bool tsdbNextDataBlock(STsdbReader *pReader);
|
||||
int32_t tsdbNextDataBlock(STsdbReader *pReader, bool *hasNext);
|
||||
int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave);
|
||||
void tsdbReleaseDataBlock(STsdbReader *pReader);
|
||||
SSDataBlock *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList);
|
||||
|
|
|
|||
|
|
@ -109,23 +109,18 @@ typedef struct {
|
|||
} STqPushEntry;
|
||||
|
||||
struct STQ {
|
||||
SVnode* pVnode;
|
||||
char* path;
|
||||
int64_t walLogLastVer;
|
||||
|
||||
SRWLatch pushLock;
|
||||
|
||||
SHashObj* pPushMgr; // consumerId -> STqPushEntry
|
||||
SHashObj* pHandle; // subKey -> STqHandle
|
||||
SHashObj* pCheckInfo; // topic -> SAlterCheckInfo
|
||||
|
||||
SVnode* pVnode;
|
||||
char* path;
|
||||
int64_t walLogLastVer;
|
||||
SRWLatch lock;
|
||||
SHashObj* pPushMgr; // consumerId -> STqPushEntry
|
||||
SHashObj* pHandle; // subKey -> STqHandle
|
||||
SHashObj* pCheckInfo; // topic -> SAlterCheckInfo
|
||||
STqOffsetStore* pOffsetStore;
|
||||
|
||||
TDB* pMetaDB;
|
||||
TTB* pExecStore;
|
||||
TTB* pCheckStore;
|
||||
|
||||
SStreamMeta* pStreamMeta;
|
||||
TDB* pMetaDB;
|
||||
TTB* pExecStore;
|
||||
TTB* pCheckStore;
|
||||
SStreamMeta* pStreamMeta;
|
||||
};
|
||||
|
||||
typedef struct {
|
||||
|
|
@ -144,8 +139,7 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs
|
|||
int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** pHeadWithCkSum, uint64_t reqId);
|
||||
|
||||
// tqExec
|
||||
int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp);
|
||||
// int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp* pRsp);
|
||||
int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp, int32_t* totalRows);
|
||||
int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision);
|
||||
int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp, int32_t type);
|
||||
int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry);
|
||||
|
|
@ -164,7 +158,7 @@ typedef struct {
|
|||
int32_t size;
|
||||
} STqOffsetHead;
|
||||
|
||||
STqOffsetStore* tqOffsetOpen();
|
||||
STqOffsetStore* tqOffsetOpen(STQ* pTq);
|
||||
void tqOffsetClose(STqOffsetStore*);
|
||||
STqOffset* tqOffsetRead(STqOffsetStore* pStore, const char* subscribeKey);
|
||||
int32_t tqOffsetWrite(STqOffsetStore* pStore, const STqOffset* pOffset);
|
||||
|
|
|
|||
|
|
@ -224,6 +224,8 @@ int32_t tsdbTbDataIterCreate(STbData *pTbData, TSDBKEY *pFrom, int8_t backward,
|
|||
void *tsdbTbDataIterDestroy(STbDataIter *pIter);
|
||||
void tsdbTbDataIterOpen(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter *pIter);
|
||||
bool tsdbTbDataIterNext(STbDataIter *pIter);
|
||||
void tsdbMemTableCountRows(SMemTable *pMemTable, SHashObj* pTableMap, int64_t *rowsNum);
|
||||
|
||||
// STbData
|
||||
int32_t tsdbGetNRowsInTbData(STbData *pTbData);
|
||||
// tsdbFile.c ==============================================================================================
|
||||
|
|
@ -687,6 +689,8 @@ typedef struct SSttBlockLoadInfo {
|
|||
STSchema *pSchema;
|
||||
int16_t *colIds;
|
||||
int32_t numOfCols;
|
||||
bool checkRemainingRow;
|
||||
bool isLast;
|
||||
bool sttBlockLoaded;
|
||||
int32_t numOfStt;
|
||||
|
||||
|
|
|
|||
|
|
@ -187,13 +187,14 @@ int32_t tsdbDeleteTableData(STsdb* pTsdb, int64_t version, tb_uid_t suid, tb_uid
|
|||
int32_t tsdbSetKeepCfg(STsdb* pTsdb, STsdbCfg* pCfg);
|
||||
|
||||
// tq
|
||||
int tqInit();
|
||||
void tqCleanUp();
|
||||
STQ* tqOpen(const char* path, SVnode* pVnode);
|
||||
void tqClose(STQ*);
|
||||
int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver);
|
||||
int tqRegisterPushEntry(STQ* pTq, void* pHandle, const SMqPollReq* pRequest, SRpcMsg* pRpcMsg, SMqDataRsp* pDataRsp, int32_t type);
|
||||
int tqRemovePushEntry(STQ* pTq, const char* pKey, int32_t keyLen, uint64_t consumerId, bool rspConsumer);
|
||||
int tqInit();
|
||||
void tqCleanUp();
|
||||
STQ* tqOpen(const char* path, SVnode* pVnode);
|
||||
void tqClose(STQ*);
|
||||
int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver);
|
||||
int tqRegisterPushEntry(STQ* pTq, void* pHandle, const SMqPollReq* pRequest, SRpcMsg* pRpcMsg, SMqDataRsp* pDataRsp,
|
||||
int32_t type);
|
||||
int tqUnregisterPushEntry(STQ* pTq, const char* pKey, int32_t keyLen, uint64_t consumerId, bool rspConsumer);
|
||||
|
||||
int tqCommit(STQ*);
|
||||
int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd);
|
||||
|
|
|
|||
|
|
@ -669,7 +669,7 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma
|
|||
#endif
|
||||
for (int32_t i = 0; i < taosArrayGetSize(pResList); ++i) {
|
||||
SSDataBlock *output = taosArrayGetP(pResList, i);
|
||||
smaDebug("result block, uid:%" PRIu64 ", groupid:%" PRIu64 ", rows:%d", output->info.id.uid,
|
||||
smaDebug("result block, uid:%" PRIu64 ", groupid:%" PRIu64 ", rows:%" PRId64, output->info.id.uid,
|
||||
output->info.id.groupId, output->info.rows);
|
||||
|
||||
STsdb *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb[0] : pSma->pRSmaTsdb[1]);
|
||||
|
|
|
|||
|
|
@ -55,7 +55,7 @@ void tqCleanUp() {
|
|||
}
|
||||
}
|
||||
|
||||
static void destroySTqHandle(void* data) {
|
||||
static void destroyTqHandle(void* data) {
|
||||
STqHandle* pData = (STqHandle*)data;
|
||||
qDestroyTask(pData->execHandle.task);
|
||||
if (pData->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
|
||||
|
|
@ -93,9 +93,9 @@ STQ* tqOpen(const char* path, SVnode* pVnode) {
|
|||
pTq->walLogLastVer = pVnode->pWal->vers.lastVer;
|
||||
|
||||
pTq->pHandle = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK);
|
||||
taosHashSetFreeFp(pTq->pHandle, destroySTqHandle);
|
||||
taosHashSetFreeFp(pTq->pHandle, destroyTqHandle);
|
||||
|
||||
taosInitRWLatch(&pTq->pushLock);
|
||||
taosInitRWLatch(&pTq->lock);
|
||||
pTq->pPushMgr = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
|
||||
taosHashSetFreeFp(pTq->pPushMgr, tqPushEntryFree);
|
||||
|
||||
|
|
@ -209,7 +209,7 @@ static int32_t doSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqData
|
|||
if (type == TMQ_MSG_TYPE__POLL_RSP) {
|
||||
tEncodeSMqDataRsp(&encoder, pRsp);
|
||||
} else if (type == TMQ_MSG_TYPE__TAOSX_RSP) {
|
||||
tEncodeSTaosxRsp(&encoder, (STaosxRsp*) pRsp);
|
||||
tEncodeSTaosxRsp(&encoder, (STaosxRsp*)pRsp);
|
||||
}
|
||||
|
||||
tEncoderClear(&encoder);
|
||||
|
|
@ -222,57 +222,11 @@ static int32_t doSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqData
|
|||
};
|
||||
|
||||
tmsgSendRsp(&rsp);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry) {
|
||||
SMqDataRsp* pRsp = pPushEntry->pDataRsp;
|
||||
|
||||
#if 0
|
||||
A(taosArrayGetSize(pRsp->blockData) == pRsp->blockNum);
|
||||
A(taosArrayGetSize(pRsp->blockDataLen) == pRsp->blockNum);
|
||||
|
||||
A(!pRsp->withSchema);
|
||||
A(taosArrayGetSize(pRsp->blockSchema) == 0);
|
||||
|
||||
if (pRsp->reqOffset.type == TMQ_OFFSET__LOG) {
|
||||
A(pRsp->rspOffset.version > pRsp->reqOffset.version);
|
||||
}
|
||||
#endif
|
||||
|
||||
// int32_t len = 0;
|
||||
// int32_t code = 0;
|
||||
// tEncodeSize(tEncodeSMqDataRsp, pRsp, len, code);
|
||||
// if (code < 0) {
|
||||
// return -1;
|
||||
// }
|
||||
//
|
||||
// int32_t tlen = sizeof(SMqRspHead) + len;
|
||||
// void* buf = rpcMallocCont(tlen);
|
||||
// if (buf == NULL) {
|
||||
// return -1;
|
||||
// }
|
||||
//
|
||||
// memcpy(buf, &pPushEntry->dataRsp.head, sizeof(SMqRspHead));
|
||||
//
|
||||
// void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
|
||||
//
|
||||
// SEncoder encoder = {0};
|
||||
// tEncoderInit(&encoder, abuf, len);
|
||||
// tEncodeSMqDataRsp(&encoder, pRsp);
|
||||
// tEncoderClear(&encoder);
|
||||
//
|
||||
// SRpcMsg rsp = {
|
||||
// .info = pPushEntry->pInfo,
|
||||
// .pCont = buf,
|
||||
// .contLen = tlen,
|
||||
// .code = 0,
|
||||
// };
|
||||
//
|
||||
// tmsgSendRsp(&rsp);
|
||||
//
|
||||
|
||||
SMqRspHead* pHeader = &pPushEntry->pDataRsp->head;
|
||||
doSendDataRsp(&pPushEntry->info, pRsp, pHeader->epoch, pHeader->consumerId, pHeader->mqMsgType);
|
||||
|
||||
|
|
@ -294,11 +248,7 @@ int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, con
|
|||
A(taosArrayGetSize(pRsp->blockSchema) == 0);
|
||||
|
||||
if (pRsp->reqOffset.type == TMQ_OFFSET__LOG) {
|
||||
if (pRsp->blockNum > 0) {
|
||||
A(pRsp->rspOffset.version > pRsp->reqOffset.version);
|
||||
} else {
|
||||
A(pRsp->rspOffset.version >= pRsp->reqOffset.version);
|
||||
}
|
||||
A(pRsp->rspOffset.version > pRsp->reqOffset.version);
|
||||
}
|
||||
#endif
|
||||
doSendDataRsp(&pMsg->info, pRsp, pReq->epoch, pReq->consumerId, type);
|
||||
|
|
@ -308,7 +258,7 @@ int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, con
|
|||
tFormatOffset(buf1, 80, &pRsp->reqOffset);
|
||||
tFormatOffset(buf2, 80, &pRsp->rspOffset);
|
||||
|
||||
tqDebug("vgId:%d consumer:0x%" PRIx64 " (epoch %d) send rsp, block num:%d, req:%s, rsp:%s, reqId:0x%"PRIx64,
|
||||
tqDebug("vgId:%d consumer:0x%" PRIx64 " (epoch %d) send rsp, block num:%d, req:%s, rsp:%s, reqId:0x%" PRIx64,
|
||||
TD_VID(pTq->pVnode), pReq->consumerId, pReq->epoch, pRsp->blockNum, buf1, buf2, pReq->reqId);
|
||||
|
||||
return 0;
|
||||
|
|
@ -321,7 +271,7 @@ static FORCE_INLINE bool tqOffsetLessOrEqual(const STqOffset* pLeft, const STqOf
|
|||
|
||||
int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) {
|
||||
STqOffset offset = {0};
|
||||
int32_t vgId = TD_VID(pTq->pVnode);
|
||||
int32_t vgId = TD_VID(pTq->pVnode);
|
||||
|
||||
SDecoder decoder;
|
||||
tDecoderInit(&decoder, (uint8_t*)msg, msgLen);
|
||||
|
|
@ -335,8 +285,8 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t sversion, char* msg, int32_t
|
|||
tqDebug("receive offset commit msg to %s on vgId:%d, offset(type:snapshot) uid:%" PRId64 ", ts:%" PRId64,
|
||||
offset.subKey, vgId, offset.val.uid, offset.val.ts);
|
||||
} else if (offset.val.type == TMQ_OFFSET__LOG) {
|
||||
tqDebug("receive offset commit msg to %s on vgId:%d, offset(type:log) version:%" PRId64, offset.subKey,
|
||||
vgId, offset.val.version);
|
||||
tqDebug("receive offset commit msg to %s on vgId:%d, offset(type:log) version:%" PRId64, offset.subKey, vgId,
|
||||
offset.val.version);
|
||||
if (offset.val.version + 1 == sversion) {
|
||||
offset.val.version += 1;
|
||||
}
|
||||
|
|
@ -449,8 +399,9 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand
|
|||
|
||||
char formatBuf[80];
|
||||
tFormatOffset(formatBuf, 80, pOffsetVal);
|
||||
tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, prev offset found, offset reset to %s and continue.",
|
||||
consumerId, pHandle->subKey, vgId, formatBuf);
|
||||
tqDebug("tmq poll: consumer:0x%" PRIx64
|
||||
", subkey %s, vgId:%d, existed offset found, offset reset to %s and continue. reqId:0x%" PRIx64,
|
||||
consumerId, pHandle->subKey, vgId, formatBuf, pRequest->reqId);
|
||||
return 0;
|
||||
} else {
|
||||
// no poll occurs in this vnode for this topic, let's seek to the right offset value.
|
||||
|
|
@ -497,7 +448,8 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand
|
|||
return code;
|
||||
}
|
||||
} else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) {
|
||||
tqError("tmq poll: subkey:%s, no offset committed for consumer:0x%" PRIx64 " in vg %d, subkey %s, reset none failed",
|
||||
tqError("tmq poll: subkey:%s, no offset committed for consumer:0x%" PRIx64
|
||||
" in vg %d, subkey %s, reset none failed",
|
||||
pHandle->subKey, consumerId, vgId, pRequest->subKey);
|
||||
terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET;
|
||||
return -1;
|
||||
|
|
@ -507,7 +459,49 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand
|
|||
return 0;
|
||||
}
|
||||
|
||||
static int32_t extractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg) {
|
||||
#define IS_OFFSET_RESET_TYPE(_t) ((_t) < 0)
|
||||
|
||||
static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest,
|
||||
SRpcMsg* pMsg, STqOffsetVal* pOffset) {
|
||||
int32_t code = 0;
|
||||
uint64_t consumerId = pRequest->consumerId;
|
||||
int32_t vgId = TD_VID(pTq->pVnode);
|
||||
|
||||
SMqDataRsp dataRsp = {0};
|
||||
tqInitDataRsp(&dataRsp, pRequest, pHandle->execHandle.subType);
|
||||
|
||||
// lock
|
||||
taosWLockLatch(&pTq->lock);
|
||||
|
||||
qSetTaskId(pHandle->execHandle.task, consumerId, pRequest->reqId);
|
||||
code = tqScanData(pTq, pHandle, &dataRsp, pOffset);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
taosWUnLockLatch(&pTq->lock);
|
||||
return code;
|
||||
}
|
||||
|
||||
// till now, all data has been transferred to consumer, new data needs to push client once arrived.
|
||||
if (dataRsp.blockNum == 0 && dataRsp.reqOffset.type == TMQ_OFFSET__LOG &&
|
||||
dataRsp.reqOffset.version == dataRsp.rspOffset.version && pHandle->consumerId == pRequest->consumerId) {
|
||||
code = tqRegisterPushEntry(pTq, pHandle, pRequest, pMsg, &dataRsp, TMQ_MSG_TYPE__POLL_RSP);
|
||||
taosWUnLockLatch(&pTq->lock);
|
||||
return code;
|
||||
}
|
||||
|
||||
taosWUnLockLatch(&pTq->lock);
|
||||
code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&dataRsp, TMQ_MSG_TYPE__POLL_RSP);
|
||||
|
||||
// NOTE: this pHandle->consumerId may have been changed already.
|
||||
tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, offset type:%d, uid/version:%" PRId64
|
||||
", ts:%" PRId64 ", reqId:0x%" PRIx64,
|
||||
consumerId, pHandle->subKey, vgId, dataRsp.blockNum, dataRsp.rspOffset.type, dataRsp.rspOffset.uid,
|
||||
dataRsp.rspOffset.ts, pRequest->reqId);
|
||||
|
||||
tDeleteSMqDataRsp(&dataRsp);
|
||||
return code;
|
||||
}
|
||||
|
||||
static int32_t doPollDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg) {
|
||||
int32_t code = -1;
|
||||
STqOffsetVal offset = {0};
|
||||
SWalCkHead* pCkHead = NULL;
|
||||
|
|
@ -517,9 +511,8 @@ static int32_t extractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq*
|
|||
uint64_t consumerId = pRequest->consumerId;
|
||||
|
||||
// 1. reset the offset if needed
|
||||
if (reqOffset.type > 0) {
|
||||
offset = reqOffset;
|
||||
} else { // handle the reset offset cases, according to the consumer's choice.
|
||||
if (IS_OFFSET_RESET_TYPE(reqOffset.type)) {
|
||||
// handle the reset offset cases, according to the consumer's choice.
|
||||
bool blockReturned = false;
|
||||
code = extractResetOffsetVal(&offset, pTq, pHandle, pRequest, pMsg, &blockReturned);
|
||||
if (code != 0) {
|
||||
|
|
@ -530,44 +523,20 @@ static int32_t extractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq*
|
|||
if (blockReturned) {
|
||||
return 0;
|
||||
}
|
||||
} else { // use the consumer specified offset
|
||||
// the offset value can not be monotonious increase??
|
||||
offset = reqOffset;
|
||||
}
|
||||
|
||||
// this is a normal subscription requirement
|
||||
// this is a normal subscribe requirement
|
||||
if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
|
||||
SMqDataRsp dataRsp = {0};
|
||||
tqInitDataRsp(&dataRsp, pRequest, pHandle->execHandle.subType);
|
||||
|
||||
// lock
|
||||
taosWLockLatch(&pTq->pushLock);
|
||||
|
||||
qSetTaskId(pHandle->execHandle.task, consumerId, pRequest->reqId);
|
||||
code = tqScanData(pTq, pHandle, &dataRsp, &offset);
|
||||
|
||||
// till now, all data has been transferred to consumer, new data needs to push client once arrived.
|
||||
if (dataRsp.blockNum == 0 && dataRsp.reqOffset.type == TMQ_OFFSET__LOG &&
|
||||
dataRsp.reqOffset.version == dataRsp.rspOffset.version && pHandle->consumerId == pRequest->consumerId) {
|
||||
code = tqRegisterPushEntry(pTq, pHandle, pRequest, pMsg, &dataRsp, TMQ_MSG_TYPE__POLL_RSP);
|
||||
taosWUnLockLatch(&pTq->pushLock);
|
||||
return code;
|
||||
}
|
||||
|
||||
taosWUnLockLatch(&pTq->pushLock);
|
||||
code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&dataRsp, TMQ_MSG_TYPE__POLL_RSP);
|
||||
|
||||
// NOTE: this pHandle->consumerId may have been changed already.
|
||||
tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, offset type:%d, uid/version:%" PRId64
|
||||
", ts:%" PRId64", reqId:0x%"PRIx64,
|
||||
consumerId, pHandle->subKey, vgId, dataRsp.blockNum, dataRsp.rspOffset.type, dataRsp.rspOffset.uid,
|
||||
dataRsp.rspOffset.ts, pRequest->reqId);
|
||||
|
||||
tDeleteSMqDataRsp(&dataRsp);
|
||||
return code;
|
||||
return extractDataAndRspForNormalSubscribe(pTq, pHandle, pRequest, pMsg, &offset);
|
||||
}
|
||||
|
||||
// todo handle the case where re-balance occurs.
|
||||
// for taosx
|
||||
SMqMetaRsp metaRsp = {0};
|
||||
STaosxRsp taosxRsp = {0};
|
||||
STaosxRsp taosxRsp = {0};
|
||||
tqInitTaosxRsp(&taosxRsp, pRequest);
|
||||
|
||||
if (offset.type != TMQ_OFFSET__LOG) {
|
||||
|
|
@ -579,8 +548,7 @@ static int32_t extractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq*
|
|||
code = tqSendMetaPollRsp(pTq, pMsg, pRequest, &metaRsp);
|
||||
tqDebug("tmq poll: consumer:0x%" PRIx64 " subkey:%s vgId:%d, send meta offset type:%d,uid:%" PRId64
|
||||
",ts:%" PRId64,
|
||||
consumerId, pHandle->subKey, vgId, metaRsp.rspOffset.type, metaRsp.rspOffset.uid,
|
||||
metaRsp.rspOffset.ts);
|
||||
consumerId, pHandle->subKey, vgId, metaRsp.rspOffset.type, metaRsp.rspOffset.uid, metaRsp.rspOffset.ts);
|
||||
taosMemoryFree(metaRsp.metaRsp);
|
||||
tDeleteSTaosxRsp(&taosxRsp);
|
||||
return code;
|
||||
|
|
@ -598,9 +566,9 @@ static int32_t extractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq*
|
|||
",version:%" PRId64,
|
||||
consumerId, pHandle->subKey, vgId, taosxRsp.blockNum, taosxRsp.rspOffset.type, taosxRsp.rspOffset.uid,
|
||||
taosxRsp.rspOffset.version);
|
||||
} else {
|
||||
}
|
||||
|
||||
// if (offset.type == TMQ_OFFSET__LOG) {
|
||||
if (offset.type == TMQ_OFFSET__LOG) {
|
||||
int64_t fetchVer = offset.version + 1;
|
||||
pCkHead = taosMemoryMalloc(sizeof(SWalCkHead) + 2048);
|
||||
if (pCkHead == NULL) {
|
||||
|
|
@ -608,9 +576,8 @@ static int32_t extractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq*
|
|||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return -1;
|
||||
}
|
||||
|
||||
walSetReaderCapacity(pHandle->pWalReader, 2048);
|
||||
|
||||
int totalRows = 0;
|
||||
while (1) {
|
||||
// todo refactor: this is not correct.
|
||||
int32_t savedEpoch = atomic_load_32(&pHandle->epoch);
|
||||
|
|
@ -633,32 +600,16 @@ static int32_t extractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq*
|
|||
tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d) iter log, vgId:%d offset %" PRId64 " msgType %d", consumerId,
|
||||
pRequest->epoch, vgId, fetchVer, pHead->msgType);
|
||||
|
||||
if (pHead->msgType == TDMT_VND_SUBMIT) {
|
||||
SPackedData submit = {
|
||||
.msgStr = POINTER_SHIFT(pHead->body, sizeof(SSubmitReq2Msg)),
|
||||
.msgLen = pHead->bodyLen - sizeof(SSubmitReq2Msg),
|
||||
.ver = pHead->version,
|
||||
};
|
||||
|
||||
if (tqTaosxScanLog(pTq, pHandle, submit, &taosxRsp) < 0) {
|
||||
tqError("tmq poll: tqTaosxScanLog error %" PRId64 ", in vgId:%d, subkey %s", consumerId, vgId,
|
||||
pRequest->subKey);
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (taosxRsp.blockNum > 0) {
|
||||
tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer);
|
||||
// process meta
|
||||
if (pHead->msgType != TDMT_VND_SUBMIT) {
|
||||
if (totalRows > 0) {
|
||||
tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer - 1);
|
||||
code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP);
|
||||
tDeleteSTaosxRsp(&taosxRsp);
|
||||
taosMemoryFreeClear(pCkHead);
|
||||
return code;
|
||||
} else {
|
||||
fetchVer++;
|
||||
}
|
||||
|
||||
} else {
|
||||
/*A(pHandle->fetchMeta);*/
|
||||
/*A(IS_META_MSG(pHead->msgType));*/
|
||||
tqDebug("fetch meta msg, ver:%" PRId64 ", type:%s", pHead->version, TMSG_INFO(pHead->msgType));
|
||||
tqOffsetResetToLog(&metaRsp.rspOffset, fetchVer);
|
||||
metaRsp.resMsgType = pHead->msgType;
|
||||
|
|
@ -675,6 +626,31 @@ static int32_t extractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq*
|
|||
tDeleteSTaosxRsp(&taosxRsp);
|
||||
return code;
|
||||
}
|
||||
|
||||
// process data
|
||||
SPackedData submit = {
|
||||
.msgStr = POINTER_SHIFT(pHead->body, sizeof(SSubmitReq2Msg)),
|
||||
.msgLen = pHead->bodyLen - sizeof(SSubmitReq2Msg),
|
||||
.ver = pHead->version,
|
||||
};
|
||||
|
||||
if (tqTaosxScanLog(pTq, pHandle, submit, &taosxRsp, &totalRows) < 0) {
|
||||
tqError("tmq poll: tqTaosxScanLog error %" PRId64 ", in vgId:%d, subkey %s", consumerId, vgId,
|
||||
pRequest->subKey);
|
||||
taosMemoryFreeClear(pCkHead);
|
||||
tDeleteSTaosxRsp(&taosxRsp);
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (totalRows >= 4096 || taosxRsp.createTableNum > 0) {
|
||||
tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer);
|
||||
code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP);
|
||||
tDeleteSTaosxRsp(&taosxRsp);
|
||||
taosMemoryFreeClear(pCkHead);
|
||||
return code;
|
||||
} else {
|
||||
fetchVer++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -684,7 +660,7 @@ static int32_t extractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq*
|
|||
}
|
||||
|
||||
int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
|
||||
SMqPollReq req = {0};
|
||||
SMqPollReq req = {0};
|
||||
if (tDeserializeSMqPollReq(pMsg->pCont, pMsg->contLen, &req) < 0) {
|
||||
tqError("tDeserializeSMqPollReq %d failed", pMsg->contLen);
|
||||
terrno = TSDB_CODE_INVALID_MSG;
|
||||
|
|
@ -705,31 +681,32 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
|
|||
}
|
||||
|
||||
// 2. check re-balance status
|
||||
taosRLockLatch(&pTq->pushLock);
|
||||
taosRLockLatch(&pTq->lock);
|
||||
if (pHandle->consumerId != consumerId) {
|
||||
tqDebug("ERROR tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64,
|
||||
consumerId, TD_VID(pTq->pVnode), req.subKey, pHandle->consumerId);
|
||||
terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH;
|
||||
taosRUnLockLatch(&pTq->pushLock);
|
||||
taosRUnLockLatch(&pTq->lock);
|
||||
return -1;
|
||||
}
|
||||
taosRUnLockLatch(&pTq->pushLock);
|
||||
taosRUnLockLatch(&pTq->lock);
|
||||
|
||||
taosWLockLatch(&pTq->pushLock);
|
||||
// 3. update the epoch value
|
||||
taosWLockLatch(&pTq->lock);
|
||||
int32_t savedEpoch = pHandle->epoch;
|
||||
if (savedEpoch < reqEpoch) {
|
||||
tqDebug("tmq poll: consumer:0x%" PRIx64 " epoch update from %d to %d by poll req", consumerId, savedEpoch, reqEpoch);
|
||||
tqDebug("tmq poll: consumer:0x%" PRIx64 " epoch update from %d to %d by poll req", consumerId, savedEpoch,
|
||||
reqEpoch);
|
||||
pHandle->epoch = reqEpoch;
|
||||
}
|
||||
taosWUnLockLatch(&pTq->pushLock);
|
||||
taosWUnLockLatch(&pTq->lock);
|
||||
|
||||
char buf[80];
|
||||
tFormatOffset(buf, 80, &reqOffset);
|
||||
tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey %s, recv poll req vgId:%d, req:%s, reqId:0x%" PRIx64,
|
||||
consumerId, req.epoch, pHandle->subKey, vgId, buf, req.reqId);
|
||||
|
||||
return extractDataForMq(pTq, pHandle, &req, pMsg);
|
||||
return doPollDataForMq(pTq, pHandle, &req, pMsg);
|
||||
}
|
||||
|
||||
int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) {
|
||||
|
|
@ -737,12 +714,12 @@ int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
|
|||
|
||||
tqDebug("vgId:%d, tq process delete sub req %s", pTq->pVnode->config.vgId, pReq->subKey);
|
||||
|
||||
taosWLockLatch(&pTq->pushLock);
|
||||
taosWLockLatch(&pTq->lock);
|
||||
int32_t code = taosHashRemove(pTq->pPushMgr, pReq->subKey, strlen(pReq->subKey));
|
||||
if (code != 0) {
|
||||
tqDebug("vgId:%d, tq remove push handle %s", pTq->pVnode->config.vgId, pReq->subKey);
|
||||
}
|
||||
taosWUnLockLatch(&pTq->pushLock);
|
||||
taosWUnLockLatch(&pTq->lock);
|
||||
|
||||
STqHandle* pHandle = taosHashGet(pTq->pHandle, pReq->subKey, strlen(pReq->subKey));
|
||||
if (pHandle) {
|
||||
|
|
@ -806,25 +783,24 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
|
|||
SVnode* pVnode = pTq->pVnode;
|
||||
int32_t vgId = TD_VID(pVnode);
|
||||
|
||||
tqDebug("vgId:%d, tq process sub req %s, Id:0x%" PRIx64 " -> Id:0x%" PRIx64, pVnode->config.vgId, req.subKey,
|
||||
tqDebug("vgId:%d, tq process sub req:%s, Id:0x%" PRIx64 " -> Id:0x%" PRIx64, pVnode->config.vgId, req.subKey,
|
||||
req.oldConsumerId, req.newConsumerId);
|
||||
|
||||
STqHandle* pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey));
|
||||
if (pHandle == NULL) {
|
||||
if (req.oldConsumerId != -1) {
|
||||
tqError("vgId:%d, build new consumer handle %s for consumer:0x%" PRIx64 ", but old consumerId is %" PRId64 "",
|
||||
tqError("vgId:%d, build new consumer handle %s for consumer:0x%" PRIx64 ", but old consumerId:0x%" PRIx64,
|
||||
req.vgId, req.subKey, req.newConsumerId, req.oldConsumerId);
|
||||
}
|
||||
|
||||
if (req.newConsumerId == -1) {
|
||||
tqError("vgId:%d, tq invalid rebalance request, new consumerId %" PRId64 "", req.vgId, req.newConsumerId);
|
||||
tqError("vgId:%d, tq invalid re-balance request, new consumerId %" PRId64 "", req.vgId, req.newConsumerId);
|
||||
taosMemoryFree(req.qmsg);
|
||||
return 0;
|
||||
}
|
||||
|
||||
STqHandle tqHandle = {0};
|
||||
pHandle = &tqHandle;
|
||||
/*taosInitRWLatch(&pExec->lock);*/
|
||||
|
||||
uint64_t oldConsumerId = pHandle->consumerId;
|
||||
memcpy(pHandle->subKey, req.subKey, TSDB_SUBSCRIBE_KEY_LEN);
|
||||
|
|
@ -858,8 +834,8 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
|
|||
pHandle->execHandle.execCol.qmsg = req.qmsg;
|
||||
req.qmsg = NULL;
|
||||
|
||||
pHandle->execHandle.task =
|
||||
qCreateQueueExecTaskInfo(pHandle->execHandle.execCol.qmsg, &handle, vgId, &pHandle->execHandle.numOfCols, NULL);
|
||||
pHandle->execHandle.task = qCreateQueueExecTaskInfo(pHandle->execHandle.execCol.qmsg, &handle, vgId,
|
||||
&pHandle->execHandle.numOfCols, req.newConsumerId);
|
||||
void* scanner = NULL;
|
||||
qExtractStreamScanner(pHandle->execHandle.task, &scanner);
|
||||
pHandle->execHandle.pExecReader = qExtractReaderFromStreamScanner(scanner);
|
||||
|
|
@ -872,7 +848,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
|
|||
buildSnapContext(handle.meta, handle.version, 0, pHandle->execHandle.subType, pHandle->fetchMeta,
|
||||
(SSnapContext**)(&handle.sContext));
|
||||
|
||||
pHandle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &handle, vgId, NULL, NULL);
|
||||
pHandle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &handle, vgId, NULL, req.newConsumerId);
|
||||
} else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) {
|
||||
pHandle->pWalReader = walOpenReader(pVnode->pWal, NULL);
|
||||
pHandle->execHandle.execTb.suid = req.suid;
|
||||
|
|
@ -890,7 +866,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
|
|||
|
||||
buildSnapContext(handle.meta, handle.version, req.suid, pHandle->execHandle.subType, pHandle->fetchMeta,
|
||||
(SSnapContext**)(&handle.sContext));
|
||||
pHandle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &handle, vgId, NULL, NULL);
|
||||
pHandle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &handle, vgId, NULL, req.newConsumerId);
|
||||
}
|
||||
|
||||
taosHashPut(pTq->pHandle, req.subKey, strlen(req.subKey), pHandle, sizeof(STqHandle));
|
||||
|
|
@ -907,28 +883,34 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
|
|||
atomic_add_fetch_32(&pHandle->epoch, 1);
|
||||
taosMemoryFree(req.qmsg);
|
||||
return tqMetaSaveHandle(pTq, req.subKey, pHandle);
|
||||
}
|
||||
} else {
|
||||
tqInfo("vgId:%d switch consumer from Id:0x%" PRIx64 " to Id:0x%" PRIx64, req.vgId, pHandle->consumerId,
|
||||
req.newConsumerId);
|
||||
|
||||
tqInfo("vgId:%d switch consumer from Id:0x%" PRIx64 " to Id:0x%" PRIx64, req.vgId, pHandle->consumerId,
|
||||
req.newConsumerId);
|
||||
// kill executing task
|
||||
qTaskInfo_t pTaskInfo = pHandle->execHandle.task;
|
||||
if (pTaskInfo != NULL) {
|
||||
qKillTask(pTaskInfo, TSDB_CODE_SUCCESS);
|
||||
}
|
||||
|
||||
taosWLockLatch(&pTq->pushLock);
|
||||
atomic_store_32(&pHandle->epoch, -1);
|
||||
taosWLockLatch(&pTq->lock);
|
||||
atomic_store_32(&pHandle->epoch, -1);
|
||||
|
||||
// remove if it has been register in the push manager, and return one empty block to consumer
|
||||
tqRemovePushEntry(pTq, req.subKey, (int32_t) strlen(req.subKey), pHandle->consumerId, true);
|
||||
// remove if it has been register in the push manager, and return one empty block to consumer
|
||||
tqUnregisterPushEntry(pTq, req.subKey, (int32_t)strlen(req.subKey), pHandle->consumerId, true);
|
||||
|
||||
atomic_store_64(&pHandle->consumerId, req.newConsumerId);
|
||||
atomic_add_fetch_32(&pHandle->epoch, 1);
|
||||
atomic_store_64(&pHandle->consumerId, req.newConsumerId);
|
||||
atomic_add_fetch_32(&pHandle->epoch, 1);
|
||||
|
||||
if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
|
||||
qStreamCloseTsdbReader(pHandle->execHandle.task);
|
||||
}
|
||||
if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
|
||||
qStreamCloseTsdbReader(pTaskInfo);
|
||||
}
|
||||
|
||||
taosWUnLockLatch(&pTq->pushLock);
|
||||
if (tqMetaSaveHandle(pTq, req.subKey, pHandle) < 0) {
|
||||
taosMemoryFree(req.qmsg);
|
||||
return -1;
|
||||
taosWUnLockLatch(&pTq->lock);
|
||||
if (tqMetaSaveHandle(pTq, req.subKey, pHandle) < 0) {
|
||||
taosMemoryFree(req.qmsg);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1008,7 +990,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
|
|||
pTask->tbSink.vnode = pTq->pVnode;
|
||||
pTask->tbSink.tbSinkFunc = tqSinkToTablePipeline2;
|
||||
|
||||
int32_t ver1 = 1;
|
||||
int32_t ver1 = 1;
|
||||
SMetaInfo info = {0};
|
||||
int32_t code = metaGetInfo(pTq->pVnode->pMeta, pTask->tbSink.stbUid, &info, NULL);
|
||||
if (code == TSDB_CODE_SUCCESS) {
|
||||
|
|
@ -1017,13 +999,14 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
|
|||
|
||||
pTask->tbSink.pTSchema =
|
||||
tBuildTSchema(pTask->tbSink.pSchemaWrapper->pSchema, pTask->tbSink.pSchemaWrapper->nCols, ver1);
|
||||
if(pTask->tbSink.pTSchema == NULL) {
|
||||
if (pTask->tbSink.pTSchema == NULL) {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
streamSetupTrigger(pTask);
|
||||
tqInfo("expand stream task on vg %d, task id %d, child id %d, level %d", vgId, pTask->taskId, pTask->selfChildId, pTask->taskLevel);
|
||||
tqInfo("expand stream task on vg %d, task id %d, child id %d, level %d", vgId, pTask->taskId, pTask->selfChildId,
|
||||
pTask->taskLevel);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -269,7 +269,7 @@ int32_t tqMetaDeleteHandle(STQ* pTq, const char* key) {
|
|||
}
|
||||
|
||||
int32_t tqMetaRestoreHandle(STQ* pTq) {
|
||||
int code = 0;
|
||||
int code = 0;
|
||||
TBC* pCur = NULL;
|
||||
if (tdbTbcOpen(pTq->pExecStore, &pCur, NULL) < 0) {
|
||||
return -1;
|
||||
|
|
@ -307,7 +307,7 @@ int32_t tqMetaRestoreHandle(STQ* pTq) {
|
|||
|
||||
if (handle.execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
|
||||
handle.execHandle.task =
|
||||
qCreateQueueExecTaskInfo(handle.execHandle.execCol.qmsg, &reader, vgId, &handle.execHandle.numOfCols, NULL);
|
||||
qCreateQueueExecTaskInfo(handle.execHandle.execCol.qmsg, &reader, vgId, &handle.execHandle.numOfCols, 0);
|
||||
if (handle.execHandle.task == NULL) {
|
||||
tqError("cannot create exec task for %s", handle.subKey);
|
||||
code = -1;
|
||||
|
|
@ -332,7 +332,7 @@ int32_t tqMetaRestoreHandle(STQ* pTq) {
|
|||
|
||||
buildSnapContext(reader.meta, reader.version, 0, handle.execHandle.subType, handle.fetchMeta,
|
||||
(SSnapContext**)(&reader.sContext));
|
||||
handle.execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, NULL);
|
||||
handle.execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, 0);
|
||||
} else if (handle.execHandle.subType == TOPIC_SUB_TYPE__TABLE) {
|
||||
handle.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL);
|
||||
|
||||
|
|
@ -341,7 +341,7 @@ int32_t tqMetaRestoreHandle(STQ* pTq) {
|
|||
tqDebug("vgId:%d, tq try to get all ctb, suid:%" PRId64, pTq->pVnode->config.vgId, handle.execHandle.execTb.suid);
|
||||
for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) {
|
||||
int64_t tbUid = *(int64_t*)taosArrayGet(tbUidList, i);
|
||||
tqDebug("vgId:%d, idx %d, uid:%" PRId64, TD_VID(pTq->pVnode), i, tbUid);
|
||||
tqDebug("vgId:%d, idx %d, uid:%" PRId64, vgId, i, tbUid);
|
||||
}
|
||||
handle.execHandle.pExecReader = tqOpenReader(pTq->pVnode);
|
||||
tqReaderSetTbUidList(handle.execHandle.pExecReader, tbUidList);
|
||||
|
|
@ -349,9 +349,9 @@ int32_t tqMetaRestoreHandle(STQ* pTq) {
|
|||
|
||||
buildSnapContext(reader.meta, reader.version, handle.execHandle.execTb.suid, handle.execHandle.subType,
|
||||
handle.fetchMeta, (SSnapContext**)(&reader.sContext));
|
||||
handle.execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, NULL);
|
||||
handle.execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, 0);
|
||||
}
|
||||
tqDebug("tq restore %s consumer %" PRId64 " vgId:%d", handle.subKey, handle.consumerId, TD_VID(pTq->pVnode));
|
||||
tqDebug("tq restore %s consumer %" PRId64 " vgId:%d", handle.subKey, handle.consumerId, vgId);
|
||||
taosHashPut(pTq->pHandle, pKey, kLen, &handle, sizeof(STqHandle));
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -206,38 +206,110 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_
|
|||
}
|
||||
#endif
|
||||
|
||||
int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) {
|
||||
typedef struct {
|
||||
void* pKey;
|
||||
int64_t keyLen;
|
||||
} SItem;
|
||||
|
||||
static void recordPushedEntry(SArray* cachedKey, void* pIter);
|
||||
|
||||
static void freeItem(void* param) {
|
||||
SItem* p = (SItem*)param;
|
||||
taosMemoryFree(p->pKey);
|
||||
}
|
||||
|
||||
static void doRemovePushedEntry(SArray* pCachedKeys, STQ* pTq) {
|
||||
int32_t vgId = TD_VID(pTq->pVnode);
|
||||
int32_t numOfKeys = (int32_t)taosArrayGetSize(pCachedKeys);
|
||||
|
||||
for (int32_t i = 0; i < numOfKeys; i++) {
|
||||
SItem* pItem = taosArrayGet(pCachedKeys, i);
|
||||
if (taosHashRemove(pTq->pPushMgr, pItem->pKey, pItem->keyLen) != 0) {
|
||||
tqError("vgId:%d, tq push hash remove key error, key: %s", vgId, (char*)pItem->pKey);
|
||||
}
|
||||
}
|
||||
|
||||
if (numOfKeys > 0) {
|
||||
tqDebug("vgId:%d, pushed %d items and remain:%d", vgId, numOfKeys, (int32_t)taosHashGetSize(pTq->pPushMgr));
|
||||
}
|
||||
}
|
||||
|
||||
static void doPushDataForEntry(void* pIter, STqExecHandle* pExec, STQ* pTq, int64_t ver, int32_t vgId, char* pData,
|
||||
int32_t dataLen, SArray* pCachedKey) {
|
||||
STqPushEntry* pPushEntry = *(STqPushEntry**)pIter;
|
||||
|
||||
SMqDataRsp* pRsp = pPushEntry->pDataRsp;
|
||||
if (pRsp->reqOffset.version >= ver) {
|
||||
tqDebug("vgId:%d, push entry req version %" PRId64 ", while push version %" PRId64 ", skip", vgId,
|
||||
pRsp->reqOffset.version, ver);
|
||||
return;
|
||||
}
|
||||
|
||||
qTaskInfo_t pTaskInfo = pExec->task;
|
||||
|
||||
// prepare scan mem data
|
||||
SPackedData submit = {.msgStr = pData, .msgLen = dataLen, .ver = ver};
|
||||
|
||||
if (qStreamSetScanMemData(pTaskInfo, submit) != 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// here start to scan submit block to extract the subscribed data
|
||||
int32_t totalRows = 0;
|
||||
|
||||
while (1) {
|
||||
SSDataBlock* pDataBlock = NULL;
|
||||
uint64_t ts = 0;
|
||||
if (qExecTask(pTaskInfo, &pDataBlock, &ts) < 0) {
|
||||
tqDebug("vgId:%d, tq exec error since %s", vgId, terrstr());
|
||||
}
|
||||
|
||||
if (pDataBlock == NULL) {
|
||||
break;
|
||||
}
|
||||
|
||||
tqAddBlockDataToRsp(pDataBlock, pRsp, pExec->numOfCols, pTq->pVnode->config.tsdbCfg.precision);
|
||||
pRsp->blockNum++;
|
||||
totalRows += pDataBlock->info.rows;
|
||||
}
|
||||
|
||||
tqDebug("vgId:%d, tq handle push, subkey:%s, block num:%d, rows:%d", vgId, pPushEntry->subKey, pRsp->blockNum,
|
||||
totalRows);
|
||||
|
||||
if (pRsp->blockNum > 0) {
|
||||
tqOffsetResetToLog(&pRsp->rspOffset, ver);
|
||||
tqPushDataRsp(pTq, pPushEntry);
|
||||
recordPushedEntry(pCachedKey, pIter);
|
||||
}
|
||||
}
|
||||
|
||||
int32_t tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) {
|
||||
void* pReq = POINTER_SHIFT(msg, sizeof(SSubmitReq2Msg));
|
||||
int32_t len = msgLen - sizeof(SSubmitReq2Msg);
|
||||
int32_t vgId = TD_VID(pTq->pVnode);
|
||||
|
||||
if (msgType == TDMT_VND_SUBMIT) {
|
||||
// lock push mgr to avoid potential msg lost
|
||||
taosWLockLatch(&pTq->pushLock);
|
||||
taosWLockLatch(&pTq->lock);
|
||||
|
||||
int32_t numOfRegisteredPush = taosHashGetSize(pTq->pPushMgr);
|
||||
if (numOfRegisteredPush > 0) {
|
||||
tqDebug("vgId:%d tq push msg version:%" PRId64 " type:%s, head:%p, body:%p len:%d, numOfPushed consumers:%d",
|
||||
vgId, ver, TMSG_INFO(msgType), msg, pReq, len, numOfRegisteredPush);
|
||||
vgId, ver, TMSG_INFO(msgType), msg, pReq, len, numOfRegisteredPush);
|
||||
|
||||
SArray* cachedKeys = taosArrayInit(0, sizeof(void*));
|
||||
SArray* cachedKeyLens = taosArrayInit(0, sizeof(size_t));
|
||||
|
||||
void* data = taosMemoryMalloc(len);
|
||||
void* data = taosMemoryMalloc(len);
|
||||
if (data == NULL) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
tqError("failed to copy data for stream since out of memory");
|
||||
taosArrayDestroyP(cachedKeys, (FDelete)taosMemoryFree);
|
||||
taosArrayDestroy(cachedKeyLens);
|
||||
|
||||
// unlock
|
||||
taosWUnLockLatch(&pTq->pushLock);
|
||||
tqError("failed to copy data for stream since out of memory, vgId:%d", vgId);
|
||||
taosWUnLockLatch(&pTq->lock);
|
||||
return -1;
|
||||
}
|
||||
|
||||
memcpy(data, pReq, len);
|
||||
|
||||
void* pIter = NULL;
|
||||
SArray* cachedKey = taosArrayInit(0, sizeof(SItem));
|
||||
void* pIter = NULL;
|
||||
|
||||
while (1) {
|
||||
pIter = taosHashIterate(pTq->pPushMgr, pIter);
|
||||
if (pIter == NULL) {
|
||||
|
|
@ -248,83 +320,30 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver)
|
|||
|
||||
STqHandle* pHandle = taosHashGet(pTq->pHandle, pPushEntry->subKey, strlen(pPushEntry->subKey));
|
||||
if (pHandle == NULL) {
|
||||
tqDebug("vgId:%d, cannot find handle %s", pTq->pVnode->config.vgId, pPushEntry->subKey);
|
||||
continue;
|
||||
}
|
||||
|
||||
SMqDataRsp* pRsp = pPushEntry->pDataRsp;
|
||||
if (pRsp->reqOffset.version >= ver) {
|
||||
tqDebug("vgId:%d, push entry req version %" PRId64 ", while push version %" PRId64 ", skip", vgId,
|
||||
pRsp->reqOffset.version, ver);
|
||||
tqDebug("vgId:%d, failed to find handle %s in pushing data to consumer, ignore", pTq->pVnode->config.vgId,
|
||||
pPushEntry->subKey);
|
||||
continue;
|
||||
}
|
||||
|
||||
STqExecHandle* pExec = &pHandle->execHandle;
|
||||
qTaskInfo_t task = pExec->task;
|
||||
|
||||
// prepare scan mem data
|
||||
SPackedData submit = {
|
||||
.msgStr = data,
|
||||
.msgLen = len,
|
||||
.ver = ver,
|
||||
};
|
||||
if(qStreamSetScanMemData(task, submit) != 0){
|
||||
continue;
|
||||
}
|
||||
|
||||
// here start to scan submit block to extract the subscribed data
|
||||
while (1) {
|
||||
SSDataBlock* pDataBlock = NULL;
|
||||
uint64_t ts = 0;
|
||||
if (qExecTask(task, &pDataBlock, &ts) < 0) {
|
||||
tqDebug("vgId:%d, tq exec error since %s", vgId, terrstr());
|
||||
}
|
||||
|
||||
if (pDataBlock == NULL) {
|
||||
break;
|
||||
}
|
||||
|
||||
tqAddBlockDataToRsp(pDataBlock, pRsp, pExec->numOfCols, pTq->pVnode->config.tsdbCfg.precision);
|
||||
pRsp->blockNum++;
|
||||
}
|
||||
|
||||
tqDebug("vgId:%d, tq handle push, subkey:%s, block num:%d", vgId, pPushEntry->subKey, pRsp->blockNum);
|
||||
if (pRsp->blockNum > 0) {
|
||||
// set offset
|
||||
tqOffsetResetToLog(&pRsp->rspOffset, ver);
|
||||
|
||||
// remove from hash
|
||||
size_t kLen;
|
||||
void* key = taosHashGetKey(pIter, &kLen);
|
||||
void* keyCopy = taosMemoryCalloc(1, kLen + 1);
|
||||
memcpy(keyCopy, key, kLen);
|
||||
|
||||
taosArrayPush(cachedKeys, &keyCopy);
|
||||
taosArrayPush(cachedKeyLens, &kLen);
|
||||
|
||||
tqPushDataRsp(pTq, pPushEntry);
|
||||
}
|
||||
doPushDataForEntry(pIter, pExec, pTq, ver, vgId, data, len, cachedKey);
|
||||
}
|
||||
|
||||
// delete entry
|
||||
for (int32_t i = 0; i < taosArrayGetSize(cachedKeys); i++) {
|
||||
void* key = taosArrayGetP(cachedKeys, i);
|
||||
size_t kLen = *(size_t*)taosArrayGet(cachedKeyLens, i);
|
||||
if (taosHashRemove(pTq->pPushMgr, key, kLen) != 0) {
|
||||
tqError("vgId:%d, tq push hash remove key error, key: %s", pTq->pVnode->config.vgId, (char*)key);
|
||||
}
|
||||
}
|
||||
|
||||
taosArrayDestroyP(cachedKeys, (FDelete)taosMemoryFree);
|
||||
taosArrayDestroy(cachedKeyLens);
|
||||
doRemovePushedEntry(cachedKey, pTq);
|
||||
taosArrayDestroyEx(cachedKey, freeItem);
|
||||
taosMemoryFree(data);
|
||||
}
|
||||
|
||||
// unlock
|
||||
taosWUnLockLatch(&pTq->pushLock);
|
||||
taosWUnLockLatch(&pTq->lock);
|
||||
}
|
||||
|
||||
// push data for stream processing
|
||||
if (!tsDisableStream && vnodeIsRoleLeader(pTq->pVnode)) {
|
||||
if (taosHashGetSize(pTq->pStreamMeta->pTasks) == 0) return 0;
|
||||
if (taosHashGetSize(pTq->pStreamMeta->pTasks) == 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (msgType == TDMT_VND_SUBMIT) {
|
||||
void* data = taosMemoryMalloc(len);
|
||||
if (data == NULL) {
|
||||
|
|
@ -332,12 +351,9 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver)
|
|||
tqError("failed to copy data for stream since out of memory");
|
||||
return -1;
|
||||
}
|
||||
|
||||
memcpy(data, pReq, len);
|
||||
SPackedData submit = {
|
||||
.msgStr = data,
|
||||
.msgLen = len,
|
||||
.ver = ver,
|
||||
};
|
||||
SPackedData submit = {.msgStr = data, .msgLen = len, .ver = ver};
|
||||
|
||||
tqDebug("tq copy write msg %p %d %" PRId64 " from %p", data, len, ver, pReq);
|
||||
tqProcessSubmitReq(pTq, submit);
|
||||
|
|
@ -351,8 +367,15 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver)
|
|||
return 0;
|
||||
}
|
||||
|
||||
int32_t tqRegisterPushEntry(STQ* pTq, void* pHandle, const SMqPollReq* pRequest, SRpcMsg* pRpcMsg,
|
||||
SMqDataRsp* pDataRsp, int32_t type) {
|
||||
void recordPushedEntry(SArray* cachedKey, void* pIter) {
|
||||
size_t kLen = 0;
|
||||
void* key = taosHashGetKey(pIter, &kLen);
|
||||
SItem item = {.pKey = strndup(key, kLen), .keyLen = kLen};
|
||||
taosArrayPush(cachedKey, &item);
|
||||
}
|
||||
|
||||
int32_t tqRegisterPushEntry(STQ* pTq, void* pHandle, const SMqPollReq* pRequest, SRpcMsg* pRpcMsg, SMqDataRsp* pDataRsp,
|
||||
int32_t type) {
|
||||
uint64_t consumerId = pRequest->consumerId;
|
||||
int32_t vgId = TD_VID(pTq->pVnode);
|
||||
STqHandle* pTqHandle = pHandle;
|
||||
|
|
@ -369,7 +392,7 @@ int32_t tqRegisterPushEntry(STQ* pTq, void* pHandle, const SMqPollReq* pRequest,
|
|||
memcpy(pPushEntry->subKey, pTqHandle->subKey, TSDB_SUBSCRIBE_KEY_LEN);
|
||||
|
||||
if (type == TMQ_MSG_TYPE__TAOSX_RSP) {
|
||||
pPushEntry->pDataRsp = taosMemoryCalloc(1, sizeof(STaosxRsp));
|
||||
pPushEntry->pDataRsp = taosMemoryCalloc(1, sizeof(STaosxRsp));
|
||||
memcpy(pPushEntry->pDataRsp, pDataRsp, sizeof(STaosxRsp));
|
||||
} else if (type == TMQ_MSG_TYPE__POLL_RSP) {
|
||||
pPushEntry->pDataRsp = taosMemoryCalloc(1, sizeof(SMqDataRsp));
|
||||
|
|
@ -383,13 +406,13 @@ int32_t tqRegisterPushEntry(STQ* pTq, void* pHandle, const SMqPollReq* pRequest,
|
|||
|
||||
taosHashPut(pTq->pPushMgr, pTqHandle->subKey, strlen(pTqHandle->subKey), &pPushEntry, sizeof(void*));
|
||||
|
||||
tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s offset:%" PRId64 ", vgId:%d save handle to push mgr, total:%d", consumerId,
|
||||
pTqHandle->subKey, pDataRsp->reqOffset.version, vgId, taosHashGetSize(pTq->pPushMgr));
|
||||
tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s offset:%" PRId64 ", vgId:%d save handle to push mgr, total:%d",
|
||||
consumerId, pTqHandle->subKey, pDataRsp->reqOffset.version, vgId, taosHashGetSize(pTq->pPushMgr));
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tqRemovePushEntry(STQ* pTq, const char* pKey, int32_t keyLen, uint64_t consumerId, bool rspConsumer) {
|
||||
int32_t vgId = TD_VID(pTq->pVnode);
|
||||
int32_t tqUnregisterPushEntry(STQ* pTq, const char* pKey, int32_t keyLen, uint64_t consumerId, bool rspConsumer) {
|
||||
int32_t vgId = TD_VID(pTq->pVnode);
|
||||
STqPushEntry** pEntry = taosHashGet(pTq->pPushMgr, pKey, keyLen);
|
||||
|
||||
if (pEntry != NULL) {
|
||||
|
|
@ -399,7 +422,7 @@ int32_t tqRemovePushEntry(STQ* pTq, const char* pKey, int32_t keyLen, uint64_t c
|
|||
tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s vgId:%d remove from push mgr, remains:%d", consumerId,
|
||||
(*pEntry)->subKey, vgId, taosHashGetSize(pTq->pPushMgr) - 1);
|
||||
|
||||
if (rspConsumer) { // rsp the old consumer with empty block.
|
||||
if (rspConsumer) { // rsp the old consumer with empty block.
|
||||
tqPushDataRsp(pTq, *pEntry);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -340,7 +340,7 @@ int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) {
|
|||
continue;
|
||||
}
|
||||
ret->fetchType = FETCH_TYPE__DATA;
|
||||
tqDebug("return data rows %d", ret->data.info.rows);
|
||||
tqDebug("return data rows %" PRId64, ret->data.info.rows);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -18,7 +18,9 @@
|
|||
int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision) {
|
||||
int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);
|
||||
void* buf = taosMemoryCalloc(1, dataStrLen);
|
||||
if (buf == NULL) return -1;
|
||||
if (buf == NULL) {
|
||||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
|
||||
SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)buf;
|
||||
pRetrieve->useconds = 0;
|
||||
|
|
@ -31,7 +33,8 @@ int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t
|
|||
actualLen += sizeof(SRetrieveTableRsp);
|
||||
taosArrayPush(pRsp->blockDataLen, &actualLen);
|
||||
taosArrayPush(pRsp->blockData, &buf);
|
||||
return 0;
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, STaosxRsp* pRsp) {
|
||||
|
|
@ -62,69 +65,84 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, STaosxRsp* pRsp, in
|
|||
}
|
||||
|
||||
int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset) {
|
||||
const STqExecHandle* pExec = &pHandle->execHandle;
|
||||
const int32_t MAX_ROWS_TO_RETURN = 4096;
|
||||
int32_t vgId = TD_VID(pTq->pVnode);
|
||||
int32_t code = 0;
|
||||
int32_t totalRows = 0;
|
||||
|
||||
qTaskInfo_t task = pExec->task;
|
||||
const STqExecHandle* pExec = &pHandle->execHandle;
|
||||
qTaskInfo_t task = pExec->task;
|
||||
|
||||
if (qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType) < 0) {
|
||||
tqDebug("prepare scan failed, return");
|
||||
tqDebug("prepare scan failed, vgId:%d, consumer:0x%" PRIx64, vgId, pHandle->consumerId);
|
||||
if (pOffset->type == TMQ_OFFSET__LOG) {
|
||||
pRsp->rspOffset = *pOffset;
|
||||
return 0;
|
||||
return code;
|
||||
} else {
|
||||
tqOffsetResetToLog(pOffset, pHandle->snapshotVer);
|
||||
if (qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType) < 0) {
|
||||
tqDebug("prepare scan failed, return");
|
||||
tqDebug("prepare scan failed, vgId:%d, consumer:0x%" PRIx64, vgId, pHandle->consumerId);
|
||||
pRsp->rspOffset = *pOffset;
|
||||
return 0;
|
||||
return code;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int32_t rowCnt = 0;
|
||||
while (1) {
|
||||
SSDataBlock* pDataBlock = NULL;
|
||||
uint64_t ts = 0;
|
||||
|
||||
tqDebug("vgId:%d, tmq task start to execute", pTq->pVnode->config.vgId);
|
||||
if (qExecTask(task, &pDataBlock, &ts) < 0) {
|
||||
tqError("vgId:%d, task exec error since %s", pTq->pVnode->config.vgId, terrstr());
|
||||
return -1;
|
||||
tqDebug("vgId:%d, tmq task start to execute, consumer:0x%" PRIx64, vgId, pHandle->consumerId);
|
||||
|
||||
code = qExecTask(task, &pDataBlock, &ts);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
tqError("vgId:%d, task exec error since %s, consumer:0x%" PRIx64, vgId, terrstr(), pHandle->consumerId);
|
||||
return code;
|
||||
}
|
||||
|
||||
tqDebug("consumer:0x%"PRIx64" vgId:%d, tmq task executed, get %p", pHandle->consumerId, pTq->pVnode->config.vgId, pDataBlock);
|
||||
|
||||
// current scan should be stopped asap, since the rebalance occurs.
|
||||
// current scan should be stopped ASAP, since the re-balance occurs.
|
||||
if (pDataBlock == NULL) {
|
||||
break;
|
||||
}
|
||||
|
||||
tqAddBlockDataToRsp(pDataBlock, pRsp, pExec->numOfCols, pTq->pVnode->config.tsdbCfg.precision);
|
||||
code = tqAddBlockDataToRsp(pDataBlock, pRsp, pExec->numOfCols, pTq->pVnode->config.tsdbCfg.precision);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
tqError("vgId:%d, failed to add block to rsp msg", vgId);
|
||||
return code;
|
||||
}
|
||||
|
||||
pRsp->blockNum++;
|
||||
|
||||
tqDebug("vgId:%d, consumer:0x%" PRIx64 " tmq task executed, rows:%" PRId64 ", total blocks:%d", vgId,
|
||||
pHandle->consumerId, pDataBlock->info.rows, pRsp->blockNum);
|
||||
|
||||
if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) {
|
||||
rowCnt += pDataBlock->info.rows;
|
||||
if (rowCnt >= 4096) {
|
||||
totalRows += pDataBlock->info.rows;
|
||||
if (totalRows >= MAX_ROWS_TO_RETURN) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (qStreamExtractOffset(task, &pRsp->rspOffset) < 0) {
|
||||
return -1;
|
||||
}
|
||||
qStreamExtractOffset(task, &pRsp->rspOffset);
|
||||
|
||||
if (pRsp->rspOffset.type == 0) {
|
||||
tqError("expected rsp offset: type %d %" PRId64 " %" PRId64 " %" PRId64, pRsp->rspOffset.type, pRsp->rspOffset.ts,
|
||||
pRsp->rspOffset.uid, pRsp->rspOffset.version);
|
||||
return -1;
|
||||
code = TSDB_CODE_INVALID_PARA;
|
||||
tqError("vgId:%d, expected rsp offset: type %d %" PRId64 " %" PRId64 " %" PRId64, vgId, pRsp->rspOffset.type,
|
||||
pRsp->rspOffset.ts, pRsp->rspOffset.uid, pRsp->rspOffset.version);
|
||||
return code;
|
||||
}
|
||||
|
||||
if (pRsp->withTbName || pRsp->withSchema) {
|
||||
tqError("get column should not with meta:%d,%d", pRsp->withTbName, pRsp->withSchema);
|
||||
return -1;
|
||||
code = TSDB_CODE_INVALID_PARA;
|
||||
tqError("vgId:%d, get column should not with meta:%d,%d", vgId, pRsp->withTbName, pRsp->withSchema);
|
||||
return code;
|
||||
}
|
||||
return 0;
|
||||
|
||||
tqDebug("vgId:%d, consumer:0x%" PRIx64 " tmq task executed, total blocks:%d, rows:%d", vgId, pHandle->consumerId,
|
||||
pRsp->blockNum, totalRows);
|
||||
|
||||
return code;
|
||||
}
|
||||
|
||||
int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* pOffset) {
|
||||
|
|
@ -230,23 +248,15 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta
|
|||
return 0;
|
||||
}
|
||||
|
||||
int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp) {
|
||||
int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp, int32_t* totalRows) {
|
||||
STqExecHandle* pExec = &pHandle->execHandle;
|
||||
/*A(pExec->subType != TOPIC_SUB_TYPE__COLUMN);*/
|
||||
|
||||
SArray* pBlocks = taosArrayInit(0, sizeof(SSDataBlock));
|
||||
SArray* pSchemas = taosArrayInit(0, sizeof(void*));
|
||||
SArray* pBlocks = taosArrayInit(0, sizeof(SSDataBlock));
|
||||
SArray* pSchemas = taosArrayInit(0, sizeof(void*));
|
||||
|
||||
if (pExec->subType == TOPIC_SUB_TYPE__TABLE) {
|
||||
STqReader* pReader = pExec->pExecReader;
|
||||
/*tqReaderSetDataMsg(pReader, pReq, 0);*/
|
||||
tqReaderSetSubmitReq2(pReader, submit.msgStr, submit.msgLen, submit.ver);
|
||||
while (tqNextDataBlock2(pReader)) {
|
||||
/*SSDataBlock block = {0};*/
|
||||
/*if (tqRetrieveDataBlock(&block, pReader) < 0) {*/
|
||||
/*if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue;*/
|
||||
/*}*/
|
||||
|
||||
taosArrayClear(pBlocks);
|
||||
taosArrayClear(pSchemas);
|
||||
SSubmitTbData* pSubmitTbDataRet = NULL;
|
||||
|
|
@ -254,7 +264,6 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR
|
|||
if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue;
|
||||
}
|
||||
if (pRsp->withTbName) {
|
||||
/*int64_t uid = pExec->pExecReader->msgIter.uid;*/
|
||||
int64_t uid = pExec->pExecReader->lastBlkUid;
|
||||
if (tqAddTbNameToRsp(pTq, uid, pRsp, taosArrayGetSize(pBlocks)) < 0) {
|
||||
taosArrayDestroyEx(pBlocks, (FDelete)blockDataFreeRes);
|
||||
|
|
@ -276,7 +285,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR
|
|||
if (TSDB_CODE_SUCCESS != code) {
|
||||
continue;
|
||||
}
|
||||
void* createReq = taosMemoryCalloc(1, len);
|
||||
void* createReq = taosMemoryCalloc(1, len);
|
||||
SEncoder encoder = {0};
|
||||
tEncoderInit(&encoder, createReq, len);
|
||||
code = tEncodeSVCreateTbReq(&encoder, pSubmitTbDataRet->pCreateTbReq);
|
||||
|
|
@ -296,6 +305,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR
|
|||
SSDataBlock* pBlock = taosArrayGet(pBlocks, i);
|
||||
tqAddBlockDataToRsp(pBlock, (SMqDataRsp*)pRsp, taosArrayGetSize(pBlock->pDataBlock),
|
||||
pTq->pVnode->config.tsdbCfg.precision);
|
||||
totalRows += pBlock->info.rows;
|
||||
blockDataFreeRes(pBlock);
|
||||
SSchemaWrapper* pSW = taosArrayGetP(pSchemas, i);
|
||||
taosArrayPush(pRsp->blockSchema, &pSW);
|
||||
|
|
@ -304,13 +314,8 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR
|
|||
}
|
||||
} else if (pExec->subType == TOPIC_SUB_TYPE__DB) {
|
||||
STqReader* pReader = pExec->pExecReader;
|
||||
/*tqReaderSetDataMsg(pReader, pReq, 0);*/
|
||||
tqReaderSetSubmitReq2(pReader, submit.msgStr, submit.msgLen, submit.ver);
|
||||
while (tqNextDataBlockFilterOut2(pReader, pExec->execDb.pFilterOutTbUid)) {
|
||||
/*SSDataBlock block = {0};*/
|
||||
/*if (tqRetrieveDataBlock(&block, pReader) < 0) {*/
|
||||
/*if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue;*/
|
||||
/*}*/
|
||||
taosArrayClear(pBlocks);
|
||||
taosArrayClear(pSchemas);
|
||||
SSubmitTbData* pSubmitTbDataRet = NULL;
|
||||
|
|
@ -339,7 +344,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR
|
|||
if (TSDB_CODE_SUCCESS != code) {
|
||||
continue;
|
||||
}
|
||||
void* createReq = taosMemoryCalloc(1, len);
|
||||
void* createReq = taosMemoryCalloc(1, len);
|
||||
SEncoder encoder = {0};
|
||||
tEncoderInit(&encoder, createReq, len);
|
||||
code = tEncodeSVCreateTbReq(&encoder, pSubmitTbDataRet->pCreateTbReq);
|
||||
|
|
@ -355,15 +360,11 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR
|
|||
|
||||
tEncoderClear(&encoder);
|
||||
}
|
||||
/*tqAddBlockDataToRsp(&block, (SMqDataRsp*)pRsp, taosArrayGetSize(block.pDataBlock),*/
|
||||
/*pTq->pVnode->config.tsdbCfg.precision);*/
|
||||
/*blockDataFreeRes(&block);*/
|
||||
/*tqAddBlockSchemaToRsp(pExec, (SMqDataRsp*)pRsp);*/
|
||||
/*pRsp->blockNum++;*/
|
||||
for (int32_t i = 0; i < taosArrayGetSize(pBlocks); i++) {
|
||||
SSDataBlock* pBlock = taosArrayGet(pBlocks, i);
|
||||
tqAddBlockDataToRsp(pBlock, (SMqDataRsp*)pRsp, taosArrayGetSize(pBlock->pDataBlock),
|
||||
pTq->pVnode->config.tsdbCfg.precision);
|
||||
*totalRows += pBlock->info.rows;
|
||||
blockDataFreeRes(pBlock);
|
||||
SSchemaWrapper* pSW = taosArrayGetP(pSchemas, i);
|
||||
taosArrayPush(pRsp->blockSchema, &pSW);
|
||||
|
|
@ -373,9 +374,5 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR
|
|||
}
|
||||
taosArrayDestroy(pBlocks);
|
||||
taosArrayDestroy(pSchemas);
|
||||
// if (pRsp->blockNum == 0) {
|
||||
// return -1;
|
||||
// }
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -590,6 +590,7 @@ typedef struct {
|
|||
SDataFReader **pDataFReader;
|
||||
TSDBROW row;
|
||||
|
||||
bool checkRemainingRow;
|
||||
SMergeTree mergeTree;
|
||||
SMergeTree *pMergeTree;
|
||||
SSttBlockLoadInfo *pLoadInfo;
|
||||
|
|
@ -633,12 +634,25 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa
|
|||
if (code) goto _err;
|
||||
}
|
||||
|
||||
state->pLoadInfo->colIds = aCols;
|
||||
state->pLoadInfo->numOfCols = nCols;
|
||||
for (int i = 0; i < state->pLoadInfo->numOfStt; ++i) {
|
||||
state->pLoadInfo[i].colIds = aCols;
|
||||
state->pLoadInfo[i].numOfCols = nCols;
|
||||
state->pLoadInfo[i].isLast = isLast;
|
||||
}
|
||||
tMergeTreeOpen(&state->mergeTree, 1, *state->pDataFReader, state->suid, state->uid,
|
||||
&(STimeWindow){.skey = state->lastTs, .ekey = TSKEY_MAX},
|
||||
&(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, state->pLoadInfo, false, NULL, true);
|
||||
state->pMergeTree = &state->mergeTree;
|
||||
state->state = SFSLASTNEXTROW_BLOCKROW;
|
||||
}
|
||||
case SFSLASTNEXTROW_BLOCKROW: {
|
||||
if (nCols != state->pLoadInfo->numOfCols) {
|
||||
for (int i = 0; i < state->pLoadInfo->numOfStt; ++i) {
|
||||
state->pLoadInfo[i].numOfCols = nCols;
|
||||
|
||||
state->pLoadInfo[i].checkRemainingRow = state->checkRemainingRow;
|
||||
}
|
||||
}
|
||||
bool hasVal = tMergeTreeNext(&state->mergeTree);
|
||||
if (!hasVal) {
|
||||
if (tMergeTreeIgnoreEarlierTs(&state->mergeTree)) {
|
||||
|
|
@ -649,76 +663,23 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa
|
|||
state->state = SFSLASTNEXTROW_FILESET;
|
||||
goto _next_fileset;
|
||||
}
|
||||
state->state = SFSLASTNEXTROW_BLOCKROW;
|
||||
checkRemainingRow = false;
|
||||
}
|
||||
case SFSLASTNEXTROW_BLOCKROW: {
|
||||
bool skipRow = false;
|
||||
do {
|
||||
bool hasVal = false;
|
||||
state->row = tMergeTreeGetRow(&state->mergeTree);
|
||||
*ppRow = &state->row;
|
||||
if (nCols != state->pLoadInfo->numOfCols) {
|
||||
state->pLoadInfo->numOfCols = nCols;
|
||||
}
|
||||
hasVal = tMergeTreeNext(&state->mergeTree);
|
||||
if (TSDBROW_TS(&state->row) <= state->lastTs) {
|
||||
*pIgnoreEarlierTs = true;
|
||||
*ppRow = NULL;
|
||||
return code;
|
||||
}
|
||||
state->row = tMergeTreeGetRow(&state->mergeTree);
|
||||
*ppRow = &state->row;
|
||||
|
||||
*pIgnoreEarlierTs = false;
|
||||
if (!hasVal) {
|
||||
state->state = SFSLASTNEXTROW_FILESET;
|
||||
break;
|
||||
}
|
||||
if (TSDBROW_TS(&state->row) <= state->lastTs) {
|
||||
*pIgnoreEarlierTs = true;
|
||||
*ppRow = NULL;
|
||||
return code;
|
||||
}
|
||||
|
||||
if (checkRemainingRow) {
|
||||
bool skipBlock = true;
|
||||
|
||||
SBlockData *pBlockData = state->row.pBlockData;
|
||||
|
||||
for (int inputColIndex = 0; inputColIndex < nCols; ++inputColIndex) {
|
||||
for (int colIndex = 0; colIndex < pBlockData->nColData; ++colIndex) {
|
||||
SColData *pColData = &pBlockData->aColData[colIndex];
|
||||
int16_t cid = pColData->cid;
|
||||
|
||||
if (cid == aCols[inputColIndex]) {
|
||||
if (isLast && (pColData->flag & HAS_VALUE)) {
|
||||
skipBlock = false;
|
||||
break;
|
||||
} else if (pColData->flag & (HAS_VALUE | HAS_NULL)) {
|
||||
skipBlock = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
/*
|
||||
for (int colIndex = 0; colIndex < pBlockData->nColData; ++colIndex) {
|
||||
SColData *pColData = &pBlockData->aColData[colIndex];
|
||||
int16_t cid = pColData->cid;
|
||||
|
||||
if (inputColIndex < nCols && cid == aCols[inputColIndex]) {
|
||||
if (isLast && (pColData->flag & HAS_VALUE)) {
|
||||
skipBlock = false;
|
||||
break;
|
||||
} else if (pColData->flag & (HAS_VALUE | HAS_NULL)) {
|
||||
skipBlock = false;
|
||||
break;
|
||||
}
|
||||
|
||||
++inputColIndex;
|
||||
}
|
||||
}
|
||||
*/
|
||||
if (skipBlock) {
|
||||
skipRow = true;
|
||||
}
|
||||
}
|
||||
} while (skipRow);
|
||||
*pIgnoreEarlierTs = false;
|
||||
if (!hasVal) {
|
||||
state->state = SFSLASTNEXTROW_FILESET;
|
||||
}
|
||||
|
||||
if (!state->checkRemainingRow) {
|
||||
state->checkRemainingRow = true;
|
||||
}
|
||||
return code;
|
||||
}
|
||||
default:
|
||||
|
|
|
|||
|
|
@ -282,6 +282,40 @@ bool tsdbTbDataIterNext(STbDataIter *pIter) {
|
|||
return true;
|
||||
}
|
||||
|
||||
int64_t tsdbCountTbDataRows(STbData *pTbData) {
|
||||
SMemSkipListNode *pNode = pTbData->sl.pHead;
|
||||
int64_t rowsNum = 0;
|
||||
|
||||
while (NULL != pNode) {
|
||||
pNode = SL_GET_NODE_FORWARD(pNode, 0);
|
||||
if (pNode == pTbData->sl.pTail) {
|
||||
return rowsNum;
|
||||
}
|
||||
|
||||
rowsNum++;
|
||||
}
|
||||
|
||||
return rowsNum;
|
||||
}
|
||||
|
||||
void tsdbMemTableCountRows(SMemTable *pMemTable, SHashObj* pTableMap, int64_t *rowsNum) {
|
||||
taosRLockLatch(&pMemTable->latch);
|
||||
for (int32_t i = 0; i < pMemTable->nBucket; ++i) {
|
||||
STbData *pTbData = pMemTable->aBucket[i];
|
||||
while (pTbData) {
|
||||
void* p = taosHashGet(pTableMap, &pTbData->uid, sizeof(pTbData->uid));
|
||||
if (p == NULL) {
|
||||
pTbData = pTbData->next;
|
||||
continue;
|
||||
}
|
||||
|
||||
*rowsNum += tsdbCountTbDataRows(pTbData);
|
||||
pTbData = pTbData->next;
|
||||
}
|
||||
}
|
||||
taosRUnLockLatch(&pMemTable->latch);
|
||||
}
|
||||
|
||||
static int32_t tsdbMemTableRehash(SMemTable *pMemTable) {
|
||||
int32_t code = 0;
|
||||
|
||||
|
|
|
|||
|
|
@ -504,9 +504,34 @@ bool tLDataIterNextRow(SLDataIter *pIter, const char *idStr) {
|
|||
pIter->iRow += step;
|
||||
|
||||
while (1) {
|
||||
bool skipBlock = false;
|
||||
|
||||
findNextValidRow(pIter, idStr);
|
||||
|
||||
if (pIter->iRow >= pBlockData->nRow || pIter->iRow < 0) {
|
||||
if (pIter->pBlockLoadInfo->checkRemainingRow) {
|
||||
skipBlock = true;
|
||||
int16_t *aCols = pIter->pBlockLoadInfo->colIds;
|
||||
int nCols = pIter->pBlockLoadInfo->numOfCols;
|
||||
bool isLast = pIter->pBlockLoadInfo->isLast;
|
||||
for (int inputColIndex = 0; inputColIndex < nCols; ++inputColIndex) {
|
||||
for (int colIndex = 0; colIndex < pBlockData->nColData; ++colIndex) {
|
||||
SColData *pColData = &pBlockData->aColData[colIndex];
|
||||
int16_t cid = pColData->cid;
|
||||
|
||||
if (cid == aCols[inputColIndex]) {
|
||||
if (isLast && (pColData->flag & HAS_VALUE)) {
|
||||
skipBlock = false;
|
||||
break;
|
||||
} else if (pColData->flag & (HAS_VALUE | HAS_NULL)) {
|
||||
skipBlock = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (skipBlock || pIter->iRow >= pBlockData->nRow || pIter->iRow < 0) {
|
||||
tLDataIterNextBlock(pIter, idStr);
|
||||
if (pIter->pSttBlk == NULL) { // no more data
|
||||
goto _exit;
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -156,13 +156,10 @@ int vnodeShouldCommit(SVnode *pVnode) {
|
|||
bool needCommit = false;
|
||||
|
||||
taosThreadMutexLock(&pVnode->mutex);
|
||||
if (!pVnode->inUse || !diskAvail) {
|
||||
goto _out;
|
||||
if (pVnode->inUse && diskAvail) {
|
||||
needCommit =
|
||||
((pVnode->inUse->size > pVnode->inUse->node.size) && (pSched->commitMs + SYNC_VND_COMMIT_MIN_MS < nowMs));
|
||||
}
|
||||
needCommit =
|
||||
(((pVnode->inUse->size > pVnode->inUse->node.size) && (pSched->commitMs + SYNC_VND_COMMIT_MIN_MS < nowMs)) ||
|
||||
(pVnode->inUse->size > 0 && pSched->commitMs + pSched->maxWaitMs < nowMs));
|
||||
_out:
|
||||
taosThreadMutexUnlock(&pVnode->mutex);
|
||||
return needCommit;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -335,6 +335,7 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot *
|
|||
|
||||
// commit json
|
||||
if (!rollback) {
|
||||
pWriter->info.state.committed = pWriter->ever;
|
||||
pVnode->config = pWriter->info.config;
|
||||
pVnode->state = (SVState){.committed = pWriter->info.state.committed,
|
||||
.applied = pWriter->info.state.committed,
|
||||
|
|
|
|||
|
|
@ -378,7 +378,7 @@ static int32_t vnodeSyncEqCtrlMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) {
|
|||
return -1;
|
||||
}
|
||||
|
||||
int32_t code = tmsgPutToQueue(msgcb, SYNC_CTRL_QUEUE, pMsg);
|
||||
int32_t code = tmsgPutToQueue(msgcb, SYNC_RD_QUEUE, pMsg);
|
||||
if (code != 0) {
|
||||
rpcFreeCont(pMsg->pCont);
|
||||
pMsg->pCont = NULL;
|
||||
|
|
|
|||
|
|
@ -865,19 +865,19 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName
|
|||
|
||||
vgInfo = taosArraySearch(dbInfo->vgArray, &hashValue, ctgHashValueComp, TD_EQ);
|
||||
|
||||
/*
|
||||
void* pIter = taosHashIterate(dbInfo->vgHash, NULL);
|
||||
while (pIter) {
|
||||
vgInfo = pIter;
|
||||
if (hashValue >= vgInfo->hashBegin && hashValue <= vgInfo->hashEnd) {
|
||||
taosHashCancelIterate(dbInfo->vgHash, pIter);
|
||||
break;
|
||||
}
|
||||
/*
|
||||
void* pIter = taosHashIterate(dbInfo->vgHash, NULL);
|
||||
while (pIter) {
|
||||
vgInfo = pIter;
|
||||
if (hashValue >= vgInfo->hashBegin && hashValue <= vgInfo->hashEnd) {
|
||||
taosHashCancelIterate(dbInfo->vgHash, pIter);
|
||||
break;
|
||||
}
|
||||
|
||||
pIter = taosHashIterate(dbInfo->vgHash, pIter);
|
||||
vgInfo = NULL;
|
||||
}
|
||||
*/
|
||||
pIter = taosHashIterate(dbInfo->vgHash, pIter);
|
||||
vgInfo = NULL;
|
||||
}
|
||||
*/
|
||||
|
||||
if (NULL == vgInfo) {
|
||||
ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, db,
|
||||
|
|
@ -902,7 +902,7 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo*
|
|||
|
||||
CTG_ERR_RET(ctgMakeVgArray(dbInfo));
|
||||
|
||||
int32_t vgNum = taosArrayGetSize(dbInfo->vgArray);
|
||||
int32_t vgNum = taosArrayGetSize(dbInfo->vgArray);
|
||||
if (vgNum <= 0) {
|
||||
ctgError("db vgroup cache invalid, db:%s, vgroup number:%d", dbFName, vgNum);
|
||||
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
|
||||
|
|
@ -982,43 +982,43 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo*
|
|||
CTG_RET(code);
|
||||
}
|
||||
|
||||
int32_t ctgGetVgIdsFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, char* dbFName, const char* pTbs[], int32_t tbNum, int32_t* vgId) {
|
||||
int32_t code = 0;
|
||||
CTG_ERR_RET(ctgMakeVgArray(dbInfo));
|
||||
int32_t ctgGetVgIdsFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, char* dbFName, const char* pTbs[], int32_t tbNum,
|
||||
int32_t* vgId) {
|
||||
int32_t code = 0;
|
||||
CTG_ERR_RET(ctgMakeVgArray(dbInfo));
|
||||
|
||||
int32_t vgNum = taosArrayGetSize(dbInfo->vgArray);
|
||||
int32_t vgNum = taosArrayGetSize(dbInfo->vgArray);
|
||||
|
||||
if (vgNum <= 0) {
|
||||
ctgError("db vgroup cache invalid, db:%s, vgroup number:%d", dbFName, vgNum);
|
||||
CTG_ERR_RET(TSDB_CODE_TSC_DB_NOT_SELECTED);
|
||||
}
|
||||
if (vgNum <= 0) {
|
||||
ctgError("db vgroup cache invalid, db:%s, vgroup number:%d", dbFName, vgNum);
|
||||
CTG_ERR_RET(TSDB_CODE_TSC_DB_NOT_SELECTED);
|
||||
}
|
||||
|
||||
SVgroupInfo* vgInfo = NULL;
|
||||
char tbFullName[TSDB_TABLE_FNAME_LEN];
|
||||
snprintf(tbFullName, sizeof(tbFullName), "%s.", dbFName);
|
||||
int32_t offset = strlen(tbFullName);
|
||||
SVgroupInfo* vgInfo = NULL;
|
||||
char tbFullName[TSDB_TABLE_FNAME_LEN];
|
||||
snprintf(tbFullName, sizeof(tbFullName), "%s.", dbFName);
|
||||
int32_t offset = strlen(tbFullName);
|
||||
|
||||
for (int32_t i = 0; i < tbNum; ++i) {
|
||||
snprintf(tbFullName + offset, sizeof(tbFullName) - offset, "%s", pTbs[i]);
|
||||
uint32_t hashValue = taosGetTbHashVal(tbFullName, (uint32_t)strlen(tbFullName), dbInfo->hashMethod,
|
||||
dbInfo->hashPrefix, dbInfo->hashSuffix);
|
||||
for (int32_t i = 0; i < tbNum; ++i) {
|
||||
snprintf(tbFullName + offset, sizeof(tbFullName) - offset, "%s", pTbs[i]);
|
||||
uint32_t hashValue = taosGetTbHashVal(tbFullName, (uint32_t)strlen(tbFullName), dbInfo->hashMethod,
|
||||
dbInfo->hashPrefix, dbInfo->hashSuffix);
|
||||
|
||||
vgInfo = taosArraySearch(dbInfo->vgArray, &hashValue, ctgHashValueComp, TD_EQ);
|
||||
if (NULL == vgInfo) {
|
||||
ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, dbFName,
|
||||
(int32_t)taosArrayGetSize(dbInfo->vgArray));
|
||||
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
|
||||
}
|
||||
vgInfo = taosArraySearch(dbInfo->vgArray, &hashValue, ctgHashValueComp, TD_EQ);
|
||||
if (NULL == vgInfo) {
|
||||
ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, dbFName,
|
||||
(int32_t)taosArrayGetSize(dbInfo->vgArray));
|
||||
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
|
||||
}
|
||||
|
||||
vgId[i] = vgInfo->vgId;
|
||||
vgId[i] = vgInfo->vgId;
|
||||
|
||||
ctgDebug("Got tb %s vgId:%d", tbFullName, vgInfo->vgId);
|
||||
}
|
||||
ctgDebug("Got tb %s vgId:%d", tbFullName, vgInfo->vgId);
|
||||
}
|
||||
|
||||
CTG_RET(code);
|
||||
CTG_RET(code);
|
||||
}
|
||||
|
||||
|
||||
int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2) {
|
||||
if (*(uint64_t*)key1 < ((SSTableVersion*)key2)->suid) {
|
||||
return -1;
|
||||
|
|
@ -1070,7 +1070,7 @@ int32_t ctgMakeVgArray(SDBVgInfo* dbInfo) {
|
|||
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
|
||||
}
|
||||
|
||||
void* pIter = taosHashIterate(dbInfo->vgHash, NULL);
|
||||
void* pIter = taosHashIterate(dbInfo->vgHash, NULL);
|
||||
while (pIter) {
|
||||
taosArrayPush(dbInfo->vgArray, pIter);
|
||||
pIter = taosHashIterate(dbInfo->vgHash, pIter);
|
||||
|
|
@ -1082,7 +1082,6 @@ int32_t ctgMakeVgArray(SDBVgInfo* dbInfo) {
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
int32_t ctgCloneVgInfo(SDBVgInfo* src, SDBVgInfo** dst) {
|
||||
CTG_ERR_RET(ctgMakeVgArray(src));
|
||||
|
||||
|
|
@ -1429,10 +1428,10 @@ void ctgGetClusterCacheStat(SCatalog* pCtg) {
|
|||
pCtg->cacheStat.cacheNum[i] = 0;
|
||||
}
|
||||
|
||||
SCtgDBCache *dbCache = NULL;
|
||||
void *pIter = taosHashIterate(pCtg->dbCache, NULL);
|
||||
SCtgDBCache* dbCache = NULL;
|
||||
void* pIter = taosHashIterate(pCtg->dbCache, NULL);
|
||||
while (pIter) {
|
||||
dbCache = (SCtgDBCache *)pIter;
|
||||
dbCache = (SCtgDBCache*)pIter;
|
||||
|
||||
for (int32_t i = 0; i < CTG_CI_MAX_VALUE; ++i) {
|
||||
if (0 == (gCtgStatItem[i].flag & CTG_CI_FLAG_LEVEL_DB)) {
|
||||
|
|
@ -1446,7 +1445,7 @@ void ctgGetClusterCacheStat(SCatalog* pCtg) {
|
|||
}
|
||||
}
|
||||
|
||||
void ctgSummaryClusterCacheStat(SCatalog *pCtg) {
|
||||
void ctgSummaryClusterCacheStat(SCatalog* pCtg) {
|
||||
for (int32_t i = 0; i < CTG_CI_MAX_VALUE; ++i) {
|
||||
if (gCtgStatItem[i].flag & CTG_CI_FLAG_LEVEL_GLOBAL) {
|
||||
continue;
|
||||
|
|
@ -1458,7 +1457,7 @@ void ctgSummaryClusterCacheStat(SCatalog *pCtg) {
|
|||
}
|
||||
}
|
||||
|
||||
void ctgGetGlobalCacheStat(SCtgCacheStat *pStat) {
|
||||
void ctgGetGlobalCacheStat(SCtgCacheStat* pStat) {
|
||||
for (int32_t i = 0; i < CTG_CI_MAX_VALUE; ++i) {
|
||||
if (gCtgStatItem[i].flag & CTG_CI_FLAG_LEVEL_GLOBAL) {
|
||||
continue;
|
||||
|
|
@ -1469,10 +1468,10 @@ void ctgGetGlobalCacheStat(SCtgCacheStat *pStat) {
|
|||
gCtgMgmt.statInfo.cache.cacheNHit[i] = 0;
|
||||
}
|
||||
|
||||
SCatalog *pCtg = NULL;
|
||||
void *pIter = taosHashIterate(gCtgMgmt.pCluster, NULL);
|
||||
SCatalog* pCtg = NULL;
|
||||
void* pIter = taosHashIterate(gCtgMgmt.pCluster, NULL);
|
||||
while (pIter) {
|
||||
pCtg = *(SCatalog **)pIter;
|
||||
pCtg = *(SCatalog**)pIter;
|
||||
|
||||
if (pCtg) {
|
||||
ctgGetClusterCacheStat(pCtg);
|
||||
|
|
@ -1484,4 +1483,3 @@ void ctgGetGlobalCacheStat(SCtgCacheStat *pStat) {
|
|||
|
||||
memcpy(pStat, &gCtgMgmt.statInfo.cache, sizeof(gCtgMgmt.statInfo.cache));
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -107,6 +107,7 @@ int32_t tableListGetGroupList(const STableListInfo* pTableList, int32_t
|
|||
uint64_t tableListGetSize(const STableListInfo* pTableList);
|
||||
uint64_t tableListGetSuid(const STableListInfo* pTableList);
|
||||
STableKeyInfo* tableListGetInfo(const STableListInfo* pTableList, int32_t index);
|
||||
int32_t tableListFind(const STableListInfo* pTableList, uint64_t uid, int32_t startIndex);
|
||||
|
||||
size_t getResultRowSize(struct SqlFunctionCtx* pCtx, int32_t numOfOutput);
|
||||
void initResultRowInfo(SResultRowInfo* pResultRowInfo);
|
||||
|
|
|
|||
|
|
@ -143,10 +143,7 @@ typedef struct {
|
|||
SQueryTableDataCond tableCond;
|
||||
int64_t fillHistoryVer1;
|
||||
int64_t fillHistoryVer2;
|
||||
|
||||
// int8_t triggerSaved;
|
||||
// int64_t deleteMarkSaved;
|
||||
SStreamState* pState;
|
||||
SStreamState* pState;
|
||||
} SStreamTaskInfo;
|
||||
|
||||
typedef struct {
|
||||
|
|
@ -168,26 +165,25 @@ typedef struct STaskStopInfo {
|
|||
} STaskStopInfo;
|
||||
|
||||
struct SExecTaskInfo {
|
||||
STaskIdInfo id;
|
||||
uint32_t status;
|
||||
STimeWindow window;
|
||||
STaskCostInfo cost;
|
||||
int64_t owner; // if it is in execution
|
||||
int32_t code;
|
||||
int32_t qbufQuota; // total available buffer (in KB) during execution query
|
||||
|
||||
int64_t version; // used for stream to record wal version, why not move to sschemainfo
|
||||
STaskIdInfo id;
|
||||
uint32_t status;
|
||||
STimeWindow window;
|
||||
STaskCostInfo cost;
|
||||
int64_t owner; // if it is in execution
|
||||
int32_t code;
|
||||
int32_t qbufQuota; // total available buffer (in KB) during execution query
|
||||
int64_t version; // used for stream to record wal version, why not move to sschemainfo
|
||||
SStreamTaskInfo streamInfo;
|
||||
SSchemaInfo schemaInfo;
|
||||
STableListInfo* pTableInfoList; // this is a table list
|
||||
const char* sql; // query sql string
|
||||
jmp_buf env; // jump to this position when error happens.
|
||||
EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model]
|
||||
const char* sql; // query sql string
|
||||
jmp_buf env; // jump to this position when error happens.
|
||||
EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model]
|
||||
SSubplan* pSubplan;
|
||||
struct SOperatorInfo* pRoot;
|
||||
SLocalFetch localFetch;
|
||||
SArray* pResultBlockList; // result block list
|
||||
STaskStopInfo stopInfo;
|
||||
SRWLatch lock; // secure the access of STableListInfo
|
||||
};
|
||||
|
||||
enum {
|
||||
|
|
@ -326,6 +322,8 @@ typedef struct STableScanBase {
|
|||
int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan
|
||||
int32_t dataBlockLoadFlag;
|
||||
SLimitInfo limitInfo;
|
||||
// there are more than one table list exists in one task, if only one vnode exists.
|
||||
STableListInfo* pTableListInfo;
|
||||
} STableScanBase;
|
||||
|
||||
typedef struct STableScanInfo {
|
||||
|
|
@ -339,6 +337,7 @@ typedef struct STableScanInfo {
|
|||
int8_t scanMode;
|
||||
int8_t assignBlockUid;
|
||||
bool hasGroupByTag;
|
||||
bool countOnly;
|
||||
} STableScanInfo;
|
||||
|
||||
typedef struct STableMergeScanInfo {
|
||||
|
|
@ -365,11 +364,12 @@ typedef struct STableMergeScanInfo {
|
|||
} STableMergeScanInfo;
|
||||
|
||||
typedef struct STagScanInfo {
|
||||
SColumnInfo* pCols;
|
||||
SSDataBlock* pRes;
|
||||
SColMatchInfo matchInfo;
|
||||
int32_t curPos;
|
||||
SReadHandle readHandle;
|
||||
SColumnInfo* pCols;
|
||||
SSDataBlock* pRes;
|
||||
SColMatchInfo matchInfo;
|
||||
int32_t curPos;
|
||||
SReadHandle readHandle;
|
||||
STableListInfo* pTableListInfo;
|
||||
} STagScanInfo;
|
||||
|
||||
typedef enum EStreamScanMode {
|
||||
|
|
@ -485,16 +485,11 @@ typedef struct SStreamScanInfo {
|
|||
} SStreamScanInfo;
|
||||
|
||||
typedef struct {
|
||||
// int8_t subType;
|
||||
// bool withMeta;
|
||||
// int64_t suid;
|
||||
// int64_t snapVersion;
|
||||
// void *metaInfo;
|
||||
// void *dataInfo;
|
||||
SVnode* vnode;
|
||||
SSDataBlock pRes; // result SSDataBlock
|
||||
STsdbReader* dataReader;
|
||||
SSnapContext* sContext;
|
||||
SVnode* vnode;
|
||||
SSDataBlock pRes; // result SSDataBlock
|
||||
STsdbReader* dataReader;
|
||||
SSnapContext* sContext;
|
||||
STableListInfo* pTableListInfo;
|
||||
} SStreamRawScanInfo;
|
||||
|
||||
typedef struct STableCountScanSupp {
|
||||
|
|
@ -523,6 +518,16 @@ typedef struct SOptrBasicInfo {
|
|||
bool mergeResultBlock;
|
||||
} SOptrBasicInfo;
|
||||
|
||||
typedef struct SAggOperatorInfo {
|
||||
SOptrBasicInfo binfo;
|
||||
SAggSupporter aggSup;
|
||||
STableQueryInfo* current;
|
||||
uint64_t groupId;
|
||||
SGroupResInfo groupResInfo;
|
||||
SExprSupp scalarExprSup;
|
||||
bool groupKeyOptimized;
|
||||
} SAggOperatorInfo;
|
||||
|
||||
typedef struct SIntervalAggOperatorInfo {
|
||||
SOptrBasicInfo binfo; // basic info
|
||||
SAggSupporter aggSup; // aggregate supporter
|
||||
|
|
@ -559,6 +564,7 @@ typedef struct SStreamIntervalOperatorInfo {
|
|||
STimeWindowAggSupp twAggSup;
|
||||
bool invertible;
|
||||
bool ignoreExpiredData;
|
||||
bool ignoreExpiredDataSaved;
|
||||
SArray* pDelWins; // SWinRes
|
||||
int32_t delIndex;
|
||||
SSDataBlock* pDelRes;
|
||||
|
|
@ -620,6 +626,7 @@ typedef struct SStreamSessionAggOperatorInfo {
|
|||
SPhysiNode* pPhyNode; // create new child
|
||||
bool isFinal;
|
||||
bool ignoreExpiredData;
|
||||
bool ignoreExpiredDataSaved;
|
||||
SArray* pUpdated;
|
||||
SSHashObj* pStUpdated;
|
||||
} SStreamSessionAggOperatorInfo;
|
||||
|
|
@ -637,6 +644,7 @@ typedef struct SStreamStateAggOperatorInfo {
|
|||
void* pDelIterator;
|
||||
SArray* pChildren; // cache for children's result;
|
||||
bool ignoreExpiredData;
|
||||
bool ignoreExpiredDataSaved;
|
||||
SArray* pUpdated;
|
||||
SSHashObj* pSeUpdated;
|
||||
} SStreamStateAggOperatorInfo;
|
||||
|
|
@ -690,6 +698,9 @@ typedef struct SStreamFillOperatorInfo {
|
|||
#define OPTR_IS_OPENED(_optr) (((_optr)->status & OP_OPENED) == OP_OPENED)
|
||||
#define OPTR_SET_OPENED(_optr) ((_optr)->status |= OP_OPENED)
|
||||
|
||||
SExecTaskInfo* doCreateExecTaskInfo(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model,
|
||||
char* dbFName);
|
||||
|
||||
SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t cleanup,
|
||||
__optr_close_fn_t closeFn, __optr_reqBuf_fn_t reqBufFn, __optr_explain_fn_t explain);
|
||||
int32_t optrDummyOpenFn(SOperatorInfo* pOperator);
|
||||
|
|
@ -734,8 +745,9 @@ void updateLoadRemoteInfo(SLoadRemoteDataInfo* pInfo, int64_t numOfRows, int3
|
|||
|
||||
STimeWindow getFirstQualifiedTimeWindow(int64_t ts, STimeWindow* pWindow, SInterval* pInterval, int32_t order);
|
||||
|
||||
int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag, bool inheritUsOrder);
|
||||
int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz);
|
||||
SOperatorInfo* extractOperatorInTree(SOperatorInfo* pOperator, int32_t type, const char* id);
|
||||
int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag, bool inheritUsOrder);
|
||||
int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz);
|
||||
|
||||
extern void doDestroyExchangeOperatorInfo(void* param);
|
||||
|
||||
|
|
@ -756,11 +768,11 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR
|
|||
// clang-format off
|
||||
SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode* pExNode, SExecTaskInfo* pTaskInfo);
|
||||
|
||||
SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, SExecTaskInfo* pTaskInfo);
|
||||
SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, STableListInfo* pTableList, SExecTaskInfo* pTaskInfo);
|
||||
|
||||
SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle, SExecTaskInfo* pTaskInfo);
|
||||
SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo);
|
||||
|
||||
SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo);
|
||||
SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo);
|
||||
|
||||
SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode* pScanPhyNode, const char* pUser, SExecTaskInfo* pTaskInfo);
|
||||
|
||||
|
|
@ -776,7 +788,7 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode*
|
|||
|
||||
SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** dowStreams, size_t numStreams, SMergePhysiNode* pMergePhysiNode, SExecTaskInfo* pTaskInfo);
|
||||
|
||||
SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pTableScanNode, SReadHandle* readHandle, SExecTaskInfo* pTaskInfo);
|
||||
SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pTableScanNode, SReadHandle* readHandle, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo);
|
||||
|
||||
SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo);
|
||||
|
||||
|
|
@ -790,9 +802,9 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionW
|
|||
|
||||
SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* pAggNode, SExecTaskInfo* pTaskInfo);
|
||||
|
||||
SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDistScanPhysiNode* pBlockScanNode, SExecTaskInfo* pTaskInfo);
|
||||
SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDistScanPhysiNode* pBlockScanNode, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo);
|
||||
|
||||
SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SNode* pTagCond, SExecTaskInfo* pTaskInfo);
|
||||
SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SNode* pTagCond, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo);
|
||||
|
||||
SOperatorInfo* createRawScanOperatorInfo(SReadHandle* pHandle, SExecTaskInfo* pTaskInfo);
|
||||
|
||||
|
|
@ -837,9 +849,12 @@ void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status);
|
|||
|
||||
char* buildTaskId(uint64_t taskId, uint64_t queryId);
|
||||
|
||||
SArray* getTableListInfo(const SExecTaskInfo* pTaskInfo);
|
||||
|
||||
int32_t createExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId,
|
||||
int32_t vgId, char* sql, EOPTR_EXEC_MODEL model);
|
||||
int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pTaskInfo, SReadHandle* readHandle);
|
||||
int32_t vgId, char* sql, EOPTR_EXEC_MODEL model);
|
||||
int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, STableListInfo* pTableListInfo,
|
||||
SReadHandle* readHandle);
|
||||
int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SArray* pExecInfoList);
|
||||
|
||||
STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval,
|
||||
|
|
@ -853,7 +868,6 @@ bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap);
|
|||
bool functionNeedToExecute(SqlFunctionCtx* pCtx);
|
||||
bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup);
|
||||
bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup);
|
||||
bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup);
|
||||
bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SStreamState* pState, STimeWindowAggSupp* pTwSup);
|
||||
void appendOneRowToStreamSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid,
|
||||
uint64_t* pGp, void* pTbName);
|
||||
|
|
|
|||
|
|
@ -92,8 +92,8 @@ typedef struct SResultRowData {
|
|||
|
||||
typedef struct SStreamFillLinearInfo {
|
||||
TSKEY nextEnd;
|
||||
SArray* pDeltaVal; // double. value for Fill(linear).
|
||||
SArray* pNextDeltaVal; // double. value for Fill(linear).
|
||||
SArray* pEndPoints;
|
||||
SArray* pNextEndPoints;
|
||||
int64_t winIndex;
|
||||
bool hasNext;
|
||||
} SStreamFillLinearInfo;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,563 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "filter.h"
|
||||
#include "function.h"
|
||||
#include "functionMgt.h"
|
||||
#include "os.h"
|
||||
#include "querynodes.h"
|
||||
#include "tfill.h"
|
||||
#include "tname.h"
|
||||
|
||||
#include "tdatablock.h"
|
||||
#include "tglobal.h"
|
||||
#include "tmsg.h"
|
||||
#include "ttime.h"
|
||||
|
||||
#include "executorimpl.h"
|
||||
#include "index.h"
|
||||
#include "query.h"
|
||||
#include "tcompare.h"
|
||||
#include "thash.h"
|
||||
#include "ttypes.h"
|
||||
#include "vnode.h"
|
||||
|
||||
typedef struct {
|
||||
bool hasAgg;
|
||||
int32_t numOfRows;
|
||||
int32_t startOffset;
|
||||
} SFunctionCtxStatus;
|
||||
|
||||
static void destroyAggOperatorInfo(void* param);
|
||||
static void setExecutionContext(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t groupId);
|
||||
|
||||
static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBlock** ppBlock);
|
||||
static void destroyDataBlockForEmptyInput(bool blockAllocated, SSDataBlock** ppBlock);
|
||||
|
||||
static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator);
|
||||
static int32_t doAggregateImpl(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx);
|
||||
static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator);
|
||||
|
||||
static int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, size_t keyBufSize,
|
||||
const char* pKey);
|
||||
|
||||
static int32_t addNewResultRowBuf(SResultRow* pWindowRes, SDiskbasedBuf* pResultBuf, uint32_t size);
|
||||
|
||||
static void doSetTableGroupOutputBuf(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t groupId);
|
||||
|
||||
static void functionCtxSave(SqlFunctionCtx* pCtx, SFunctionCtxStatus* pStatus);
|
||||
static void functionCtxRestore(SqlFunctionCtx* pCtx, SFunctionCtxStatus* pStatus);
|
||||
|
||||
SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* pAggNode,
|
||||
SExecTaskInfo* pTaskInfo) {
|
||||
SAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SAggOperatorInfo));
|
||||
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
|
||||
if (pInfo == NULL || pOperator == NULL) {
|
||||
goto _error;
|
||||
}
|
||||
|
||||
SSDataBlock* pResBlock = createDataBlockFromDescNode(pAggNode->node.pOutputDataBlockDesc);
|
||||
initBasicInfo(&pInfo->binfo, pResBlock);
|
||||
|
||||
size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
|
||||
initResultSizeInfo(&pOperator->resultInfo, 4096);
|
||||
|
||||
int32_t num = 0;
|
||||
SExprInfo* pExprInfo = createExprInfo(pAggNode->pAggFuncs, pAggNode->pGroupKeys, &num);
|
||||
int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str,
|
||||
pTaskInfo->streamInfo.pState);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
goto _error;
|
||||
}
|
||||
|
||||
int32_t numOfScalarExpr = 0;
|
||||
SExprInfo* pScalarExprInfo = NULL;
|
||||
if (pAggNode->pExprs != NULL) {
|
||||
pScalarExprInfo = createExprInfo(pAggNode->pExprs, NULL, &numOfScalarExpr);
|
||||
}
|
||||
|
||||
code = initExprSupp(&pInfo->scalarExprSup, pScalarExprInfo, numOfScalarExpr);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
goto _error;
|
||||
}
|
||||
|
||||
code = filterInitFromNode((SNode*)pAggNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
goto _error;
|
||||
}
|
||||
|
||||
pInfo->binfo.mergeResultBlock = pAggNode->mergeDataBlock;
|
||||
pInfo->groupKeyOptimized = pAggNode->groupKeyOptimized;
|
||||
pInfo->groupId = UINT64_MAX;
|
||||
|
||||
setOperatorInfo(pOperator, "TableAggregate", QUERY_NODE_PHYSICAL_PLAN_HASH_AGG, true, OP_NOT_OPENED, pInfo,
|
||||
pTaskInfo);
|
||||
pOperator->fpSet = createOperatorFpSet(doOpenAggregateOptr, getAggregateResult, NULL, destroyAggOperatorInfo,
|
||||
optrDefaultBufFn, NULL);
|
||||
|
||||
if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) {
|
||||
STableScanInfo* pTableScanInfo = downstream->info;
|
||||
pTableScanInfo->base.pdInfo.pExprSup = &pOperator->exprSupp;
|
||||
pTableScanInfo->base.pdInfo.pAggSup = &pInfo->aggSup;
|
||||
}
|
||||
|
||||
code = appendDownstream(pOperator, &downstream, 1);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
goto _error;
|
||||
}
|
||||
|
||||
return pOperator;
|
||||
|
||||
_error:
|
||||
if (pInfo != NULL) {
|
||||
destroyAggOperatorInfo(pInfo);
|
||||
}
|
||||
|
||||
if (pOperator != NULL) {
|
||||
cleanupExprSupp(&pOperator->exprSupp);
|
||||
}
|
||||
|
||||
taosMemoryFreeClear(pOperator);
|
||||
pTaskInfo->code = code;
|
||||
return NULL;
|
||||
}
|
||||
|
||||
void destroyAggOperatorInfo(void* param) {
|
||||
SAggOperatorInfo* pInfo = (SAggOperatorInfo*)param;
|
||||
cleanupBasicInfo(&pInfo->binfo);
|
||||
|
||||
cleanupAggSup(&pInfo->aggSup);
|
||||
cleanupExprSupp(&pInfo->scalarExprSup);
|
||||
cleanupGroupResInfo(&pInfo->groupResInfo);
|
||||
taosMemoryFreeClear(param);
|
||||
}
|
||||
|
||||
// this is a blocking operator
|
||||
int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) {
|
||||
if (OPTR_IS_OPENED(pOperator)) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
SAggOperatorInfo* pAggInfo = pOperator->info;
|
||||
|
||||
SExprSupp* pSup = &pOperator->exprSupp;
|
||||
SOperatorInfo* downstream = pOperator->pDownstream[0];
|
||||
|
||||
int64_t st = taosGetTimestampUs();
|
||||
|
||||
int32_t order = TSDB_ORDER_ASC;
|
||||
int32_t scanFlag = MAIN_SCAN;
|
||||
|
||||
bool hasValidBlock = false;
|
||||
bool blockAllocated = false;
|
||||
|
||||
while (1) {
|
||||
SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
|
||||
if (pBlock == NULL) {
|
||||
if (!hasValidBlock) {
|
||||
createDataBlockForEmptyInput(pOperator, &pBlock);
|
||||
if (pBlock == NULL) {
|
||||
break;
|
||||
}
|
||||
blockAllocated = true;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
hasValidBlock = true;
|
||||
|
||||
int32_t code = getTableScanInfo(pOperator, &order, &scanFlag, false);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
destroyDataBlockForEmptyInput(blockAllocated, &pBlock);
|
||||
T_LONG_JMP(pTaskInfo->env, code);
|
||||
}
|
||||
|
||||
// there is an scalar expression that needs to be calculated before apply the group aggregation.
|
||||
if (pAggInfo->scalarExprSup.pExprInfo != NULL && !blockAllocated) {
|
||||
SExprSupp* pSup1 = &pAggInfo->scalarExprSup;
|
||||
code = projectApplyFunctions(pSup1->pExprInfo, pBlock, pBlock, pSup1->pCtx, pSup1->numOfExprs, NULL);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
destroyDataBlockForEmptyInput(blockAllocated, &pBlock);
|
||||
T_LONG_JMP(pTaskInfo->env, code);
|
||||
}
|
||||
}
|
||||
|
||||
// the pDataBlock are always the same one, no need to call this again
|
||||
setExecutionContext(pOperator, pOperator->exprSupp.numOfExprs, pBlock->info.id.groupId);
|
||||
setInputDataBlock(pSup, pBlock, order, scanFlag, true);
|
||||
code = doAggregateImpl(pOperator, pSup->pCtx);
|
||||
if (code != 0) {
|
||||
destroyDataBlockForEmptyInput(blockAllocated, &pBlock);
|
||||
T_LONG_JMP(pTaskInfo->env, code);
|
||||
}
|
||||
|
||||
destroyDataBlockForEmptyInput(blockAllocated, &pBlock);
|
||||
}
|
||||
|
||||
// the downstream operator may return with error code, so let's check the code before generating results.
|
||||
if (pTaskInfo->code != TSDB_CODE_SUCCESS) {
|
||||
T_LONG_JMP(pTaskInfo->env, pTaskInfo->code);
|
||||
}
|
||||
|
||||
initGroupedResultInfo(&pAggInfo->groupResInfo, pAggInfo->aggSup.pResultRowHashTable, 0);
|
||||
OPTR_SET_OPENED(pOperator);
|
||||
|
||||
pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;
|
||||
return pTaskInfo->code;
|
||||
}
|
||||
|
||||
SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) {
|
||||
SAggOperatorInfo* pAggInfo = pOperator->info;
|
||||
SOptrBasicInfo* pInfo = &pAggInfo->binfo;
|
||||
|
||||
if (pOperator->status == OP_EXEC_DONE) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
pTaskInfo->code = pOperator->fpSet._openFn(pOperator);
|
||||
if (pTaskInfo->code != TSDB_CODE_SUCCESS) {
|
||||
setOperatorCompleted(pOperator);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
|
||||
while (1) {
|
||||
doBuildResultDatablock(pOperator, pInfo, &pAggInfo->groupResInfo, pAggInfo->aggSup.pResultBuf);
|
||||
doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL);
|
||||
|
||||
if (!hasRemainResults(&pAggInfo->groupResInfo)) {
|
||||
setOperatorCompleted(pOperator);
|
||||
break;
|
||||
}
|
||||
|
||||
if (pInfo->pRes->info.rows > 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
size_t rows = blockDataGetNumOfRows(pInfo->pRes);
|
||||
pOperator->resultInfo.totalRows += rows;
|
||||
|
||||
return (rows == 0) ? NULL : pInfo->pRes;
|
||||
}
|
||||
|
||||
int32_t doAggregateImpl(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx) {
|
||||
for (int32_t k = 0; k < pOperator->exprSupp.numOfExprs; ++k) {
|
||||
if (functionNeedToExecute(&pCtx[k])) {
|
||||
// todo add a dummy funtion to avoid process check
|
||||
if (pCtx[k].fpSet.process == NULL) {
|
||||
continue;
|
||||
}
|
||||
|
||||
int32_t code = pCtx[k].fpSet.process(&pCtx[k]);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s aggregate function error happens, code: %s", GET_TASKID(pOperator->pTaskInfo), tstrerror(code));
|
||||
return code;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBlock** ppBlock) {
|
||||
if (!tsCountAlwaysReturnValue) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
SAggOperatorInfo* pAggInfo = pOperator->info;
|
||||
if (pAggInfo->groupKeyOptimized) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
SOperatorInfo* downstream = pOperator->pDownstream[0];
|
||||
if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_PARTITION ||
|
||||
(downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN &&
|
||||
((STableScanInfo*)downstream->info)->hasGroupByTag == true)) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx;
|
||||
bool hasCountFunc = false;
|
||||
|
||||
for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) {
|
||||
const char* pName = pCtx[i].pExpr->pExpr->_function.functionName;
|
||||
if ((strcmp(pName, "count") == 0) || (strcmp(pName, "hyperloglog") == 0) ||
|
||||
(strcmp(pName, "_hyperloglog_partial") == 0) || (strcmp(pName, "_hyperloglog_merge") == 0)) {
|
||||
hasCountFunc = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!hasCountFunc) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
SSDataBlock* pBlock = createDataBlock();
|
||||
pBlock->info.rows = 1;
|
||||
pBlock->info.capacity = 0;
|
||||
|
||||
for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) {
|
||||
SColumnInfoData colInfo = {0};
|
||||
colInfo.hasNull = true;
|
||||
colInfo.info.type = TSDB_DATA_TYPE_NULL;
|
||||
colInfo.info.bytes = 1;
|
||||
|
||||
SExprInfo* pOneExpr = &pOperator->exprSupp.pExprInfo[i];
|
||||
for (int32_t j = 0; j < pOneExpr->base.numOfParams; ++j) {
|
||||
SFunctParam* pFuncParam = &pOneExpr->base.pParam[j];
|
||||
if (pFuncParam->type == FUNC_PARAM_TYPE_COLUMN) {
|
||||
int32_t slotId = pFuncParam->pCol->slotId;
|
||||
int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
|
||||
if (slotId >= numOfCols) {
|
||||
taosArrayEnsureCap(pBlock->pDataBlock, slotId + 1);
|
||||
for (int32_t k = numOfCols; k < slotId + 1; ++k) {
|
||||
taosArrayPush(pBlock->pDataBlock, &colInfo);
|
||||
}
|
||||
}
|
||||
} else if (pFuncParam->type == FUNC_PARAM_TYPE_VALUE) {
|
||||
// do nothing
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
blockDataEnsureCapacity(pBlock, pBlock->info.rows);
|
||||
*ppBlock = pBlock;
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
void destroyDataBlockForEmptyInput(bool blockAllocated, SSDataBlock** ppBlock) {
|
||||
if (!blockAllocated) {
|
||||
return;
|
||||
}
|
||||
|
||||
blockDataDestroy(*ppBlock);
|
||||
*ppBlock = NULL;
|
||||
}
|
||||
|
||||
void setExecutionContext(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t groupId) {
|
||||
SAggOperatorInfo* pAggInfo = pOperator->info;
|
||||
if (pAggInfo->groupId != UINT64_MAX && pAggInfo->groupId == groupId) {
|
||||
return;
|
||||
}
|
||||
|
||||
doSetTableGroupOutputBuf(pOperator, numOfOutput, groupId);
|
||||
|
||||
// record the current active group id
|
||||
pAggInfo->groupId = groupId;
|
||||
}
|
||||
|
||||
void doSetTableGroupOutputBuf(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t groupId) {
|
||||
// for simple group by query without interval, all the tables belong to one group result.
|
||||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
SAggOperatorInfo* pAggInfo = pOperator->info;
|
||||
|
||||
SResultRowInfo* pResultRowInfo = &pAggInfo->binfo.resultRowInfo;
|
||||
SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx;
|
||||
int32_t* rowEntryInfoOffset = pOperator->exprSupp.rowEntryInfoOffset;
|
||||
|
||||
SResultRow* pResultRow = doSetResultOutBufByKey(pAggInfo->aggSup.pResultBuf, pResultRowInfo, (char*)&groupId,
|
||||
sizeof(groupId), true, groupId, pTaskInfo, false, &pAggInfo->aggSup, true);
|
||||
/*
|
||||
* not assign result buffer yet, add new result buffer
|
||||
* all group belong to one result set, and each group result has different group id so set the id to be one
|
||||
*/
|
||||
if (pResultRow->pageId == -1) {
|
||||
int32_t ret = addNewResultRowBuf(pResultRow, pAggInfo->aggSup.pResultBuf, pAggInfo->binfo.pRes->info.rowSize);
|
||||
if (ret != TSDB_CODE_SUCCESS) {
|
||||
T_LONG_JMP(pTaskInfo->env, terrno);
|
||||
}
|
||||
}
|
||||
|
||||
setResultRowInitCtx(pResultRow, pCtx, numOfOutput, rowEntryInfoOffset);
|
||||
}
|
||||
|
||||
// a new buffer page for each table. Needs to opt this design
|
||||
int32_t addNewResultRowBuf(SResultRow* pWindowRes, SDiskbasedBuf* pResultBuf, uint32_t size) {
|
||||
if (pWindowRes->pageId != -1) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
SFilePage* pData = NULL;
|
||||
|
||||
// in the first scan, new space needed for results
|
||||
int32_t pageId = -1;
|
||||
SArray* list = getDataBufPagesIdList(pResultBuf);
|
||||
|
||||
if (taosArrayGetSize(list) == 0) {
|
||||
pData = getNewBufPage(pResultBuf, &pageId);
|
||||
pData->num = sizeof(SFilePage);
|
||||
} else {
|
||||
SPageInfo* pi = getLastPageInfo(list);
|
||||
pData = getBufPage(pResultBuf, getPageId(pi));
|
||||
if (pData == NULL) {
|
||||
qError("failed to get buffer, code:%s", tstrerror(terrno));
|
||||
return terrno;
|
||||
}
|
||||
|
||||
pageId = getPageId(pi);
|
||||
|
||||
if (pData->num + size > getBufPageSize(pResultBuf)) {
|
||||
// release current page first, and prepare the next one
|
||||
releaseBufPageInfo(pResultBuf, pi);
|
||||
|
||||
pData = getNewBufPage(pResultBuf, &pageId);
|
||||
if (pData != NULL) {
|
||||
pData->num = sizeof(SFilePage);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (pData == NULL) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
// set the number of rows in current disk page
|
||||
if (pWindowRes->pageId == -1) { // not allocated yet, allocate new buffer
|
||||
pWindowRes->pageId = pageId;
|
||||
pWindowRes->offset = (int32_t)pData->num;
|
||||
|
||||
pData->num += size;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, size_t keyBufSize,
|
||||
const char* pKey) {
|
||||
int32_t code = 0;
|
||||
// _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
|
||||
|
||||
pAggSup->currentPageId = -1;
|
||||
pAggSup->resultRowSize = getResultRowSize(pCtx, numOfOutput);
|
||||
pAggSup->keyBuf = taosMemoryCalloc(1, keyBufSize + POINTER_BYTES + sizeof(int64_t));
|
||||
pAggSup->pResultRowHashTable = tSimpleHashInit(100, taosFastHash);
|
||||
|
||||
if (pAggSup->keyBuf == NULL || pAggSup->pResultRowHashTable == NULL) {
|
||||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
|
||||
uint32_t defaultPgsz = 0;
|
||||
uint32_t defaultBufsz = 0;
|
||||
getBufferPgSize(pAggSup->resultRowSize, &defaultPgsz, &defaultBufsz);
|
||||
|
||||
if (!osTempSpaceAvailable()) {
|
||||
code = TSDB_CODE_NO_AVAIL_DISK;
|
||||
qError("Init stream agg supporter failed since %s, %s", terrstr(code), pKey);
|
||||
return code;
|
||||
}
|
||||
|
||||
code = createDiskbasedBuf(&pAggSup->pResultBuf, defaultPgsz, defaultBufsz, pKey, tsTempDir);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("Create agg result buf failed since %s, %s", tstrerror(code), pKey);
|
||||
return code;
|
||||
}
|
||||
|
||||
return code;
|
||||
}
|
||||
|
||||
void cleanupAggSup(SAggSupporter* pAggSup) {
|
||||
taosMemoryFreeClear(pAggSup->keyBuf);
|
||||
tSimpleHashCleanup(pAggSup->pResultRowHashTable);
|
||||
destroyDiskbasedBuf(pAggSup->pResultBuf);
|
||||
}
|
||||
|
||||
int32_t initAggSup(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize,
|
||||
const char* pkey, void* pState) {
|
||||
int32_t code = initExprSupp(pSup, pExprInfo, numOfCols);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
return code;
|
||||
}
|
||||
|
||||
code = doInitAggInfoSup(pAggSup, pSup->pCtx, numOfCols, keyBufSize, pkey);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
return code;
|
||||
}
|
||||
|
||||
for (int32_t i = 0; i < numOfCols; ++i) {
|
||||
if (pState) {
|
||||
pSup->pCtx[i].saveHandle.pBuf = NULL;
|
||||
pSup->pCtx[i].saveHandle.pState = pState;
|
||||
pSup->pCtx[i].exprIdx = i;
|
||||
} else {
|
||||
pSup->pCtx[i].saveHandle.pBuf = pAggSup->pResultBuf;
|
||||
}
|
||||
}
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
void applyAggFunctionOnPartialTuples(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, SColumnInfoData* pTimeWindowData,
|
||||
int32_t offset, int32_t forwardStep, int32_t numOfTotal, int32_t numOfOutput) {
|
||||
for (int32_t k = 0; k < numOfOutput; ++k) {
|
||||
// keep it temporarily
|
||||
SFunctionCtxStatus status = {0};
|
||||
functionCtxSave(&pCtx[k], &status);
|
||||
|
||||
pCtx[k].input.startRowIndex = offset;
|
||||
pCtx[k].input.numOfRows = forwardStep;
|
||||
|
||||
// not a whole block involved in query processing, statistics data can not be used
|
||||
// NOTE: the original value of isSet have been changed here
|
||||
if (pCtx[k].input.colDataSMAIsSet && forwardStep < numOfTotal) {
|
||||
pCtx[k].input.colDataSMAIsSet = false;
|
||||
}
|
||||
|
||||
if (pCtx[k].isPseudoFunc) {
|
||||
SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(&pCtx[k]);
|
||||
|
||||
char* p = GET_ROWCELL_INTERBUF(pEntryInfo);
|
||||
|
||||
SColumnInfoData idata = {0};
|
||||
idata.info.type = TSDB_DATA_TYPE_BIGINT;
|
||||
idata.info.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes;
|
||||
idata.pData = p;
|
||||
|
||||
SScalarParam out = {.columnData = &idata};
|
||||
SScalarParam tw = {.numOfRows = 5, .columnData = pTimeWindowData};
|
||||
pCtx[k].sfp.process(&tw, 1, &out);
|
||||
pEntryInfo->numOfRes = 1;
|
||||
} else {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
if (functionNeedToExecute(&pCtx[k]) && pCtx[k].fpSet.process != NULL) {
|
||||
code = pCtx[k].fpSet.process(&pCtx[k]);
|
||||
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s apply functions error, code: %s", GET_TASKID(taskInfo), tstrerror(code));
|
||||
taskInfo->code = code;
|
||||
T_LONG_JMP(taskInfo->env, code);
|
||||
}
|
||||
}
|
||||
|
||||
// restore it
|
||||
functionCtxRestore(&pCtx[k], &status);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void functionCtxSave(SqlFunctionCtx* pCtx, SFunctionCtxStatus* pStatus) {
|
||||
pStatus->hasAgg = pCtx->input.colDataSMAIsSet;
|
||||
pStatus->numOfRows = pCtx->input.numOfRows;
|
||||
pStatus->startOffset = pCtx->input.startRowIndex;
|
||||
}
|
||||
|
||||
void functionCtxRestore(SqlFunctionCtx* pCtx, SFunctionCtxStatus* pStatus) {
|
||||
pCtx->input.colDataSMAIsSet = pStatus->hasAgg;
|
||||
pCtx->input.numOfRows = pStatus->numOfRows;
|
||||
pCtx->input.startRowIndex = pStatus->startOffset;
|
||||
}
|
||||
|
|
@ -37,6 +37,7 @@ typedef struct SCacheRowsScanInfo {
|
|||
SSDataBlock* pBufferredRes;
|
||||
SArray* pUidList;
|
||||
int32_t indexOfBufferedRes;
|
||||
STableListInfo* pTableList;
|
||||
} SCacheRowsScanInfo;
|
||||
|
||||
static SSDataBlock* doScanCache(SOperatorInfo* pOperator);
|
||||
|
|
@ -47,15 +48,17 @@ static int32_t removeRedundantTsCol(SLastRowScanPhysiNode* pScanNode, SColM
|
|||
#define SCAN_ROW_TYPE(_t) ((_t)? CACHESCAN_RETRIEVE_LAST : CACHESCAN_RETRIEVE_LAST_ROW)
|
||||
|
||||
SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SReadHandle* readHandle,
|
||||
SExecTaskInfo* pTaskInfo) {
|
||||
STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
SCacheRowsScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SCacheRowsScanInfo));
|
||||
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
|
||||
if (pInfo == NULL || pOperator == NULL) {
|
||||
code = TSDB_CODE_OUT_OF_MEMORY;
|
||||
tableListDestroy(pTableListInfo);
|
||||
goto _error;
|
||||
}
|
||||
|
||||
pInfo->pTableList = pTableListInfo;
|
||||
pInfo->readHandle = *readHandle;
|
||||
|
||||
SDataBlockDescNode* pDescNode = pScanNode->scan.node.pOutputDataBlockDesc;
|
||||
|
|
@ -75,20 +78,18 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe
|
|||
goto _error;
|
||||
}
|
||||
|
||||
STableListInfo* pTableList = pTaskInfo->pTableInfoList;
|
||||
|
||||
int32_t totalTables = tableListGetSize(pTableList);
|
||||
int32_t totalTables = tableListGetSize(pTableListInfo);
|
||||
int32_t capacity = 0;
|
||||
|
||||
pInfo->pUidList = taosArrayInit(4, sizeof(int64_t));
|
||||
|
||||
// partition by tbname
|
||||
if (oneTableForEachGroup(pTableList) || (totalTables == 1)) {
|
||||
if (oneTableForEachGroup(pTableListInfo) || (totalTables == 1)) {
|
||||
pInfo->retrieveType = CACHESCAN_RETRIEVE_TYPE_ALL | SCAN_ROW_TYPE(pScanNode->ignoreNull);
|
||||
|
||||
STableKeyInfo* pList = tableListGetInfo(pTableList, 0);
|
||||
STableKeyInfo* pList = tableListGetInfo(pTableListInfo, 0);
|
||||
|
||||
uint64_t suid = tableListGetSuid(pTableList);
|
||||
uint64_t suid = tableListGetSuid(pTableListInfo);
|
||||
code = tsdbCacherowsReaderOpen(pInfo->readHandle.vnode, pInfo->retrieveType, pList, totalTables,
|
||||
taosArrayGetSize(pInfo->matchInfo.pList), suid, &pInfo->pLastrowReader, pTaskInfo->id.str);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
|
|
@ -136,7 +137,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
|
|||
|
||||
SCacheRowsScanInfo* pInfo = pOperator->info;
|
||||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
STableListInfo* pTableList = pTaskInfo->pTableInfoList;
|
||||
STableListInfo* pTableList = pInfo->pTableList;
|
||||
|
||||
uint64_t suid = tableListGetSuid(pTableList);
|
||||
int32_t size = tableListGetSize(pTableList);
|
||||
|
|
@ -281,6 +282,7 @@ void destroyCacheScanOperator(void* param) {
|
|||
taosMemoryFree(pInfo->pSlotIds);
|
||||
taosArrayDestroy(pInfo->pUidList);
|
||||
taosArrayDestroy(pInfo->matchInfo.pList);
|
||||
tableListDestroy(pInfo->pTableList);
|
||||
|
||||
if (pInfo->pLastrowReader != NULL) {
|
||||
pInfo->pLastrowReader = tsdbCacherowsReaderClose(pInfo->pLastrowReader);
|
||||
|
|
|
|||
|
|
@ -70,7 +70,7 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn
|
|||
tsem_wait(&pExchangeInfo->ready);
|
||||
|
||||
if (isTaskKilled(pTaskInfo)) {
|
||||
longjmp(pTaskInfo->env, pTaskInfo->code);
|
||||
T_LONG_JMP(pTaskInfo->env, pTaskInfo->code);
|
||||
}
|
||||
|
||||
for (int32_t i = 0; i < totalSources; ++i) {
|
||||
|
|
@ -212,6 +212,11 @@ static SSDataBlock* loadRemoteData(SOperatorInfo* pOperator) {
|
|||
return NULL;
|
||||
}
|
||||
|
||||
doFilter(pBlock, pOperator->exprSupp.pFilterInfo, NULL);
|
||||
if (blockDataGetNumOfRows(pBlock) == 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
SLimitInfo* pLimitInfo = &pExchangeInfo->limitInfo;
|
||||
if (hasLimitOffsetInfo(pLimitInfo)) {
|
||||
int32_t status = handleLimitOffset(pOperator, pLimitInfo, pBlock, false);
|
||||
|
|
@ -303,6 +308,11 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode
|
|||
pTaskInfo);
|
||||
pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pDummyBlock->pDataBlock);
|
||||
|
||||
code = filterInitFromNode((SNode*)pExNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
goto _error;
|
||||
}
|
||||
|
||||
pOperator->fpSet =
|
||||
createOperatorFpSet(prepareLoadRemoteData, loadRemoteData, NULL, destroyExchangeOperatorInfo, optrDefaultBufFn, NULL);
|
||||
return pOperator;
|
||||
|
|
@ -566,7 +576,7 @@ int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator) {
|
|||
pOperator->status = OP_RES_TO_RETURN;
|
||||
pOperator->cost.openCost = taosGetTimestampUs() - startTs;
|
||||
if (isTaskKilled(pTaskInfo)) {
|
||||
longjmp(pTaskInfo->env, pTaskInfo->code);
|
||||
T_LONG_JMP(pTaskInfo->env, pTaskInfo->code);
|
||||
}
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
|
|
@ -619,7 +629,7 @@ int32_t seqLoadRemoteData(SOperatorInfo* pOperator) {
|
|||
doSendFetchDataRequest(pExchangeInfo, pTaskInfo, pExchangeInfo->current);
|
||||
tsem_wait(&pExchangeInfo->ready);
|
||||
if (isTaskKilled(pTaskInfo)) {
|
||||
longjmp(pTaskInfo->env, pTaskInfo->code);
|
||||
T_LONG_JMP(pTaskInfo->env, pTaskInfo->code);
|
||||
}
|
||||
|
||||
SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, pExchangeInfo->current);
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ struct STableListInfo {
|
|||
int32_t numOfOuputGroups; // the data block will be generated one by one
|
||||
int32_t* groupOffset; // keep the offset value for each group in the tableList
|
||||
SArray* pTableList;
|
||||
SHashObj* map; // speedup acquire the tableQueryInfo by table uid
|
||||
SHashObj* map; // speedup acquire the tableQueryInfo by table uid
|
||||
uint64_t suid;
|
||||
};
|
||||
|
||||
|
|
@ -571,6 +571,10 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis
|
|||
memcpy(pStart, data, len);
|
||||
pStart += len;
|
||||
} else if (IS_VAR_DATA_TYPE(pValue->info.type)) {
|
||||
if (varDataTLen(data) > pValue->info.bytes) {
|
||||
code = TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER;
|
||||
goto end;
|
||||
}
|
||||
memcpy(pStart, data, varDataTLen(data));
|
||||
pStart += varDataTLen(data);
|
||||
} else {
|
||||
|
|
@ -1800,6 +1804,21 @@ STableKeyInfo* tableListGetInfo(const STableListInfo* pTableList, int32_t index)
|
|||
return taosArrayGet(pTableList->pTableList, index);
|
||||
}
|
||||
|
||||
int32_t tableListFind(const STableListInfo* pTableList, uint64_t uid, int32_t startIndex) {
|
||||
int32_t numOfTables = taosArrayGetSize(pTableList->pTableList);
|
||||
if (startIndex >= numOfTables) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
for (int32_t i = startIndex; i < numOfTables; ++i) {
|
||||
STableKeyInfo* p = taosArrayGet(pTableList->pTableList, i);
|
||||
if (p->uid == uid) {
|
||||
return i;
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
uint64_t getTableGroupId(const STableListInfo* pTableList, uint64_t tableUid) {
|
||||
int32_t* slot = taosHashGet(pTableList->map, &tableUid, sizeof(tableUid));
|
||||
ASSERT(pTableList->map != NULL && slot != NULL);
|
||||
|
|
@ -2042,7 +2061,7 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags
|
|||
|
||||
void printDataBlock(SSDataBlock* pBlock, const char* flag) {
|
||||
if (!pBlock || pBlock->info.rows == 0) {
|
||||
qDebug("===stream===printDataBlock: Block is Null or Empty");
|
||||
qDebug("===stream===%s: Block is Null or Empty", flag);
|
||||
return;
|
||||
}
|
||||
char* pBuf = NULL;
|
||||
|
|
|
|||
|
|
@ -242,30 +242,28 @@ int32_t qSetSMAInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks,
|
|||
return code;
|
||||
}
|
||||
|
||||
qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int32_t vgId, int32_t* numOfCols, SSchemaWrapper** pSchema) {
|
||||
if (msg == NULL) {
|
||||
// create raw scan
|
||||
SExecTaskInfo* pTaskInfo = taosMemoryCalloc(1, sizeof(SExecTaskInfo));
|
||||
qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int32_t vgId, int32_t* numOfCols,
|
||||
uint64_t id) {
|
||||
if (msg == NULL) { // create raw scan
|
||||
SExecTaskInfo* pTaskInfo = doCreateExecTaskInfo(0, id, vgId, OPTR_EXEC_MODEL_QUEUE, "");
|
||||
if (NULL == pTaskInfo) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return NULL;
|
||||
}
|
||||
|
||||
setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED);
|
||||
|
||||
pTaskInfo->cost.created = taosGetTimestampUs();
|
||||
pTaskInfo->execModel = OPTR_EXEC_MODEL_QUEUE;
|
||||
pTaskInfo->pRoot = createRawScanOperatorInfo(pReaderHandle, pTaskInfo);
|
||||
if (NULL == pTaskInfo->pRoot) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
taosMemoryFree(pTaskInfo);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
qDebug("create raw scan task info completed, vgId:%d, %s", vgId, GET_TASKID(pTaskInfo));
|
||||
return pTaskInfo;
|
||||
}
|
||||
|
||||
struct SSubplan* pPlan = NULL;
|
||||
int32_t code = qStringToSubplan(msg, &pPlan);
|
||||
|
||||
int32_t code = qStringToSubplan(msg, &pPlan);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
terrno = code;
|
||||
return NULL;
|
||||
|
|
@ -292,9 +290,6 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int3
|
|||
}
|
||||
}
|
||||
|
||||
if (pSchema) {
|
||||
*pSchema = tCloneSSchemaWrapper(((SExecTaskInfo*)pTaskInfo)->schemaInfo.qsw);
|
||||
}
|
||||
return pTaskInfo;
|
||||
}
|
||||
|
||||
|
|
@ -409,7 +404,8 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo
|
|||
}
|
||||
}
|
||||
|
||||
STableListInfo* pTableListInfo = pTaskInfo->pTableInfoList;
|
||||
STableListInfo* pTableListInfo = ((STableScanInfo*)pScanInfo->pTableScanOp->info)->base.pTableListInfo;
|
||||
taosWLockLatch(&pTaskInfo->lock);
|
||||
|
||||
for (int32_t i = 0; i < numOfQualifiedTables; ++i) {
|
||||
uint64_t* uid = taosArrayGet(qa, i);
|
||||
|
|
@ -424,6 +420,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo
|
|||
if (code != TSDB_CODE_SUCCESS) {
|
||||
taosMemoryFree(keyBuf);
|
||||
taosArrayDestroy(qa);
|
||||
taosWUnLockLatch(&pTaskInfo->lock);
|
||||
return code;
|
||||
}
|
||||
}
|
||||
|
|
@ -445,6 +442,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo
|
|||
tableListAddTableInfo(pTableListInfo, keyInfo.uid, keyInfo.groupId);
|
||||
}
|
||||
|
||||
taosWUnLockLatch(&pTaskInfo->lock);
|
||||
if (keyBuf != NULL) {
|
||||
taosMemoryFree(keyBuf);
|
||||
}
|
||||
|
|
@ -452,7 +450,9 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo
|
|||
taosArrayDestroy(qa);
|
||||
} else { // remove the table id in current list
|
||||
qDebug(" %d remove child tables from the stream scanner", (int32_t)taosArrayGetSize(tableIdList));
|
||||
taosWLockLatch(&pTaskInfo->lock);
|
||||
code = tqReaderRemoveTbUidList(pScanInfo->tqReader, tableIdList);
|
||||
taosWUnLockLatch(&pTaskInfo->lock);
|
||||
}
|
||||
|
||||
return code;
|
||||
|
|
@ -485,9 +485,7 @@ int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* table
|
|||
|
||||
int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, SSubplan* pSubplan,
|
||||
qTaskInfo_t* pTaskInfo, DataSinkHandle* handle, char* sql, EOPTR_EXEC_MODEL model) {
|
||||
assert(pSubplan != NULL);
|
||||
SExecTaskInfo** pTask = (SExecTaskInfo**)pTaskInfo;
|
||||
|
||||
taosThreadOnce(&initPoolOnce, initRefPool);
|
||||
|
||||
qDebug("start to create task, TID:0x%" PRIx64 " QID:0x%" PRIx64 ", vgId:%d", taskId, pSubplan->id.queryId, vgId);
|
||||
|
|
@ -507,7 +505,12 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId,
|
|||
|
||||
if (handle) {
|
||||
void* pSinkParam = NULL;
|
||||
code = createDataSinkParam(pSubplan->pDataSink, &pSinkParam, pTaskInfo, readHandle);
|
||||
|
||||
SArray* pInfoList = getTableListInfo(*pTask);
|
||||
STableListInfo* pTableListInfo = taosArrayGetP(pInfoList, 0);
|
||||
taosArrayDestroy(pInfoList);
|
||||
|
||||
code = createDataSinkParam(pSubplan->pDataSink, &pSinkParam, pTableListInfo, readHandle);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("failed to createDataSinkParam, vgId:%d, code:%s, %s", vgId, tstrerror(code), (*pTask)->id.str);
|
||||
goto _error;
|
||||
|
|
@ -749,6 +752,23 @@ int32_t qAsyncKillTask(qTaskInfo_t qinfo, int32_t rspCode) {
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t qKillTask(qTaskInfo_t tinfo, int32_t rspCode) {
|
||||
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
|
||||
if (pTaskInfo == NULL) {
|
||||
return TSDB_CODE_QRY_INVALID_QHANDLE;
|
||||
}
|
||||
|
||||
qDebug("%s sync killed execTask", GET_TASKID(pTaskInfo));
|
||||
setTaskKilled(pTaskInfo, TSDB_CODE_TSC_QUERY_KILLED);
|
||||
|
||||
while (qTaskIsExecuting(pTaskInfo)) {
|
||||
taosMsleep(10);
|
||||
}
|
||||
|
||||
pTaskInfo->code = rspCode;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
bool qTaskIsExecuting(qTaskInfo_t qinfo) {
|
||||
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qinfo;
|
||||
if (NULL == pTaskInfo) {
|
||||
|
|
@ -888,7 +908,8 @@ int32_t qStreamSetParamForRecover(qTaskInfo_t tinfo) {
|
|||
pInfo->twAggSup.deleteMarkSaved = pInfo->twAggSup.deleteMark;
|
||||
pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
|
||||
pInfo->twAggSup.deleteMark = INT64_MAX;
|
||||
|
||||
pInfo->ignoreExpiredDataSaved = pInfo->ignoreExpiredData;
|
||||
pInfo->ignoreExpiredData = false;
|
||||
} else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION ||
|
||||
pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION ||
|
||||
pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) {
|
||||
|
|
@ -904,6 +925,8 @@ int32_t qStreamSetParamForRecover(qTaskInfo_t tinfo) {
|
|||
pInfo->twAggSup.deleteMarkSaved = pInfo->twAggSup.deleteMark;
|
||||
pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
|
||||
pInfo->twAggSup.deleteMark = INT64_MAX;
|
||||
pInfo->ignoreExpiredDataSaved = pInfo->ignoreExpiredData;
|
||||
pInfo->ignoreExpiredData = false;
|
||||
} else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) {
|
||||
SStreamStateAggOperatorInfo* pInfo = pOperator->info;
|
||||
ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE ||
|
||||
|
|
@ -917,6 +940,8 @@ int32_t qStreamSetParamForRecover(qTaskInfo_t tinfo) {
|
|||
pInfo->twAggSup.deleteMarkSaved = pInfo->twAggSup.deleteMark;
|
||||
pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
|
||||
pInfo->twAggSup.deleteMark = INT64_MAX;
|
||||
pInfo->ignoreExpiredDataSaved = pInfo->ignoreExpiredData;
|
||||
pInfo->ignoreExpiredData = false;
|
||||
}
|
||||
|
||||
// iterate operator tree
|
||||
|
|
@ -944,35 +969,23 @@ int32_t qStreamRestoreParam(qTaskInfo_t tinfo) {
|
|||
pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL ||
|
||||
pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) {
|
||||
SStreamIntervalOperatorInfo* pInfo = pOperator->info;
|
||||
/*ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE);*/
|
||||
/*ASSERT(pInfo->twAggSup.deleteMark == INT64_MAX);*/
|
||||
|
||||
pInfo->twAggSup.calTrigger = pInfo->twAggSup.calTriggerSaved;
|
||||
pInfo->twAggSup.deleteMark = pInfo->twAggSup.deleteMarkSaved;
|
||||
/*ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE ||*/
|
||||
/*pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE);*/
|
||||
pInfo->ignoreExpiredData = pInfo->ignoreExpiredDataSaved;
|
||||
qInfo("restore stream param for interval: %d, %" PRId64, pInfo->twAggSup.calTrigger, pInfo->twAggSup.deleteMark);
|
||||
} else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION ||
|
||||
pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION ||
|
||||
pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) {
|
||||
SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
|
||||
/*ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE);*/
|
||||
/*ASSERT(pInfo->twAggSup.deleteMark == INT64_MAX);*/
|
||||
|
||||
pInfo->twAggSup.calTrigger = pInfo->twAggSup.calTriggerSaved;
|
||||
pInfo->twAggSup.deleteMark = pInfo->twAggSup.deleteMarkSaved;
|
||||
/*ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE ||*/
|
||||
/*pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE);*/
|
||||
pInfo->ignoreExpiredData = pInfo->ignoreExpiredDataSaved;
|
||||
qInfo("restore stream param for session: %d, %" PRId64, pInfo->twAggSup.calTrigger, pInfo->twAggSup.deleteMark);
|
||||
} else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) {
|
||||
SStreamStateAggOperatorInfo* pInfo = pOperator->info;
|
||||
/*ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE);*/
|
||||
/*ASSERT(pInfo->twAggSup.deleteMark == INT64_MAX);*/
|
||||
|
||||
pInfo->twAggSup.calTrigger = pInfo->twAggSup.calTriggerSaved;
|
||||
pInfo->twAggSup.deleteMark = pInfo->twAggSup.deleteMarkSaved;
|
||||
/*ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE ||*/
|
||||
/*pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE);*/
|
||||
pInfo->ignoreExpiredData = pInfo->ignoreExpiredDataSaved;
|
||||
qInfo("restore stream param for state: %d, %" PRId64, pInfo->twAggSup.calTrigger, pInfo->twAggSup.deleteMark);
|
||||
}
|
||||
|
||||
|
|
@ -990,6 +1003,7 @@ int32_t qStreamRestoreParam(qTaskInfo_t tinfo) {
|
|||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool qStreamRecoverScanFinished(qTaskInfo_t tinfo) {
|
||||
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
|
||||
return pTaskInfo->streamInfo.recoverScanFinished;
|
||||
|
|
@ -1059,7 +1073,7 @@ int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* s
|
|||
|
||||
int32_t qStreamSetScanMemData(qTaskInfo_t tinfo, SPackedData submit) {
|
||||
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
|
||||
if((pTaskInfo->execModel != OPTR_EXEC_MODEL_QUEUE) || (pTaskInfo->streamInfo.submit.msgStr != NULL)){
|
||||
if ((pTaskInfo->execModel != OPTR_EXEC_MODEL_QUEUE) || (pTaskInfo->streamInfo.submit.msgStr != NULL)) {
|
||||
qError("qStreamSetScanMemData err:%d,%p", pTaskInfo->execModel, pTaskInfo->streamInfo.submit.msgStr);
|
||||
return -1;
|
||||
}
|
||||
|
|
@ -1072,6 +1086,8 @@ int32_t qStreamSetScanMemData(qTaskInfo_t tinfo, SPackedData submit) {
|
|||
int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subType) {
|
||||
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
|
||||
SOperatorInfo* pOperator = pTaskInfo->pRoot;
|
||||
const char* id = GET_TASKID(pTaskInfo);
|
||||
|
||||
pTaskInfo->streamInfo.prepareStatus = *pOffset;
|
||||
pTaskInfo->streamInfo.returned = 0;
|
||||
|
||||
|
|
@ -1081,24 +1097,20 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
|
|||
|
||||
if (subType == TOPIC_SUB_TYPE__COLUMN) {
|
||||
pOperator->status = OP_OPENED;
|
||||
|
||||
// TODO add more check
|
||||
if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
|
||||
if(pOperator->numOfDownstream != 1){
|
||||
qError("pOperator->numOfDownstream != 1:%d", pOperator->numOfDownstream);
|
||||
return -1;
|
||||
}
|
||||
pOperator = pOperator->pDownstream[0];
|
||||
}
|
||||
pOperator = extractOperatorInTree(pOperator, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, id);
|
||||
|
||||
SStreamScanInfo* pInfo = pOperator->info;
|
||||
STableScanInfo* pScanInfo = pInfo->pTableScanOp->info;
|
||||
STableScanBase* pScanBaseInfo = &pScanInfo->base;
|
||||
STableListInfo* pTableListInfo = pScanBaseInfo->pTableListInfo;
|
||||
|
||||
if (pOffset->type == TMQ_OFFSET__LOG) {
|
||||
STableScanInfo* pTSInfo = pInfo->pTableScanOp->info;
|
||||
tsdbReaderClose(pTSInfo->base.dataReader);
|
||||
pTSInfo->base.dataReader = NULL;
|
||||
tsdbReaderClose(pScanBaseInfo->dataReader);
|
||||
pScanBaseInfo->dataReader = NULL;
|
||||
|
||||
// let's seek to the next version in wal file
|
||||
if (tqSeekVer(pInfo->tqReader, pOffset->version + 1, pTaskInfo->id.str) < 0) {
|
||||
qError("tqSeekVer failed ver:%" PRId64, pOffset->version + 1);
|
||||
qError("tqSeekVer failed ver:%" PRId64 ", %s", pOffset->version + 1, id);
|
||||
return -1;
|
||||
}
|
||||
} else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) {
|
||||
|
|
@ -1106,127 +1118,138 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
|
|||
// those data are from the snapshot in tsdb, besides the data in the wal file.
|
||||
int64_t uid = pOffset->uid;
|
||||
int64_t ts = pOffset->ts;
|
||||
int32_t index = 0;
|
||||
|
||||
// this value may be changed if new tables are created
|
||||
taosRLockLatch(&pTaskInfo->lock);
|
||||
int32_t numOfTables = tableListGetSize(pTableListInfo);
|
||||
|
||||
if (uid == 0) {
|
||||
if (tableListGetSize(pTaskInfo->pTableInfoList) != 0) {
|
||||
STableKeyInfo* pTableInfo = tableListGetInfo(pTaskInfo->pTableInfoList, 0);
|
||||
if (numOfTables != 0) {
|
||||
STableKeyInfo* pTableInfo = tableListGetInfo(pTableListInfo, 0);
|
||||
uid = pTableInfo->uid;
|
||||
ts = INT64_MIN;
|
||||
pScanInfo->currentTable = 0;
|
||||
} else {
|
||||
qError("uid == 0 and tablelist size is 0");
|
||||
taosRUnLockLatch(&pTaskInfo->lock);
|
||||
qError("no table in table list, %s", id);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
/*if (pTaskInfo->streamInfo.lastStatus.type != TMQ_OFFSET__SNAPSHOT_DATA ||*/
|
||||
/*pTaskInfo->streamInfo.lastStatus.uid != uid || pTaskInfo->streamInfo.lastStatus.ts != ts) {*/
|
||||
STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info;
|
||||
int32_t numOfTables = tableListGetSize(pTaskInfo->pTableInfoList);
|
||||
|
||||
qDebug("switch to table uid:%" PRId64 " ts:%" PRId64 "% "PRId64 " rows returned", uid, ts, pInfo->pTableScanOp->resultInfo.totalRows);
|
||||
pInfo->pTableScanOp->resultInfo.totalRows = 0;
|
||||
|
||||
bool found = false;
|
||||
for (int32_t i = 0; i < numOfTables; i++) {
|
||||
STableKeyInfo* pTableInfo = tableListGetInfo(pTaskInfo->pTableInfoList, i);
|
||||
if (pTableInfo->uid == uid) {
|
||||
found = true;
|
||||
pTableScanInfo->currentTable = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
// start from current accessed position
|
||||
// we cannot start from the pScanInfo->currentTable, since the commit offset may cause the rollback of the start
|
||||
// position, let's find it from the beginning.
|
||||
index = tableListFind(pTableListInfo, uid, 0);
|
||||
taosRUnLockLatch(&pTaskInfo->lock);
|
||||
|
||||
// TODO after dropping table, table may not found
|
||||
if(!found){
|
||||
qError("uid not found in tablelist %" PRId64, uid);
|
||||
if (index >= 0) {
|
||||
pScanInfo->currentTable = index;
|
||||
} else {
|
||||
qError("vgId:%d uid:%" PRIu64 " not found in table list, total:%d, index:%d %s", pTaskInfo->id.vgId, uid,
|
||||
numOfTables, pScanInfo->currentTable, id);
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (pTableScanInfo->base.dataReader == NULL) {
|
||||
STableKeyInfo* pList = tableListGetInfo(pTaskInfo->pTableInfoList, 0);
|
||||
int32_t num = tableListGetSize(pTaskInfo->pTableInfoList);
|
||||
STableKeyInfo keyInfo = {.uid = uid};
|
||||
int64_t oldSkey = pScanBaseInfo->cond.twindows.skey;
|
||||
|
||||
if (tsdbReaderOpen(pTableScanInfo->base.readHandle.vnode, &pTableScanInfo->base.cond, pList, num,
|
||||
pTableScanInfo->pResBlock, &pTableScanInfo->base.dataReader, NULL) < 0 ||
|
||||
pTableScanInfo->base.dataReader == NULL) {
|
||||
qError("tsdbReaderOpen failed. uid:%" PRIi64, pOffset->uid);
|
||||
// let's start from the next ts that returned to consumer.
|
||||
pScanBaseInfo->cond.twindows.skey = ts + 1;
|
||||
pScanInfo->scanTimes = 0;
|
||||
|
||||
if (pScanBaseInfo->dataReader == NULL) {
|
||||
int32_t code = tsdbReaderOpen(pScanBaseInfo->readHandle.vnode, &pScanBaseInfo->cond, &keyInfo, 1,
|
||||
pScanInfo->pResBlock, &pScanBaseInfo->dataReader, id, false);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("prepare read tsdb snapshot failed, uid:%" PRId64 ", code:%s %s", pOffset->uid, tstrerror(code), id);
|
||||
terrno = code;
|
||||
return -1;
|
||||
}
|
||||
|
||||
qDebug("tsdb reader created with offset(snapshot) uid:%" PRId64 " ts:%" PRId64 " table index:%d, total:%d, %s",
|
||||
uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->currentTable, numOfTables, id);
|
||||
} else {
|
||||
tsdbSetTableList(pScanBaseInfo->dataReader, &keyInfo, 1);
|
||||
tsdbReaderReset(pScanBaseInfo->dataReader, &pScanBaseInfo->cond);
|
||||
qDebug("tsdb reader offset seek snapshot to uid:%" PRId64 " ts %" PRId64 " table index:%d numOfTable:%d, %s",
|
||||
uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->currentTable, numOfTables, id);
|
||||
}
|
||||
|
||||
STableKeyInfo tki = {.uid = uid};
|
||||
tsdbSetTableList(pTableScanInfo->base.dataReader, &tki, 1);
|
||||
int64_t oldSkey = pTableScanInfo->base.cond.twindows.skey;
|
||||
pTableScanInfo->base.cond.twindows.skey = ts + 1;
|
||||
tsdbReaderReset(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond);
|
||||
pTableScanInfo->base.cond.twindows.skey = oldSkey;
|
||||
pTableScanInfo->scanTimes = 0;
|
||||
|
||||
qDebug("tsdb reader offset seek snapshot to uid:%" PRId64 " ts %" PRId64 ", table cur set to %d , all table num %d", uid,
|
||||
ts, pTableScanInfo->currentTable, numOfTables);
|
||||
// restore the key value
|
||||
pScanBaseInfo->cond.twindows.skey = oldSkey;
|
||||
} else {
|
||||
qError("invalid pOffset->type:%d", pOffset->type);
|
||||
return -1;
|
||||
}
|
||||
} else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) {
|
||||
SStreamRawScanInfo* pInfo = pOperator->info;
|
||||
SSnapContext* sContext = pInfo->sContext;
|
||||
if (setForSnapShot(sContext, pOffset->uid) != 0) {
|
||||
qError("setDataForSnapShot error. uid:%" PRIi64, pOffset->uid);
|
||||
qError("invalid pOffset->type:%d, %s", pOffset->type, id);
|
||||
return -1;
|
||||
}
|
||||
|
||||
SMetaTableInfo mtInfo = getUidfromSnapShot(sContext);
|
||||
tsdbReaderClose(pInfo->dataReader);
|
||||
pInfo->dataReader = NULL;
|
||||
} else { // subType == TOPIC_SUB_TYPE__TABLE/TOPIC_SUB_TYPE__DB
|
||||
|
||||
cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond);
|
||||
tableListClear(pTaskInfo->pTableInfoList);
|
||||
if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) {
|
||||
SStreamRawScanInfo* pInfo = pOperator->info;
|
||||
SSnapContext* sContext = pInfo->sContext;
|
||||
|
||||
if (mtInfo.uid == 0) {
|
||||
return 0; // no data
|
||||
SOperatorInfo* p = extractOperatorInTree(pOperator, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, id);
|
||||
STableListInfo* pTableListInfo = ((SStreamRawScanInfo*)(p->info))->pTableListInfo;
|
||||
|
||||
if (setForSnapShot(sContext, pOffset->uid) != 0) {
|
||||
qError("setDataForSnapShot error. uid:%" PRId64 " , %s", pOffset->uid, id);
|
||||
return -1;
|
||||
}
|
||||
|
||||
SMetaTableInfo mtInfo = getUidfromSnapShot(sContext);
|
||||
tsdbReaderClose(pInfo->dataReader);
|
||||
pInfo->dataReader = NULL;
|
||||
|
||||
cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond);
|
||||
tableListClear(pTableListInfo);
|
||||
|
||||
if (mtInfo.uid == 0) {
|
||||
return 0; // no data
|
||||
}
|
||||
|
||||
initQueryTableDataCondForTmq(&pTaskInfo->streamInfo.tableCond, sContext, &mtInfo);
|
||||
pTaskInfo->streamInfo.tableCond.twindows.skey = pOffset->ts;
|
||||
|
||||
tableListAddTableInfo(pTableListInfo, mtInfo.uid, 0);
|
||||
|
||||
STableKeyInfo* pList = tableListGetInfo(pTableListInfo, 0);
|
||||
int32_t size = tableListGetSize(pTableListInfo);
|
||||
|
||||
tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL, &pInfo->dataReader, NULL,
|
||||
false);
|
||||
|
||||
cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond);
|
||||
strcpy(pTaskInfo->streamInfo.tbName, mtInfo.tbName);
|
||||
tDeleteSSchemaWrapper(pTaskInfo->streamInfo.schema);
|
||||
pTaskInfo->streamInfo.schema = mtInfo.schema;
|
||||
|
||||
qDebug("tmqsnap qStreamPrepareScan snapshot data uid:%" PRId64 " ts %" PRId64 " %s", mtInfo.uid, pOffset->ts, id);
|
||||
} else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_META) {
|
||||
SStreamRawScanInfo* pInfo = pOperator->info;
|
||||
SSnapContext* sContext = pInfo->sContext;
|
||||
if (setForSnapShot(sContext, pOffset->uid) != 0) {
|
||||
qError("setForSnapShot error. uid:%" PRIu64 " ,version:%" PRId64, pOffset->uid, pOffset->version);
|
||||
return -1;
|
||||
}
|
||||
qDebug("tmqsnap qStreamPrepareScan snapshot meta uid:%" PRId64 " ts %" PRId64 " %s", pOffset->uid, pOffset->ts,
|
||||
id);
|
||||
} else if (pOffset->type == TMQ_OFFSET__LOG) {
|
||||
SStreamRawScanInfo* pInfo = pOperator->info;
|
||||
tsdbReaderClose(pInfo->dataReader);
|
||||
pInfo->dataReader = NULL;
|
||||
qDebug("tmqsnap qStreamPrepareScan snapshot log, %s", id);
|
||||
}
|
||||
|
||||
initQueryTableDataCondForTmq(&pTaskInfo->streamInfo.tableCond, sContext, &mtInfo);
|
||||
pTaskInfo->streamInfo.tableCond.twindows.skey = pOffset->ts;
|
||||
|
||||
if (pTaskInfo->pTableInfoList == NULL) {
|
||||
pTaskInfo->pTableInfoList = tableListCreate();
|
||||
}
|
||||
|
||||
tableListAddTableInfo(pTaskInfo->pTableInfoList, mtInfo.uid, 0);
|
||||
|
||||
STableKeyInfo* pList = tableListGetInfo(pTaskInfo->pTableInfoList, 0);
|
||||
int32_t size = tableListGetSize(pTaskInfo->pTableInfoList);
|
||||
|
||||
tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL, &pInfo->dataReader, NULL);
|
||||
|
||||
cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond);
|
||||
strcpy(pTaskInfo->streamInfo.tbName, mtInfo.tbName);
|
||||
tDeleteSSchemaWrapper(pTaskInfo->streamInfo.schema);
|
||||
pTaskInfo->streamInfo.schema = mtInfo.schema;
|
||||
|
||||
qDebug("tmqsnap qStreamPrepareScan snapshot data uid:%" PRId64 " ts %" PRId64, mtInfo.uid, pOffset->ts);
|
||||
} else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_META) {
|
||||
SStreamRawScanInfo* pInfo = pOperator->info;
|
||||
SSnapContext* sContext = pInfo->sContext;
|
||||
if (setForSnapShot(sContext, pOffset->uid) != 0) {
|
||||
qError("setForSnapShot error. uid:%" PRIu64 " ,version:%" PRId64, pOffset->uid, pOffset->version);
|
||||
return -1;
|
||||
}
|
||||
qDebug("tmqsnap qStreamPrepareScan snapshot meta uid:%" PRId64 " ts %" PRId64, pOffset->uid, pOffset->ts);
|
||||
} else if (pOffset->type == TMQ_OFFSET__LOG) {
|
||||
SStreamRawScanInfo* pInfo = pOperator->info;
|
||||
tsdbReaderClose(pInfo->dataReader);
|
||||
pInfo->dataReader = NULL;
|
||||
qDebug("tmqsnap qStreamPrepareScan snapshot log");
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
void qProcessRspMsg(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) {
|
||||
SMsgSendInfo* pSendInfo = (SMsgSendInfo*)pMsg->info.ahandle;
|
||||
if(pMsg->info.ahandle == NULL){
|
||||
if (pMsg->info.ahandle == NULL) {
|
||||
qError("pMsg->info.ahandle is NULL");
|
||||
return;
|
||||
}
|
||||
|
|
@ -1247,4 +1270,3 @@ void qProcessRspMsg(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) {
|
|||
rpcFreeCont(pMsg->pCont);
|
||||
destroySendMsgInfo(pSendInfo);
|
||||
}
|
||||
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue