xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

other: merge 3.0

This commit is contained in:
Haojun Liao 2024-02-19 18:27:18 +08:00
parent a3b04c3c58 1bf885d810
commit 931b9f0b51
26 changed files with 2834 additions and 1401 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

667
docs/zh/07-develop/07-tmq.mdx
View File

@ -91,6 +91,7 @@ TDengine 会为 WAL 文件自动创建索引以支持快速随机访问，并提
不同语言下， TMQ 订阅相关的 API 及数据结构如下（详细的接口说明可以参考连接器章节，注意consumer结构不是线程安全的，在一个线程使用consumer时，不要在另一个线程close这个consumer）： 不同语言下， TMQ 订阅相关的 API 及数据结构如下（详细的接口说明可以参考连接器章节，注意consumer结构不是线程安全的，在一个线程使用consumer时，不要在另一个线程close这个consumer）：
<Tabs defaultValue="java" groupId="lang"> <Tabs defaultValue="java" groupId="lang">
<TabItem value="c" label="C"> <TabItem value="c" label="C">
```c ```c
@ -101,17 +102,17 @@ TDengine 会为 WAL 文件自动创建索引以支持快速随机访问，并提
typedef void(tmq_commit_cb(tmq_t *tmq, int32_t code, void *param)); typedef void(tmq_commit_cb(tmq_t *tmq, int32_t code, void *param));
typedef enum tmq_conf_res_t { typedef enum tmq_conf_res_t {
TMQ_CONF_UNKNOWN = -2, TMQ_CONF_UNKNOWN = -2,
TMQ_CONF_INVALID = -1, TMQ_CONF_INVALID = -1,
TMQ_CONF_OK = 0, TMQ_CONF_OK = 0,
} tmq_conf_res_t; } tmq_conf_res_t;
typedef struct tmq_topic_assignment { typedef struct tmq_topic_assignment {
int32_t vgId; int32_t vgId;
int64_t currentOffset; int64_t currentOffset;
int64_t begin; int64_t begin;
int64_t end; int64_t end;
} tmq_topic_assignment; } tmq_topic_assignment;
DLL_EXPORT tmq_conf_t *tmq_conf_new(); DLL_EXPORT tmq_conf_t *tmq_conf_new();
DLL_EXPORT tmq_conf_res_t tmq_conf_set(tmq_conf_t *conf, const char *key, const char *value); DLL_EXPORT tmq_conf_res_t tmq_conf_set(tmq_conf_t *conf, const char *key, const char *value);
@ -146,7 +147,6 @@ TDengine 会为 WAL 文件自动创建索引以支持快速随机访问，并提
DLL_EXPORT int64_t tmq_get_vgroup_offset(TAOS_RES* res); DLL_EXPORT int64_t tmq_get_vgroup_offset(TAOS_RES* res);
DLL_EXPORT const char *tmq_err2str(int32_t code); DLL_EXPORT const char *tmq_err2str(int32_t code);
``` ```
</TabItem> </TabItem>
<TabItem value="java" label="Java"> <TabItem value="java" label="Java">
@ -250,281 +250,280 @@ TDengine 会为 WAL 文件自动创建索引以支持快速随机访问，并提
+ futures::Stream< + futures::Stream<
Item = Result<(Self::Offset, MessageSet<Self::Meta, Self::Data>), Self::Error>, Item = Result<(Self::Offset, MessageSet<Self::Meta, Self::Data>), Self::Error>,
>, >,
>, >,
>; >;
async fn commit(&self, offset: Self::Offset) -> Result<(), Self::Error>; async fn commit(&self, offset: Self::Offset) -> Result<(), Self::Error>;
async fn unsubscribe(self); async fn unsubscribe(self);
``` ```
可在 <https://docs.rs/taos> 上查看详细 API 说明。 可在 <https://docs.rs/taos> 上查看详细 API 说明。
</TabItem> </TabItem>
<TabItem label="Node.JS" value="Node.JS"> <TabItem label="Node.JS" value="Node.JS">
```js ```js
function TMQConsumer(config) function TMQConsumer(config)
function subscribe(topic)
function consume(timeout)
function subscription()
function unsubscribe()
function commit(msg)
function close()
```
function subscribe(topic) </TabItem>
function consume(timeout) <TabItem value="C#" label="C#">
function subscription() ```csharp
class ConsumerBuilder<TValue>
ConsumerBuilder(IEnumerable<KeyValuePair<string, string>> config)
public IConsumer<TValue> Build()
void Subscribe(IEnumerable<string> topics)
void Subscribe(string topic)
ConsumeResult<TValue> Consume(int millisecondsTimeout)
List<string> Subscription()
void Unsubscribe()
List<TopicPartitionOffset> Commit()
void Close()
```
</TabItem>
</Tabs>
function unsubscribe() # 数据订阅示例
## 写入数据
function commit(msg) 首先完成建库、建一张超级表和多张子表操作，然后就可以写入数据了，比如：
function close() ```sql
``` DROP DATABASE IF EXISTS tmqdb;
CREATE DATABASE tmqdb WAL_RETENTION_PERIOD 3600;
</TabItem> CREATE TABLE tmqdb.stb (ts TIMESTAMP, c1 INT, c2 FLOAT, c3 VARCHAR(16)) TAGS(t1 INT, t3 VARCHAR(16));
CREATE TABLE tmqdb.ctb0 USING tmqdb.stb TAGS(0, "subtable0");
<TabItem value="C#" label="C#"> CREATE TABLE tmqdb.ctb1 USING tmqdb.stb TAGS(1, "subtable1");
INSERT INTO tmqdb.ctb0 VALUES(now, 0, 0, 'a0')(now+1s, 0, 0, 'a00');
```csharp
class ConsumerBuilder<TValue>
ConsumerBuilder(IEnumerable<KeyValuePair<string, string>> config)
public IConsumer<TValue> Build()
void Subscribe(IEnumerable<string> topics)
void Subscribe(string topic)
ConsumeResult<TValue> Consume(int millisecondsTimeout)
List<string> Subscription()
void Unsubscribe()
List<TopicPartitionOffset> Commit()
void Close()
```
</TabItem>
</Tabs>
# 数据订阅示例
## 写入数据
首先完成建库、建一张超级表和多张子表操作，然后就可以写入数据了，比如：
```sql
DROP DATABASE IF EXISTS tmqdb;
CREATE DATABASE tmqdb WAL_RETENTION_PERIOD 3600;
CREATE TABLE tmqdb.stb (ts TIMESTAMP, c1 INT, c2 FLOAT, c3 VARCHAR(16)) TAGS(t1 INT, t3 VARCHAR(16));
CREATE TABLE tmqdb.ctb0 USING tmqdb.stb TAGS(0, "subtable0");
CREATE TABLE tmqdb.ctb1 USING tmqdb.stb TAGS(1, "subtable1");
INSERT INTO tmqdb.ctb0 VALUES(now, 0, 0, 'a0')(now+1s, 0, 0, 'a00');
INSERT INTO tmqdb.ctb1 VALUES(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11'); INSERT INTO tmqdb.ctb1 VALUES(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11');
``` ```
## 创建 topic ## 创建 topic
使用 SQL 创建一个 topic： 使用 SQL 创建一个 topic：
```sql ```sql
CREATE TOPIC topic_name AS SELECT ts, c1, c2, c3 FROM tmqdb.stb WHERE c1 > 1; CREATE TOPIC topic_name AS SELECT ts, c1, c2, c3 FROM tmqdb.stb WHERE c1 > 1;
``` ```
## 创建消费者 *consumer* ## 创建消费者 *consumer*
对于不同编程语言，其设置方式如下： 对于不同编程语言，其设置方式如下：
<Tabs defaultValue="java" groupId="lang"> <Tabs defaultValue="java" groupId="lang">
<TabItem value="c" label="C">
```c <TabItem value="c" label="C">
/* 根据需要，设置消费组 (group.id)、自动提交 (enable.auto.commit)、 ```c
自动提交时间间隔 (auto.commit.interval.ms)、用户名 (td.connect.user)、密码 (td.connect.pass) 等参数 */ /* 根据需要，设置消费组 (group.id)、自动提交 (enable.auto.commit)、
tmq_conf_t* conf = tmq_conf_new(); 自动提交时间间隔 (auto.commit.interval.ms)、用户名 (td.connect.user)、密码 (td.connect.pass) 等参数 */
tmq_conf_set(conf, "enable.auto.commit", "true"); tmq_conf_t* conf = tmq_conf_new();
tmq_conf_set(conf, "auto.commit.interval.ms", "1000"); tmq_conf_set(conf, "enable.auto.commit", "true");
tmq_conf_set(conf, "group.id", "cgrpName"); tmq_conf_set(conf, "auto.commit.interval.ms", "1000");
tmq_conf_set(conf, "td.connect.user", "root"); tmq_conf_set(conf, "group.id", "cgrpName");
tmq_conf_set(conf, "td.connect.pass", "taosdata"); tmq_conf_set(conf, "td.connect.user", "root");
tmq_conf_set(conf, "auto.offset.reset", "latest"); tmq_conf_set(conf, "td.connect.pass", "taosdata");
tmq_conf_set(conf, "msg.with.table.name", "true"); tmq_conf_set(conf, "auto.offset.reset", "latest");
tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); tmq_conf_set(conf, "msg.with.table.name", "true");
tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL);
tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); tmq_t* tmq = tmq_consumer_new(conf, NULL, 0);
tmq_conf_destroy(conf); tmq_conf_destroy(conf);
```
</TabItem>
<TabItem value="java" label="Java">
对于 Java 程序，还可以使用如下配置项：
| 参数名称 | 类型 | 参数说明 |
| ----------------------------- | ------ | ----------------------------------------------------------------------------------------------------------------------------- |
| `td.connect.type` | string | 连接类型，"jni" 指原生连接，"ws" 指 websocket 连接，默认值为 "jni" |
| `bootstrap.servers` | string | 连接地址，如 `localhost:6030` |
| `value.deserializer` | string | 值解析方法，使用此方法应实现 `com.taosdata.jdbc.tmq.Deserializer` 接口或继承 `com.taosdata.jdbc.tmq.ReferenceDeserializer` 类 |
| `value.deserializer.encoding` | string | 指定字符串解析的字符集 | |
需要注意：此处使用 `bootstrap.servers` 替代 `td.connect.ip` 和 `td.connect.port`，以提供与 Kafka 一致的接口。
```java
Properties properties = new Properties();
properties.setProperty("enable.auto.commit", "true");
properties.setProperty("auto.commit.interval.ms", "1000");
properties.setProperty("group.id", "cgrpName");
properties.setProperty("bootstrap.servers", "127.0.0.1:6030");
properties.setProperty("td.connect.user", "root");
properties.setProperty("td.connect.pass", "taosdata");
properties.setProperty("auto.offset.reset", "latest");
properties.setProperty("msg.with.table.name", "true");
properties.setProperty("value.deserializer", "com.taos.example.MetersDeserializer");
TaosConsumer<Meters> consumer = new TaosConsumer<>(properties);
/* value deserializer definition. */
import com.taosdata.jdbc.tmq.ReferenceDeserializer;
public class MetersDeserializer extends ReferenceDeserializer<Meters> {
}
```
</TabItem>
<TabItem label="Go" value="Go">
```go
conf := &tmq.ConfigMap{
"group.id": "test",
"auto.offset.reset": "latest",
"td.connect.ip": "127.0.0.1",
"td.connect.user": "root",
"td.connect.pass": "taosdata",
"td.connect.port": "6030",
"client.id": "test_tmq_c",
"enable.auto.commit": "false",
"msg.with.table.name": "true",
}
consumer, err := NewConsumer(conf)
```
</TabItem>
<TabItem label="Rust" value="Rust">
```rust
let mut dsn: Dsn = "taos://".parse()?;
dsn.set("group.id", "group1");
dsn.set("client.id", "test");
dsn.set("auto.offset.reset", "latest");
let tmq = TmqBuilder::from_dsn(dsn)?;
let mut consumer = tmq.build()?;
```
</TabItem>
<TabItem value="Python" label="Python">
Python 语言下引入 `taos` 库的 `Consumer` 类，创建一个 Consumer 示例：
```python
from taos.tmq import Consumer
# Syntax: `consumer = Consumer(configs)`
#
# Example:
consumer = Consumer(
{
"group.id": "local",
"client.id": "1",
"enable.auto.commit": "true",
"auto.commit.interval.ms": "1000",
"td.connect.ip": "127.0.0.1",
"td.connect.user": "root",
"td.connect.pass": "taosdata",
"auto.offset.reset": "latest",
"msg.with.table.name": "true",
}
)
```
</TabItem>
<TabItem label="Node.JS" value="Node.JS">
```js
// 根据需要，设置消费组 (group.id)、自动提交 (enable.auto.commit)、
// 自动提交时间间隔 (auto.commit.interval.ms)、用户名 (td.connect.user)、密码 (td.connect.pass) 等参数
let consumer = taos.consumer({
'enable.auto.commit': 'true',
'auto.commit.interval.ms','1000',
'group.id': 'tg2',
'td.connect.user': 'root',
'td.connect.pass': 'taosdata',
'auto.offset.reset','latest',
'msg.with.table.name': 'true',
'td.connect.ip','127.0.0.1',
'td.connect.port','6030'
});
``` ```
</TabItem>
</TabItem> <TabItem value="java" label="Java">
<TabItem value="C#" label="C#"> 对于 Java 程序，还可以使用如下配置项：
```csharp | 参数名称 | 类型 | 参数说明 |
var cfg = new Dictionary<string, string>() | ----------------------------- | ------ | ----------------------------------------------------------------------------------------------------------------------------- |
{ | `td.connect.type` | string | 连接类型，"jni" 指原生连接，"ws" 指 websocket 连接，默认值为 "jni" |
{ "group.id", "group1" }, | `bootstrap.servers` | string | 连接地址，如 `localhost:6030` |
{ "auto.offset.reset", "latest" }, | `value.deserializer` | string | 值解析方法，使用此方法应实现 `com.taosdata.jdbc.tmq.Deserializer` 接口或继承 `com.taosdata.jdbc.tmq.ReferenceDeserializer` 类 |
{ "td.connect.ip", "127.0.0.1" }, | `value.deserializer.encoding` | string | 指定字符串解析的字符集 | |
{ "td.connect.user", "root" },
{ "td.connect.pass", "taosdata" }, 需要注意：此处使用 `bootstrap.servers` 替代 `td.connect.ip` 和 `td.connect.port`，以提供与 Kafka 一致的接口。
{ "td.connect.port", "6030" },
{ "client.id", "tmq_example" }, ```java
{ "enable.auto.commit", "true" }, Properties properties = new Properties();
{ "msg.with.table.name", "false" }, properties.setProperty("enable.auto.commit", "true");
}; properties.setProperty("auto.commit.interval.ms", "1000");
var consumer = new ConsumerBuilder<Dictionary<string, object>>(cfg).Build(); properties.setProperty("group.id", "cgrpName");
properties.setProperty("bootstrap.servers", "127.0.0.1:6030");
properties.setProperty("td.connect.user", "root");
properties.setProperty("td.connect.pass", "taosdata");
properties.setProperty("auto.offset.reset", "latest");
properties.setProperty("msg.with.table.name", "true");
properties.setProperty("value.deserializer", "com.taos.example.MetersDeserializer");
TaosConsumer<Meters> consumer = new TaosConsumer<>(properties);
/* value deserializer definition. */
import com.taosdata.jdbc.tmq.ReferenceDeserializer;
public class MetersDeserializer extends ReferenceDeserializer<Meters> {
}
``` ```
</TabItem>
</TabItem> <TabItem label="Go" value="Go">
```go
conf := &tmq.ConfigMap{
"group.id": "test",
"auto.offset.reset": "latest",
"td.connect.ip": "127.0.0.1",
"td.connect.user": "root",
"td.connect.pass": "taosdata",
"td.connect.port": "6030",
"client.id": "test_tmq_c",
"enable.auto.commit": "false",
"msg.with.table.name": "true",
}
consumer, err := NewConsumer(conf)
```
</TabItem>
</Tabs> <TabItem label="Rust" value="Rust">
```rust
let mut dsn: Dsn = "taos://".parse()?;
dsn.set("group.id", "group1");
dsn.set("client.id", "test");
dsn.set("auto.offset.reset", "latest");
let tmq = TmqBuilder::from_dsn(dsn)?;
let mut consumer = tmq.build()?;
```
</TabItem>
上述配置中包括 consumer group ID，如果多个 consumer 指定的 consumer group ID 一样，则自动形成一个 consumer group，共享消费进度。 <TabItem value="Python" label="Python">
Python 语言下引入 `taos` 库的 `Consumer` 类，创建一个 Consumer 示例：
```python
from taos.tmq import Consumer
# Syntax: `consumer = Consumer(configs)`
#
# Example:
consumer = Consumer(
{
"group.id": "local",
"client.id": "1",
"enable.auto.commit": "true",
"auto.commit.interval.ms": "1000",
"td.connect.ip": "127.0.0.1",
"td.connect.user": "root",
"td.connect.pass": "taosdata",
"auto.offset.reset": "latest",
"msg.with.table.name": "true",
}
)
```
</TabItem>
## 订阅 *topics* <TabItem label="Node.JS" value="Node.JS">
```js
// 根据需要，设置消费组 (group.id)、自动提交 (enable.auto.commit)、
// 自动提交时间间隔 (auto.commit.interval.ms)、用户名 (td.connect.user)、密码 (td.connect.pass) 等参数
let consumer = taos.consumer({
'enable.auto.commit': 'true',
'auto.commit.interval.ms','1000',
'group.id': 'tg2',
'td.connect.user': 'root',
'td.connect.pass': 'taosdata',
'auto.offset.reset','latest',
'msg.with.table.name': 'true',
'td.connect.ip','127.0.0.1',
'td.connect.port','6030'
});
```
</TabItem>
一个 consumer 支持同时订阅多个 topic。 <TabItem value="C#" label="C#">
```csharp
var cfg = new Dictionary<string, string>()
{
{ "group.id", "group1" },
{ "auto.offset.reset", "latest" },
{ "td.connect.ip", "127.0.0.1" },
{ "td.connect.user", "root" },
{ "td.connect.pass", "taosdata" },
{ "td.connect.port", "6030" },
{ "client.id", "tmq_example" },
{ "enable.auto.commit", "true" },
{ "msg.with.table.name", "false" },
};
var consumer = new ConsumerBuilder<Dictionary<string, object>>(cfg).Build();
```
</TabItem>
</Tabs>
上述配置中包括 consumer group ID，如果多个 consumer 指定的 consumer group ID 一样，则自动形成一个 consumer group，共享消费进度。
## 订阅 *topics*
<Tabs defaultValue="java" groupId="lang"> 一个 consumer 支持同时订阅多个 topic。
<TabItem value="c" label="C">
<Tabs defaultValue="java" groupId="lang">
```c <TabItem value="c" label="C">
```c
// 创建订阅 topics 列表 // 创建订阅 topics 列表
tmq_list_t* topicList = tmq_list_new(); tmq_list_t* topicList = tmq_list_new();
tmq_list_append(topicList, "topicName"); tmq_list_append(topicList, "topicName");
// 启动订阅 // 启动订阅
tmq_subscribe(tmq, topicList); tmq_subscribe(tmq, topicList);
tmq_list_destroy(topicList); tmq_list_destroy(topicList);
``` ```
</TabItem> </TabItem>
<TabItem value="java" label="Java"> <TabItem value="java" label="Java">
```java ```java
List<String> topics = new ArrayList<>(); List<String> topics = new ArrayList<>();
topics.add("tmq_topic"); topics.add("tmq_topic");
consumer.subscribe(topics); consumer.subscribe(topics);
``` ```
</TabItem> </TabItem>
<TabItem value="Go" label="Go"> <TabItem value="Go" label="Go">
```go ```go
err = consumer.Subscribe("example_tmq_topic", nil) err = consumer.Subscribe("example_tmq_topic", nil)
@ -533,26 +532,26 @@ INSERT INTO tmqdb.ctb1 VALUES(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11');
} }
``` ```
</TabItem> </TabItem>
<TabItem value="Rust" label="Rust"> <TabItem value="Rust" label="Rust">
```rust ```rust
consumer.subscribe(["tmq_meters"]).await?; consumer.subscribe(["tmq_meters"]).await?;
``` ```
</TabItem> </TabItem>
<TabItem value="Python" label="Python"> <TabItem value="Python" label="Python">
```python ```python
consumer.subscribe(['topic1', 'topic2']) consumer.subscribe(['topic1', 'topic2'])
``` ```
</TabItem> </TabItem>
<TabItem label="Node.JS" value="Node.JS"> <TabItem label="Node.JS" value="Node.JS">
```js ```js
// 创建订阅 topics 列表 // 创建订阅 topics 列表
let topics = ['topic_test'] let topics = ['topic_test']
@ -560,11 +559,11 @@ INSERT INTO tmqdb.ctb1 VALUES(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11');
consumer.subscribe(topics); consumer.subscribe(topics);
``` ```
</TabItem> </TabItem>
<TabItem value="C#" label="C#"> <TabItem value="C#" label="C#">
```csharp ```csharp
// 创建订阅 topics 列表 // 创建订阅 topics 列表
List<String> topics = new List<string>(); List<String> topics = new List<string>();
topics.add("tmq_topic"); topics.add("tmq_topic");
@ -572,18 +571,19 @@ INSERT INTO tmqdb.ctb1 VALUES(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11');
consumer.Subscribe(topics); consumer.Subscribe(topics);
``` ```
</TabItem> </TabItem>
</Tabs> </Tabs>
## 消费 ## 消费
以下代码展示了不同语言下如何对 TMQ 消息进行消费。 以下代码展示了不同语言下如何对 TMQ 消息进行消费。
<Tabs defaultValue="java" groupId="lang"> <Tabs defaultValue="java" groupId="lang">
<TabItem value="c" label="C">
```c <TabItem value="c" label="C">
```c
// 消费数据 // 消费数据
while (running) { while (running) {
TAOS_RES* msg = tmq_consumer_poll(tmq, timeOut); TAOS_RES* msg = tmq_consumer_poll(tmq, timeOut);
@ -591,11 +591,11 @@ INSERT INTO tmqdb.ctb1 VALUES(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11');
} }
``` ```
这里是一个 **while** 循环，每调用一次 tmq_consumer_poll()，获取一个消息，该消息与普通查询返回的结果集完全相同，可以使用相同的解析 API 完成消息内容的解析。 这里是一个 **while** 循环，每调用一次 tmq_consumer_poll()，获取一个消息，该消息与普通查询返回的结果集完全相同，可以使用相同的解析 API 完成消息内容的解析。
</TabItem>
<TabItem value="java" label="Java">
</TabItem>
<TabItem value="java" label="Java">
```java ```java
while(running){ while(running){
ConsumerRecords<Meters> meters = consumer.poll(Duration.ofMillis(100)); ConsumerRecords<Meters> meters = consumer.poll(Duration.ofMillis(100));
@ -604,10 +604,10 @@ INSERT INTO tmqdb.ctb1 VALUES(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11');
} }
} }
``` ```
</TabItem>
</TabItem> <TabItem value="Go" label="Go">
<TabItem value="Go" label="Go">
```go ```go
for { for {
@ -625,9 +625,9 @@ INSERT INTO tmqdb.ctb1 VALUES(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11');
} }
``` ```
</TabItem> </TabItem>
<TabItem value="Rust" label="Rust"> <TabItem value="Rust" label="Rust">
```rust ```rust
{ {
@ -660,8 +660,8 @@ INSERT INTO tmqdb.ctb1 VALUES(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11');
} }
``` ```
</TabItem> </TabItem>
<TabItem value="Python" label="Python"> <TabItem value="Python" label="Python">
```python ```python
while True: while True:
@ -677,9 +677,9 @@ INSERT INTO tmqdb.ctb1 VALUES(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11');
print(block.fetchall()) print(block.fetchall())
``` ```
</TabItem> </TabItem>
<TabItem label="Node.JS" value="Node.JS"> <TabItem label="Node.JS" value="Node.JS">
```js ```js
while(true){ while(true){
@ -691,11 +691,11 @@ INSERT INTO tmqdb.ctb1 VALUES(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11');
} }
``` ```
</TabItem> </TabItem>
<TabItem value="C#" label="C#"> <TabItem value="C#" label="C#">
```csharp ```csharp
// 消费数据 // 消费数据
while (true) while (true)
{ {
@ -708,18 +708,19 @@ INSERT INTO tmqdb.ctb1 VALUES(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11');
} }
``` ```
</TabItem> </TabItem>
</Tabs> </Tabs>
## 结束消费 ## 结束消费
消费结束后，应当取消订阅。 消费结束后，应当取消订阅。
<Tabs defaultValue="java" groupId="lang"> <Tabs defaultValue="java" groupId="lang">
<TabItem value="c" label="C">
<TabItem value="c" label="C">
```c ```c
/* 取消订阅 */ /* 取消订阅 */
tmq_unsubscribe(tmq); tmq_unsubscribe(tmq);
@ -727,10 +728,10 @@ INSERT INTO tmqdb.ctb1 VALUES(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11');
tmq_consumer_close(tmq); tmq_consumer_close(tmq);
``` ```
</TabItem> </TabItem>
<TabItem value="java" label="Java"> <TabItem value="java" label="Java">
```java ```java
/* 取消订阅 */ /* 取消订阅 */
consumer.unsubscribe(); consumer.unsubscribe();
@ -738,11 +739,11 @@ INSERT INTO tmqdb.ctb1 VALUES(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11');
consumer.close(); consumer.close();
``` ```
</TabItem> </TabItem>
<TabItem value="Go" label="Go">
<TabItem value="Go" label="Go"> ```go
```go
/* Unsubscribe */ /* Unsubscribe */
_ = consumer.Unsubscribe() _ = consumer.Unsubscribe()
@ -750,38 +751,38 @@ INSERT INTO tmqdb.ctb1 VALUES(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11');
_ = consumer.Close() _ = consumer.Close()
``` ```
</TabItem> </TabItem>
<TabItem value="Rust" label="Rust">
<TabItem value="Rust" label="Rust"> ```rust
```rust
consumer.unsubscribe().await; consumer.unsubscribe().await;
``` ```
</TabItem> </TabItem>
<TabItem value="Python" label="Python"> <TabItem value="Python" label="Python">
```py ```py
# 取消订阅 # 取消订阅
consumer.unsubscribe() consumer.unsubscribe()
# 关闭消费 # 关闭消费
consumer.close() consumer.close()
``` ```
</TabItem> </TabItem>
<TabItem label="Node.JS" value="Node.JS"> <TabItem label="Node.JS" value="Node.JS">
```js ```js
consumer.unsubscribe(); consumer.unsubscribe();
consumer.close(); consumer.close();
``` ```
</TabItem> </TabItem>
<TabItem value="C#" label="C#"> <TabItem value="C#" label="C#">
```csharp ```csharp
// 取消订阅 // 取消订阅
consumer.Unsubscribe(); consumer.Unsubscribe();
@ -789,7 +790,7 @@ INSERT INTO tmqdb.ctb1 VALUES(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11');
consumer.Close(); consumer.Close();
``` ```
</TabItem> </TabItem>
</Tabs> </Tabs>
@ -799,41 +800,41 @@ INSERT INTO tmqdb.ctb1 VALUES(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11');
<Tabs defaultValue="java" groupId="lang"> <Tabs defaultValue="java" groupId="lang">
<TabItem label="C" value="c"> <TabItem label="C" value="c">
<CDemo /> <CDemo />
</TabItem> </TabItem>
<TabItem label="Java" value="java"> <TabItem label="Java" value="java">
<Tabs defaultValue="native"> <Tabs defaultValue="native">
<TabItem value="native" label="本地连接"> <TabItem value="native" label="本地连接">
<Java /> <Java />
</TabItem> </TabItem>
<TabItem value="ws" label="WebSocket 连接"> <TabItem value="ws" label="WebSocket 连接">
<JavaWS /> <JavaWS />
</TabItem> </TabItem>
</Tabs> </Tabs>
</TabItem> </TabItem>
<TabItem label="Go" value="Go"> <TabItem label="Go" value="Go">
<Go/> <Go/>
</TabItem> </TabItem>
<TabItem label="Rust" value="Rust"> <TabItem label="Rust" value="Rust">
<Rust /> <Rust />
</TabItem> </TabItem>
<TabItem label="Python" value="Python"> <TabItem label="Python" value="Python">
<Python /> <Python />
</TabItem> </TabItem>
<TabItem label="Node.JS" value="Node.JS"> <TabItem label="Node.JS" value="Node.JS">
<Node/> <Node/>
</TabItem> </TabItem>
<TabItem label="C#" value="C#">
<CSharp/>
</TabItem>
<TabItem label="C#" value="C#">
<CSharp/>
</TabItem>
</Tabs> </Tabs>
#订阅高级功能 #订阅高级功能

28
include/common/tmsg.h
View File

@ -1400,7 +1400,7 @@ void tFreeSCompactDbReq(SCompactDbReq* pReq);
typedef struct { typedef struct {
int32_t compactId; int32_t compactId;
int8_t bAccepted; int8_t bAccepted;
} SCompactDbRsp; } SCompactDbRsp;
int32_t tSerializeSCompactDbRsp(void* buf, int32_t bufLen, SCompactDbRsp* pRsp); int32_t tSerializeSCompactDbRsp(void* buf, int32_t bufLen, SCompactDbRsp* pRsp);
@ -1414,7 +1414,7 @@ typedef struct {
int32_t tSerializeSKillCompactReq(void* buf, int32_t bufLen, SKillCompactReq* pReq); int32_t tSerializeSKillCompactReq(void* buf, int32_t bufLen, SKillCompactReq* pReq);
int32_t tDeserializeSKillCompactReq(void* buf, int32_t bufLen, SKillCompactReq* pReq); int32_t tDeserializeSKillCompactReq(void* buf, int32_t bufLen, SKillCompactReq* pReq);
void tFreeSKillCompactReq(SKillCompactReq *pReq); void tFreeSKillCompactReq(SKillCompactReq* pReq);
typedef struct { typedef struct {
char name[TSDB_FUNC_NAME_LEN]; char name[TSDB_FUNC_NAME_LEN];
@ -1751,9 +1751,9 @@ int32_t tSerializeSCompactVnodeReq(void* buf, int32_t bufLen, SCompactVnodeReq*
int32_t tDeserializeSCompactVnodeReq(void* buf, int32_t bufLen, SCompactVnodeReq* pReq); int32_t tDeserializeSCompactVnodeReq(void* buf, int32_t bufLen, SCompactVnodeReq* pReq);
typedef struct { typedef struct {
int32_t compactId; int32_t compactId;
int32_t vgId; int32_t vgId;
int32_t dnodeId; int32_t dnodeId;
} SVKillCompactReq; } SVKillCompactReq;
int32_t tSerializeSVKillCompactReq(void* buf, int32_t bufLen, SVKillCompactReq* pReq); int32_t tSerializeSVKillCompactReq(void* buf, int32_t bufLen, SVKillCompactReq* pReq);
@ -1954,9 +1954,9 @@ typedef struct {
char db[TSDB_DB_FNAME_LEN]; char db[TSDB_DB_FNAME_LEN];
char tb[TSDB_TABLE_NAME_LEN]; char tb[TSDB_TABLE_NAME_LEN];
char user[TSDB_USER_LEN]; char user[TSDB_USER_LEN];
char filterTb[TSDB_TABLE_NAME_LEN]; // for ins_columns char filterTb[TSDB_TABLE_NAME_LEN]; // for ins_columns
int64_t showId; int64_t showId;
int64_t compactId; // for compact int64_t compactId; // for compact
} SRetrieveTableReq; } SRetrieveTableReq;
typedef struct SSysTableSchema { typedef struct SSysTableSchema {
@ -3784,12 +3784,12 @@ typedef struct {
} SMqHbReq; } SMqHbReq;
typedef struct { typedef struct {
char topic[TSDB_TOPIC_FNAME_LEN]; char topic[TSDB_TOPIC_FNAME_LEN];
int8_t noPrivilege; int8_t noPrivilege;
} STopicPrivilege; } STopicPrivilege;
typedef struct { typedef struct {
SArray* topicPrivileges; // SArray<STopicPrivilege> SArray* topicPrivileges; // SArray<STopicPrivilege>
} SMqHbRsp; } SMqHbRsp;
typedef struct { typedef struct {
@ -3927,8 +3927,8 @@ int32_t tDeserializeSMqSeekReq(void* buf, int32_t bufLen, SMqSeekReq* pReq);
#define SUBMIT_REQ_COLUMN_DATA_FORMAT 0x2 #define SUBMIT_REQ_COLUMN_DATA_FORMAT 0x2
#define SUBMIT_REQ_FROM_FILE 0x4 #define SUBMIT_REQ_FROM_FILE 0x4
#define SOURCE_NULL 0 #define SOURCE_NULL 0
#define SOURCE_TAOSX 1 #define SOURCE_TAOSX 1
typedef struct { typedef struct {
int32_t flags; int32_t flags;
@ -3940,8 +3940,8 @@ typedef struct {
SArray* aRowP; SArray* aRowP;
SArray* aCol; SArray* aCol;
}; };
int64_t ctimeMs; int64_t ctimeMs;
int8_t source; int8_t source;
} SSubmitTbData; } SSubmitTbData;
typedef struct { typedef struct {

104
include/libs/stream/tstream.h
View File

@ -50,21 +50,21 @@ extern "C" {
(_t)->hTaskInfo.id.streamId = 0; \ (_t)->hTaskInfo.id.streamId = 0; \
} while (0) } while (0)
#define STREAM_EXEC_T_EXTRACT_WAL_DATA (-1) #define STREAM_EXEC_T_EXTRACT_WAL_DATA (-1)
#define STREAM_EXEC_T_START_ALL_TASKS (-2) #define STREAM_EXEC_T_START_ALL_TASKS (-2)
#define STREAM_EXEC_T_START_ONE_TASK (-3) #define STREAM_EXEC_T_START_ONE_TASK (-3)
#define STREAM_EXEC_T_RESTART_ALL_TASKS (-4) #define STREAM_EXEC_T_RESTART_ALL_TASKS (-4)
#define STREAM_EXEC_T_STOP_ALL_TASKS (-5) #define STREAM_EXEC_T_STOP_ALL_TASKS (-5)
#define STREAM_EXEC_T_RESUME_TASK (-6) #define STREAM_EXEC_T_RESUME_TASK (-6)
#define STREAM_EXEC_T_UPDATE_TASK_EPSET (-7) #define STREAM_EXEC_T_UPDATE_TASK_EPSET (-7)
typedef struct SStreamTask SStreamTask; typedef struct SStreamTask SStreamTask;
typedef struct SStreamQueue SStreamQueue; typedef struct SStreamQueue SStreamQueue;
typedef struct SStreamTaskSM SStreamTaskSM; typedef struct SStreamTaskSM SStreamTaskSM;
#define SSTREAM_TASK_VER 3 #define SSTREAM_TASK_VER 3
#define SSTREAM_TASK_INCOMPATIBLE_VER 1 #define SSTREAM_TASK_INCOMPATIBLE_VER 1
#define SSTREAM_TASK_NEED_CONVERT_VER 2 #define SSTREAM_TASK_NEED_CONVERT_VER 2
#define SSTREAM_TASK_SUBTABLE_CHANGED_VER 3 #define SSTREAM_TASK_SUBTABLE_CHANGED_VER 3
enum { enum {
@ -405,8 +405,8 @@ typedef struct SHistoryTaskInfo {
int32_t tickCount; int32_t tickCount;
int32_t retryTimes; int32_t retryTimes;
int32_t waitInterval; int32_t waitInterval;
int64_t haltVer; // offset in wal when halt the stream task int64_t haltVer; // offset in wal when halt the stream task
bool operatorOpen; // false by default bool operatorOpen; // false by default
} SHistoryTaskInfo; } SHistoryTaskInfo;
typedef struct STaskOutputInfo { typedef struct STaskOutputInfo {
@ -463,21 +463,22 @@ struct SStreamTask {
struct SStreamMeta* pMeta; struct SStreamMeta* pMeta;
SSHashObj* pNameMap; SSHashObj* pNameMap;
void* pBackend; void* pBackend;
char reserve[256]; int8_t subtableWithoutMd5;
char reserve[255];
}; };
typedef int32_t (*startComplete_fn_t)(struct SStreamMeta*); typedef int32_t (*startComplete_fn_t)(struct SStreamMeta*);
typedef struct STaskStartInfo { typedef struct STaskStartInfo {
int64_t startTs; int64_t startTs;
int64_t readyTs; int64_t readyTs;
int32_t tasksWillRestart; int32_t tasksWillRestart;
int32_t taskStarting; // restart flag, sentinel to guard the restart procedure. int32_t taskStarting; // restart flag, sentinel to guard the restart procedure.
SHashObj* pReadyTaskSet; // tasks that are all ready for running stream processing SHashObj* pReadyTaskSet; // tasks that are all ready for running stream processing
SHashObj* pFailedTaskSet; // tasks that are done the check downstream process, may be successful or failed SHashObj* pFailedTaskSet; // tasks that are done the check downstream process, may be successful or failed
int64_t elapsedTime; int64_t elapsedTime;
int32_t restartCount; // restart task counter int32_t restartCount; // restart task counter
startComplete_fn_t completeFn; // complete callback function startComplete_fn_t completeFn; // complete callback function
} STaskStartInfo; } STaskStartInfo;
typedef struct STaskUpdateInfo { typedef struct STaskUpdateInfo {
@ -504,7 +505,7 @@ typedef struct SStreamMeta {
int32_t vgId; int32_t vgId;
int64_t stage; int64_t stage;
int32_t role; int32_t role;
bool sendMsgBeforeClosing; // send hb to mnode before close all tasks when switch to follower. bool sendMsgBeforeClosing; // send hb to mnode before close all tasks when switch to follower.
STaskStartInfo startInfo; STaskStartInfo startInfo;
TdThreadRwlock lock; TdThreadRwlock lock;
SScanWalInfo scanInfo; SScanWalInfo scanInfo;
@ -532,7 +533,7 @@ int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo)
int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo); int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo);
SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, bool fillHistory, int64_t triggerParam, SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, bool fillHistory, int64_t triggerParam,
SArray* pTaskList, bool hasFillhistory); SArray* pTaskList, bool hasFillhistory, int8_t subtableWithoutMd5);
int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask); int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask);
int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask); int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask);
void tFreeStreamTask(SStreamTask* pTask); void tFreeStreamTask(SStreamTask* pTask);
@ -678,18 +679,18 @@ typedef struct STaskStatusEntry {
int32_t statusLastDuration; // to record the last duration of current status int32_t statusLastDuration; // to record the last duration of current status
int64_t stage; int64_t stage;
int32_t nodeId; int32_t nodeId;
int64_t verStart; // start version in WAL, only valid for source task int64_t verStart; // start version in WAL, only valid for source task
int64_t verEnd; // end version in WAL, only valid for source task int64_t verEnd; // end version in WAL, only valid for source task
int64_t processedVer; // only valid for source task int64_t processedVer; // only valid for source task
int64_t checkpointId; // current active checkpoint id int64_t checkpointId; // current active checkpoint id
int32_t chkpointTransId; // checkpoint trans id int32_t chkpointTransId; // checkpoint trans id
int8_t checkpointFailed; // denote if the checkpoint is failed or not int8_t checkpointFailed; // denote if the checkpoint is failed or not
bool inputQChanging; // inputQ is changing or not bool inputQChanging; // inputQ is changing or not
int64_t inputQUnchangeCounter; int64_t inputQUnchangeCounter;
double inputQUsed; // in MiB double inputQUsed; // in MiB
double inputRate; double inputRate;
double sinkQuota; // existed quota size for sink task double sinkQuota; // existed quota size for sink task
double sinkDataSize; // sink to dst data size double sinkDataSize; // sink to dst data size
} STaskStatusEntry; } STaskStatusEntry;
typedef struct SStreamHbMsg { typedef struct SStreamHbMsg {
@ -720,8 +721,8 @@ int32_t tEncodeStreamTaskUpdateMsg(SEncoder* pEncoder, const SStreamTaskNodeUpda
int32_t tDecodeStreamTaskUpdateMsg(SDecoder* pDecoder, SStreamTaskNodeUpdateMsg* pMsg); int32_t tDecodeStreamTaskUpdateMsg(SDecoder* pDecoder, SStreamTaskNodeUpdateMsg* pMsg);
typedef struct SStreamTaskState { typedef struct SStreamTaskState {
ETaskStatus state; ETaskStatus state;
char* name; char* name;
} SStreamTaskState; } SStreamTaskState;
int32_t tEncodeStreamTaskCheckReq(SEncoder* pEncoder, const SStreamTaskCheckReq* pReq); int32_t tEncodeStreamTaskCheckReq(SEncoder* pEncoder, const SStreamTaskCheckReq* pReq);
@ -826,7 +827,8 @@ SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st);
// stream task meta // stream task meta
void streamMetaInit(); void streamMetaInit();
void streamMetaCleanup(); void streamMetaCleanup();
SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc, int32_t vgId, int64_t stage, startComplete_fn_t fn); SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc, int32_t vgId, int64_t stage,
startComplete_fn_t fn);
void streamMetaClose(SStreamMeta* streamMeta); void streamMetaClose(SStreamMeta* streamMeta);
int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask); // save to stream meta store int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask); // save to stream meta store
int32_t streamMetaRemoveTask(SStreamMeta* pMeta, STaskId* pKey); int32_t streamMetaRemoveTask(SStreamMeta* pMeta, STaskId* pKey);
@ -845,22 +847,22 @@ void streamMetaNotifyClose(SStreamMeta* pMeta);
void streamMetaStartHb(SStreamMeta* pMeta); void streamMetaStartHb(SStreamMeta* pMeta);
bool streamMetaTaskInTimer(SStreamMeta* pMeta); bool streamMetaTaskInTimer(SStreamMeta* pMeta);
int32_t streamMetaAddTaskLaunchResult(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, int64_t startTs, int32_t streamMetaAddTaskLaunchResult(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, int64_t startTs,
int64_t endTs, bool ready); int64_t endTs, bool ready);
int32_t streamMetaResetTaskStatus(SStreamMeta* pMeta); int32_t streamMetaResetTaskStatus(SStreamMeta* pMeta);
void streamMetaRLock(SStreamMeta* pMeta); void streamMetaRLock(SStreamMeta* pMeta);
void streamMetaRUnLock(SStreamMeta* pMeta); void streamMetaRUnLock(SStreamMeta* pMeta);
void streamMetaWLock(SStreamMeta* pMeta); void streamMetaWLock(SStreamMeta* pMeta);
void streamMetaWUnLock(SStreamMeta* pMeta); void streamMetaWUnLock(SStreamMeta* pMeta);
void streamMetaResetStartInfo(STaskStartInfo* pMeta); void streamMetaResetStartInfo(STaskStartInfo* pMeta);
SArray* streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta); SArray* streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta);
void streamMetaUpdateStageRole(SStreamMeta* pMeta, int64_t stage, bool isLeader); void streamMetaUpdateStageRole(SStreamMeta* pMeta, int64_t stage, bool isLeader);
int32_t streamMetaLoadAllTasks(SStreamMeta* pMeta); int32_t streamMetaLoadAllTasks(SStreamMeta* pMeta);
int32_t streamMetaStartAllTasks(SStreamMeta* pMeta); int32_t streamMetaStartAllTasks(SStreamMeta* pMeta);
int32_t streamMetaStopAllTasks(SStreamMeta* pMeta); int32_t streamMetaStopAllTasks(SStreamMeta* pMeta);
int32_t streamMetaStartOneTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId); int32_t streamMetaStartOneTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId);
bool streamMetaAllTasksReady(const SStreamMeta* pMeta); bool streamMetaAllTasksReady(const SStreamMeta* pMeta);
tmr_h streamTimerGetInstance(); tmr_h streamTimerGetInstance();
// checkpoint // checkpoint
int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq); int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq);

10
source/client/src/clientEnv.c
View File

@ -681,8 +681,9 @@ void taos_init_imp(void) {
snprintf(logDirName, 64, "taoslog"); snprintf(logDirName, 64, "taoslog");
#endif #endif
if (taosCreateLog(logDirName, 10, configDir, NULL, NULL, NULL, NULL, 1) != 0) { if (taosCreateLog(logDirName, 10, configDir, NULL, NULL, NULL, NULL, 1) != 0) {
// ignore create log failed, only print
printf(" WARING: Create %s failed:%s. configDir=%s\n", logDirName, strerror(errno), configDir); printf(" WARING: Create %s failed:%s. configDir=%s\n", logDirName, strerror(errno), configDir);
tscInitRes = -1;
return;
} }
if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1) != 0) { if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1) != 0) {
@ -750,8 +751,11 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
tstrncpy(configDir, str, PATH_MAX); tstrncpy(configDir, str, PATH_MAX);
tscInfo("set cfg:%s to %s", configDir, str); tscInfo("set cfg:%s to %s", configDir, str);
return 0; return 0;
} else { }
taos_init(); // initialize global config
// initialize global config
if (taos_init() != 0) {
return -1;
} }
SConfig *pCfg = taosGetCfg(); SConfig *pCfg = taosGetCfg();

7
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -699,6 +699,7 @@ typedef struct {
int64_t checkpointId; int64_t checkpointId;
int32_t indexForMultiAggBalance; int32_t indexForMultiAggBalance;
int8_t subTableWithoutMd5;
char reserve[256]; char reserve[256];
} SStreamObj; } SStreamObj;
@ -776,8 +777,8 @@ typedef enum {
GRANT_STATE_REASON_MAX, GRANT_STATE_REASON_MAX,
} EGrantStateReason; } EGrantStateReason;
#define GRANT_STATE_NUM 30 #define GRANT_STATE_NUM 30
#define GRANT_ACTIVE_NUM 10 #define GRANT_ACTIVE_NUM 10
#define GRANT_ACTIVE_HEAD_LEN 30 #define GRANT_ACTIVE_HEAD_LEN 30
typedef struct { typedef struct {
@ -812,7 +813,7 @@ typedef struct {
int64_t id : 24; int64_t id : 24;
}; };
}; };
char machine[TSDB_MACHINE_ID_LEN + 1]; char machine[TSDB_MACHINE_ID_LEN + 1];
} SGrantMachine; } SGrantMachine;
typedef struct { typedef struct {

2
source/dnode/mnode/impl/inc/mndStream.h
View File

@ -24,7 +24,7 @@ extern "C" {
#endif #endif
#define MND_STREAM_RESERVE_SIZE 64 #define MND_STREAM_RESERVE_SIZE 64
#define MND_STREAM_VER_NUMBER 4 #define MND_STREAM_VER_NUMBER 5
#define MND_STREAM_CREATE_NAME "stream-create" #define MND_STREAM_CREATE_NAME "stream-create"
#define MND_STREAM_CHECKPOINT_NAME "stream-checkpoint" #define MND_STREAM_CHECKPOINT_NAME "stream-checkpoint"

5
source/dnode/mnode/impl/src/mndDef.c
View File

@ -85,6 +85,7 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) {
// 3.0.50 ver = 3 // 3.0.50 ver = 3
if (tEncodeI64(pEncoder, pObj->checkpointId) < 0) return -1; if (tEncodeI64(pEncoder, pObj->checkpointId) < 0) return -1;
if (tEncodeI8(pEncoder, pObj->subTableWithoutMd5) < 0) return -1;
if (tEncodeCStrWithLen(pEncoder, pObj->reserve, sizeof(pObj->reserve) - 1) < 0) return -1; if (tEncodeCStrWithLen(pEncoder, pObj->reserve, sizeof(pObj->reserve) - 1) < 0) return -1;
@ -168,6 +169,10 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj, int32_t sver) {
if (sver >= 3) { if (sver >= 3) {
if (tDecodeI64(pDecoder, &pObj->checkpointId) < 0) return -1; if (tDecodeI64(pDecoder, &pObj->checkpointId) < 0) return -1;
} }
if (sver >= 5) {
if (tDecodeI8(pDecoder, &pObj->subTableWithoutMd5) < 0) return -1;
}
if (tDecodeCStrTo(pDecoder, pObj->reserve) < 0) return -1; if (tDecodeCStrTo(pDecoder, pObj->reserve) < 0) return -1;
tEndDecode(pDecoder); tEndDecode(pDecoder);

124
source/dnode/mnode/impl/src/mndScheduler.c
View File

@ -14,13 +14,13 @@
*/ */
#include "mndScheduler.h" #include "mndScheduler.h"
#include "tmisce.h"
#include "mndMnode.h"
#include "mndDb.h" #include "mndDb.h"
#include "mndMnode.h"
#include "mndSnode.h" #include "mndSnode.h"
#include "mndVgroup.h" #include "mndVgroup.h"
#include "parser.h" #include "parser.h"
#include "tcompare.h" #include "tcompare.h"
#include "tmisce.h"
#include "tname.h" #include "tname.h"
#include "tuuid.h" #include "tuuid.h"
@ -189,7 +189,7 @@ SVgObj* mndSchedFetchOneVg(SMnode* pMnode, SStreamObj* pStream) {
return NULL; return NULL;
} }
if(pStream->indexForMultiAggBalance == -1){ if (pStream->indexForMultiAggBalance == -1) {
taosSeedRand(taosSafeRand()); taosSeedRand(taosSafeRand());
pStream->indexForMultiAggBalance = taosRand() % pDbObj->cfg.numOfVgroups; pStream->indexForMultiAggBalance = taosRand() % pDbObj->cfg.numOfVgroups;
} }
@ -204,7 +204,7 @@ SVgObj* mndSchedFetchOneVg(SMnode* pMnode, SStreamObj* pStream) {
sdbRelease(pMnode->pSdb, pVgroup); sdbRelease(pMnode->pSdb, pVgroup);
continue; continue;
} }
if (index++ == pStream->indexForMultiAggBalance){ if (index++ == pStream->indexForMultiAggBalance) {
pStream->indexForMultiAggBalance++; pStream->indexForMultiAggBalance++;
pStream->indexForMultiAggBalance %= pDbObj->cfg.numOfVgroups; pStream->indexForMultiAggBalance %= pDbObj->cfg.numOfVgroups;
sdbCancelFetch(pMnode->pSdb, pIter); sdbCancelFetch(pMnode->pSdb, pIter);
@ -217,12 +217,12 @@ SVgObj* mndSchedFetchOneVg(SMnode* pMnode, SStreamObj* pStream) {
return pVgroup; return pVgroup;
} }
static int32_t doAddSinkTask(SStreamObj* pStream, SMnode* pMnode, SVgObj* pVgroup, static int32_t doAddSinkTask(SStreamObj* pStream, SMnode* pMnode, SVgObj* pVgroup, SEpSet* pEpset, bool isFillhistory) {
SEpSet* pEpset, bool isFillhistory) { int64_t uid = (isFillhistory) ? pStream->hTaskUid : pStream->uid;
int64_t uid = (isFillhistory) ? pStream->hTaskUid : pStream->uid;
SArray** pTaskList = (isFillhistory) ? taosArrayGetLast(pStream->pHTasksList) : taosArrayGetLast(pStream->tasks); SArray** pTaskList = (isFillhistory) ? taosArrayGetLast(pStream->pHTasksList) : taosArrayGetLast(pStream->tasks);
SStreamTask* pTask = tNewStreamTask(uid, TASK_LEVEL__SINK, pEpset, isFillhistory, 0, *pTaskList, pStream->conf.fillHistory); SStreamTask* pTask = tNewStreamTask(uid, TASK_LEVEL__SINK, pEpset, isFillhistory, 0, *pTaskList,
pStream->conf.fillHistory, pStream->subTableWithoutMd5);
if (pTask == NULL) { if (pTask == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return terrno; return terrno;
@ -235,12 +235,12 @@ static int32_t doAddSinkTask(SStreamObj* pStream, SMnode* pMnode, SVgObj* pVgrou
return mndSetSinkTaskInfo(pStream, pTask); return mndSetSinkTaskInfo(pStream, pTask);
} }
static int32_t doAddSinkTaskToVg(SMnode* pMnode, SStreamObj* pStream, SEpSet* pEpset, SVgObj* vgObj){ static int32_t doAddSinkTaskToVg(SMnode* pMnode, SStreamObj* pStream, SEpSet* pEpset, SVgObj* vgObj) {
int32_t code = doAddSinkTask(pStream, pMnode, vgObj, pEpset, false); int32_t code = doAddSinkTask(pStream, pMnode, vgObj, pEpset, false);
if (code != 0) { if (code != 0) {
return code; return code;
} }
if(pStream->conf.fillHistory){ if (pStream->conf.fillHistory) {
code = doAddSinkTask(pStream, pMnode, vgObj, pEpset, true); code = doAddSinkTask(pStream, pMnode, vgObj, pEpset, true);
if (code != 0) { if (code != 0) {
return code; return code;
@ -267,7 +267,7 @@ static int32_t doAddShuffleSinkTask(SMnode* pMnode, SStreamObj* pStream, SEpSet*
} }
int32_t code = doAddSinkTaskToVg(pMnode, pStream, pEpset, pVgroup); int32_t code = doAddSinkTaskToVg(pMnode, pStream, pEpset, pVgroup);
if(code != 0){ if (code != 0) {
sdbRelease(pSdb, pVgroup); sdbRelease(pSdb, pVgroup);
return code; return code;
} }
@ -279,7 +279,7 @@ static int32_t doAddShuffleSinkTask(SMnode* pMnode, SStreamObj* pStream, SEpSet*
} }
static int64_t getVgroupLastVer(const SArray* pList, int32_t vgId) { static int64_t getVgroupLastVer(const SArray* pList, int32_t vgId) {
for(int32_t i = 0; i < taosArrayGetSize(pList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pList); ++i) {
SVgroupVer* pVer = taosArrayGet(pList, i); SVgroupVer* pVer = taosArrayGet(pList, i);
if (pVer->vgId == vgId) { if (pVer->vgId == vgId) {
return pVer->ver; return pVer->ver;
@ -315,19 +315,18 @@ static void streamTaskSetDataRange(SStreamTask* pTask, int64_t skey, SArray* pVe
pRange->range.minVer = latestVer + 1; pRange->range.minVer = latestVer + 1;
pRange->range.maxVer = INT64_MAX; pRange->range.maxVer = INT64_MAX;
mDebug("add source task 0x%x timeWindow:%" PRId64 "-%" PRId64 " verRange:%" PRId64 "-%" PRId64, mDebug("add source task 0x%x timeWindow:%" PRId64 "-%" PRId64 " verRange:%" PRId64 "-%" PRId64, pTask->id.taskId,
pTask->id.taskId, pWindow->skey, pWindow->ekey, pRange->range.minVer, pRange->range.maxVer); pWindow->skey, pWindow->ekey, pRange->range.minVer, pRange->range.maxVer);
} }
} }
static SStreamTask* buildSourceTask(SStreamObj* pStream, SEpSet* pEpset, static SStreamTask* buildSourceTask(SStreamObj* pStream, SEpSet* pEpset, bool isFillhistory, bool useTriggerParam) {
bool isFillhistory, bool useTriggerParam) {
uint64_t uid = (isFillhistory) ? pStream->hTaskUid : pStream->uid; uint64_t uid = (isFillhistory) ? pStream->hTaskUid : pStream->uid;
SArray** pTaskList = (isFillhistory) ? taosArrayGetLast(pStream->pHTasksList) : taosArrayGetLast(pStream->tasks); SArray** pTaskList = (isFillhistory) ? taosArrayGetLast(pStream->pHTasksList) : taosArrayGetLast(pStream->tasks);
SStreamTask* pTask = tNewStreamTask(uid, TASK_LEVEL__SOURCE, pEpset, SStreamTask* pTask =
isFillhistory, useTriggerParam ? pStream->conf.triggerParam : 0, tNewStreamTask(uid, TASK_LEVEL__SOURCE, pEpset, isFillhistory, useTriggerParam ? pStream->conf.triggerParam : 0,
*pTaskList, pStream->conf.fillHistory); *pTaskList, pStream->conf.fillHistory, pStream->subTableWithoutMd5);
if (pTask == NULL) { if (pTask == NULL) {
return NULL; return NULL;
} }
@ -335,7 +334,7 @@ static SStreamTask* buildSourceTask(SStreamObj* pStream, SEpSet* pEpset,
return pTask; return pTask;
} }
static void addNewTaskList(SStreamObj* pStream){ static void addNewTaskList(SStreamObj* pStream) {
SArray* pTaskList = taosArrayInit(0, POINTER_BYTES); SArray* pTaskList = taosArrayInit(0, POINTER_BYTES);
taosArrayPush(pStream->tasks, &pTaskList); taosArrayPush(pStream->tasks, &pTaskList);
if (pStream->conf.fillHistory) { if (pStream->conf.fillHistory) {
@ -364,11 +363,11 @@ static void setHTasksId(SStreamObj* pStream) {
} }
} }
static int32_t doAddSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStream, SEpSet* pEpset, static int32_t doAddSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStream, SEpSet* pEpset, int64_t skey,
int64_t skey, SArray* pVerList, SVgObj* pVgroup, bool isFillhistory, bool useTriggerParam ){ SArray* pVerList, SVgObj* pVgroup, bool isFillhistory, bool useTriggerParam) {
// new stream task // new stream task
SStreamTask* pTask = buildSourceTask(pStream, pEpset, isFillhistory, useTriggerParam); SStreamTask* pTask = buildSourceTask(pStream, pEpset, isFillhistory, useTriggerParam);
if(pTask == NULL){ if (pTask == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return terrno; return terrno;
} }
@ -377,15 +376,15 @@ static int32_t doAddSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStre
streamTaskSetDataRange(pTask, skey, pVerList, pVgroup->vgId); streamTaskSetDataRange(pTask, skey, pVerList, pVgroup->vgId);
int32_t code = mndAssignStreamTaskToVgroup(pMnode, pTask, plan, pVgroup); int32_t code = mndAssignStreamTaskToVgroup(pMnode, pTask, plan, pVgroup);
if(code != 0){ if (code != 0) {
terrno = code; terrno = code;
return terrno; return terrno;
} }
return TDB_CODE_SUCCESS; return TDB_CODE_SUCCESS;
} }
static SSubplan* getScanSubPlan(const SQueryPlan* pPlan){ static SSubplan* getScanSubPlan(const SQueryPlan* pPlan) {
int32_t numOfPlanLevel = LIST_LENGTH(pPlan->pSubplans); int32_t numOfPlanLevel = LIST_LENGTH(pPlan->pSubplans);
SNodeListNode* inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, numOfPlanLevel - 1); SNodeListNode* inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, numOfPlanLevel - 1);
if (LIST_LENGTH(inner->pNodeList) != 1) { if (LIST_LENGTH(inner->pNodeList) != 1) {
terrno = TSDB_CODE_QRY_INVALID_INPUT; terrno = TSDB_CODE_QRY_INVALID_INPUT;
@ -400,7 +399,7 @@ static SSubplan* getScanSubPlan(const SQueryPlan* pPlan){
return plan; return plan;
} }
static SSubplan* getAggSubPlan(const SQueryPlan* pPlan, int index){ static SSubplan* getAggSubPlan(const SQueryPlan* pPlan, int index) {
SNodeListNode* inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, index); SNodeListNode* inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, index);
if (LIST_LENGTH(inner->pNodeList) != 1) { if (LIST_LENGTH(inner->pNodeList) != 1) {
terrno = TSDB_CODE_QRY_INVALID_INPUT; terrno = TSDB_CODE_QRY_INVALID_INPUT;
@ -415,8 +414,8 @@ static SSubplan* getAggSubPlan(const SQueryPlan* pPlan, int index){
return plan; return plan;
} }
static int32_t addSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStream, static int32_t addSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStream, SEpSet* pEpset,
SEpSet* pEpset, int64_t nextWindowSkey, SArray* pVerList, bool useTriggerParam) { int64_t nextWindowSkey, SArray* pVerList, bool useTriggerParam) {
addNewTaskList(pStream); addNewTaskList(pStream);
void* pIter = NULL; void* pIter = NULL;
@ -433,15 +432,16 @@ static int32_t addSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStream
continue; continue;
} }
int code = doAddSourceTask(pMnode, plan, pStream, pEpset, nextWindowSkey, pVerList, pVgroup, false, useTriggerParam); int code =
if(code != 0){ doAddSourceTask(pMnode, plan, pStream, pEpset, nextWindowSkey, pVerList, pVgroup, false, useTriggerParam);
if (code != 0) {
sdbRelease(pSdb, pVgroup); sdbRelease(pSdb, pVgroup);
return code; return code;
} }
if (pStream->conf.fillHistory) { if (pStream->conf.fillHistory) {
code = doAddSourceTask(pMnode, plan, pStream, pEpset, nextWindowSkey, pVerList, pVgroup, true, useTriggerParam); code = doAddSourceTask(pMnode, plan, pStream, pEpset, nextWindowSkey, pVerList, pVgroup, true, useTriggerParam);
if(code != 0){ if (code != 0) {
sdbRelease(pSdb, pVgroup); sdbRelease(pSdb, pVgroup);
return code; return code;
} }
@ -461,9 +461,9 @@ static SStreamTask* buildAggTask(SStreamObj* pStream, SEpSet* pEpset, bool isFil
uint64_t uid = (isFillhistory) ? pStream->hTaskUid : pStream->uid; uint64_t uid = (isFillhistory) ? pStream->hTaskUid : pStream->uid;
SArray** pTaskList = (isFillhistory) ? taosArrayGetLast(pStream->pHTasksList) : taosArrayGetLast(pStream->tasks); SArray** pTaskList = (isFillhistory) ? taosArrayGetLast(pStream->pHTasksList) : taosArrayGetLast(pStream->tasks);
SStreamTask* pAggTask = tNewStreamTask(uid, TASK_LEVEL__AGG, pEpset, isFillhistory, SStreamTask* pAggTask =
useTriggerParam ? pStream->conf.triggerParam : 0, tNewStreamTask(uid, TASK_LEVEL__AGG, pEpset, isFillhistory, useTriggerParam ? pStream->conf.triggerParam : 0,
*pTaskList, pStream->conf.fillHistory); *pTaskList, pStream->conf.fillHistory, pStream->subTableWithoutMd5);
if (pAggTask == NULL) { if (pAggTask == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return NULL;
@ -472,8 +472,8 @@ static SStreamTask* buildAggTask(SStreamObj* pStream, SEpSet* pEpset, bool isFil
return pAggTask; return pAggTask;
} }
static int32_t doAddAggTask(SStreamObj* pStream, SMnode* pMnode, SSubplan* plan, SEpSet* pEpset, static int32_t doAddAggTask(SStreamObj* pStream, SMnode* pMnode, SSubplan* plan, SEpSet* pEpset, SVgObj* pVgroup,
SVgObj* pVgroup, SSnodeObj* pSnode, bool isFillhistory, bool useTriggerParam){ SSnodeObj* pSnode, bool isFillhistory, bool useTriggerParam) {
int32_t code = 0; int32_t code = 0;
SStreamTask* pTask = buildAggTask(pStream, pEpset, isFillhistory, useTriggerParam); SStreamTask* pTask = buildAggTask(pStream, pEpset, isFillhistory, useTriggerParam);
if (pTask == NULL) { if (pTask == NULL) {
@ -490,7 +490,7 @@ static int32_t doAddAggTask(SStreamObj* pStream, SMnode* pMnode, SSubplan* plan,
return code; return code;
} }
static int32_t addAggTask(SStreamObj* pStream, SMnode* pMnode, SSubplan* plan, SEpSet* pEpset, bool useTriggerParam){ static int32_t addAggTask(SStreamObj* pStream, SMnode* pMnode, SSubplan* plan, SEpSet* pEpset, bool useTriggerParam) {
SVgObj* pVgroup = NULL; SVgObj* pVgroup = NULL;
SSnodeObj* pSnode = NULL; SSnodeObj* pSnode = NULL;
int32_t code = 0; int32_t code = 0;
@ -504,20 +504,20 @@ static int32_t addAggTask(SStreamObj* pStream, SMnode* pMnode, SSubplan* plan, S
} }
code = doAddAggTask(pStream, pMnode, plan, pEpset, pVgroup, pSnode, false, useTriggerParam); code = doAddAggTask(pStream, pMnode, plan, pEpset, pVgroup, pSnode, false, useTriggerParam);
if(code != 0){ if (code != 0) {
goto END; goto END;
} }
if (pStream->conf.fillHistory) { if (pStream->conf.fillHistory) {
code = doAddAggTask(pStream, pMnode, plan, pEpset, pVgroup, pSnode, true, useTriggerParam); code = doAddAggTask(pStream, pMnode, plan, pEpset, pVgroup, pSnode, true, useTriggerParam);
if(code != 0){ if (code != 0) {
goto END; goto END;
} }
setHTasksId(pStream); setHTasksId(pStream);
} }
END: END:
if (pSnode != NULL) { if (pSnode != NULL) {
sdbRelease(pMnode->pSdb, pSnode); sdbRelease(pMnode->pSdb, pSnode);
} else { } else {
@ -526,7 +526,7 @@ static int32_t addAggTask(SStreamObj* pStream, SMnode* pMnode, SSubplan* plan, S
return code; return code;
} }
static int32_t addSinkTask(SMnode* pMnode, SStreamObj* pStream, SEpSet* pEpset){ static int32_t addSinkTask(SMnode* pMnode, SStreamObj* pStream, SEpSet* pEpset) {
int32_t code = 0; int32_t code = 0;
addNewTaskList(pStream); addNewTaskList(pStream);
@ -548,9 +548,9 @@ static int32_t addSinkTask(SMnode* pMnode, SStreamObj* pStream, SEpSet* pEpset){
return TDB_CODE_SUCCESS; return TDB_CODE_SUCCESS;
} }
static void bindTaskToSinkTask(SStreamObj* pStream, SMnode* pMnode, SArray* pSinkTaskList, SStreamTask* task){ static void bindTaskToSinkTask(SStreamObj* pStream, SMnode* pMnode, SArray* pSinkTaskList, SStreamTask* task) {
mndAddDispatcherForInternalTask(pMnode, pStream, pSinkTaskList, task); mndAddDispatcherForInternalTask(pMnode, pStream, pSinkTaskList, task);
for(int32_t k = 0; k < taosArrayGetSize(pSinkTaskList); k++) { for (int32_t k = 0; k < taosArrayGetSize(pSinkTaskList); k++) {
SStreamTask* pSinkTask = taosArrayGetP(pSinkTaskList, k); SStreamTask* pSinkTask = taosArrayGetP(pSinkTaskList, k);
streamTaskSetUpstreamInfo(pSinkTask, task); streamTaskSetUpstreamInfo(pSinkTask, task);
} }
@ -558,10 +558,10 @@ static void bindTaskToSinkTask(SStreamObj* pStream, SMnode* pMnode, SArray* pSin
} }
static void bindAggSink(SStreamObj* pStream, SMnode* pMnode, SArray* tasks) { static void bindAggSink(SStreamObj* pStream, SMnode* pMnode, SArray* tasks) {
SArray* pSinkTaskList = taosArrayGetP(tasks, SINK_NODE_LEVEL); SArray* pSinkTaskList = taosArrayGetP(tasks, SINK_NODE_LEVEL);
SArray** pAggTaskList = taosArrayGetLast(tasks); SArray** pAggTaskList = taosArrayGetLast(tasks);
for(int i = 0; i < taosArrayGetSize(*pAggTaskList); i++){ for (int i = 0; i < taosArrayGetSize(*pAggTaskList); i++) {
SStreamTask* pAggTask = taosArrayGetP(*pAggTaskList, i); SStreamTask* pAggTask = taosArrayGetP(*pAggTaskList, i);
bindTaskToSinkTask(pStream, pMnode, pSinkTaskList, pAggTask); bindTaskToSinkTask(pStream, pMnode, pSinkTaskList, pAggTask);
mDebug("bindAggSink taskId:%s to sink task list", pAggTask->id.idStr); mDebug("bindAggSink taskId:%s to sink task list", pAggTask->id.idStr);
@ -572,7 +572,7 @@ static void bindSourceSink(SStreamObj* pStream, SMnode* pMnode, SArray* tasks, b
SArray* pSinkTaskList = taosArrayGetP(tasks, SINK_NODE_LEVEL); SArray* pSinkTaskList = taosArrayGetP(tasks, SINK_NODE_LEVEL);
SArray* pSourceTaskList = taosArrayGetP(tasks, hasExtraSink ? SINK_NODE_LEVEL + 1 : SINK_NODE_LEVEL); SArray* pSourceTaskList = taosArrayGetP(tasks, hasExtraSink ? SINK_NODE_LEVEL + 1 : SINK_NODE_LEVEL);
for(int i = 0; i < taosArrayGetSize(pSourceTaskList); i++){ for (int i = 0; i < taosArrayGetSize(pSourceTaskList); i++) {
SStreamTask* pSourceTask = taosArrayGetP(pSourceTaskList, i); SStreamTask* pSourceTask = taosArrayGetP(pSourceTaskList, i);
mDebug("bindSourceSink taskId:%s to sink task list", pSourceTask->id.idStr); mDebug("bindSourceSink taskId:%s to sink task list", pSourceTask->id.idStr);
@ -591,8 +591,8 @@ static void bindTwoLevel(SArray* tasks, int32_t begin, int32_t end) {
SArray* pUpTaskList = taosArrayGetP(tasks, size - 2); SArray* pUpTaskList = taosArrayGetP(tasks, size - 2);
SStreamTask** pDownTask = taosArrayGetLast(pDownTaskList); SStreamTask** pDownTask = taosArrayGetLast(pDownTaskList);
end = end > taosArrayGetSize(pUpTaskList) ? taosArrayGetSize(pUpTaskList): end; end = end > taosArrayGetSize(pUpTaskList) ? taosArrayGetSize(pUpTaskList) : end;
for(int i = begin; i < end; i++){ for (int i = begin; i < end; i++) {
SStreamTask* pUpTask = taosArrayGetP(pUpTaskList, i); SStreamTask* pUpTask = taosArrayGetP(pUpTaskList, i);
pUpTask->info.selfChildId = i - begin; pUpTask->info.selfChildId = i - begin;
streamTaskSetFixedDownstreamInfo(pUpTask, *pDownTask); streamTaskSetFixedDownstreamInfo(pUpTask, *pDownTask);
@ -616,8 +616,8 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
bool multiTarget = (pDbObj->cfg.numOfVgroups > 1); bool multiTarget = (pDbObj->cfg.numOfVgroups > 1);
sdbRelease(pSdb, pDbObj); sdbRelease(pSdb, pDbObj);
mDebug("doScheduleStream numOfPlanLevel:%d, exDb:%d, multiTarget:%d, fix vgId:%d, physicalPlan:%s", mDebug("doScheduleStream numOfPlanLevel:%d, exDb:%d, multiTarget:%d, fix vgId:%d, physicalPlan:%s", numOfPlanLevel,
numOfPlanLevel, externalTargetDB, multiTarget, pStream->fixedSinkVgId, pStream->physicalPlan); externalTargetDB, multiTarget, pStream->fixedSinkVgId, pStream->physicalPlan);
pStream->tasks = taosArrayInit(numOfPlanLevel + 1, POINTER_BYTES); pStream->tasks = taosArrayInit(numOfPlanLevel + 1, POINTER_BYTES);
pStream->pHTasksList = taosArrayInit(numOfPlanLevel + 1, POINTER_BYTES); pStream->pHTasksList = taosArrayInit(numOfPlanLevel + 1, POINTER_BYTES);
@ -632,7 +632,7 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
pStream->totalLevel = numOfPlanLevel + hasExtraSink; pStream->totalLevel = numOfPlanLevel + hasExtraSink;
SSubplan* plan = getScanSubPlan(pPlan); // source plan SSubplan* plan = getScanSubPlan(pPlan); // source plan
if (plan == NULL) { if (plan == NULL) {
return terrno; return terrno;
} }
@ -649,32 +649,32 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
return TDB_CODE_SUCCESS; return TDB_CODE_SUCCESS;
} }
if(numOfPlanLevel == 3){ if (numOfPlanLevel == 3) {
plan = getAggSubPlan(pPlan, 1); // middle agg plan plan = getAggSubPlan(pPlan, 1); // middle agg plan
if (plan == NULL) { if (plan == NULL) {
return terrno; return terrno;
} }
do{ do {
SArray** list = taosArrayGetLast(pStream->tasks); SArray** list = taosArrayGetLast(pStream->tasks);
float size = (float)taosArrayGetSize(*list); float size = (float)taosArrayGetSize(*list);
size_t cnt = (size_t)ceil(size/tsStreamAggCnt); size_t cnt = (size_t)ceil(size / tsStreamAggCnt);
if(cnt <= 1) break; if (cnt <= 1) break;
mDebug("doScheduleStream add middle agg, size:%d, cnt:%d", (int)size, (int)cnt); mDebug("doScheduleStream add middle agg, size:%d, cnt:%d", (int)size, (int)cnt);
addNewTaskList(pStream); addNewTaskList(pStream);
for(int j = 0; j < cnt; j++){ for (int j = 0; j < cnt; j++) {
code = addAggTask(pStream, pMnode, plan, pEpset, false); code = addAggTask(pStream, pMnode, plan, pEpset, false);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;
} }
bindTwoLevel(pStream->tasks, j*tsStreamAggCnt, (j+1)*tsStreamAggCnt); bindTwoLevel(pStream->tasks, j * tsStreamAggCnt, (j + 1) * tsStreamAggCnt);
if (pStream->conf.fillHistory) { if (pStream->conf.fillHistory) {
bindTwoLevel(pStream->pHTasksList, j*tsStreamAggCnt, (j+1)*tsStreamAggCnt); bindTwoLevel(pStream->pHTasksList, j * tsStreamAggCnt, (j + 1) * tsStreamAggCnt);
} }
} }
}while(1); } while (1);
} }
plan = getAggSubPlan(pPlan, 0); plan = getAggSubPlan(pPlan, 0);
@ -684,7 +684,7 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
mDebug("doScheduleStream add final agg"); mDebug("doScheduleStream add final agg");
SArray** list = taosArrayGetLast(pStream->tasks); SArray** list = taosArrayGetLast(pStream->tasks);
size_t size = taosArrayGetSize(*list); size_t size = taosArrayGetSize(*list);
addNewTaskList(pStream); addNewTaskList(pStream);
code = addAggTask(pStream, pMnode, plan, pEpset, true); code = addAggTask(pStream, pMnode, plan, pEpset, true);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {

11
source/dnode/mnode/impl/src/mndSma.c
View File

@ -567,6 +567,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea
streamObj.conf.triggerParam = pCreate->maxDelay; streamObj.conf.triggerParam = pCreate->maxDelay;
streamObj.ast = taosStrdup(smaObj.ast); streamObj.ast = taosStrdup(smaObj.ast);
streamObj.indexForMultiAggBalance = -1; streamObj.indexForMultiAggBalance = -1;
streamObj.subTableWithoutMd5 = 1;
// check the maxDelay // check the maxDelay
if (streamObj.conf.triggerParam < TSDB_MIN_ROLLUP_MAX_DELAY) { if (streamObj.conf.triggerParam < TSDB_MIN_ROLLUP_MAX_DELAY) {
@ -898,11 +899,11 @@ _OVER:
} }
int32_t mndDropSmasByStb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { int32_t mndDropSmasByStb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
SSmaObj *pSma = NULL; SSmaObj *pSma = NULL;
void *pIter = NULL; void *pIter = NULL;
SVgObj *pVgroup = NULL; SVgObj *pVgroup = NULL;
int32_t code = -1; int32_t code = -1;
while (1) { while (1) {
pIter = sdbFetch(pSdb, SDB_SMA, pIter, (void **)&pSma); pIter = sdbFetch(pSdb, SDB_SMA, pIter, (void **)&pSma);

34
source/dnode/mnode/impl/src/mndStream.c
View File

@ -27,7 +27,7 @@
#include "tmisce.h" #include "tmisce.h"
#include "tname.h" #include "tname.h"
#define MND_STREAM_MAX_NUM 60 #define MND_STREAM_MAX_NUM 60
typedef struct SMStreamNodeCheckMsg { typedef struct SMStreamNodeCheckMsg {
int8_t placeHolder; // // to fix windows compile error, define place holder int8_t placeHolder; // // to fix windows compile error, define place holder
@ -192,7 +192,7 @@ SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw) {
STREAM_DECODE_OVER: STREAM_DECODE_OVER:
taosMemoryFreeClear(buf); taosMemoryFreeClear(buf);
if (terrno != TSDB_CODE_SUCCESS) { if (terrno != TSDB_CODE_SUCCESS) {
char* p = (pStream == NULL) ? "null" : pStream->name; char *p = (pStream == NULL) ? "null" : pStream->name;
mError("stream:%s, failed to decode from raw:%p since %s", p, pRaw, terrstr()); mError("stream:%s, failed to decode from raw:%p since %s", p, pRaw, terrstr());
taosMemoryFreeClear(pRow); taosMemoryFreeClear(pRow);
return NULL; return NULL;
@ -802,8 +802,7 @@ static int32_t mndProcessStreamCheckpointTmr(SRpcMsg *pReq) {
} }
static int32_t mndBuildStreamCheckpointSourceReq(void **pBuf, int32_t *pLen, int32_t nodeId, int64_t checkpointId, static int32_t mndBuildStreamCheckpointSourceReq(void **pBuf, int32_t *pLen, int32_t nodeId, int64_t checkpointId,
int64_t streamId, int32_t taskId, int32_t transId, int64_t streamId, int32_t taskId, int32_t transId, int8_t mndTrigger) {
int8_t mndTrigger) {
SStreamCheckpointSourceReq req = {0}; SStreamCheckpointSourceReq req = {0};
req.checkpointId = checkpointId; req.checkpointId = checkpointId;
req.nodeId = nodeId; req.nodeId = nodeId;
@ -878,11 +877,10 @@ static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStre
int32_t code = -1; int32_t code = -1;
int64_t ts = taosGetTimestampMs(); int64_t ts = taosGetTimestampMs();
if (mndTrigger == 1 && (ts - pStream->checkpointFreq < tsStreamCheckpointInterval * 1000)) { if (mndTrigger == 1 && (ts - pStream->checkpointFreq < tsStreamCheckpointInterval * 1000)) {
// mWarn("checkpoint interval less than the threshold, ignore it"); // mWarn("checkpoint interval less than the threshold, ignore it");
return -1; return -1;
} }
bool conflict = mndStreamTransConflictCheck(pMnode, pStream->uid, MND_STREAM_CHECKPOINT_NAME, lock); bool conflict = mndStreamTransConflictCheck(pMnode, pStream->uid, MND_STREAM_CHECKPOINT_NAME, lock);
if (conflict) { if (conflict) {
mndAddtoCheckpointWaitingList(pStream, checkpointId); mndAddtoCheckpointWaitingList(pStream, checkpointId);
@ -1144,7 +1142,7 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
return -1; return -1;
} }
STrans* pTrans = doCreateTrans(pMnode, pStream, pReq, MND_STREAM_DROP_NAME, "drop stream"); STrans *pTrans = doCreateTrans(pMnode, pStream, pReq, MND_STREAM_DROP_NAME, "drop stream");
if (pTrans == NULL) { if (pTrans == NULL) {
mError("stream:%s, failed to drop since %s", dropReq.name, terrstr()); mError("stream:%s, failed to drop since %s", dropReq.name, terrstr());
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
@ -1570,7 +1568,7 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) {
return -1; return -1;
} }
STrans* pTrans = doCreateTrans(pMnode, pStream, pReq, MND_STREAM_PAUSE_NAME, "pause the stream"); STrans *pTrans = doCreateTrans(pMnode, pStream, pReq, MND_STREAM_PAUSE_NAME, "pause the stream");
if (pTrans == NULL) { if (pTrans == NULL) {
mError("stream:%s failed to pause stream since %s", pauseReq.name, terrstr()); mError("stream:%s failed to pause stream since %s", pauseReq.name, terrstr());
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
@ -1590,7 +1588,7 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) {
// pause stream // pause stream
taosWLockLatch(&pStream->lock); taosWLockLatch(&pStream->lock);
pStream->status = STREAM_STATUS__PAUSE; pStream->status = STREAM_STATUS__PAUSE;
if (mndPersistTransLog(pStream, pTrans,SDB_STATUS_READY) < 0) { if (mndPersistTransLog(pStream, pTrans, SDB_STATUS_READY) < 0) {
taosWUnLockLatch(&pStream->lock); taosWUnLockLatch(&pStream->lock);
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
@ -1617,7 +1615,7 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
SStreamObj *pStream = NULL; SStreamObj *pStream = NULL;
if(grantCheckExpire(TSDB_GRANT_STREAMS) < 0){ if (grantCheckExpire(TSDB_GRANT_STREAMS) < 0) {
terrno = TSDB_CODE_GRANT_EXPIRED; terrno = TSDB_CODE_GRANT_EXPIRED;
return -1; return -1;
} }
@ -1659,7 +1657,7 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) {
return -1; return -1;
} }
STrans* pTrans = doCreateTrans(pMnode, pStream, pReq, MND_STREAM_RESUME_NAME, "resume the stream"); STrans *pTrans = doCreateTrans(pMnode, pStream, pReq, MND_STREAM_RESUME_NAME, "resume the stream");
if (pTrans == NULL) { if (pTrans == NULL) {
mError("stream:%s, failed to resume stream since %s", resumeReq.name, terrstr()); mError("stream:%s, failed to resume stream since %s", resumeReq.name, terrstr());
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
@ -2106,10 +2104,10 @@ void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) {
ASSERT(taosHashGetSize(pExecNode->pTaskMap) == taosArrayGetSize(pExecNode->pTaskList)); ASSERT(taosHashGetSize(pExecNode->pTaskMap) == taosArrayGetSize(pExecNode->pTaskList));
} }
static void doAddTaskId(SArray* pList, int32_t taskId, int64_t uid, int32_t numOfTotal) { static void doAddTaskId(SArray *pList, int32_t taskId, int64_t uid, int32_t numOfTotal) {
int32_t num = taosArrayGetSize(pList); int32_t num = taosArrayGetSize(pList);
for(int32_t i = 0; i < num; ++i) { for (int32_t i = 0; i < num; ++i) {
int32_t* pId = taosArrayGet(pList, i); int32_t *pId = taosArrayGet(pList, i);
if (taskId == *pId) { if (taskId == *pId) {
return; return;
} }
@ -2122,7 +2120,7 @@ static void doAddTaskId(SArray* pList, int32_t taskId, int64_t uid, int32_t numO
} }
int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) { int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
SStreamTaskCheckpointReq req = {0}; SStreamTaskCheckpointReq req = {0};
SDecoder decoder = {0}; SDecoder decoder = {0};
@ -2143,7 +2141,7 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) {
SStreamObj *pStream = mndGetStreamObj(pMnode, req.streamId); SStreamObj *pStream = mndGetStreamObj(pMnode, req.streamId);
if (pStream == NULL) { if (pStream == NULL) {
mError("failed to find the stream:0x%"PRIx64" not handle the checkpoint req", req.streamId); mError("failed to find the stream:0x%" PRIx64 " not handle the checkpoint req", req.streamId);
terrno = TSDB_CODE_MND_STREAM_NOT_EXIST; terrno = TSDB_CODE_MND_STREAM_NOT_EXIST;
taosThreadMutexUnlock(&execInfo.lock); taosThreadMutexUnlock(&execInfo.lock);
@ -2151,13 +2149,13 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) {
} }
int32_t numOfTasks = mndGetNumOfStreamTasks(pStream); int32_t numOfTasks = mndGetNumOfStreamTasks(pStream);
SArray **pReqTaskList = (SArray**)taosHashGet(execInfo.pTransferStateStreams, &req.streamId, sizeof(req.streamId)); SArray **pReqTaskList = (SArray **)taosHashGet(execInfo.pTransferStateStreams, &req.streamId, sizeof(req.streamId));
if (pReqTaskList == NULL) { if (pReqTaskList == NULL) {
SArray *pList = taosArrayInit(4, sizeof(int32_t)); SArray *pList = taosArrayInit(4, sizeof(int32_t));
doAddTaskId(pList, req.taskId, pStream->uid, numOfTasks); doAddTaskId(pList, req.taskId, pStream->uid, numOfTasks);
taosHashPut(execInfo.pTransferStateStreams, &req.streamId, sizeof(int64_t), &pList, sizeof(void *)); taosHashPut(execInfo.pTransferStateStreams, &req.streamId, sizeof(int64_t), &pList, sizeof(void *));
pReqTaskList = (SArray**)taosHashGet(execInfo.pTransferStateStreams, &req.streamId, sizeof(req.streamId)); pReqTaskList = (SArray **)taosHashGet(execInfo.pTransferStateStreams, &req.streamId, sizeof(req.streamId));
} else { } else {
doAddTaskId(*pReqTaskList, req.taskId, pStream->uid, numOfTasks); doAddTaskId(*pReqTaskList, req.taskId, pStream->uid, numOfTasks);
} }

97
source/dnode/vnode/src/tq/tqSink.c
View File

@ -33,15 +33,18 @@ static int32_t doBuildAndSendDeleteMsg(SVnode* pVnode, char* stbFullName, SSData
int64_t suid); int64_t suid);
static int32_t doBuildAndSendSubmitMsg(SVnode* pVnode, SStreamTask* pTask, SSubmitReq2* pReq, int32_t numOfBlocks); static int32_t doBuildAndSendSubmitMsg(SVnode* pVnode, SStreamTask* pTask, SSubmitReq2* pReq, int32_t numOfBlocks);
static int32_t buildSubmitMsgImpl(SSubmitReq2* pSubmitReq, int32_t vgId, void** pMsg, int32_t* msgLen); static int32_t buildSubmitMsgImpl(SSubmitReq2* pSubmitReq, int32_t vgId, void** pMsg, int32_t* msgLen);
static int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDataBlock* pDataBlock, const char* id); static int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDataBlock* pDataBlock,
const char* id);
static int32_t doWaitForDstTableCreated(SVnode* pVnode, SStreamTask* pTask, STableSinkInfo* pTableSinkInfo, static int32_t doWaitForDstTableCreated(SVnode* pVnode, SStreamTask* pTask, STableSinkInfo* pTableSinkInfo,
const char* dstTableName, int64_t* uid); const char* dstTableName, int64_t* uid);
static int32_t doPutIntoCache(SSHashObj* pSinkTableMap, STableSinkInfo* pTableSinkInfo, uint64_t groupId, const char* id); static int32_t doPutIntoCache(SSHashObj* pSinkTableMap, STableSinkInfo* pTableSinkInfo, uint64_t groupId,
const char* id);
static bool isValidDstChildTable(SMetaReader* pReader, int32_t vgId, const char* ctbName, int64_t suid); static bool isValidDstChildTable(SMetaReader* pReader, int32_t vgId, const char* ctbName, int64_t suid);
static int32_t initCreateTableMsg(SVCreateTbReq* pCreateTableReq, uint64_t suid, const char* stbFullName, int32_t numOfTags); static int32_t initCreateTableMsg(SVCreateTbReq* pCreateTableReq, uint64_t suid, const char* stbFullName,
int32_t numOfTags);
static SArray* createDefaultTagColName(); static SArray* createDefaultTagColName();
static void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDataBlock, const char* stbFullName, static void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDataBlock, const char* stbFullName,
int64_t gid, bool newSubTableRule); int64_t gid, bool newSubTableRule);
int32_t tqBuildDeleteReq(STQ* pTq, const char* stbFullName, const SSDataBlock* pDataBlock, SBatchDeleteReq* deleteReq, int32_t tqBuildDeleteReq(STQ* pTq, const char* stbFullName, const SSDataBlock* pDataBlock, SBatchDeleteReq* deleteReq,
const char* pIdStr, bool newSubTableRule) { const char* pIdStr, bool newSubTableRule) {
@ -68,10 +71,7 @@ int32_t tqBuildDeleteReq(STQ* pTq, const char* stbFullName, const SSDataBlock* p
if (varTbName != NULL && varTbName != (void*)-1) { if (varTbName != NULL && varTbName != (void*)-1) {
name = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN); name = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN);
memcpy(name, varDataVal(varTbName), varDataLen(varTbName)); memcpy(name, varDataVal(varTbName), varDataLen(varTbName));
if(newSubTableRule && if (newSubTableRule && !isAutoTableName(name) && !alreadyAddGroupId(name) && groupId != 0) {
!isAutoTableName(name) &&
!alreadyAddGroupId(name) &&
groupId != 0) {
buildCtbNameAddGruopId(name, groupId); buildCtbNameAddGruopId(name, groupId);
} }
} else if (stbFullName) { } else if (stbFullName) {
@ -134,7 +134,7 @@ end:
return ret; return ret;
} }
static bool tqGetTableInfo(SSHashObj* pTableInfoMap,uint64_t groupId, STableSinkInfo** pInfo) { static bool tqGetTableInfo(SSHashObj* pTableInfoMap, uint64_t groupId, STableSinkInfo** pInfo) {
void* pVal = tSimpleHashGet(pTableInfoMap, &groupId, sizeof(uint64_t)); void* pVal = tSimpleHashGet(pTableInfoMap, &groupId, sizeof(uint64_t));
if (pVal) { if (pVal) {
*pInfo = *(STableSinkInfo**)pVal; *pInfo = *(STableSinkInfo**)pVal;
@ -149,7 +149,7 @@ static int32_t tqPutReqToQueue(SVnode* pVnode, SVCreateTbBatchReq* pReqs) {
int32_t tlen = 0; int32_t tlen = 0;
encodeCreateChildTableForRPC(pReqs, TD_VID(pVnode), &buf, &tlen); encodeCreateChildTableForRPC(pReqs, TD_VID(pVnode), &buf, &tlen);
SRpcMsg msg = { .msgType = TDMT_VND_CREATE_TABLE, .pCont = buf, .contLen = tlen }; SRpcMsg msg = {.msgType = TDMT_VND_CREATE_TABLE, .pCont = buf, .contLen = tlen};
if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) {
tqError("failed to put into write-queue since %s", terrstr()); tqError("failed to put into write-queue since %s", terrstr());
} }
@ -181,14 +181,12 @@ SArray* createDefaultTagColName() {
void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDataBlock, const char* stbFullName, void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDataBlock, const char* stbFullName,
int64_t gid, bool newSubTableRule) { int64_t gid, bool newSubTableRule) {
if (pDataBlock->info.parTbName[0]) { if (pDataBlock->info.parTbName[0]) {
if(newSubTableRule && if (newSubTableRule && !isAutoTableName(pDataBlock->info.parTbName) &&
!isAutoTableName(pDataBlock->info.parTbName) && !alreadyAddGroupId(pDataBlock->info.parTbName) && gid != 0) {
!alreadyAddGroupId(pDataBlock->info.parTbName) &&
gid != 0) {
pCreateTableReq->name = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN); pCreateTableReq->name = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN);
strcpy(pCreateTableReq->name, pDataBlock->info.parTbName); strcpy(pCreateTableReq->name, pDataBlock->info.parTbName);
buildCtbNameAddGruopId(pCreateTableReq->name, gid); buildCtbNameAddGruopId(pCreateTableReq->name, gid);
}else{ } else {
pCreateTableReq->name = taosStrdup(pDataBlock->info.parTbName); pCreateTableReq->name = taosStrdup(pDataBlock->info.parTbName);
} }
} else { } else {
@ -196,17 +194,18 @@ void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDa
} }
} }
static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* pDataBlock, SStreamTask* pTask, static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* pDataBlock,
int64_t suid) { SStreamTask* pTask, int64_t suid) {
tqDebug("s-task:%s build create table msg", pTask->id.idStr); tqDebug("s-task:%s build create table msg", pTask->id.idStr);
STSchema* pTSchema = pTask->outputInfo.tbSink.pTSchema; STSchema* pTSchema = pTask->outputInfo.tbSink.pTSchema;
int32_t rows = pDataBlock->info.rows; int32_t rows = pDataBlock->info.rows;
SArray* tagArray = taosArrayInit(4, sizeof(STagVal));; SArray* tagArray = taosArrayInit(4, sizeof(STagVal));
int32_t code = 0; ;
int32_t code = 0;
SVCreateTbBatchReq reqs = {0}; SVCreateTbBatchReq reqs = {0};
SArray* crTblArray = reqs.pArray = taosArrayInit(1, sizeof(SVCreateTbReq)); SArray* crTblArray = reqs.pArray = taosArrayInit(1, sizeof(SVCreateTbReq));
if (NULL == reqs.pArray) { if (NULL == reqs.pArray) {
tqError("s-task:%s failed to init create table msg, code:%s", pTask->id.idStr, tstrerror(terrno)); tqError("s-task:%s failed to init create table msg, code:%s", pTask->id.idStr, tstrerror(terrno));
goto _end; goto _end;
@ -262,7 +261,8 @@ static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, S
ASSERT(gid == *(int64_t*)pGpIdData); ASSERT(gid == *(int64_t*)pGpIdData);
} }
setCreateTableMsgTableName(pCreateTbReq, pDataBlock, stbFullName, gid, pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER); setCreateTableMsgTableName(pCreateTbReq, pDataBlock, stbFullName, gid,
pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1);
taosArrayPush(reqs.pArray, pCreateTbReq); taosArrayPush(reqs.pArray, pCreateTbReq);
tqDebug("s-task:%s build create table:%s msg complete", pTask->id.idStr, pCreateTbReq->name); tqDebug("s-task:%s build create table:%s msg complete", pTask->id.idStr, pCreateTbReq->name);
@ -274,7 +274,7 @@ static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, S
tqError("s-task:%s failed to send create table msg", pTask->id.idStr); tqError("s-task:%s failed to send create table msg", pTask->id.idStr);
} }
_end: _end:
taosArrayDestroy(tagArray); taosArrayDestroy(tagArray);
taosArrayDestroyEx(crTblArray, (FDelete)tdDestroySVCreateTbReq); taosArrayDestroyEx(crTblArray, (FDelete)tdDestroySVCreateTbReq);
return code; return code;
@ -361,7 +361,8 @@ int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, c
pExisted->aRowP = pFinal; pExisted->aRowP = pFinal;
tqTrace("s-task:%s rows merged, final rows:%d, uid:%" PRId64 ", existed auto-create table:%d, new-block:%d", id, tqTrace("s-task:%s rows merged, final rows:%d, uid:%" PRId64 ", existed auto-create table:%d, new-block:%d", id,
(int32_t)taosArrayGetSize(pFinal), pExisted->uid, (pExisted->pCreateTbReq != NULL), (pNew->pCreateTbReq != NULL)); (int32_t)taosArrayGetSize(pFinal), pExisted->uid, (pExisted->pCreateTbReq != NULL),
(pNew->pCreateTbReq != NULL));
tdDestroySVCreateTbReq(pNew->pCreateTbReq); tdDestroySVCreateTbReq(pNew->pCreateTbReq);
taosMemoryFree(pNew->pCreateTbReq); taosMemoryFree(pNew->pCreateTbReq);
@ -373,7 +374,7 @@ int32_t doBuildAndSendDeleteMsg(SVnode* pVnode, char* stbFullName, SSDataBlock*
SBatchDeleteReq deleteReq = {.suid = suid, .deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq))}; SBatchDeleteReq deleteReq = {.suid = suid, .deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq))};
int32_t code = tqBuildDeleteReq(pVnode->pTq, stbFullName, pDataBlock, &deleteReq, pTask->id.idStr, int32_t code = tqBuildDeleteReq(pVnode->pTq, stbFullName, pDataBlock, &deleteReq, pTask->id.idStr,
pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER); pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;
} }
@ -416,8 +417,8 @@ bool isValidDstChildTable(SMetaReader* pReader, int32_t vgId, const char* ctbNam
} }
if (pReader->me.ctbEntry.suid != suid) { if (pReader->me.ctbEntry.suid != suid) {
tqError("vgId:%d, failed to write into %s, since suid mismatch, expect suid:%" PRId64 ", actual:%" PRId64, tqError("vgId:%d, failed to write into %s, since suid mismatch, expect suid:%" PRId64 ", actual:%" PRId64, vgId,
vgId, ctbName, suid, pReader->me.ctbEntry.suid); ctbName, suid, pReader->me.ctbEntry.suid);
terrno = TSDB_CODE_TDB_TABLE_IN_OTHER_STABLE; terrno = TSDB_CODE_TDB_TABLE_IN_OTHER_STABLE;
return false; return false;
} }
@ -437,10 +438,10 @@ SVCreateTbReq* buildAutoCreateTableReq(const char* stbFullName, int64_t suid, in
taosArrayClear(pTagArray); taosArrayClear(pTagArray);
initCreateTableMsg(pCreateTbReq, suid, stbFullName, 1); initCreateTableMsg(pCreateTbReq, suid, stbFullName, 1);
STagVal tagVal = { .cid = numOfCols, .type = TSDB_DATA_TYPE_UBIGINT, .i64 = pDataBlock->info.id.groupId}; STagVal tagVal = {.cid = numOfCols, .type = TSDB_DATA_TYPE_UBIGINT, .i64 = pDataBlock->info.id.groupId};
taosArrayPush(pTagArray, &tagVal); taosArrayPush(pTagArray, &tagVal);
tTagNew(pTagArray, 1, false, (STag**) &pCreateTbReq->ctb.pTag); tTagNew(pTagArray, 1, false, (STag**)&pCreateTbReq->ctb.pTag);
if (pCreateTbReq->ctb.pTag == NULL) { if (pCreateTbReq->ctb.pTag == NULL) {
tdDestroySVCreateTbReq(pCreateTbReq); tdDestroySVCreateTbReq(pCreateTbReq);
@ -513,13 +514,13 @@ int32_t buildSubmitMsgImpl(SSubmitReq2* pSubmitReq, int32_t vgId, void** pMsg, i
} }
int32_t tsAscendingSortFn(const void* p1, const void* p2) { int32_t tsAscendingSortFn(const void* p1, const void* p2) {
SRow* pRow1 = *(SRow**) p1; SRow* pRow1 = *(SRow**)p1;
SRow* pRow2 = *(SRow**) p2; SRow* pRow2 = *(SRow**)p2;
if (pRow1->ts == pRow2->ts) { if (pRow1->ts == pRow2->ts) {
return 0; return 0;
} else { } else {
return pRow1->ts > pRow2->ts? 1:-1; return pRow1->ts > pRow2->ts ? 1 : -1;
} }
} }
@ -563,7 +564,7 @@ int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDat
void* colData = colDataGetData(pColData, j); void* colData = colDataGetData(pColData, j);
if (IS_STR_DATA_TYPE(pCol->type)) { if (IS_STR_DATA_TYPE(pCol->type)) {
// address copy, no value // address copy, no value
SValue sv = (SValue){.nData = varDataLen(colData), .pData = (uint8_t*) varDataVal(colData)}; SValue sv = (SValue){.nData = varDataLen(colData), .pData = (uint8_t*)varDataVal(colData)};
SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv); SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv);
taosArrayPush(pVals, &cv); taosArrayPush(pVals, &cv);
} else { } else {
@ -666,11 +667,9 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
if (dstTableName[0] == 0) { if (dstTableName[0] == 0) {
memset(dstTableName, 0, TSDB_TABLE_NAME_LEN); memset(dstTableName, 0, TSDB_TABLE_NAME_LEN);
buildCtbNameByGroupIdImpl(stbFullName, groupId, dstTableName); buildCtbNameByGroupIdImpl(stbFullName, groupId, dstTableName);
}else{ } else {
if(pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && if (pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1 &&
!isAutoTableName(dstTableName) && !isAutoTableName(dstTableName) && !alreadyAddGroupId(dstTableName) && groupId != 0) {
!alreadyAddGroupId(dstTableName) &&
groupId != 0) {
buildCtbNameAddGruopId(dstTableName, groupId); buildCtbNameAddGruopId(dstTableName, groupId);
} }
} }
@ -693,7 +692,7 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
tqTrace("s-task:%s cached tableInfo uid is invalid, acquire it from meta", id); tqTrace("s-task:%s cached tableInfo uid is invalid, acquire it from meta", id);
return doWaitForDstTableCreated(pVnode, pTask, pTableSinkInfo, dstTableName, &pTableData->uid); return doWaitForDstTableCreated(pVnode, pTask, pTableSinkInfo, dstTableName, &pTableData->uid);
} else { } else {
tqTrace("s-task:%s set the dstTable uid from cache:%"PRId64, id, pTableData->uid); tqTrace("s-task:%s set the dstTable uid from cache:%" PRId64, id, pTableData->uid);
} }
} else { } else {
// The auto-create option will always set to be open for those submit messages, which arrive during the period // The auto-create option will always set to be open for those submit messages, which arrive during the period
@ -714,7 +713,8 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
pTableData->flags = SUBMIT_REQ_AUTO_CREATE_TABLE; pTableData->flags = SUBMIT_REQ_AUTO_CREATE_TABLE;
pTableData->pCreateTbReq = pTableData->pCreateTbReq =
buildAutoCreateTableReq(stbFullName, suid, pTSchema->numOfCols + 1, pDataBlock, pTagArray, pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER); buildAutoCreateTableReq(stbFullName, suid, pTSchema->numOfCols + 1, pDataBlock, pTagArray,
pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1);
taosArrayDestroy(pTagArray); taosArrayDestroy(pTagArray);
if (pTableData->pCreateTbReq == NULL) { if (pTableData->pCreateTbReq == NULL) {
@ -746,12 +746,12 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
return TDB_CODE_SUCCESS; return TDB_CODE_SUCCESS;
} }
int32_t tqSetDstTableDataPayload(uint64_t suid, const STSchema *pTSchema, int32_t blockIndex, SSDataBlock* pDataBlock, int32_t tqSetDstTableDataPayload(uint64_t suid, const STSchema* pTSchema, int32_t blockIndex, SSDataBlock* pDataBlock,
SSubmitTbData* pTableData, const char* id) { SSubmitTbData* pTableData, const char* id) {
int32_t numOfRows = pDataBlock->info.rows; int32_t numOfRows = pDataBlock->info.rows;
tqDebug("s-task:%s sink data pipeline, build submit msg from %dth resBlock, including %d rows, dst suid:%" PRId64, tqDebug("s-task:%s sink data pipeline, build submit msg from %dth resBlock, including %d rows, dst suid:%" PRId64, id,
id, blockIndex + 1, numOfRows, suid); blockIndex + 1, numOfRows, suid);
char* dstTableName = pDataBlock->info.parTbName; char* dstTableName = pDataBlock->info.parTbName;
// convert all rows // convert all rows
@ -767,14 +767,14 @@ int32_t tqSetDstTableDataPayload(uint64_t suid, const STSchema *pTSchema, int32_
} }
bool hasOnlySubmitData(const SArray* pBlocks, int32_t numOfBlocks) { bool hasOnlySubmitData(const SArray* pBlocks, int32_t numOfBlocks) {
for(int32_t i = 0; i < numOfBlocks; ++i) { for (int32_t i = 0; i < numOfBlocks; ++i) {
SSDataBlock* p = taosArrayGet(pBlocks, i); SSDataBlock* p = taosArrayGet(pBlocks, i);
if (p->info.type == STREAM_DELETE_RESULT || p->info.type == STREAM_CREATE_CHILD_TABLE) { if (p->info.type == STREAM_DELETE_RESULT || p->info.type == STREAM_CREATE_CHILD_TABLE) {
return false; return false;
} }
} }
return true; return true;
} }
void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) { void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
@ -793,7 +793,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
tqDebug("vgId:%d, s-task:%s write %d stream resBlock(s) into table, has delete block, submit one-by-one", vgId, id, tqDebug("vgId:%d, s-task:%s write %d stream resBlock(s) into table, has delete block, submit one-by-one", vgId, id,
numOfBlocks); numOfBlocks);
for(int32_t i = 0; i < numOfBlocks; ++i) { for (int32_t i = 0; i < numOfBlocks; ++i) {
if (streamTaskShouldStop(pTask)) { if (streamTaskShouldStop(pTask)) {
return; return;
} }
@ -832,7 +832,8 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
} }
} else { } else {
tqDebug("vgId:%d, s-task:%s write %d stream resBlock(s) into table, merge submit msg", vgId, id, numOfBlocks); tqDebug("vgId:%d, s-task:%s write %d stream resBlock(s) into table, merge submit msg", vgId, id, numOfBlocks);
SHashObj* pTableIndexMap = taosHashInit(numOfBlocks, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); SHashObj* pTableIndexMap =
taosHashInit(numOfBlocks, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
SSubmitReq2 submitReq = {.aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData))}; SSubmitReq2 submitReq = {.aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData))};
if (submitReq.aSubmitTbData == NULL) { if (submitReq.aSubmitTbData == NULL) {

2
source/libs/parser/inc/sql.y
View File

@ -516,7 +516,7 @@ cmd ::= SHOW VNODES.
// show alive // show alive
cmd ::= SHOW db_name_cond_opt(A) ALIVE. { pCxt->pRootNode = createShowAliveStmt(pCxt, A, QUERY_NODE_SHOW_DB_ALIVE_STMT); } cmd ::= SHOW db_name_cond_opt(A) ALIVE. { pCxt->pRootNode = createShowAliveStmt(pCxt, A, QUERY_NODE_SHOW_DB_ALIVE_STMT); }
cmd ::= SHOW CLUSTER ALIVE. { pCxt->pRootNode = createShowAliveStmt(pCxt, NULL, QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT); } cmd ::= SHOW CLUSTER ALIVE. { pCxt->pRootNode = createShowAliveStmt(pCxt, NULL, QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT); }
cmd ::= SHOW db_name_cond_opt(A) VIEWS. { pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_VIEWS_STMT, A, NULL, OP_TYPE_LIKE); } cmd ::= SHOW db_name_cond_opt(A) VIEWS like_pattern_opt(B). { pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_VIEWS_STMT, A, B, OP_TYPE_LIKE); }
cmd ::= SHOW CREATE VIEW full_table_name(A). { pCxt->pRootNode = createShowCreateViewStmt(pCxt, QUERY_NODE_SHOW_CREATE_VIEW_STMT, A); } cmd ::= SHOW CREATE VIEW full_table_name(A). { pCxt->pRootNode = createShowCreateViewStmt(pCxt, QUERY_NODE_SHOW_CREATE_VIEW_STMT, A); }
cmd ::= SHOW COMPACTS. { pCxt->pRootNode = createShowCompactsStmt(pCxt, QUERY_NODE_SHOW_COMPACTS_STMT); } cmd ::= SHOW COMPACTS. { pCxt->pRootNode = createShowCompactsStmt(pCxt, QUERY_NODE_SHOW_COMPACTS_STMT); }
cmd ::= SHOW COMPACT NK_INTEGER(A). { pCxt->pRootNode = createShowCompactDetailsStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &A)); } cmd ::= SHOW COMPACT NK_INTEGER(A). { pCxt->pRootNode = createShowCompactDetailsStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &A)); }

15
source/libs/parser/src/parTranslater.c
View File

@ -9399,7 +9399,20 @@ static int32_t createOperatorNode(EOperatorType opType, const char* pColName, SN
} }
static const char* getTbNameColName(ENodeType type) { static const char* getTbNameColName(ENodeType type) {
return (QUERY_NODE_SHOW_STABLES_STMT == type ? "stable_name" : "table_name"); const char* colName;
switch (type)
{
case QUERY_NODE_SHOW_VIEWS_STMT:
colName = "view_name";
break;
case QUERY_NODE_SHOW_STABLES_STMT:
colName = "stable_name";
break;
default:
colName = "table_name";
break;
}
return colName;
} }
static int32_t createLogicCondNode(SNode* pCond1, SNode* pCond2, SNode** pCond, ELogicConditionType logicCondType) { static int32_t createLogicCondNode(SNode* pCond1, SNode* pCond2, SNode** pCond, ELogicConditionType logicCondType) {

2521
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

2
source/libs/scheduler/src/schJob.c
View File

@ -66,7 +66,7 @@ FORCE_INLINE bool schJobNeedToStop(SSchJob *pJob, int8_t *pStatus) {
return true; return true;
} }
if ((*pJob->chkKillFp)(pJob->chkKillParam)) { if (pJob->chkKillFp && (*pJob->chkKillFp)(pJob->chkKillParam)) {
schUpdateJobErrCode(pJob, TSDB_CODE_TSC_QUERY_KILLED); schUpdateJobErrCode(pJob, TSDB_CODE_TSC_QUERY_KILLED);
return true; return true;
} }

479
source/libs/scheduler/test/schedulerTests.cpp
View File

@ -54,9 +54,8 @@
namespace { namespace {
extern "C" int32_t schHandleResponseMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *msg, int32_t msgSize, extern "C" int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDataBuf *pMsg, int32_t rspCode);
int32_t rspCode); extern "C" int32_t schHandleCallback(void *param, const SDataBuf *pMsg, int32_t rspCode);
extern "C" int32_t schHandleCallback(void *param, const SDataBuf *pMsg, int32_t msgType, int32_t rspCode);
int64_t insertJobRefId = 0; int64_t insertJobRefId = 0;
int64_t queryJobRefId = 0; int64_t queryJobRefId = 0;
@ -67,7 +66,7 @@ uint64_t schtQueryId = 1;
bool schtTestStop = false; bool schtTestStop = false;
bool schtTestDeadLoop = false; bool schtTestDeadLoop = false;
int32_t schtTestMTRunSec = 10; int32_t schtTestMTRunSec = 1;
int32_t schtTestPrintNum = 1000; int32_t schtTestPrintNum = 1000;
int32_t schtStartFetch = 0; int32_t schtStartFetch = 0;
@ -85,10 +84,69 @@ void schtInitLogFile() {
} }
void schtQueryCb(SExecResult *pResult, void *param, int32_t code) { void schtQueryCb(SExecResult *pResult, void *param, int32_t code) {
assert(TSDB_CODE_SUCCESS == code);
*(int32_t *)param = 1; *(int32_t *)param = 1;
} }
int32_t schtBuildQueryRspMsg(uint32_t *msize, void** rspMsg) {
SQueryTableRsp rsp = {0};
rsp.code = 0;
rsp.affectedRows = 0;
rsp.tbVerInfo = NULL;
int32_t msgSize = tSerializeSQueryTableRsp(NULL, 0, &rsp);
if (msgSize < 0) {
qError("tSerializeSQueryTableRsp failed");
return TSDB_CODE_OUT_OF_MEMORY;
}
void *pRsp = taosMemoryCalloc(msgSize, 1);
if (NULL == pRsp) {
qError("rpcMallocCont %d failed", msgSize);
return TSDB_CODE_OUT_OF_MEMORY;
}
if (tSerializeSQueryTableRsp(pRsp, msgSize, &rsp) < 0) {
qError("tSerializeSQueryTableRsp %d failed", msgSize);
return TSDB_CODE_OUT_OF_MEMORY;
}
*rspMsg = pRsp;
*msize = msgSize;
return TSDB_CODE_SUCCESS;
}
int32_t schtBuildFetchRspMsg(uint32_t *msize, void** rspMsg) {
SRetrieveTableRsp* rsp = (SRetrieveTableRsp*)taosMemoryCalloc(sizeof(SRetrieveTableRsp), 1);
rsp->completed = 1;
rsp->numOfRows = 10;
rsp->compLen = 0;
*rspMsg = rsp;
*msize = sizeof(SRetrieveTableRsp);
return TSDB_CODE_SUCCESS;
}
int32_t schtBuildSubmitRspMsg(uint32_t *msize, void** rspMsg) {
SSubmitRsp2 submitRsp = {0};
int32_t msgSize = 0, ret = 0;
SEncoder ec = {0};
tEncodeSize(tEncodeSSubmitRsp2, &submitRsp, msgSize, ret);
void* msg = taosMemoryCalloc(1, msgSize);
tEncoderInit(&ec, (uint8_t*)msg, msgSize);
tEncodeSSubmitRsp2(&ec, &submitRsp);
tEncoderClear(&ec);
*rspMsg = msg;
*msize = msgSize;
return TSDB_CODE_SUCCESS;
}
void schtBuildQueryDag(SQueryPlan *dag) { void schtBuildQueryDag(SQueryPlan *dag) {
uint64_t qId = schtQueryId; uint64_t qId = schtQueryId;
@ -98,8 +156,8 @@ void schtBuildQueryDag(SQueryPlan *dag) {
SNodeListNode *scan = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST); SNodeListNode *scan = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST);
SNodeListNode *merge = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST); SNodeListNode *merge = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST);
SSubplan *scanPlan = (SSubplan *)taosMemoryCalloc(1, sizeof(SSubplan)); SSubplan *scanPlan = (SSubplan*)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN);
SSubplan *mergePlan = (SSubplan *)taosMemoryCalloc(1, sizeof(SSubplan)); SSubplan *mergePlan = (SSubplan*)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN);
scanPlan->id.queryId = qId; scanPlan->id.queryId = qId;
scanPlan->id.groupId = 0x0000000000000002; scanPlan->id.groupId = 0x0000000000000002;
@ -113,7 +171,7 @@ void schtBuildQueryDag(SQueryPlan *dag) {
scanPlan->pChildren = NULL; scanPlan->pChildren = NULL;
scanPlan->level = 1; scanPlan->level = 1;
scanPlan->pParents = nodesMakeList(); scanPlan->pParents = nodesMakeList();
scanPlan->pNode = (SPhysiNode *)taosMemoryCalloc(1, sizeof(SPhysiNode)); scanPlan->pNode = (SPhysiNode *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN);
scanPlan->msgType = TDMT_SCH_QUERY; scanPlan->msgType = TDMT_SCH_QUERY;
mergePlan->id.queryId = qId; mergePlan->id.queryId = qId;
@ -125,7 +183,7 @@ void schtBuildQueryDag(SQueryPlan *dag) {
mergePlan->pChildren = nodesMakeList(); mergePlan->pChildren = nodesMakeList();
mergePlan->pParents = NULL; mergePlan->pParents = NULL;
mergePlan->pNode = (SPhysiNode *)taosMemoryCalloc(1, sizeof(SPhysiNode)); mergePlan->pNode = (SPhysiNode *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_MERGE);
mergePlan->msgType = TDMT_SCH_QUERY; mergePlan->msgType = TDMT_SCH_QUERY;
merge->pNodeList = nodesMakeList(); merge->pNodeList = nodesMakeList();
@ -151,8 +209,7 @@ void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) {
SNodeListNode *scan = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST); SNodeListNode *scan = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST);
SNodeListNode *merge = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST); SNodeListNode *merge = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST);
SSubplan *scanPlan = (SSubplan *)taosMemoryCalloc(scanPlanNum, sizeof(SSubplan)); SSubplan *mergePlan = (SSubplan*)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN);
SSubplan *mergePlan = (SSubplan *)taosMemoryCalloc(1, sizeof(SSubplan));
merge->pNodeList = nodesMakeList(); merge->pNodeList = nodesMakeList();
scan->pNodeList = nodesMakeList(); scan->pNodeList = nodesMakeList();
@ -160,29 +217,30 @@ void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) {
mergePlan->pChildren = nodesMakeList(); mergePlan->pChildren = nodesMakeList();
for (int32_t i = 0; i < scanPlanNum; ++i) { for (int32_t i = 0; i < scanPlanNum; ++i) {
scanPlan[i].id.queryId = qId; SSubplan *scanPlan = (SSubplan*)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN);
scanPlan[i].id.groupId = 0x0000000000000002; scanPlan->id.queryId = qId;
scanPlan[i].id.subplanId = 0x0000000000000003 + i; scanPlan->id.groupId = 0x0000000000000002;
scanPlan[i].subplanType = SUBPLAN_TYPE_SCAN; scanPlan->id.subplanId = 0x0000000000000003 + i;
scanPlan->subplanType = SUBPLAN_TYPE_SCAN;
scanPlan[i].execNode.nodeId = 1 + i; scanPlan->execNode.nodeId = 1 + i;
scanPlan[i].execNode.epSet.inUse = 0; scanPlan->execNode.epSet.inUse = 0;
scanPlan[i].execNodeStat.tableNum = taosRand() % 30; scanPlan->execNodeStat.tableNum = taosRand() % 30;
addEpIntoEpSet(&scanPlan[i].execNode.epSet, "ep0", 6030); addEpIntoEpSet(&scanPlan->execNode.epSet, "ep0", 6030);
addEpIntoEpSet(&scanPlan[i].execNode.epSet, "ep1", 6030); addEpIntoEpSet(&scanPlan->execNode.epSet, "ep1", 6030);
addEpIntoEpSet(&scanPlan[i].execNode.epSet, "ep2", 6030); addEpIntoEpSet(&scanPlan->execNode.epSet, "ep2", 6030);
scanPlan[i].execNode.epSet.inUse = taosRand() % 3; scanPlan->execNode.epSet.inUse = taosRand() % 3;
scanPlan[i].pChildren = NULL; scanPlan->pChildren = NULL;
scanPlan[i].level = 1; scanPlan->level = 1;
scanPlan[i].pParents = nodesMakeList(); scanPlan->pParents = nodesMakeList();
scanPlan[i].pNode = (SPhysiNode *)taosMemoryCalloc(1, sizeof(SPhysiNode)); scanPlan->pNode = (SPhysiNode *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN);
scanPlan[i].msgType = TDMT_SCH_QUERY; scanPlan->msgType = TDMT_SCH_QUERY;
nodesListAppend(scanPlan[i].pParents, (SNode *)mergePlan); nodesListAppend(scanPlan->pParents, (SNode *)mergePlan);
nodesListAppend(mergePlan->pChildren, (SNode *)(scanPlan + i)); nodesListAppend(mergePlan->pChildren, (SNode *)scanPlan);
nodesListAppend(scan->pNodeList, (SNode *)(scanPlan + i)); nodesListAppend(scan->pNodeList, (SNode *)scanPlan);
} }
mergePlan->id.queryId = qId; mergePlan->id.queryId = qId;
@ -193,7 +251,7 @@ void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) {
mergePlan->execNode.epSet.numOfEps = 0; mergePlan->execNode.epSet.numOfEps = 0;
mergePlan->pParents = NULL; mergePlan->pParents = NULL;
mergePlan->pNode = (SPhysiNode *)taosMemoryCalloc(1, sizeof(SPhysiNode)); mergePlan->pNode = (SPhysiNode *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_MERGE);
mergePlan->msgType = TDMT_SCH_QUERY; mergePlan->msgType = TDMT_SCH_QUERY;
nodesListAppend(merge->pNodeList, (SNode *)mergePlan); nodesListAppend(merge->pNodeList, (SNode *)mergePlan);
@ -211,45 +269,50 @@ void schtBuildInsertDag(SQueryPlan *dag) {
dag->numOfSubplans = 2; dag->numOfSubplans = 2;
dag->pSubplans = nodesMakeList(); dag->pSubplans = nodesMakeList();
SNodeListNode *inserta = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST); SNodeListNode *inserta = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST);
SSubplan *insertPlan = (SSubplan *)taosMemoryCalloc(2, sizeof(SSubplan));
insertPlan[0].id.queryId = qId;
insertPlan[0].id.groupId = 0x0000000000000003;
insertPlan[0].id.subplanId = 0x0000000000000004;
insertPlan[0].subplanType = SUBPLAN_TYPE_MODIFY;
insertPlan[0].level = 0;
insertPlan[0].execNode.nodeId = 1;
insertPlan[0].execNode.epSet.inUse = 0;
addEpIntoEpSet(&insertPlan[0].execNode.epSet, "ep0", 6030);
insertPlan[0].pChildren = NULL;
insertPlan[0].pParents = NULL;
insertPlan[0].pNode = NULL;
insertPlan[0].pDataSink = (SDataSinkNode *)taosMemoryCalloc(1, sizeof(SDataSinkNode));
insertPlan[0].msgType = TDMT_VND_SUBMIT;
insertPlan[1].id.queryId = qId;
insertPlan[1].id.groupId = 0x0000000000000003;
insertPlan[1].id.subplanId = 0x0000000000000005;
insertPlan[1].subplanType = SUBPLAN_TYPE_MODIFY;
insertPlan[1].level = 0;
insertPlan[1].execNode.nodeId = 1;
insertPlan[1].execNode.epSet.inUse = 0;
addEpIntoEpSet(&insertPlan[1].execNode.epSet, "ep0", 6030);
insertPlan[1].pChildren = NULL;
insertPlan[1].pParents = NULL;
insertPlan[1].pNode = NULL;
insertPlan[1].pDataSink = (SDataSinkNode *)taosMemoryCalloc(1, sizeof(SDataSinkNode));
insertPlan[1].msgType = TDMT_VND_SUBMIT;
inserta->pNodeList = nodesMakeList(); inserta->pNodeList = nodesMakeList();
SSubplan *insertPlan = (SSubplan*)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN);
insertPlan->id.queryId = qId;
insertPlan->id.groupId = 0x0000000000000003;
insertPlan->id.subplanId = 0x0000000000000004;
insertPlan->subplanType = SUBPLAN_TYPE_MODIFY;
insertPlan->level = 0;
insertPlan->execNode.nodeId = 1;
insertPlan->execNode.epSet.inUse = 0;
addEpIntoEpSet(&insertPlan->execNode.epSet, "ep0", 6030);
insertPlan->pChildren = NULL;
insertPlan->pParents = NULL;
insertPlan->pNode = NULL;
insertPlan->pDataSink = (SDataSinkNode*)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_INSERT);
((SDataInserterNode*)insertPlan->pDataSink)->size = 1;
((SDataInserterNode*)insertPlan->pDataSink)->pData = taosMemoryCalloc(1, 1);
insertPlan->msgType = TDMT_VND_SUBMIT;
nodesListAppend(inserta->pNodeList, (SNode *)insertPlan); nodesListAppend(inserta->pNodeList, (SNode *)insertPlan);
insertPlan += 1;
insertPlan = (SSubplan*)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN);
insertPlan->id.queryId = qId;
insertPlan->id.groupId = 0x0000000000000003;
insertPlan->id.subplanId = 0x0000000000000005;
insertPlan->subplanType = SUBPLAN_TYPE_MODIFY;
insertPlan->level = 0;
insertPlan->execNode.nodeId = 1;
insertPlan->execNode.epSet.inUse = 0;
addEpIntoEpSet(&insertPlan->execNode.epSet, "ep0", 6030);
insertPlan->pChildren = NULL;
insertPlan->pParents = NULL;
insertPlan->pNode = NULL;
insertPlan->pDataSink = (SDataSinkNode*)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_INSERT);
((SDataInserterNode*)insertPlan->pDataSink)->size = 1;
((SDataInserterNode*)insertPlan->pDataSink)->pData = taosMemoryCalloc(1, 1);
insertPlan->msgType = TDMT_VND_SUBMIT;
nodesListAppend(inserta->pNodeList, (SNode *)insertPlan); nodesListAppend(inserta->pNodeList, (SNode *)insertPlan);
nodesListAppend(dag->pSubplans, (SNode *)inserta); nodesListAppend(dag->pSubplans, (SNode *)inserta);
@ -325,7 +388,7 @@ void schtSetRpcSendRequest() {
} }
} }
int32_t schtAsyncSendMsgToServer(void *pTransporter, SEpSet *epSet, int64_t *pTransporterId, SMsgSendInfo *pInfo) { int32_t schtAsyncSendMsgToServer(void *pTransporter, SEpSet *epSet, int64_t *pTransporterId, SMsgSendInfo *pInfo, bool persistHandle, void* rpcCtx) {
if (pInfo) { if (pInfo) {
taosMemoryFreeClear(pInfo->param); taosMemoryFreeClear(pInfo->param);
taosMemoryFreeClear(pInfo->msgInfo.pData); taosMemoryFreeClear(pInfo->msgInfo.pData);
@ -336,17 +399,17 @@ int32_t schtAsyncSendMsgToServer(void *pTransporter, SEpSet *epSet, int64_t *pTr
void schtSetAsyncSendMsgToServer() { void schtSetAsyncSendMsgToServer() {
static Stub stub; static Stub stub;
stub.set(asyncSendMsgToServer, schtAsyncSendMsgToServer); stub.set(asyncSendMsgToServerExt, schtAsyncSendMsgToServer);
{ {
#ifdef WINDOWS #ifdef WINDOWS
AddrAny any; AddrAny any;
std::map<std::string, void *> result; std::map<std::string, void *> result;
any.get_func_addr("asyncSendMsgToServer", result); any.get_func_addr("asyncSendMsgToServerExt", result);
#endif #endif
#ifdef LINUX #ifdef LINUX
AddrAny any("libtransport.so"); AddrAny any("libtransport.so");
std::map<std::string, void *> result; std::map<std::string, void *> result;
any.get_global_func_addr_dynsym("^asyncSendMsgToServer$", result); any.get_global_func_addr_dynsym("^asyncSendMsgToServerExt$", result);
#endif #endif
for (const auto &f : result) { for (const auto &f : result) {
stub.set(f.second, schtAsyncSendMsgToServer); stub.set(f.second, schtAsyncSendMsgToServer);
@ -374,9 +437,13 @@ void *schtSendRsp(void *param) {
while (pIter) { while (pIter) {
SSchTask *task = *(SSchTask **)pIter; SSchTask *task = *(SSchTask **)pIter;
SSubmitRsp rsp = {0}; SDataBuf msg = {0};
rsp.affectedRows = 10; void* rmsg = NULL;
schHandleResponseMsg(pJob, task, TDMT_VND_SUBMIT_RSP, (char *)&rsp, sizeof(rsp), 0); schtBuildSubmitRspMsg(&msg.len, &rmsg);
msg.msgType = TDMT_VND_SUBMIT_RSP;
msg.pData = rmsg;
schHandleResponseMsg(pJob, task, task->execId, &msg, 0);
pIter = taosHashIterate(pJob->execTasks, pIter); pIter = taosHashIterate(pJob->execTasks, pIter);
} }
@ -393,11 +460,13 @@ void *schtCreateFetchRspThread(void *param) {
taosSsleep(1); taosSsleep(1);
int32_t code = 0; int32_t code = 0;
SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, sizeof(SRetrieveTableRsp)); SDataBuf msg = {0};
rsp->completed = 1; void* rmsg = NULL;
rsp->numOfRows = 10; schtBuildFetchRspMsg(&msg.len, &rmsg);
msg.msgType = TDMT_SCH_MERGE_FETCH_RSP;
code = schHandleResponseMsg(pJob, pJob->fetchTask, TDMT_SCH_FETCH_RSP, (char *)rsp, sizeof(*rsp), 0); msg.pData = rmsg;
code = schHandleResponseMsg(pJob, pJob->fetchTask, pJob->fetchTask->execId, &msg, 0);
schReleaseJob(job); schReleaseJob(job);
@ -414,7 +483,7 @@ void *schtFetchRspThread(void *aa) {
continue; continue;
} }
taosUsleep(1); taosUsleep(100);
param = (SSchTaskCallbackParam *)taosMemoryCalloc(1, sizeof(*param)); param = (SSchTaskCallbackParam *)taosMemoryCalloc(1, sizeof(*param));
@ -426,10 +495,11 @@ void *schtFetchRspThread(void *aa) {
rsp->completed = 1; rsp->completed = 1;
rsp->numOfRows = 10; rsp->numOfRows = 10;
dataBuf.msgType = TDMT_SCH_FETCH_RSP;
dataBuf.pData = rsp; dataBuf.pData = rsp;
dataBuf.len = sizeof(*rsp); dataBuf.len = sizeof(*rsp);
code = schHandleCallback(param, &dataBuf, TDMT_SCH_FETCH_RSP, 0); code = schHandleCallback(param, &dataBuf, 0);
assert(code == 0 || code); assert(code == 0 || code);
} }
@ -456,7 +526,7 @@ void *schtRunJobThread(void *aa) {
char *dbname = "1.db1"; char *dbname = "1.db1";
char *tablename = "table1"; char *tablename = "table1";
SVgroupInfo vgInfo = {0}; SVgroupInfo vgInfo = {0};
SQueryPlan dag; SQueryPlan* dag = (SQueryPlan*)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN);
schtInitLogFile(); schtInitLogFile();
@ -470,19 +540,19 @@ void *schtRunJobThread(void *aa) {
SSchJob *pJob = NULL; SSchJob *pJob = NULL;
SSchTaskCallbackParam *param = NULL; SSchTaskCallbackParam *param = NULL;
SHashObj *execTasks = NULL; SHashObj *execTasks = NULL;
SDataBuf dataBuf = {0};
uint32_t jobFinished = 0; uint32_t jobFinished = 0;
int32_t queryDone = 0; int32_t queryDone = 0;
while (!schtTestStop) { while (!schtTestStop) {
schtBuildQueryDag(&dag); schtBuildQueryDag(dag);
SArray *qnodeList = taosArrayInit(1, sizeof(SEp)); SArray *qnodeList = taosArrayInit(1, sizeof(SQueryNodeLoad));
SEp qnodeAddr = {0}; SQueryNodeLoad load = {0};
strcpy(qnodeAddr.fqdn, "qnode0.ep"); load.addr.epSet.numOfEps = 1;
qnodeAddr.port = 6031; strcpy(load.addr.epSet.eps[0].fqdn, "qnode0.ep");
taosArrayPush(qnodeList, &qnodeAddr); load.addr.epSet.eps[0].port = 6031;
taosArrayPush(qnodeList, &load);
queryDone = 0; queryDone = 0;
@ -492,7 +562,7 @@ void *schtRunJobThread(void *aa) {
req.syncReq = false; req.syncReq = false;
req.pConn = &conn; req.pConn = &conn;
req.pNodeList = qnodeList; req.pNodeList = qnodeList;
req.pDag = &dag; req.pDag = dag;
req.sql = "select * from tb"; req.sql = "select * from tb";
req.execFp = schtQueryCb; req.execFp = schtQueryCb;
req.cbParam = &queryDone; req.cbParam = &queryDone;
@ -503,7 +573,7 @@ void *schtRunJobThread(void *aa) {
pJob = schAcquireJob(queryJobRefId); pJob = schAcquireJob(queryJobRefId);
if (NULL == pJob) { if (NULL == pJob) {
taosArrayDestroy(qnodeList); taosArrayDestroy(qnodeList);
schtFreeQueryDag(&dag); schtFreeQueryDag(dag);
continue; continue;
} }
@ -526,11 +596,14 @@ void *schtRunJobThread(void *aa) {
SSchTask *task = (SSchTask *)pIter; SSchTask *task = (SSchTask *)pIter;
param->taskId = task->taskId; param->taskId = task->taskId;
SQueryTableRsp rsp = {0};
dataBuf.pData = &rsp;
dataBuf.len = sizeof(rsp);
code = schHandleCallback(param, &dataBuf, TDMT_SCH_QUERY_RSP, 0); SDataBuf msg = {0};
void* rmsg = NULL;
schtBuildQueryRspMsg(&msg.len, &rmsg);
msg.msgType = TDMT_SCH_QUERY_RSP;
msg.pData = rmsg;
code = schHandleCallback(param, &msg, 0);
assert(code == 0 || code); assert(code == 0 || code);
pIter = taosHashIterate(execTasks, pIter); pIter = taosHashIterate(execTasks, pIter);
@ -545,11 +618,13 @@ void *schtRunJobThread(void *aa) {
SSchTask *task = (SSchTask *)pIter; SSchTask *task = (SSchTask *)pIter;
param->taskId = task->taskId - 1; param->taskId = task->taskId - 1;
SQueryTableRsp rsp = {0}; SDataBuf msg = {0};
dataBuf.pData = &rsp; void* rmsg = NULL;
dataBuf.len = sizeof(rsp); schtBuildQueryRspMsg(&msg.len, &rmsg);
msg.msgType = TDMT_SCH_QUERY_RSP;
msg.pData = rmsg;
code = schHandleCallback(param, &dataBuf, TDMT_SCH_QUERY_RSP, 0); code = schHandleCallback(param, &msg, 0);
assert(code == 0 || code); assert(code == 0 || code);
pIter = taosHashIterate(execTasks, pIter); pIter = taosHashIterate(execTasks, pIter);
@ -575,7 +650,6 @@ void *schtRunJobThread(void *aa) {
if (0 == code) { if (0 == code) {
SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)data; SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)data;
assert(pRsp->completed == 1); assert(pRsp->completed == 1);
assert(pRsp->numOfRows == 10);
} }
data = NULL; data = NULL;
@ -587,7 +661,7 @@ void *schtRunJobThread(void *aa) {
taosHashCleanup(execTasks); taosHashCleanup(execTasks);
taosArrayDestroy(qnodeList); taosArrayDestroy(qnodeList);
schtFreeQueryDag(&dag); schtFreeQueryDag(dag);
if (++jobFinished % schtTestPrintNum == 0) { if (++jobFinished % schtTestPrintNum == 0) {
printf("jobFinished:%d\n", jobFinished); printf("jobFinished:%d\n", jobFinished);
@ -609,6 +683,7 @@ void *schtFreeJobThread(void *aa) {
return NULL; return NULL;
} }
} // namespace } // namespace
TEST(queryTest, normalCase) { TEST(queryTest, normalCase) {
@ -618,21 +693,20 @@ TEST(queryTest, normalCase) {
char *tablename = "table1"; char *tablename = "table1";
SVgroupInfo vgInfo = {0}; SVgroupInfo vgInfo = {0};
int64_t job = 0; int64_t job = 0;
SQueryPlan dag; SQueryPlan* dag = (SQueryPlan*)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN);
memset(&dag, 0, sizeof(dag)); SArray *qnodeList = taosArrayInit(1, sizeof(SQueryNodeLoad));
SArray *qnodeList = taosArrayInit(1, sizeof(SEp)); SQueryNodeLoad load = {0};
load.addr.epSet.numOfEps = 1;
SEp qnodeAddr = {0}; strcpy(load.addr.epSet.eps[0].fqdn, "qnode0.ep");
strcpy(qnodeAddr.fqdn, "qnode0.ep"); load.addr.epSet.eps[0].port = 6031;
qnodeAddr.port = 6031; taosArrayPush(qnodeList, &load);
taosArrayPush(qnodeList, &qnodeAddr);
int32_t code = schedulerInit(); int32_t code = schedulerInit();
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
schtBuildQueryDag(&dag); schtBuildQueryDag(dag);
schtSetPlanToString(); schtSetPlanToString();
schtSetExecNode(); schtSetExecNode();
@ -645,7 +719,7 @@ TEST(queryTest, normalCase) {
SSchedulerReq req = {0}; SSchedulerReq req = {0};
req.pConn = &conn; req.pConn = &conn;
req.pNodeList = qnodeList; req.pNodeList = qnodeList;
req.pDag = &dag; req.pDag = dag;
req.sql = "select * from tb"; req.sql = "select * from tb";
req.execFp = schtQueryCb; req.execFp = schtQueryCb;
req.cbParam = &queryDone; req.cbParam = &queryDone;
@ -659,9 +733,14 @@ TEST(queryTest, normalCase) {
while (pIter) { while (pIter) {
SSchTask *task = *(SSchTask **)pIter; SSchTask *task = *(SSchTask **)pIter;
SQueryTableRsp rsp = {0}; SDataBuf msg = {0};
code = schHandleResponseMsg(pJob, task, TDMT_SCH_QUERY_RSP, (char *)&rsp, sizeof(rsp), 0); void* rmsg = NULL;
schtBuildQueryRspMsg(&msg.len, &rmsg);
msg.msgType = TDMT_SCH_QUERY_RSP;
msg.pData = rmsg;
code = schHandleResponseMsg(pJob, task, task->execId, &msg, 0);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
pIter = taosHashIterate(pJob->execTasks, pIter); pIter = taosHashIterate(pJob->execTasks, pIter);
} }
@ -669,11 +748,18 @@ TEST(queryTest, normalCase) {
pIter = taosHashIterate(pJob->execTasks, NULL); pIter = taosHashIterate(pJob->execTasks, NULL);
while (pIter) { while (pIter) {
SSchTask *task = *(SSchTask **)pIter; SSchTask *task = *(SSchTask **)pIter;
if (JOB_TASK_STATUS_EXEC == task->status) {
SDataBuf msg = {0};
void* rmsg = NULL;
schtBuildQueryRspMsg(&msg.len, &rmsg);
msg.msgType = TDMT_SCH_QUERY_RSP;
msg.pData = rmsg;
code = schHandleResponseMsg(pJob, task, task->execId, &msg, 0);
ASSERT_EQ(code, 0);
}
SQueryTableRsp rsp = {0};
code = schHandleResponseMsg(pJob, task, TDMT_SCH_QUERY_RSP, (char *)&rsp, sizeof(rsp), 0);
ASSERT_EQ(code, 0);
pIter = taosHashIterate(pJob->execTasks, pIter); pIter = taosHashIterate(pJob->execTasks, pIter);
} }
@ -703,18 +789,12 @@ TEST(queryTest, normalCase) {
ASSERT_EQ(pRsp->numOfRows, 10); ASSERT_EQ(pRsp->numOfRows, 10);
taosMemoryFreeClear(data); taosMemoryFreeClear(data);
data = NULL;
code = schedulerFetchRows(job, &req);
ASSERT_EQ(code, 0);
ASSERT_TRUE(data == NULL);
schReleaseJob(job); schReleaseJob(job);
schedulerDestroy();
schedulerFreeJob(&job, 0); schedulerFreeJob(&job, 0);
schtFreeQueryDag(&dag);
schedulerDestroy();
} }
TEST(queryTest, readyFirstCase) { TEST(queryTest, readyFirstCase) {
@ -724,21 +804,20 @@ TEST(queryTest, readyFirstCase) {
char *tablename = "table1"; char *tablename = "table1";
SVgroupInfo vgInfo = {0}; SVgroupInfo vgInfo = {0};
int64_t job = 0; int64_t job = 0;
SQueryPlan dag; SQueryPlan* dag = (SQueryPlan*)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN);
memset(&dag, 0, sizeof(dag)); SArray *qnodeList = taosArrayInit(1, sizeof(SQueryNodeLoad));
SArray *qnodeList = taosArrayInit(1, sizeof(SEp)); SQueryNodeLoad load = {0};
load.addr.epSet.numOfEps = 1;
SEp qnodeAddr = {0}; strcpy(load.addr.epSet.eps[0].fqdn, "qnode0.ep");
strcpy(qnodeAddr.fqdn, "qnode0.ep"); load.addr.epSet.eps[0].port = 6031;
qnodeAddr.port = 6031; taosArrayPush(qnodeList, &load);
taosArrayPush(qnodeList, &qnodeAddr);
int32_t code = schedulerInit(); int32_t code = schedulerInit();
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
schtBuildQueryDag(&dag); schtBuildQueryDag(dag);
schtSetPlanToString(); schtSetPlanToString();
schtSetExecNode(); schtSetExecNode();
@ -751,7 +830,7 @@ TEST(queryTest, readyFirstCase) {
SSchedulerReq req = {0}; SSchedulerReq req = {0};
req.pConn = &conn; req.pConn = &conn;
req.pNodeList = qnodeList; req.pNodeList = qnodeList;
req.pDag = &dag; req.pDag = dag;
req.sql = "select * from tb"; req.sql = "select * from tb";
req.execFp = schtQueryCb; req.execFp = schtQueryCb;
req.cbParam = &queryDone; req.cbParam = &queryDone;
@ -764,8 +843,13 @@ TEST(queryTest, readyFirstCase) {
while (pIter) { while (pIter) {
SSchTask *task = *(SSchTask **)pIter; SSchTask *task = *(SSchTask **)pIter;
SQueryTableRsp rsp = {0}; SDataBuf msg = {0};
code = schHandleResponseMsg(pJob, task, TDMT_SCH_QUERY_RSP, (char *)&rsp, sizeof(rsp), 0); void* rmsg = NULL;
schtBuildQueryRspMsg(&msg.len, &rmsg);
msg.msgType = TDMT_SCH_QUERY_RSP;
msg.pData = rmsg;
code = schHandleResponseMsg(pJob, task, task->execId, &msg, 0);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
pIter = taosHashIterate(pJob->execTasks, pIter); pIter = taosHashIterate(pJob->execTasks, pIter);
@ -775,10 +859,18 @@ TEST(queryTest, readyFirstCase) {
while (pIter) { while (pIter) {
SSchTask *task = *(SSchTask **)pIter; SSchTask *task = *(SSchTask **)pIter;
SQueryTableRsp rsp = {0}; if (JOB_TASK_STATUS_EXEC == task->status) {
code = schHandleResponseMsg(pJob, task, TDMT_SCH_QUERY_RSP, (char *)&rsp, sizeof(rsp), 0); SDataBuf msg = {0};
void* rmsg = NULL;
schtBuildQueryRspMsg(&msg.len, &rmsg);
msg.msgType = TDMT_SCH_QUERY_RSP;
msg.pData = rmsg;
code = schHandleResponseMsg(pJob, task, task->execId, &msg, 0);
ASSERT_EQ(code, 0);
}
ASSERT_EQ(code, 0);
pIter = taosHashIterate(pJob->execTasks, pIter); pIter = taosHashIterate(pJob->execTasks, pIter);
} }
@ -807,18 +899,11 @@ TEST(queryTest, readyFirstCase) {
ASSERT_EQ(pRsp->numOfRows, 10); ASSERT_EQ(pRsp->numOfRows, 10);
taosMemoryFreeClear(data); taosMemoryFreeClear(data);
data = NULL;
code = schedulerFetchRows(job, &req);
ASSERT_EQ(code, 0);
ASSERT_TRUE(data == NULL);
schReleaseJob(job); schReleaseJob(job);
schedulerFreeJob(&job, 0);
schtFreeQueryDag(&dag);
schedulerDestroy(); schedulerDestroy();
schedulerFreeJob(&job, 0);
} }
TEST(queryTest, flowCtrlCase) { TEST(queryTest, flowCtrlCase) {
@ -828,35 +913,39 @@ TEST(queryTest, flowCtrlCase) {
char *tablename = "table1"; char *tablename = "table1";
SVgroupInfo vgInfo = {0}; SVgroupInfo vgInfo = {0};
int64_t job = 0; int64_t job = 0;
SQueryPlan dag; SQueryPlan* dag = (SQueryPlan*)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN);
schtInitLogFile(); schtInitLogFile();
taosSeedRand(taosGetTimestampSec()); taosSeedRand(taosGetTimestampSec());
SArray *qnodeList = taosArrayInit(1, sizeof(SEp)); SArray *qnodeList = taosArrayInit(1, sizeof(SQueryNodeLoad));
SQueryNodeLoad load = {0};
load.addr.epSet.numOfEps = 1;
strcpy(load.addr.epSet.eps[0].fqdn, "qnode0.ep");
load.addr.epSet.eps[0].port = 6031;
taosArrayPush(qnodeList, &load);
SEp qnodeAddr = {0};
strcpy(qnodeAddr.fqdn, "qnode0.ep");
qnodeAddr.port = 6031;
taosArrayPush(qnodeList, &qnodeAddr);
int32_t code = schedulerInit(); int32_t code = schedulerInit();
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
schtBuildQueryFlowCtrlDag(&dag); schtBuildQueryFlowCtrlDag(dag);
schtSetPlanToString(); schtSetPlanToString();
schtSetExecNode(); schtSetExecNode();
schtSetAsyncSendMsgToServer(); schtSetAsyncSendMsgToServer();
initTaskQueue();
int32_t queryDone = 0; int32_t queryDone = 0;
SRequestConnInfo conn = {0}; SRequestConnInfo conn = {0};
conn.pTrans = mockPointer; conn.pTrans = mockPointer;
SSchedulerReq req = {0}; SSchedulerReq req = {0};
req.pConn = &conn; req.pConn = &conn;
req.pNodeList = qnodeList; req.pNodeList = qnodeList;
req.pDag = &dag; req.pDag = dag;
req.sql = "select * from tb"; req.sql = "select * from tb";
req.execFp = schtQueryCb; req.execFp = schtQueryCb;
req.cbParam = &queryDone; req.cbParam = &queryDone;
@ -866,41 +955,27 @@ TEST(queryTest, flowCtrlCase) {
SSchJob *pJob = schAcquireJob(job); SSchJob *pJob = schAcquireJob(job);
bool qDone = false; while (!queryDone) {
while (!qDone) {
void *pIter = taosHashIterate(pJob->execTasks, NULL); void *pIter = taosHashIterate(pJob->execTasks, NULL);
if (NULL == pIter) {
break;
}
while (pIter) { while (pIter) {
SSchTask *task = *(SSchTask **)pIter; SSchTask *task = *(SSchTask **)pIter;
taosHashCancelIterate(pJob->execTasks, pIter); if (JOB_TASK_STATUS_EXEC == task->status && 0 != task->lastMsgType) {
SDataBuf msg = {0};
if (task->lastMsgType == TDMT_SCH_QUERY) { void* rmsg = NULL;
SQueryTableRsp rsp = {0}; schtBuildQueryRspMsg(&msg.len, &rmsg);
code = schHandleResponseMsg(pJob, task, TDMT_SCH_QUERY_RSP, (char *)&rsp, sizeof(rsp), 0); msg.msgType = TDMT_SCH_QUERY_RSP;
msg.pData = rmsg;
code = schHandleResponseMsg(pJob, task, task->execId, &msg, 0);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
} else {
qDone = true;
break;
} }
pIter = NULL; pIter = taosHashIterate(pJob->execTasks, pIter);
} }
} }
while (true) {
if (queryDone) {
break;
}
taosUsleep(10000);
}
TdThreadAttr thattr; TdThreadAttr thattr;
taosThreadAttrInit(&thattr); taosThreadAttrInit(&thattr);
@ -918,18 +993,11 @@ TEST(queryTest, flowCtrlCase) {
ASSERT_EQ(pRsp->numOfRows, 10); ASSERT_EQ(pRsp->numOfRows, 10);
taosMemoryFreeClear(data); taosMemoryFreeClear(data);
data = NULL;
code = schedulerFetchRows(job, &req);
ASSERT_EQ(code, 0);
ASSERT_TRUE(data == NULL);
schReleaseJob(job); schReleaseJob(job);
schedulerFreeJob(&job, 0);
schtFreeQueryDag(&dag);
schedulerDestroy(); schedulerDestroy();
schedulerFreeJob(&job, 0);
} }
TEST(insertTest, normalCase) { TEST(insertTest, normalCase) {
@ -938,20 +1006,21 @@ TEST(insertTest, normalCase) {
char *dbname = "1.db1"; char *dbname = "1.db1";
char *tablename = "table1"; char *tablename = "table1";
SVgroupInfo vgInfo = {0}; SVgroupInfo vgInfo = {0};
SQueryPlan dag; SQueryPlan* dag = (SQueryPlan*)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN);
uint64_t numOfRows = 0; uint64_t numOfRows = 0;
SArray *qnodeList = taosArrayInit(1, sizeof(SEp)); SArray *qnodeList = taosArrayInit(1, sizeof(SQueryNodeLoad));
SEp qnodeAddr = {0}; SQueryNodeLoad load = {0};
strcpy(qnodeAddr.fqdn, "qnode0.ep"); load.addr.epSet.numOfEps = 1;
qnodeAddr.port = 6031; strcpy(load.addr.epSet.eps[0].fqdn, "qnode0.ep");
taosArrayPush(qnodeList, &qnodeAddr); load.addr.epSet.eps[0].port = 6031;
taosArrayPush(qnodeList, &load);
int32_t code = schedulerInit(); int32_t code = schedulerInit();
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
schtBuildInsertDag(&dag); schtBuildInsertDag(dag);
schtSetPlanToString(); schtSetPlanToString();
schtSetAsyncSendMsgToServer(); schtSetAsyncSendMsgToServer();
@ -962,21 +1031,19 @@ TEST(insertTest, normalCase) {
TdThread thread1; TdThread thread1;
taosThreadCreate(&(thread1), &thattr, schtSendRsp, &insertJobRefId); taosThreadCreate(&(thread1), &thattr, schtSendRsp, &insertJobRefId);
SExecResult res = {0}; int32_t queryDone = 0;
SRequestConnInfo conn = {0}; SRequestConnInfo conn = {0};
conn.pTrans = mockPointer; conn.pTrans = mockPointer;
SSchedulerReq req = {0}; SSchedulerReq req = {0};
req.pConn = &conn; req.pConn = &conn;
req.pNodeList = qnodeList; req.pNodeList = qnodeList;
req.pDag = &dag; req.pDag = dag;
req.sql = "insert into tb values(now,1)"; req.sql = "insert into tb values(now,1)";
req.execFp = schtQueryCb; req.execFp = schtQueryCb;
req.cbParam = NULL; req.cbParam = &queryDone;
code = schedulerExecJob(&req, &insertJobRefId); code = schedulerExecJob(&req, &insertJobRefId);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
ASSERT_EQ(res.numOfRows, 20);
schedulerFreeJob(&insertJobRefId, 0); schedulerFreeJob(&insertJobRefId, 0);
@ -989,7 +1056,7 @@ TEST(multiThread, forceFree) {
TdThread thread1, thread2, thread3; TdThread thread1, thread2, thread3;
taosThreadCreate(&(thread1), &thattr, schtRunJobThread, NULL); taosThreadCreate(&(thread1), &thattr, schtRunJobThread, NULL);
taosThreadCreate(&(thread2), &thattr, schtFreeJobThread, NULL); // taosThreadCreate(&(thread2), &thattr, schtFreeJobThread, NULL);
taosThreadCreate(&(thread3), &thattr, schtFetchRspThread, NULL); taosThreadCreate(&(thread3), &thattr, schtFetchRspThread, NULL);
while (true) { while (true) {
@ -1002,7 +1069,7 @@ TEST(multiThread, forceFree) {
} }
schtTestStop = true; schtTestStop = true;
taosSsleep(3); //taosSsleep(3);
} }
int main(int argc, char **argv) { int main(int argc, char **argv) {

1
source/libs/stream/src/streamDispatch.c
View File

@ -571,6 +571,7 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S
char ctbName[TSDB_TABLE_FNAME_LEN] = {0}; char ctbName[TSDB_TABLE_FNAME_LEN] = {0};
if (pDataBlock->info.parTbName[0]) { if (pDataBlock->info.parTbName[0]) {
if(pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && if(pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER &&
pTask->subtableWithoutMd5 != 1 &&
!isAutoTableName(pDataBlock->info.parTbName) && !isAutoTableName(pDataBlock->info.parTbName) &&
!alreadyAddGroupId(pDataBlock->info.parTbName) && !alreadyAddGroupId(pDataBlock->info.parTbName) &&
groupId != 0){ groupId != 0){

17
source/libs/stream/src/streamTask.c
View File

@ -80,7 +80,7 @@ static SStreamChildEpInfo* createStreamTaskEpInfo(const SStreamTask* pTask) {
} }
SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, bool fillHistory, int64_t triggerParam, SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, bool fillHistory, int64_t triggerParam,
SArray* pTaskList, bool hasFillhistory) { SArray* pTaskList, bool hasFillhistory, int8_t subtableWithoutMd5) {
SStreamTask* pTask = (SStreamTask*)taosMemoryCalloc(1, sizeof(SStreamTask)); SStreamTask* pTask = (SStreamTask*)taosMemoryCalloc(1, sizeof(SStreamTask));
if (pTask == NULL) { if (pTask == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -96,6 +96,7 @@ SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset,
pTask->info.taskLevel = taskLevel; pTask->info.taskLevel = taskLevel;
pTask->info.fillHistory = fillHistory; pTask->info.fillHistory = fillHistory;
pTask->info.triggerParam = triggerParam; pTask->info.triggerParam = triggerParam;
pTask->subtableWithoutMd5 = subtableWithoutMd5;
pTask->status.pSM = streamCreateStateMachine(pTask); pTask->status.pSM = streamCreateStateMachine(pTask);
if (pTask->status.pSM == NULL) { if (pTask->status.pSM == NULL) {
@ -205,6 +206,7 @@ int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) {
if (tEncodeCStr(pEncoder, pTask->outputInfo.shuffleDispatcher.stbFullName) < 0) return -1; if (tEncodeCStr(pEncoder, pTask->outputInfo.shuffleDispatcher.stbFullName) < 0) return -1;
} }
if (tEncodeI64(pEncoder, pTask->info.triggerParam) < 0) return -1; if (tEncodeI64(pEncoder, pTask->info.triggerParam) < 0) return -1;
if (tEncodeI8(pEncoder, pTask->subtableWithoutMd5) < 0) return -1;
if (tEncodeCStrWithLen(pEncoder, pTask->reserve, sizeof(pTask->reserve) - 1) < 0) return -1; if (tEncodeCStrWithLen(pEncoder, pTask->reserve, sizeof(pTask->reserve) - 1) < 0) return -1;
tEndEncode(pEncoder); tEndEncode(pEncoder);
@ -287,6 +289,7 @@ int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask) {
if (tDecodeCStrTo(pDecoder, pTask->outputInfo.shuffleDispatcher.stbFullName) < 0) return -1; if (tDecodeCStrTo(pDecoder, pTask->outputInfo.shuffleDispatcher.stbFullName) < 0) return -1;
} }
if (tDecodeI64(pDecoder, &pTask->info.triggerParam) < 0) return -1; if (tDecodeI64(pDecoder, &pTask->info.triggerParam) < 0) return -1;
if (tDecodeI8(pDecoder, &pTask->subtableWithoutMd5) < 0) return -1;
if (tDecodeCStrTo(pDecoder, pTask->reserve) < 0) return -1; if (tDecodeCStrTo(pDecoder, pTask->reserve) < 0) return -1;
tEndDecode(pDecoder); tEndDecode(pDecoder);
@ -482,7 +485,7 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i
pTask->execInfo.created = taosGetTimestampMs(); pTask->execInfo.created = taosGetTimestampMs();
SCheckpointInfo* pChkInfo = &pTask->chkInfo; SCheckpointInfo* pChkInfo = &pTask->chkInfo;
SDataRange* pRange = &pTask->dataRange; SDataRange* pRange = &pTask->dataRange;
// only set the version info for stream tasks without fill-history task // only set the version info for stream tasks without fill-history task
if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
@ -756,8 +759,8 @@ int8_t streamTaskSetSchedStatusInactive(SStreamTask* pTask) {
} }
int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, bool metaLock) { int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, bool metaLock) {
SStreamMeta* pMeta = pTask->pMeta; SStreamMeta* pMeta = pTask->pMeta;
STaskId sTaskId = {.streamId = pTask->streamTaskId.streamId, .taskId = pTask->streamTaskId.taskId}; STaskId sTaskId = {.streamId = pTask->streamTaskId.streamId, .taskId = pTask->streamTaskId.taskId};
if (pTask->info.fillHistory == 0) { if (pTask->info.fillHistory == 0) {
return 0; return 0;
} }
@ -864,7 +867,7 @@ void streamTaskPause(SStreamMeta* pMeta, SStreamTask* pTask) {
void streamTaskResume(SStreamTask* pTask) { void streamTaskResume(SStreamTask* pTask) {
SStreamTaskState prevState = *streamTaskGetStatus(pTask); SStreamTaskState prevState = *streamTaskGetStatus(pTask);
SStreamMeta* pMeta = pTask->pMeta; SStreamMeta* pMeta = pTask->pMeta;
if (prevState.state == TASK_STATUS__PAUSE || prevState.state == TASK_STATUS__HALT) { if (prevState.state == TASK_STATUS__PAUSE || prevState.state == TASK_STATUS__HALT) {
streamTaskRestoreStatus(pTask); streamTaskRestoreStatus(pTask);
@ -881,9 +884,7 @@ void streamTaskResume(SStreamTask* pTask) {
} }
} }
bool streamTaskIsSinkTask(const SStreamTask* pTask) { bool streamTaskIsSinkTask(const SStreamTask* pTask) { return pTask->info.taskLevel == TASK_LEVEL__SINK; }
return pTask->info.taskLevel == TASK_LEVEL__SINK;
}
int32_t streamTaskSendCheckpointReq(SStreamTask* pTask) { int32_t streamTaskSendCheckpointReq(SStreamTask* pTask) {
int32_t code; int32_t code;

1
source/libs/stream/src/streamUpdate.c
View File

@ -261,6 +261,7 @@ void updateInfoDestroy(SUpdateInfo *pInfo) {
taosArrayDestroy(pInfo->pTsSBFs); taosArrayDestroy(pInfo->pTsSBFs);
taosHashCleanup(pInfo->pMap); taosHashCleanup(pInfo->pMap);
updateInfoDestoryColseWinSBF(pInfo);
taosMemoryFree(pInfo); taosMemoryFree(pInfo);
} }

6
source/libs/transport/src/trans.c
View File

@ -140,11 +140,7 @@ void* rpcMallocCont(int64_t contLen) {
return start + sizeof(STransMsgHead); return start + sizeof(STransMsgHead);
} }
void rpcFreeCont(void* cont) { void rpcFreeCont(void* cont) { transFreeMsg(cont); }
if (cont == NULL) return;
taosMemoryFree((char*)cont - TRANS_MSG_OVERHEAD);
tTrace("rpc free cont:%p", (char*)cont - TRANS_MSG_OVERHEAD);
}
void* rpcReallocCont(void* ptr, int64_t contLen) { void* rpcReallocCont(void* ptr, int64_t contLen) {
if (ptr == NULL) return rpcMallocCont(contLen); if (ptr == NULL) return rpcMallocCont(contLen);

13
source/libs/transport/src/transCli.c
View File

@ -218,7 +218,6 @@ static void (*cliAsyncHandle[])(SCliMsg* pMsg, SCliThrd* pThrd) = {cliHandleReq,
/// static void (*cliAsyncHandle[])(SCliMsg* pMsg, SCliThrd* pThrd) = {cliHandleReq, cliHandleQuit, cliHandleRelease, /// static void (*cliAsyncHandle[])(SCliMsg* pMsg, SCliThrd* pThrd) = {cliHandleReq, cliHandleQuit, cliHandleRelease,
/// NULL,cliHandleUpdate}; /// NULL,cliHandleUpdate};
static FORCE_INLINE void destroyUserdata(STransMsg* userdata);
static FORCE_INLINE void destroyCmsg(void* cmsg); static FORCE_INLINE void destroyCmsg(void* cmsg);
static FORCE_INLINE void destroyCmsgAndAhandle(void* cmsg); static FORCE_INLINE void destroyCmsgAndAhandle(void* cmsg);
static FORCE_INLINE int cliRBChoseIdx(STrans* pTransInst); static FORCE_INLINE int cliRBChoseIdx(STrans* pTransInst);
@ -1950,14 +1949,6 @@ _err:
return NULL; return NULL;
} }
static FORCE_INLINE void destroyUserdata(STransMsg* userdata) {
if (userdata->pCont == NULL) {
return;
}
transFreeMsg(userdata->pCont);
userdata->pCont = NULL;
}
static FORCE_INLINE void destroyCmsg(void* arg) { static FORCE_INLINE void destroyCmsg(void* arg) {
SCliMsg* pMsg = arg; SCliMsg* pMsg = arg;
if (pMsg == NULL) { if (pMsg == NULL) {
@ -1965,7 +1956,7 @@ static FORCE_INLINE void destroyCmsg(void* arg) {
} }
transDestroyConnCtx(pMsg->ctx); transDestroyConnCtx(pMsg->ctx);
destroyUserdata(&pMsg->msg); transFreeMsg(pMsg->msg.pCont);
taosMemoryFree(pMsg); taosMemoryFree(pMsg);
} }
@ -1984,7 +1975,7 @@ static FORCE_INLINE void destroyCmsgAndAhandle(void* param) {
tDebug("destroy Ahandle C"); tDebug("destroy Ahandle C");
transDestroyConnCtx(pMsg->ctx); transDestroyConnCtx(pMsg->ctx);
destroyUserdata(&pMsg->msg); transFreeMsg(pMsg->msg.pCont);
taosMemoryFree(pMsg); taosMemoryFree(pMsg);
} }

1
source/libs/transport/src/transComm.c
View File

@ -87,6 +87,7 @@ void transFreeMsg(void* msg) {
if (msg == NULL) { if (msg == NULL) {
return; return;
} }
tTrace("rpc free cont:%p", (char*)msg - TRANS_MSG_OVERHEAD);
taosMemoryFree((char*)msg - sizeof(STransMsgHead)); taosMemoryFree((char*)msg - sizeof(STransMsgHead));
} }
int transSockInfo2Str(struct sockaddr* sockname, char* dst) { int transSockInfo2Str(struct sockaddr* sockname, char* dst) {

1
tests/parallel_test/cases.task
View File

@ -575,6 +575,7 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ts_3405_3398_3423.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ts_3405_3398_3423.py -N 3 -n 3
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ts-4348-td-27939.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ts-4348-td-27939.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/backslash_g.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/backslash_g.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/test_ts4467.py
,,n,system-test,python3 ./test.py -f 2-query/queryQnode.py ,,n,system-test,python3 ./test.py -f 2-query/queryQnode.py
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode1mnode.py ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode1mnode.py

16
tests/script/tsim/parser/like.sim
View File

@ -51,6 +51,22 @@ if $rows != 1 then
return -1 return -1
endi endi
$view1 = view1_name
$view2 = view2_name
sql CREATE VIEW $view1 as select * from $table1
sql CREATE VIEW $view2 AS select * from $table2
sql show views like 'view%'
if $rows != 2 then
return -1
endi
sql show views like 'view1%'
if $rows != 1 then
return -1
endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s stop -x SIGINT

63
tests/system-test/2-query/test_ts4467.py Normal file
View File

@ -0,0 +1,63 @@
import random
import itertools
from util.log import *
from util.cases import *
from util.sql import *
from util.sqlset import *
from util import constant
from util.common import *
class TDTestCase:
"""Verify the jira TS-4467
"""
def init(self, conn, logSql, replicaVar=1):
self.replicaVar = int(replicaVar)
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor())
def prepareData(self):
# db
tdSql.execute("create database if not exists db")
tdSql.execute("use db")
# table
tdSql.execute("create table t (ts timestamp, c1 varchar(16));")
# insert data
sql = "insert into t values"
for i in range(6):
sql += f"(now+{str(i+1)}s, '{'name' + str(i+1)}')"
sql += ";"
tdSql.execute(sql)
tdLog.debug("insert data successfully")
def run(self):
self.prepareData()
# join query with order by
sql = "select * from t t1, (select * from t order by ts limit 5) t2 where t1.ts = t2.ts;"
tdSql.query(sql)
tdSql.checkRows(5)
sql = "select * from t t1, (select * from t order by ts desc limit 5) t2 where t1.ts = t2.ts;"
tdSql.query(sql)
tdSql.checkRows(5)
sql = "select * from t t1, (select * from t order by ts limit 5) t2 where t1.ts = t2.ts order by t1.ts;"
tdSql.query(sql)
res1 = tdSql.queryResult
tdLog.debug("res1: %s" % str(res1))
sql = "select * from t t1, (select * from t order by ts limit 5) t2 where t1.ts = t2.ts order by t1.ts desc;"
tdSql.query(sql)
res2 = tdSql.queryResult
tdLog.debug("res2: %s" % str(res2))
assert(len(res1) == len(res2) and res1[0][0] == res2[4][0])
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

8
tools/shell/src/shellMain.c
View File

@ -28,7 +28,7 @@ void shellCrashHandler(int signum, void *sigInfo, void *context) {
#if !defined(WINDOWS) #if !defined(WINDOWS)
taosIgnSignal(SIGBUS); taosIgnSignal(SIGBUS);
#endif #endif
taosIgnSignal(SIGABRT); taosIgnSignal(SIGABRT);
taosIgnSignal(SIGFPE); taosIgnSignal(SIGFPE);
taosIgnSignal(SIGSEGV); taosIgnSignal(SIGSEGV);
@ -82,7 +82,9 @@ int main(int argc, char *argv[]) {
#ifdef WEBSOCKET #ifdef WEBSOCKET
shellCheckConnectMode(); shellCheckConnectMode();
#endif #endif
taos_init(); if (taos_init() != 0) {
return -1;
}
// kill heart-beat thread when quit // kill heart-beat thread when quit
taos_set_hb_quit(1); taos_set_hb_quit(1);
@ -105,7 +107,7 @@ int main(int argc, char *argv[]) {
return 0; return 0;
} }
// support port feature // support port feature
shellAutoInit(); shellAutoInit();
int32_t ret = shellExecute(); int32_t ret = shellExecute();
shellAutoExit(); shellAutoExit();