xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix:conflicts from 3.0

This commit is contained in:
wangmm0220 2024-02-19 18:46:31 +08:00
parent a74f12e2e1 129da0ab5c
commit 77e2500a81
32 changed files with 3147 additions and 1675 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

597
docs/zh/07-develop/07-tmq.mdx
View File

@ -91,6 +91,7 @@ TDengine 会为 WAL 文件自动创建索引以支持快速随机访问，并提
不同语言下， TMQ 订阅相关的 API 及数据结构如下（详细的接口说明可以参考连接器章节，注意consumer结构不是线程安全的，在一个线程使用consumer时，不要在另一个线程close这个consumer）：
<Tabs defaultValue="java" groupId="lang">
<TabItem value="c" label="C">
```c
@ -101,17 +102,17 @@ TDengine 会为 WAL 文件自动创建索引以支持快速随机访问，并提
typedef void(tmq_commit_cb(tmq_t *tmq, int32_t code, void *param));
typedef enum tmq_conf_res_t {
TMQ_CONF_UNKNOWN = -2,
TMQ_CONF_INVALID = -1,
TMQ_CONF_OK = 0,
} tmq_conf_res_t;
TMQ_CONF_UNKNOWN = -2,
TMQ_CONF_INVALID = -1,
TMQ_CONF_OK = 0,
} tmq_conf_res_t;
typedef struct tmq_topic_assignment {
int32_t vgId;
int64_t currentOffset;
int64_t begin;
int64_t end;
} tmq_topic_assignment;
int32_t vgId;
int64_t currentOffset;
int64_t begin;
int64_t end;
} tmq_topic_assignment;
DLL_EXPORT tmq_conf_t *tmq_conf_new();
DLL_EXPORT tmq_conf_res_t tmq_conf_set(tmq_conf_t *conf, const char *key, const char *value);
@ -146,7 +147,6 @@ TDengine 会为 WAL 文件自动创建索引以支持快速随机访问，并提
DLL_EXPORT int64_t tmq_get_vgroup_offset(TAOS_RES* res);
DLL_EXPORT const char *tmq_err2str(int32_t code);
```
</TabItem>
<TabItem value="java" label="Java">
@ -250,261 +250,260 @@ TDengine 会为 WAL 文件自动创建索引以支持快速随机访问，并提
+ futures::Stream<
Item = Result<(Self::Offset, MessageSet<Self::Meta, Self::Data>), Self::Error>,
>,
>,
>;
async fn commit(&self, offset: Self::Offset) -> Result<(), Self::Error>;
>,
>;
async fn commit(&self, offset: Self::Offset) -> Result<(), Self::Error>;
async fn unsubscribe(self);
```
async fn unsubscribe(self);
```
可在 <https://docs.rs/taos> 上查看详细 API 说明。
可在 <https://docs.rs/taos> 上查看详细 API 说明。
</TabItem>
</TabItem>
<TabItem label="Node.JS" value="Node.JS">
<TabItem label="Node.JS" value="Node.JS">
```js
function TMQConsumer(config)
```js
function TMQConsumer(config)
function subscribe(topic)
function subscribe(topic)
function consume(timeout)
function consume(timeout)
function subscription()
function subscription()
function unsubscribe()
function unsubscribe()
function commit(msg)
function commit(msg)
function close()
```
function close()
```
</TabItem>
</TabItem>
<TabItem value="C#" label="C#">
<TabItem value="C#" label="C#">
```csharp
class ConsumerBuilder<TValue>
```csharp
class ConsumerBuilder<TValue>
ConsumerBuilder(IEnumerable<KeyValuePair<string, string>> config)
ConsumerBuilder(IEnumerable<KeyValuePair<string, string>> config)
public IConsumer<TValue> Build()
public IConsumer<TValue> Build()
void Subscribe(IEnumerable<string> topics)
void Subscribe(IEnumerable<string> topics)
void Subscribe(string topic)
void Subscribe(string topic)
ConsumeResult<TValue> Consume(int millisecondsTimeout)
ConsumeResult<TValue> Consume(int millisecondsTimeout)
List<string> Subscription()
List<string> Subscription()
void Unsubscribe()
void Unsubscribe()
List<TopicPartitionOffset> Commit()
List<TopicPartitionOffset> Commit()
void Close()
```
void Close()
```
</TabItem>
</Tabs>
</TabItem>
</Tabs>
# 数据订阅示例
## 写入数据
# 数据订阅示例
## 写入数据
首先完成建库、建一张超级表和多张子表操作，然后就可以写入数据了，比如：
首先完成建库、建一张超级表和多张子表操作，然后就可以写入数据了，比如：
```sql
DROP DATABASE IF EXISTS tmqdb;
CREATE DATABASE tmqdb WAL_RETENTION_PERIOD 3600;
CREATE TABLE tmqdb.stb (ts TIMESTAMP, c1 INT, c2 FLOAT, c3 VARCHAR(16)) TAGS(t1 INT, t3 VARCHAR(16));
CREATE TABLE tmqdb.ctb0 USING tmqdb.stb TAGS(0, "subtable0");
CREATE TABLE tmqdb.ctb1 USING tmqdb.stb TAGS(1, "subtable1");
INSERT INTO tmqdb.ctb0 VALUES(now, 0, 0, 'a0')(now+1s, 0, 0, 'a00');
```sql
DROP DATABASE IF EXISTS tmqdb;
CREATE DATABASE tmqdb WAL_RETENTION_PERIOD 3600;
CREATE TABLE tmqdb.stb (ts TIMESTAMP, c1 INT, c2 FLOAT, c3 VARCHAR(16)) TAGS(t1 INT, t3 VARCHAR(16));
CREATE TABLE tmqdb.ctb0 USING tmqdb.stb TAGS(0, "subtable0");
CREATE TABLE tmqdb.ctb1 USING tmqdb.stb TAGS(1, "subtable1");
INSERT INTO tmqdb.ctb0 VALUES(now, 0, 0, 'a0')(now+1s, 0, 0, 'a00');
INSERT INTO tmqdb.ctb1 VALUES(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11');
```
## 创建 topic
使用 SQL 创建一个 topic：
使用 SQL 创建一个 topic：
```sql
CREATE TOPIC topic_name AS SELECT ts, c1, c2, c3 FROM tmqdb.stb WHERE c1 > 1;
```
```sql
CREATE TOPIC topic_name AS SELECT ts, c1, c2, c3 FROM tmqdb.stb WHERE c1 > 1;
```
## 创建消费者 *consumer*
## 创建消费者 *consumer*
对于不同编程语言，其设置方式如下：
对于不同编程语言，其设置方式如下：
<Tabs defaultValue="java" groupId="lang">
<TabItem value="c" label="C">
<Tabs defaultValue="java" groupId="lang">
```c
/* 根据需要，设置消费组 (group.id)、自动提交 (enable.auto.commit)、
自动提交时间间隔 (auto.commit.interval.ms)、用户名 (td.connect.user)、密码 (td.connect.pass) 等参数 */
tmq_conf_t* conf = tmq_conf_new();
tmq_conf_set(conf, "enable.auto.commit", "true");
tmq_conf_set(conf, "auto.commit.interval.ms", "1000");
tmq_conf_set(conf, "group.id", "cgrpName");
tmq_conf_set(conf, "td.connect.user", "root");
tmq_conf_set(conf, "td.connect.pass", "taosdata");
tmq_conf_set(conf, "auto.offset.reset", "latest");
tmq_conf_set(conf, "msg.with.table.name", "true");
tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL);
<TabItem value="c" label="C">
```c
/* 根据需要，设置消费组 (group.id)、自动提交 (enable.auto.commit)、
自动提交时间间隔 (auto.commit.interval.ms)、用户名 (td.connect.user)、密码 (td.connect.pass) 等参数 */
tmq_conf_t* conf = tmq_conf_new();
tmq_conf_set(conf, "enable.auto.commit", "true");
tmq_conf_set(conf, "auto.commit.interval.ms", "1000");
tmq_conf_set(conf, "group.id", "cgrpName");
tmq_conf_set(conf, "td.connect.user", "root");
tmq_conf_set(conf, "td.connect.pass", "taosdata");
tmq_conf_set(conf, "auto.offset.reset", "latest");
tmq_conf_set(conf, "msg.with.table.name", "true");
tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL);
tmq_t* tmq = tmq_consumer_new(conf, NULL, 0);
tmq_conf_destroy(conf);
```
tmq_t* tmq = tmq_consumer_new(conf, NULL, 0);
tmq_conf_destroy(conf);
```
</TabItem>
</TabItem>
<TabItem value="java" label="Java">
<TabItem value="java" label="Java">
对于 Java 程序，还可以使用如下配置项：
对于 Java 程序，还可以使用如下配置项：
| 参数名称 | 类型 | 参数说明 |
| ----------------------------- | ------ | ----------------------------------------------------------------------------------------------------------------------------- |
| `td.connect.type` | string | 连接类型，"jni" 指原生连接，"ws" 指 websocket 连接，默认值为 "jni" |
| `bootstrap.servers` | string | 连接地址，如 `localhost:6030` |
| `value.deserializer` | string | 值解析方法，使用此方法应实现 `com.taosdata.jdbc.tmq.Deserializer` 接口或继承 `com.taosdata.jdbc.tmq.ReferenceDeserializer` 类 |
| `value.deserializer.encoding` | string | 指定字符串解析的字符集 | |
| 参数名称 | 类型 | 参数说明 |
| ----------------------------- | ------ | ----------------------------------------------------------------------------------------------------------------------------- |
| `td.connect.type` | string | 连接类型，"jni" 指原生连接，"ws" 指 websocket 连接，默认值为 "jni" |
| `bootstrap.servers` | string | 连接地址，如 `localhost:6030` |
| `value.deserializer` | string | 值解析方法，使用此方法应实现 `com.taosdata.jdbc.tmq.Deserializer` 接口或继承 `com.taosdata.jdbc.tmq.ReferenceDeserializer` 类 |
| `value.deserializer.encoding` | string | 指定字符串解析的字符集 | |
需要注意：此处使用 `bootstrap.servers` 替代 `td.connect.ip` 和 `td.connect.port`，以提供与 Kafka 一致的接口。
需要注意：此处使用 `bootstrap.servers` 替代 `td.connect.ip` 和 `td.connect.port`，以提供与 Kafka 一致的接口。
```java
Properties properties = new Properties();
properties.setProperty("enable.auto.commit", "true");
properties.setProperty("auto.commit.interval.ms", "1000");
properties.setProperty("group.id", "cgrpName");
properties.setProperty("bootstrap.servers", "127.0.0.1:6030");
properties.setProperty("td.connect.user", "root");
properties.setProperty("td.connect.pass", "taosdata");
properties.setProperty("auto.offset.reset", "latest");
properties.setProperty("msg.with.table.name", "true");
properties.setProperty("value.deserializer", "com.taos.example.MetersDeserializer");
```java
Properties properties = new Properties();
properties.setProperty("enable.auto.commit", "true");
properties.setProperty("auto.commit.interval.ms", "1000");
properties.setProperty("group.id", "cgrpName");
properties.setProperty("bootstrap.servers", "127.0.0.1:6030");
properties.setProperty("td.connect.user", "root");
properties.setProperty("td.connect.pass", "taosdata");
properties.setProperty("auto.offset.reset", "latest");
properties.setProperty("msg.with.table.name", "true");
properties.setProperty("value.deserializer", "com.taos.example.MetersDeserializer");
TaosConsumer<Meters> consumer = new TaosConsumer<>(properties);
TaosConsumer<Meters> consumer = new TaosConsumer<>(properties);
/* value deserializer definition. */
import com.taosdata.jdbc.tmq.ReferenceDeserializer;
/* value deserializer definition. */
import com.taosdata.jdbc.tmq.ReferenceDeserializer;
public class MetersDeserializer extends ReferenceDeserializer<Meters> {
}
```
public class MetersDeserializer extends ReferenceDeserializer<Meters> {
}
```
</TabItem>
</TabItem>
<TabItem label="Go" value="Go">
<TabItem label="Go" value="Go">
```go
conf := &tmq.ConfigMap{
"group.id": "test",
"auto.offset.reset": "latest",
"td.connect.ip": "127.0.0.1",
"td.connect.user": "root",
"td.connect.pass": "taosdata",
"td.connect.port": "6030",
"client.id": "test_tmq_c",
"enable.auto.commit": "false",
"msg.with.table.name": "true",
}
consumer, err := NewConsumer(conf)
```
</TabItem>
<TabItem label="Rust" value="Rust">
```rust
let mut dsn: Dsn = "taos://".parse()?;
dsn.set("group.id", "group1");
dsn.set("client.id", "test");
dsn.set("auto.offset.reset", "latest");
let tmq = TmqBuilder::from_dsn(dsn)?;
let mut consumer = tmq.build()?;
```
</TabItem>
<TabItem value="Python" label="Python">
Python 语言下引入 `taos` 库的 `Consumer` 类，创建一个 Consumer 示例：
```python
from taos.tmq import Consumer
# Syntax: `consumer = Consumer(configs)`
#
# Example:
consumer = Consumer(
{
"group.id": "local",
"client.id": "1",
"enable.auto.commit": "true",
"auto.commit.interval.ms": "1000",
"td.connect.ip": "127.0.0.1",
"td.connect.user": "root",
"td.connect.pass": "taosdata",
"auto.offset.reset": "latest",
"msg.with.table.name": "true",
}
)
```
</TabItem>
<TabItem label="Node.JS" value="Node.JS">
```js
// 根据需要，设置消费组 (group.id)、自动提交 (enable.auto.commit)、
// 自动提交时间间隔 (auto.commit.interval.ms)、用户名 (td.connect.user)、密码 (td.connect.pass) 等参数
let consumer = taos.consumer({
'enable.auto.commit': 'true',
'auto.commit.interval.ms','1000',
'group.id': 'tg2',
'td.connect.user': 'root',
'td.connect.pass': 'taosdata',
'auto.offset.reset','latest',
'msg.with.table.name': 'true',
'td.connect.ip','127.0.0.1',
'td.connect.port','6030'
});
```go
conf := &tmq.ConfigMap{
"group.id": "test",
"auto.offset.reset": "latest",
"td.connect.ip": "127.0.0.1",
"td.connect.user": "root",
"td.connect.pass": "taosdata",
"td.connect.port": "6030",
"client.id": "test_tmq_c",
"enable.auto.commit": "false",
"msg.with.table.name": "true",
}
consumer, err := NewConsumer(conf)
```
</TabItem>
</TabItem>
<TabItem value="C#" label="C#">
<TabItem label="Rust" value="Rust">
```csharp
var cfg = new Dictionary<string, string>()
{
{ "group.id", "group1" },
{ "auto.offset.reset", "latest" },
{ "td.connect.ip", "127.0.0.1" },
{ "td.connect.user", "root" },
{ "td.connect.pass", "taosdata" },
{ "td.connect.port", "6030" },
{ "client.id", "tmq_example" },
{ "enable.auto.commit", "true" },
{ "msg.with.table.name", "false" },
};
var consumer = new ConsumerBuilder<Dictionary<string, object>>(cfg).Build();
```rust
let mut dsn: Dsn = "taos://".parse()?;
dsn.set("group.id", "group1");
dsn.set("client.id", "test");
dsn.set("auto.offset.reset", "latest");
let tmq = TmqBuilder::from_dsn(dsn)?;
let mut consumer = tmq.build()?;
```
</TabItem>
</TabItem>
</Tabs>
<TabItem value="Python" label="Python">
上述配置中包括 consumer group ID，如果多个 consumer 指定的 consumer group ID 一样，则自动形成一个 consumer group，共享消费进度。
Python 语言下引入 `taos` 库的 `Consumer` 类，创建一个 Consumer 示例：
## 订阅 *topics*
```python
from taos.tmq import Consumer
一个 consumer 支持同时订阅多个 topic。
# Syntax: `consumer = Consumer(configs)`
#
# Example:
consumer = Consumer(
{
"group.id": "local",
"client.id": "1",
"enable.auto.commit": "true",
"auto.commit.interval.ms": "1000",
"td.connect.ip": "127.0.0.1",
"td.connect.user": "root",
"td.connect.pass": "taosdata",
"auto.offset.reset": "latest",
"msg.with.table.name": "true",
}
)
```
<Tabs defaultValue="java" groupId="lang">
<TabItem value="c" label="C">
</TabItem>
```c
<TabItem label="Node.JS" value="Node.JS">
```js
// 根据需要，设置消费组 (group.id)、自动提交 (enable.auto.commit)、
// 自动提交时间间隔 (auto.commit.interval.ms)、用户名 (td.connect.user)、密码 (td.connect.pass) 等参数
let consumer = taos.consumer({
'enable.auto.commit': 'true',
'auto.commit.interval.ms','1000',
'group.id': 'tg2',
'td.connect.user': 'root',
'td.connect.pass': 'taosdata',
'auto.offset.reset','latest',
'msg.with.table.name': 'true',
'td.connect.ip','127.0.0.1',
'td.connect.port','6030'
});
```
</TabItem>
<TabItem value="C#" label="C#">
```csharp
var cfg = new Dictionary<string, string>()
{
{ "group.id", "group1" },
{ "auto.offset.reset", "latest" },
{ "td.connect.ip", "127.0.0.1" },
{ "td.connect.user", "root" },
{ "td.connect.pass", "taosdata" },
{ "td.connect.port", "6030" },
{ "client.id", "tmq_example" },
{ "enable.auto.commit", "true" },
{ "msg.with.table.name", "false" },
};
var consumer = new ConsumerBuilder<Dictionary<string, object>>(cfg).Build();
```
</TabItem>
</Tabs>
上述配置中包括 consumer group ID，如果多个 consumer 指定的 consumer group ID 一样，则自动形成一个 consumer group，共享消费进度。
## 订阅 *topics*
一个 consumer 支持同时订阅多个 topic。
<Tabs defaultValue="java" groupId="lang">
<TabItem value="c" label="C">
```c
// 创建订阅 topics 列表
tmq_list_t* topicList = tmq_list_new();
tmq_list_append(topicList, "topicName");
@ -514,8 +513,8 @@ INSERT INTO tmqdb.ctb1 VALUES(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11');
```
</TabItem>
<TabItem value="java" label="Java">
</TabItem>
<TabItem value="java" label="Java">
```java
List<String> topics = new ArrayList<>();
@ -523,8 +522,8 @@ INSERT INTO tmqdb.ctb1 VALUES(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11');
consumer.subscribe(topics);
```
</TabItem>
<TabItem value="Go" label="Go">
</TabItem>
<TabItem value="Go" label="Go">
```go
err = consumer.Subscribe("example_tmq_topic", nil)
@ -533,26 +532,26 @@ INSERT INTO tmqdb.ctb1 VALUES(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11');
}
```
</TabItem>
<TabItem value="Rust" label="Rust">
</TabItem>
<TabItem value="Rust" label="Rust">
```rust
```rust
consumer.subscribe(["tmq_meters"]).await?;
```
</TabItem>
</TabItem>
<TabItem value="Python" label="Python">
<TabItem value="Python" label="Python">
```python
```python
consumer.subscribe(['topic1', 'topic2'])
```
</TabItem>
</TabItem>
<TabItem label="Node.JS" value="Node.JS">
<TabItem label="Node.JS" value="Node.JS">
```js
```js
// 创建订阅 topics 列表
let topics = ['topic_test']
@ -560,11 +559,11 @@ INSERT INTO tmqdb.ctb1 VALUES(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11');
consumer.subscribe(topics);
```
</TabItem>
</TabItem>
<TabItem value="C#" label="C#">
<TabItem value="C#" label="C#">
```csharp
```csharp
// 创建订阅 topics 列表
List<String> topics = new List<string>();
topics.add("tmq_topic");
@ -572,18 +571,19 @@ INSERT INTO tmqdb.ctb1 VALUES(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11');
consumer.Subscribe(topics);
```
</TabItem>
</TabItem>
</Tabs>
</Tabs>
## 消费
## 消费
以下代码展示了不同语言下如何对 TMQ 消息进行消费。
以下代码展示了不同语言下如何对 TMQ 消息进行消费。
<Tabs defaultValue="java" groupId="lang">
<TabItem value="c" label="C">
<Tabs defaultValue="java" groupId="lang">
```c
<TabItem value="c" label="C">
```c
// 消费数据
while (running) {
TAOS_RES* msg = tmq_consumer_poll(tmq, timeOut);
@ -591,10 +591,10 @@ INSERT INTO tmqdb.ctb1 VALUES(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11');
}
```
这里是一个 **while** 循环，每调用一次 tmq_consumer_poll()，获取一个消息，该消息与普通查询返回的结果集完全相同，可以使用相同的解析 API 完成消息内容的解析。
这里是一个 **while** 循环，每调用一次 tmq_consumer_poll()，获取一个消息，该消息与普通查询返回的结果集完全相同，可以使用相同的解析 API 完成消息内容的解析。
</TabItem>
<TabItem value="java" label="Java">
</TabItem>
<TabItem value="java" label="Java">
```java
while(running){
@ -605,9 +605,9 @@ INSERT INTO tmqdb.ctb1 VALUES(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11');
}
```
</TabItem>
</TabItem>
<TabItem value="Go" label="Go">
<TabItem value="Go" label="Go">
```go
for {
@ -625,9 +625,9 @@ INSERT INTO tmqdb.ctb1 VALUES(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11');
}
```
</TabItem>
</TabItem>
<TabItem value="Rust" label="Rust">
<TabItem value="Rust" label="Rust">
```rust
{
@ -660,8 +660,8 @@ INSERT INTO tmqdb.ctb1 VALUES(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11');
}
```
</TabItem>
<TabItem value="Python" label="Python">
</TabItem>
<TabItem value="Python" label="Python">
```python
while True:
@ -677,9 +677,9 @@ INSERT INTO tmqdb.ctb1 VALUES(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11');
print(block.fetchall())
```
</TabItem>
</TabItem>
<TabItem label="Node.JS" value="Node.JS">
<TabItem label="Node.JS" value="Node.JS">
```js
while(true){
@ -691,11 +691,11 @@ INSERT INTO tmqdb.ctb1 VALUES(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11');
}
```
</TabItem>
</TabItem>
<TabItem value="C#" label="C#">
<TabItem value="C#" label="C#">
```csharp
```csharp
// 消费数据
while (true)
{
@ -708,18 +708,19 @@ INSERT INTO tmqdb.ctb1 VALUES(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11');
}
```
</TabItem>
</TabItem>
</Tabs>
</Tabs>
## 结束消费
## 结束消费
消费结束后，应当取消订阅。
消费结束后，应当取消订阅。
<Tabs defaultValue="java" groupId="lang">
<TabItem value="c" label="C">
<Tabs defaultValue="java" groupId="lang">
```c
<TabItem value="c" label="C">
```c
/* 取消订阅 */
tmq_unsubscribe(tmq);
@ -727,10 +728,10 @@ INSERT INTO tmqdb.ctb1 VALUES(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11');
tmq_consumer_close(tmq);
```
</TabItem>
<TabItem value="java" label="Java">
</TabItem>
<TabItem value="java" label="Java">
```java
```java
/* 取消订阅 */
consumer.unsubscribe();
@ -738,11 +739,11 @@ INSERT INTO tmqdb.ctb1 VALUES(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11');
consumer.close();
```
</TabItem>
</TabItem>
<TabItem value="Go" label="Go">
<TabItem value="Go" label="Go">
```go
```go
/* Unsubscribe */
_ = consumer.Unsubscribe()
@ -750,38 +751,38 @@ INSERT INTO tmqdb.ctb1 VALUES(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11');
_ = consumer.Close()
```
</TabItem>
</TabItem>
<TabItem value="Rust" label="Rust">
<TabItem value="Rust" label="Rust">
```rust
```rust
consumer.unsubscribe().await;
```
</TabItem>
</TabItem>
<TabItem value="Python" label="Python">
<TabItem value="Python" label="Python">
```py
```py
# 取消订阅
consumer.unsubscribe()
# 关闭消费
consumer.close()
```
</TabItem>
<TabItem label="Node.JS" value="Node.JS">
</TabItem>
<TabItem label="Node.JS" value="Node.JS">
```js
```js
consumer.unsubscribe();
consumer.close();
```
</TabItem>
</TabItem>
<TabItem value="C#" label="C#">
<TabItem value="C#" label="C#">
```csharp
```csharp
// 取消订阅
consumer.Unsubscribe();
@ -789,7 +790,7 @@ INSERT INTO tmqdb.ctb1 VALUES(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11');
consumer.Close();
```
</TabItem>
</TabItem>
</Tabs>
@ -799,40 +800,40 @@ INSERT INTO tmqdb.ctb1 VALUES(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11');
<Tabs defaultValue="java" groupId="lang">
<TabItem label="C" value="c">
<CDemo />
</TabItem>
<TabItem label="C" value="c">
<CDemo />
</TabItem>
<TabItem label="Java" value="java">
<Tabs defaultValue="native">
<TabItem value="native" label="本地连接">
<Java />
</TabItem>
<TabItem value="ws" label="WebSocket 连接">
<JavaWS />
</TabItem>
</Tabs>
</TabItem>
<TabItem label="Java" value="java">
<Tabs defaultValue="native">
<TabItem value="native" label="本地连接">
<Java />
</TabItem>
<TabItem value="ws" label="WebSocket 连接">
<JavaWS />
</TabItem>
</Tabs>
</TabItem>
<TabItem label="Go" value="Go">
<Go/>
</TabItem>
<TabItem label="Go" value="Go">
<Go/>
</TabItem>
<TabItem label="Rust" value="Rust">
<Rust />
</TabItem>
<TabItem label="Rust" value="Rust">
<Rust />
</TabItem>
<TabItem label="Python" value="Python">
<Python />
</TabItem>
<TabItem label="Python" value="Python">
<Python />
</TabItem>
<TabItem label="Node.JS" value="Node.JS">
<Node/>
</TabItem>
<TabItem label="Node.JS" value="Node.JS">
<Node/>
</TabItem>
<TabItem label="C#" value="C#">
<CSharp/>
</TabItem>
<TabItem label="C#" value="C#">
<CSharp/>
</TabItem>
</Tabs>

5
include/libs/stream/tstream.h
View File

@ -463,7 +463,8 @@ struct SStreamTask {
struct SStreamMeta* pMeta;
SSHashObj* pNameMap;
void* pBackend;
char reserve[256];
int8_t subtableWithoutMd5;
char reserve[255];
};
typedef int32_t (*startComplete_fn_t)(struct SStreamMeta*);
@ -532,7 +533,7 @@ int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo)
int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo);
SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, bool fillHistory, int64_t triggerParam,
SArray* pTaskList, bool hasFillhistory);
SArray* pTaskList, bool hasFillhistory, int8_t subtableWithoutMd5);
int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask);
int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask);
void tFreeStreamTask(SStreamTask* pTask);

2
include/os/osFile.h
View File

@ -119,6 +119,8 @@ int32_t taosSetFileHandlesLimit();
int32_t taosLinkFile(char *src, char *dst);
bool lastErrorIsFileNotExist();
#ifdef __cplusplus
}
#endif

146
source/client/src/clientRawBlockWrite.c
View File

@ -21,8 +21,8 @@
#include "tglobal.h"
#include "tmsgtype.h"
#define LOG_ID_TAG "connId:0x%"PRIx64",reqId:0x%"PRIx64
#define LOG_ID_VALUE *(int64_t*)taos,pRequest->requestId
#define LOG_ID_TAG "connId:0x%" PRIx64 ",reqId:0x%" PRIx64
#define LOG_ID_VALUE *(int64_t*)taos, pRequest->requestId
static tb_uid_t processSuid(tb_uid_t suid, char* db) { return suid + MurmurHash3_32(db, strlen(db)); }
@ -31,8 +31,7 @@ static char* buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* sch
char* string = NULL;
cJSON* json = cJSON_CreateObject();
if (json == NULL) {
uError("create json object failed")
return NULL;
uError("create json object failed") return NULL;
}
cJSON* type = cJSON_CreateString("create");
cJSON_AddItemToObject(json, "type", type);
@ -56,7 +55,7 @@ static char* buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* sch
cJSON_AddItemToObject(column, "name", cname);
cJSON* ctype = cJSON_CreateNumber(s->type);
cJSON_AddItemToObject(column, "type", ctype);
if (s->type == TSDB_DATA_TYPE_BINARY || s->type == TSDB_DATA_TYPE_VARBINARY|| s->type == TSDB_DATA_TYPE_GEOMETRY) {
if (s->type == TSDB_DATA_TYPE_BINARY || s->type == TSDB_DATA_TYPE_VARBINARY || s->type == TSDB_DATA_TYPE_GEOMETRY) {
int32_t length = s->bytes - VARSTR_HEADER_SIZE;
cJSON* cbytes = cJSON_CreateNumber(length);
cJSON_AddItemToObject(column, "length", cbytes);
@ -131,7 +130,8 @@ static char* buildAlterSTableJson(void* alterData, int32_t alterDataLen) {
cJSON* colType = cJSON_CreateNumber(field->type);
cJSON_AddItemToObject(json, "colType", colType);
if (field->type == TSDB_DATA_TYPE_BINARY || field->type == TSDB_DATA_TYPE_VARBINARY || field->type == TSDB_DATA_TYPE_GEOMETRY) {
if (field->type == TSDB_DATA_TYPE_BINARY || field->type == TSDB_DATA_TYPE_VARBINARY ||
field->type == TSDB_DATA_TYPE_GEOMETRY) {
int32_t length = field->bytes - VARSTR_HEADER_SIZE;
cJSON* cbytes = cJSON_CreateNumber(length);
cJSON_AddItemToObject(json, "colLength", cbytes);
@ -156,7 +156,8 @@ static char* buildAlterSTableJson(void* alterData, int32_t alterDataLen) {
cJSON_AddItemToObject(json, "colName", colName);
cJSON* colType = cJSON_CreateNumber(field->type);
cJSON_AddItemToObject(json, "colType", colType);
if (field->type == TSDB_DATA_TYPE_BINARY || field->type == TSDB_DATA_TYPE_VARBINARY || field->type == TSDB_DATA_TYPE_GEOMETRY) {
if (field->type == TSDB_DATA_TYPE_BINARY || field->type == TSDB_DATA_TYPE_VARBINARY ||
field->type == TSDB_DATA_TYPE_GEOMETRY) {
int32_t length = field->bytes - VARSTR_HEADER_SIZE;
cJSON* cbytes = cJSON_CreateNumber(length);
cJSON_AddItemToObject(json, "colLength", cbytes);
@ -182,7 +183,7 @@ static char* buildAlterSTableJson(void* alterData, int32_t alterDataLen) {
}
string = cJSON_PrintUnformatted(json);
end:
end:
cJSON_Delete(json);
tFreeSMAltertbReq(&req);
return string;
@ -295,9 +296,9 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) {
cJSON* tvalue = NULL;
if (IS_VAR_DATA_TYPE(pTagVal->type)) {
char* buf = NULL;
if(pTagVal->type == TSDB_DATA_TYPE_VARBINARY){
buf = taosMemoryCalloc(pTagVal->nData*2 + 2 + 3, 1);
}else{
if (pTagVal->type == TSDB_DATA_TYPE_VARBINARY) {
buf = taosMemoryCalloc(pTagVal->nData * 2 + 2 + 3, 1);
} else {
buf = taosMemoryCalloc(pTagVal->nData + 3, 1);
}
@ -315,7 +316,7 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) {
cJSON_AddItemToArray(tags, tag);
}
end:
end:
cJSON_AddItemToObject(json, "tags", tags);
taosArrayDestroy(pTagVals);
}
@ -469,7 +470,8 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) {
cJSON* colType = cJSON_CreateNumber(vAlterTbReq.type);
cJSON_AddItemToObject(json, "colType", colType);
if (vAlterTbReq.type == TSDB_DATA_TYPE_BINARY || vAlterTbReq.type == TSDB_DATA_TYPE_VARBINARY || vAlterTbReq.type == TSDB_DATA_TYPE_GEOMETRY) {
if (vAlterTbReq.type == TSDB_DATA_TYPE_BINARY || vAlterTbReq.type == TSDB_DATA_TYPE_VARBINARY ||
vAlterTbReq.type == TSDB_DATA_TYPE_GEOMETRY) {
int32_t length = vAlterTbReq.bytes - VARSTR_HEADER_SIZE;
cJSON* cbytes = cJSON_CreateNumber(length);
cJSON_AddItemToObject(json, "colLength", cbytes);
@ -490,7 +492,8 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) {
cJSON_AddItemToObject(json, "colName", colName);
cJSON* colType = cJSON_CreateNumber(vAlterTbReq.colModType);
cJSON_AddItemToObject(json, "colType", colType);
if (vAlterTbReq.colModType == TSDB_DATA_TYPE_BINARY || vAlterTbReq.colModType == TSDB_DATA_TYPE_VARBINARY || vAlterTbReq.colModType == TSDB_DATA_TYPE_GEOMETRY) {
if (vAlterTbReq.colModType == TSDB_DATA_TYPE_BINARY || vAlterTbReq.colModType == TSDB_DATA_TYPE_VARBINARY ||
vAlterTbReq.colModType == TSDB_DATA_TYPE_GEOMETRY) {
int32_t length = vAlterTbReq.colModBytes - VARSTR_HEADER_SIZE;
cJSON* cbytes = cJSON_CreateNumber(length);
cJSON_AddItemToObject(json, "colLength", cbytes);
@ -527,9 +530,9 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) {
}
buf = parseTagDatatoJson(vAlterTbReq.pTagVal);
} else {
if(vAlterTbReq.tagType == TSDB_DATA_TYPE_VARBINARY){
buf = taosMemoryCalloc(vAlterTbReq.nTagVal*2 + 2 + 3, 1);
}else{
if (vAlterTbReq.tagType == TSDB_DATA_TYPE_VARBINARY) {
buf = taosMemoryCalloc(vAlterTbReq.nTagVal * 2 + 2 + 3, 1);
} else {
buf = taosMemoryCalloc(vAlterTbReq.nTagVal + 3, 1);
}
dataConverToStr(buf, vAlterTbReq.tagType, vAlterTbReq.pTagVal, vAlterTbReq.nTagVal, NULL);
@ -677,7 +680,7 @@ _exit:
}
static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) {
if(taos == NULL || meta == NULL) {
if (taos == NULL || meta == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
@ -692,7 +695,7 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) {
terrno = code;
return code;
}
uDebug(LOG_ID_TAG" create stable, meta:%p, metaLen:%d", LOG_ID_VALUE, meta, metaLen);
uDebug(LOG_ID_TAG " create stable, meta:%p, metaLen:%d", LOG_ID_VALUE, meta, metaLen);
pRequest->syncQuery = true;
if (!pRequest->pDb) {
code = TSDB_CODE_PAR_DB_NOT_SPECIFIED;
@ -731,8 +734,8 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) {
pReq.source = TD_REQ_FROM_TAOX;
pReq.igExists = true;
uDebug(LOG_ID_TAG" create stable name:%s suid:%" PRId64 " processSuid:%" PRId64,
LOG_ID_VALUE, req.name, req.suid, pReq.suid);
uDebug(LOG_ID_TAG " create stable name:%s suid:%" PRId64 " processSuid:%" PRId64, LOG_ID_VALUE, req.name, req.suid,
pReq.suid);
STscObj* pTscObj = pRequest->pTscObj;
SName tableName;
tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName), pReq.name);
@ -766,7 +769,7 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) {
taosMemoryFree(pCmdMsg.pMsg);
end:
uDebug(LOG_ID_TAG" create stable return, msg:%s", LOG_ID_VALUE, tstrerror(code));
uDebug(LOG_ID_TAG " create stable return, msg:%s", LOG_ID_VALUE, tstrerror(code));
destroyRequest(pRequest);
tFreeSMCreateStbReq(&pReq);
tDecoderClear(&coder);
@ -775,7 +778,7 @@ end:
}
static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) {
if(taos == NULL || meta == NULL) {
if (taos == NULL || meta == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
@ -791,7 +794,7 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) {
return code;
}
uDebug(LOG_ID_TAG" drop stable, meta:%p, metaLen:%d", LOG_ID_VALUE, meta, metaLen);
uDebug(LOG_ID_TAG " drop stable, meta:%p, metaLen:%d", LOG_ID_VALUE, meta, metaLen);
pRequest->syncQuery = true;
if (!pRequest->pDb) {
code = TSDB_CODE_PAR_DB_NOT_SPECIFIED;
@ -812,9 +815,9 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) {
goto end;
}
SRequestConnInfo conn = {.pTrans = pRequest->pTscObj->pAppInfo->pTransporter,
.requestId = pRequest->requestId,
.requestObjRefId = pRequest->self,
.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp)};
.requestId = pRequest->requestId,
.requestObjRefId = pRequest->self,
.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp)};
SName pName = {0};
toName(pRequest->pTscObj->acctId, pRequest->pDb, req.name, &pName);
STableMeta* pTableMeta = NULL;
@ -835,8 +838,8 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) {
pReq.source = TD_REQ_FROM_TAOX;
// pReq.suid = processSuid(req.suid, pRequest->pDb);
uDebug(LOG_ID_TAG" drop stable name:%s suid:%" PRId64 " new suid:%" PRId64,
LOG_ID_VALUE, req.name, req.suid, pReq.suid);
uDebug(LOG_ID_TAG " drop stable name:%s suid:%" PRId64 " new suid:%" PRId64, LOG_ID_VALUE, req.name, req.suid,
pReq.suid);
STscObj* pTscObj = pRequest->pTscObj;
SName tableName = {0};
tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName), pReq.name);
@ -870,7 +873,7 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) {
taosMemoryFree(pCmdMsg.pMsg);
end:
uDebug(LOG_ID_TAG" drop stable return, msg:%s", LOG_ID_VALUE, tstrerror(code));
uDebug(LOG_ID_TAG " drop stable return, msg:%s", LOG_ID_VALUE, tstrerror(code));
destroyRequest(pRequest);
tDecoderClear(&coder);
terrno = code;
@ -889,7 +892,7 @@ static void destroyCreateTbReqBatch(void* data) {
}
static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) {
if(taos == NULL || meta == NULL) {
if (taos == NULL || meta == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
@ -939,9 +942,9 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) {
taosHashSetFreeFp(pVgroupHashmap, destroyCreateTbReqBatch);
SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter,
.requestId = pRequest->requestId,
.requestObjRefId = pRequest->self,
.mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)};
.requestId = pRequest->requestId,
.requestObjRefId = pRequest->self,
.mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)};
pRequest->tableList = taosArrayInit(req.nReqs, sizeof(SName));
// loop to create table
@ -1033,7 +1036,7 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) {
code = pRequest->code;
end:
uDebug(LOG_ID_TAG" create table return, msg:%s", LOG_ID_VALUE, tstrerror(code));
uDebug(LOG_ID_TAG " create table return, msg:%s", LOG_ID_VALUE, tstrerror(code));
for (int32_t iReq = 0; iReq < req.nReqs; iReq++) {
pCreateReq = req.pReqs + iReq;
taosMemoryFreeClear(pCreateReq->comment);
@ -1062,7 +1065,7 @@ static void destroyDropTbReqBatch(void* data) {
}
static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) {
if(taos == NULL || meta == NULL) {
if (taos == NULL || meta == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
@ -1111,9 +1114,9 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) {
taosHashSetFreeFp(pVgroupHashmap, destroyDropTbReqBatch);
SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter,
.requestId = pRequest->requestId,
.requestObjRefId = pRequest->self,
.mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)};
.requestId = pRequest->requestId,
.requestObjRefId = pRequest->self,
.mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)};
pRequest->tableList = taosArrayInit(req.nReqs, sizeof(SName));
// loop to create table
for (int32_t iReq = 0; iReq < req.nReqs; iReq++) {
@ -1142,7 +1145,8 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) {
tb_uid_t oldSuid = pDropReq->suid;
pDropReq->suid = pTableMeta->suid;
taosMemoryFreeClear(pTableMeta);
uDebug(LOG_ID_TAG" drop table name:%s suid:%" PRId64 " new suid:%" PRId64, LOG_ID_VALUE, pDropReq->name, oldSuid, pDropReq->suid);
uDebug(LOG_ID_TAG " drop table name:%s suid:%" PRId64 " new suid:%" PRId64, LOG_ID_VALUE, pDropReq->name, oldSuid,
pDropReq->suid);
taosArrayPush(pRequest->tableList, &pName);
SVgroupDropTableBatch* pTableBatch = taosHashGet(pVgroupHashmap, &pInfo.vgId, sizeof(pInfo.vgId));
@ -1185,7 +1189,7 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) {
code = pRequest->code;
end:
uDebug(LOG_ID_TAG" drop table return, msg:%s", LOG_ID_VALUE, tstrerror(code));
uDebug(LOG_ID_TAG " drop table return, msg:%s", LOG_ID_VALUE, tstrerror(code));
taosHashCleanup(pVgroupHashmap);
destroyRequest(pRequest);
tDecoderClear(&coder);
@ -1227,16 +1231,16 @@ end:
//}
static int32_t taosDeleteData(TAOS* taos, void* meta, int32_t metaLen) {
if(taos == NULL || meta == NULL) {
if (taos == NULL || meta == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
SDeleteRes req = {0};
SDecoder coder = {0};
SDeleteRes req = {0};
SDecoder coder = {0};
char sql[256] = {0};
int32_t code = TSDB_CODE_SUCCESS;
uDebug("connId:0x%"PRIx64" delete data, meta:%p, len:%d", *(int64_t*)taos, meta, metaLen);
uDebug("connId:0x%" PRIx64 " delete data, meta:%p, len:%d", *(int64_t*)taos, meta, metaLen);
// decode and process req
void* data = POINTER_SHIFT(meta, sizeof(SMsgHead));
@ -1260,14 +1264,14 @@ static int32_t taosDeleteData(TAOS* taos, void* meta, int32_t metaLen) {
taos_free_result(res);
end:
uDebug("connId:0x%"PRIx64" delete data sql:%s, code:%s", *(int64_t*)taos, sql, tstrerror(code));
uDebug("connId:0x%" PRIx64 " delete data sql:%s, code:%s", *(int64_t*)taos, sql, tstrerror(code));
tDecoderClear(&coder);
terrno = code;
return code;
}
static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) {
if(taos == NULL || meta == NULL) {
if (taos == NULL || meta == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
@ -1312,9 +1316,9 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) {
}
SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter,
.requestId = pRequest->requestId,
.requestObjRefId = pRequest->self,
.mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)};
.requestId = pRequest->requestId,
.requestObjRefId = pRequest->self,
.mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)};
SVgroupInfo pInfo = {0};
SName pName = {0};
@ -1394,8 +1398,8 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch
return taos_write_raw_block_with_fields_with_reqid(taos, rows, pData, tbname, fields, numFields, 0);
}
int taos_write_raw_block_with_fields_with_reqid(TAOS *taos, int rows, char *pData, const char *tbname,
TAOS_FIELD *fields, int numFields, int64_t reqid){
int taos_write_raw_block_with_fields_with_reqid(TAOS* taos, int rows, char* pData, const char* tbname,
TAOS_FIELD* fields, int numFields, int64_t reqid) {
if (!taos || !pData || !tbname) {
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
@ -1410,8 +1414,8 @@ int taos_write_raw_block_with_fields_with_reqid(TAOS *taos, int rows, char *pDat
return terrno;
}
uDebug(LOG_ID_TAG " write raw block with field, rows:%d, pData:%p, tbname:%s, fields:%p, numFields:%d",
LOG_ID_VALUE, rows, pData, tbname, fields, numFields);
uDebug(LOG_ID_TAG " write raw block with field, rows:%d, pData:%p, tbname:%s, fields:%p, numFields:%d", LOG_ID_VALUE,
rows, pData, tbname, fields, numFields);
pRequest->syncQuery = true;
if (!pRequest->pDb) {
@ -1562,12 +1566,12 @@ end:
return code;
}
static void* getRawDataFromRes(void *pRetrieve){
static void* getRawDataFromRes(void* pRetrieve) {
void* rawData = NULL;
// deal with compatibility
if(*(int64_t*)pRetrieve == 0){
if (*(int64_t*)pRetrieve == 0) {
rawData = ((SRetrieveTableRsp*)pRetrieve)->data;
}else if(*(int64_t*)pRetrieve == 1){
} else if (*(int64_t*)pRetrieve == 1) {
rawData = ((SRetrieveTableRspForTmq*)pRetrieve)->data;
}
ASSERT(rawData != NULL);
@ -1575,7 +1579,7 @@ static void* getRawDataFromRes(void *pRetrieve){
}
static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) {
if(taos == NULL || data == NULL){
if (taos == NULL || data == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
@ -1644,11 +1648,11 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) {
strcpy(pName.tname, tbName);
code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta);
// if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) {
// uError("WriteRaw:catalogGetTableMeta table not exist. table name: %s", tbName);
// code = TSDB_CODE_SUCCESS;
// continue;
// }
// if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) {
// uError("WriteRaw:catalogGetTableMeta table not exist. table name: %s", tbName);
// code = TSDB_CODE_SUCCESS;
// continue;
// }
if (code != TSDB_CODE_SUCCESS) {
goto end;
}
@ -1704,7 +1708,7 @@ end:
}
static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) {
if(taos == NULL || data == NULL){
if (taos == NULL || data == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
@ -1757,7 +1761,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
}
pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
uDebug(LOG_ID_TAG" write raw metadata block num:%d", LOG_ID_VALUE, rspObj.rsp.blockNum);
uDebug(LOG_ID_TAG " write raw metadata block num:%d", LOG_ID_VALUE, rspObj.rsp.blockNum);
while (++rspObj.resIter < rspObj.rsp.blockNum) {
void* pRetrieve = taosArrayGetP(rspObj.rsp.blockData, rspObj.resIter);
if (!rspObj.rsp.withSchema) {
@ -1770,7 +1774,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
goto end;
}
uDebug(LOG_ID_TAG" write raw metadata block tbname:%s", LOG_ID_VALUE, tbName);
uDebug(LOG_ID_TAG " write raw metadata block tbname:%s", LOG_ID_VALUE, tbName);
SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}};
strcpy(pName.dbname, pRequest->pDb);
strcpy(pName.tname, tbName);
@ -1817,11 +1821,11 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
strcpy(pName.tname, pCreateReqDst->ctb.stbName);
}
code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta);
// if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) {
// uError("WriteRaw:catalogGetTableMeta table not exist. table name: %s", tbName);
// code = TSDB_CODE_SUCCESS;
// continue;
// }
// if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) {
// uError("WriteRaw:catalogGetTableMeta table not exist. table name: %s", tbName);
// code = TSDB_CODE_SUCCESS;
// continue;
// }
if (code != TSDB_CODE_SUCCESS) {
goto end;
}

547
source/client/src/clientTmq.c
View File

File diff suppressed because it is too large Load Diff

1
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -699,6 +699,7 @@ typedef struct {
int64_t checkpointId;
int32_t indexForMultiAggBalance;
int8_t subTableWithoutMd5;
char reserve[256];
} SStreamObj;

2
source/dnode/mnode/impl/inc/mndStream.h
View File

@ -24,7 +24,7 @@ extern "C" {
#endif
#define MND_STREAM_RESERVE_SIZE 64
#define MND_STREAM_VER_NUMBER 4
#define MND_STREAM_VER_NUMBER 5
#define MND_STREAM_CREATE_NAME "stream-create"
#define MND_STREAM_CHECKPOINT_NAME "stream-checkpoint"

168
source/dnode/mnode/impl/src/mndConsumer.c
View File

@ -15,20 +15,20 @@
#define _DEFAULT_SOURCE
#include "mndConsumer.h"
#include "mndPrivilege.h"
#include "mndVgroup.h"
#include "mndShow.h"
#include "mndDb.h"
#include "mndPrivilege.h"
#include "mndShow.h"
#include "mndSubscribe.h"
#include "mndTopic.h"
#include "mndTrans.h"
#include "mndVgroup.h"
#include "tcompare.h"
#include "tname.h"
#define MND_CONSUMER_VER_NUMBER 2
#define MND_CONSUMER_RESERVE_SIZE 64
#define MND_MAX_GROUP_PER_TOPIC 100
#define MND_MAX_GROUP_PER_TOPIC 100
static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer);
static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer);
@ -56,7 +56,7 @@ int32_t mndInitConsumer(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_MND_TMQ_SUBSCRIBE, mndProcessSubscribeReq);
mndSetMsgHandle(pMnode, TDMT_MND_TMQ_HB, mndProcessMqHbReq);
mndSetMsgHandle(pMnode, TDMT_MND_TMQ_ASK_EP, mndProcessAskEpReq);
// mndSetMsgHandle(pMnode, TDMT_MND_TMQ_TIMER, mndProcessMqTimerMsg);
// mndSetMsgHandle(pMnode, TDMT_MND_TMQ_TIMER, mndProcessMqTimerMsg);
mndSetMsgHandle(pMnode, TDMT_MND_TMQ_CONSUMER_RECOVER, mndProcessConsumerRecoverMsg);
mndSetMsgHandle(pMnode, TDMT_MND_TMQ_LOST_CONSUMER_CLEAR, mndProcessConsumerClearMsg);
@ -68,10 +68,11 @@ int32_t mndInitConsumer(SMnode *pMnode) {
void mndCleanupConsumer(SMnode *pMnode) {}
void mndDropConsumerFromSdb(SMnode *pMnode, int64_t consumerId, SRpcHandleInfo* info){
void mndDropConsumerFromSdb(SMnode *pMnode, int64_t consumerId, SRpcHandleInfo *info) {
SMqConsumerClearMsg *pClearMsg = rpcMallocCont(sizeof(SMqConsumerClearMsg));
if (pClearMsg == NULL) {
mError("consumer:0x%"PRIx64" failed to clear consumer due to out of memory. alloc size:%d", consumerId, (int32_t)sizeof(SMqConsumerClearMsg));
mError("consumer:0x%" PRIx64 " failed to clear consumer due to out of memory. alloc size:%d", consumerId,
(int32_t)sizeof(SMqConsumerClearMsg));
return;
}
@ -88,13 +89,14 @@ void mndDropConsumerFromSdb(SMnode *pMnode, int64_t consumerId, SRpcHandleInfo*
return;
}
static int32_t validateTopics(STrans *pTrans, const SArray *pTopicList, SMnode *pMnode, const char *pUser, bool enableReplay) {
static int32_t validateTopics(STrans *pTrans, const SArray *pTopicList, SMnode *pMnode, const char *pUser,
bool enableReplay) {
SMqTopicObj *pTopic = NULL;
int32_t code = 0;
int32_t code = 0;
int32_t numOfTopics = taosArrayGetSize(pTopicList);
for (int32_t i = 0; i < numOfTopics; i++) {
char *pOneTopic = taosArrayGetP(pTopicList, i);
char *pOneTopic = taosArrayGetP(pTopicList, i);
pTopic = mndAcquireTopic(pMnode, pOneTopic);
if (pTopic == NULL) { // terrno has been set by callee function
code = -1;
@ -112,11 +114,11 @@ static int32_t validateTopics(STrans *pTrans, const SArray *pTopicList, SMnode *
goto FAILED;
}
if(enableReplay){
if(pTopic->subType != TOPIC_SUB_TYPE__COLUMN){
if (enableReplay) {
if (pTopic->subType != TOPIC_SUB_TYPE__COLUMN) {
code = TSDB_CODE_TMQ_REPLAY_NOT_SUPPORT;
goto FAILED;
}else if(pTopic->ntbUid == 0 && pTopic->ctbStbUid == 0) {
} else if (pTopic->ntbUid == 0 && pTopic->ctbStbUid == 0) {
SDbObj *pDb = mndAcquireDb(pMnode, pTopic->db);
if (pDb == NULL) {
code = -1;
@ -132,7 +134,7 @@ static int32_t validateTopics(STrans *pTrans, const SArray *pTopicList, SMnode *
}
mndTransSetDbName(pTrans, pOneTopic, NULL);
if(mndTransCheckConflict(pMnode, pTrans) != 0){
if (mndTransCheckConflict(pMnode, pTrans) != 0) {
code = -1;
goto FAILED;
}
@ -172,7 +174,7 @@ static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) {
if (pTrans == NULL) {
goto FAIL;
}
if(validateTopics(pTrans, pConsumer->assignedTopics, pMnode, pMsg->info.conn.user, false) != 0){
if (validateTopics(pTrans, pConsumer->assignedTopics, pMnode, pMsg->info.conn.user, false) != 0) {
goto FAIL;
}
@ -197,7 +199,7 @@ static int32_t mndProcessConsumerClearMsg(SRpcMsg *pMsg) {
SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pClearMsg->consumerId);
if (pConsumer == NULL) {
mError("consumer:0x%"PRIx64" failed to be found to clear it", pClearMsg->consumerId);
mError("consumer:0x%" PRIx64 " failed to be found to clear it", pClearMsg->consumerId);
return 0;
}
@ -226,15 +228,15 @@ FAIL:
return -1;
}
static int32_t checkPrivilege(SMnode *pMnode, SMqConsumerObj *pConsumer, SMqHbRsp *rsp, char* user){
static int32_t checkPrivilege(SMnode *pMnode, SMqConsumerObj *pConsumer, SMqHbRsp *rsp, char *user) {
rsp->topicPrivileges = taosArrayInit(taosArrayGetSize(pConsumer->currentTopics), sizeof(STopicPrivilege));
if(rsp->topicPrivileges == NULL){
if (rsp->topicPrivileges == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
for(int32_t i = 0; i < taosArrayGetSize(pConsumer->currentTopics); i++){
char *topic = taosArrayGetP(pConsumer->currentTopics, i);
SMqTopicObj* pTopic = mndAcquireTopic(pMnode, topic);
for (int32_t i = 0; i < taosArrayGetSize(pConsumer->currentTopics); i++) {
char *topic = taosArrayGetP(pConsumer->currentTopics, i);
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic);
if (pTopic == NULL) { // terrno has been set by callee function
continue;
}
@ -252,10 +254,10 @@ static int32_t checkPrivilege(SMnode *pMnode, SMqConsumerObj *pConsumer, SMqHbR
}
static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) {
int32_t code = 0;
SMnode *pMnode = pMsg->info.node;
SMqHbReq req = {0};
SMqHbRsp rsp = {0};
int32_t code = 0;
SMnode *pMnode = pMsg->info.node;
SMqHbReq req = {0};
SMqHbRsp rsp = {0};
SMqConsumerObj *pConsumer = NULL;
if (tDeserializeSMqHbReq(pMsg->pCont, pMsg->contLen, &req) < 0) {
@ -264,7 +266,7 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) {
goto end;
}
int64_t consumerId = req.consumerId;
int64_t consumerId = req.consumerId;
pConsumer = mndAcquireConsumer(pMnode, consumerId);
if (pConsumer == NULL) {
mError("consumer:0x%" PRIx64 " not exist", consumerId);
@ -273,7 +275,7 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) {
goto end;
}
code = checkPrivilege(pMnode, pConsumer, &rsp, pMsg->info.conn.user);
if(code != 0){
if (code != 0) {
goto end;
}
@ -295,12 +297,12 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) {
tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &pRpcMsg);
}
for(int i = 0; i < taosArrayGetSize(req.topics); i++){
TopicOffsetRows* data = taosArrayGet(req.topics, i);
for (int i = 0; i < taosArrayGetSize(req.topics); i++) {
TopicOffsetRows *data = taosArrayGet(req.topics, i);
mInfo("heartbeat report offset rows.%s:%s", pConsumer->cgroup, data->topicName);
SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, data->topicName);
if(pSub == NULL){
if (pSub == NULL) {
#ifdef TMQ_DEBUG
ASSERT(0);
#endif
@ -308,7 +310,7 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) {
}
taosWLockLatch(&pSub->lock);
SMqConsumerEp *pConsumerEp = taosHashGet(pSub->consumerHash, &consumerId, sizeof(int64_t));
if(pConsumerEp){
if (pConsumerEp) {
taosArrayDestroy(pConsumerEp->offsetRows);
pConsumerEp->offsetRows = data->offsetRows;
data->offsetRows = NULL;
@ -413,7 +415,7 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) {
char *topic = taosArrayGetP(pConsumer->currentTopics, i);
SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, topic);
// txn guarantees pSub is created
if(pSub == NULL) {
if (pSub == NULL) {
#ifdef TMQ_DEBUG
ASSERT(0);
#endif
@ -426,7 +428,7 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) {
// 2.1 fetch topic schema
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic);
if(pTopic == NULL) {
if (pTopic == NULL) {
#ifdef TMQ_DEBUG
ASSERT(0);
#endif
@ -460,12 +462,12 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) {
for (int32_t j = 0; j < vgNum; j++) {
SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, j);
// char offsetKey[TSDB_PARTITION_KEY_LEN];
// mndMakePartitionKey(offsetKey, pConsumer->cgroup, topic, pVgEp->vgId);
// char offsetKey[TSDB_PARTITION_KEY_LEN];
// mndMakePartitionKey(offsetKey, pConsumer->cgroup, topic, pVgEp->vgId);
if(epoch == -1){
if (epoch == -1) {
SVgObj *pVgroup = mndAcquireVgroup(pMnode, pVgEp->vgId);
if(pVgroup){
if (pVgroup) {
pVgEp->epSet = mndGetVgroupEpset(pMnode, pVgroup);
mndReleaseVgroup(pMnode, pVgroup);
}
@ -495,7 +497,7 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) {
goto FAIL;
}
SMqRspHead* pHead = buf;
SMqRspHead *pHead = buf;
pHead->mqMsgType = TMQ_MSG_TYPE__EP_RSP;
pHead->epoch = serverEpoch;
@ -503,7 +505,6 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) {
pHead->walsver = 0;
pHead->walever = 0;
void *abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
tEncodeSMqAskEpRsp(&abuf, &rsp);
@ -560,21 +561,20 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
SCMSubscribeReq subscribe = {0};
tDeserializeSCMSubscribeReq(msgStr, &subscribe);
int64_t consumerId = subscribe.consumerId;
int64_t consumerId = subscribe.consumerId;
char *cgroup = subscribe.cgroup;
SMqConsumerObj *pExistedConsumer = NULL;
SMqConsumerObj *pConsumerNew = NULL;
STrans *pTrans = NULL;
STrans *pTrans = NULL;
SArray *pTopicList = subscribe.topicNames;
taosArraySort(pTopicList, taosArrayCompareString);
taosArrayRemoveDuplicate(pTopicList, taosArrayCompareString, freeItem);
int32_t newTopicNum = taosArrayGetSize(pTopicList);
for(int i = 0; i < newTopicNum; i++){
int32_t gNum = mndGetGroupNumByTopic(pMnode, (const char*)taosArrayGetP(pTopicList, i));
if(gNum >= MND_MAX_GROUP_PER_TOPIC){
for (int i = 0; i < newTopicNum; i++) {
int32_t gNum = mndGetGroupNumByTopic(pMnode, (const char *)taosArrayGetP(pTopicList, i));
if (gNum >= MND_MAX_GROUP_PER_TOPIC) {
terrno = TSDB_CODE_TMQ_GROUP_OUT_OF_RANGE;
code = terrno;
goto _over;
@ -605,7 +605,7 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
pConsumerNew->autoCommitInterval = subscribe.autoCommitInterval;
pConsumerNew->resetOffsetCfg = subscribe.resetOffsetCfg;
// pConsumerNew->updateType = CONSUMER_UPDATE_SUB; // use insert logic
// pConsumerNew->updateType = CONSUMER_UPDATE_SUB; // use insert logic
taosArrayDestroy(pConsumerNew->assignedTopics);
pConsumerNew->assignedTopics = taosArrayDup(pTopicList, topicNameDup);
@ -792,16 +792,15 @@ CM_DECODE_OVER:
}
static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer) {
mInfo("consumer:0x%" PRIx64 " sub insert, cgroup:%s status:%d(%s) epoch:%d",
pConsumer->consumerId, pConsumer->cgroup, pConsumer->status, mndConsumerStatusName(pConsumer->status),
pConsumer->epoch);
mInfo("consumer:0x%" PRIx64 " sub insert, cgroup:%s status:%d(%s) epoch:%d", pConsumer->consumerId, pConsumer->cgroup,
pConsumer->status, mndConsumerStatusName(pConsumer->status), pConsumer->epoch);
pConsumer->subscribeTime = pConsumer->createTime;
return 0;
}
static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer) {
mInfo("consumer:0x%" PRIx64 " perform delete action, status:(%d)%s", pConsumer->consumerId, pConsumer->status,
mndConsumerStatusName(pConsumer->status));
mndConsumerStatusName(pConsumer->status));
tDeleteSMqConsumerObj(pConsumer, false);
return 0;
}
@ -828,7 +827,7 @@ static void removeFromNewTopicList(SMqConsumerObj *pConsumer, const char *pTopic
taosMemoryFree(p);
mInfo("consumer:0x%" PRIx64 " remove new topic:%s in the topic list, remain newTopics:%d", pConsumer->consumerId,
pTopic, (int)taosArrayGetSize(pConsumer->rebNewTopics));
pTopic, (int)taosArrayGetSize(pConsumer->rebNewTopics));
break;
}
}
@ -844,7 +843,7 @@ static void removeFromRemoveTopicList(SMqConsumerObj *pConsumer, const char *pTo
taosMemoryFree(p);
mInfo("consumer:0x%" PRIx64 " remove topic:%s in the removed topic list, remain removedTopics:%d",
pConsumer->consumerId, pTopic, (int)taosArrayGetSize(pConsumer->rebRemovedTopics));
pConsumer->consumerId, pTopic, (int)taosArrayGetSize(pConsumer->rebRemovedTopics));
break;
}
}
@ -859,13 +858,13 @@ static void removeFromCurrentTopicList(SMqConsumerObj *pConsumer, const char *pT
taosMemoryFree(topic);
mInfo("consumer:0x%" PRIx64 " remove topic:%s in the current topic list, remain currentTopics:%d",
pConsumer->consumerId, pTopic, (int)taosArrayGetSize(pConsumer->currentTopics));
pConsumer->consumerId, pTopic, (int)taosArrayGetSize(pConsumer->currentTopics));
break;
}
}
}
static bool existInCurrentTopicList(const SMqConsumerObj* pConsumer, const char* pTopic) {
static bool existInCurrentTopicList(const SMqConsumerObj *pConsumer, const char *pTopic) {
bool existing = false;
int32_t size = taosArrayGetSize(pConsumer->currentTopics);
for (int32_t i = 0; i < size; i++) {
@ -882,7 +881,7 @@ static bool existInCurrentTopicList(const SMqConsumerObj* pConsumer, const char*
static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, SMqConsumerObj *pNewConsumer) {
mInfo("consumer:0x%" PRIx64 " perform update action, update type:%d, subscribe-time:%" PRId64 ", createTime:%" PRId64,
pOldConsumer->consumerId, pNewConsumer->updateType, pOldConsumer->subscribeTime, pOldConsumer->createTime);
pOldConsumer->consumerId, pNewConsumer->updateType, pOldConsumer->subscribeTime, pOldConsumer->createTime);
taosWLockLatch(&pOldConsumer->lock);
@ -893,19 +892,21 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
pOldConsumer->subscribeTime = taosGetTimestampMs();
pOldConsumer->status = MQ_CONSUMER_STATUS_REBALANCE;
mInfo("consumer:0x%" PRIx64 " sub update, modify existed consumer",pOldConsumer->consumerId);
// } else if (pNewConsumer->updateType == CONSUMER_UPDATE_TIMER_LOST) {
// int32_t sz = taosArrayGetSize(pOldConsumer->currentTopics);
// for (int32_t i = 0; i < sz; i++) {
// char *topic = taosStrdup(taosArrayGetP(pOldConsumer->currentTopics, i));
// taosArrayPush(pOldConsumer->rebRemovedTopics, &topic);
// }
//
// int32_t prevStatus = pOldConsumer->status;
// pOldConsumer->status = MQ_CONSUMER_STATUS_LOST;
// mInfo("consumer:0x%" PRIx64 " timer update, timer lost. state %s -> %s, reb-time:%" PRId64 ", reb-removed-topics:%d",
// pOldConsumer->consumerId, mndConsumerStatusName(prevStatus), mndConsumerStatusName(pOldConsumer->status),
// pOldConsumer->rebalanceTime, (int)taosArrayGetSize(pOldConsumer->rebRemovedTopics));
mInfo("consumer:0x%" PRIx64 " sub update, modify existed consumer", pOldConsumer->consumerId);
// } else if (pNewConsumer->updateType == CONSUMER_UPDATE_TIMER_LOST) {
// int32_t sz = taosArrayGetSize(pOldConsumer->currentTopics);
// for (int32_t i = 0; i < sz; i++) {
// char *topic = taosStrdup(taosArrayGetP(pOldConsumer->currentTopics, i));
// taosArrayPush(pOldConsumer->rebRemovedTopics, &topic);
// }
//
// int32_t prevStatus = pOldConsumer->status;
// pOldConsumer->status = MQ_CONSUMER_STATUS_LOST;
// mInfo("consumer:0x%" PRIx64 " timer update, timer lost. state %s -> %s, reb-time:%" PRId64 ",
// reb-removed-topics:%d",
// pOldConsumer->consumerId, mndConsumerStatusName(prevStatus),
// mndConsumerStatusName(pOldConsumer->status), pOldConsumer->rebalanceTime,
// (int)taosArrayGetSize(pOldConsumer->rebRemovedTopics));
} else if (pNewConsumer->updateType == CONSUMER_UPDATE_REC) {
int32_t sz = taosArrayGetSize(pOldConsumer->assignedTopics);
for (int32_t i = 0; i < sz; i++) {
@ -914,7 +915,7 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
}
pOldConsumer->status = MQ_CONSUMER_STATUS_REBALANCE;
mInfo("consumer:0x%" PRIx64 " timer update, timer recover",pOldConsumer->consumerId);
mInfo("consumer:0x%" PRIx64 " timer update, timer recover", pOldConsumer->consumerId);
} else if (pNewConsumer->updateType == CONSUMER_UPDATE_REB) {
atomic_add_fetch_32(&pOldConsumer->epoch, 1);
@ -945,11 +946,11 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
atomic_add_fetch_32(&pOldConsumer->epoch, 1);
mInfo("consumer:0x%" PRIx64 " reb update add, state (%d)%s -> (%d)%s, new epoch:%d, reb-time:%" PRId64
", current topics:%d, newTopics:%d, removeTopics:%d",
pOldConsumer->consumerId, status, mndConsumerStatusName(status), pOldConsumer->status,
mndConsumerStatusName(pOldConsumer->status), pOldConsumer->epoch, pOldConsumer->rebalanceTime,
(int)taosArrayGetSize(pOldConsumer->currentTopics), (int)taosArrayGetSize(pOldConsumer->rebNewTopics),
(int)taosArrayGetSize(pOldConsumer->rebRemovedTopics));
", current topics:%d, newTopics:%d, removeTopics:%d",
pOldConsumer->consumerId, status, mndConsumerStatusName(status), pOldConsumer->status,
mndConsumerStatusName(pOldConsumer->status), pOldConsumer->epoch, pOldConsumer->rebalanceTime,
(int)taosArrayGetSize(pOldConsumer->currentTopics), (int)taosArrayGetSize(pOldConsumer->rebNewTopics),
(int)taosArrayGetSize(pOldConsumer->rebRemovedTopics));
} else if (pNewConsumer->updateType == CONSUMER_REMOVE_REB) {
char *removedTopic = taosArrayGetP(pNewConsumer->rebRemovedTopics, 0);
@ -968,11 +969,11 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
atomic_add_fetch_32(&pOldConsumer->epoch, 1);
mInfo("consumer:0x%" PRIx64 " reb update remove, state (%d)%s -> (%d)%s, new epoch:%d, reb-time:%" PRId64
", current topics:%d, newTopics:%d, removeTopics:%d",
pOldConsumer->consumerId, status, mndConsumerStatusName(status), pOldConsumer->status,
mndConsumerStatusName(pOldConsumer->status), pOldConsumer->epoch, pOldConsumer->rebalanceTime,
(int)taosArrayGetSize(pOldConsumer->currentTopics), (int)taosArrayGetSize(pOldConsumer->rebNewTopics),
(int)taosArrayGetSize(pOldConsumer->rebRemovedTopics));
", current topics:%d, newTopics:%d, removeTopics:%d",
pOldConsumer->consumerId, status, mndConsumerStatusName(status), pOldConsumer->status,
mndConsumerStatusName(pOldConsumer->status), pOldConsumer->epoch, pOldConsumer->rebalanceTime,
(int)taosArrayGetSize(pOldConsumer->currentTopics), (int)taosArrayGetSize(pOldConsumer->rebNewTopics),
(int)taosArrayGetSize(pOldConsumer->rebRemovedTopics));
}
taosWUnLockLatch(&pOldConsumer->lock);
@ -1030,8 +1031,8 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *
int32_t cols = 0;
// consumer id
char consumerIdHex[32] = {0};
sprintf(varDataVal(consumerIdHex), "0x%"PRIx64, pConsumer->consumerId);
char consumerIdHex[32] = {0};
sprintf(varDataVal(consumerIdHex), "0x%" PRIx64, pConsumer->consumerId);
varDataSetLen(consumerIdHex, strlen(varDataVal(consumerIdHex)));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
@ -1086,12 +1087,13 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->rebalanceTime, pConsumer->rebalanceTime == 0);
char buf[TSDB_OFFSET_LEN] = {0};
char buf[TSDB_OFFSET_LEN] = {0};
STqOffsetVal pVal = {.type = pConsumer->resetOffsetCfg};
tFormatOffset(buf, TSDB_OFFSET_LEN, &pVal);
char parasStr[64 + TSDB_OFFSET_LEN + VARSTR_HEADER_SIZE] = {0};
sprintf(varDataVal(parasStr), "tbname:%d,commit:%d,interval:%dms,reset:%s", pConsumer->withTbName, pConsumer->autoCommit, pConsumer->autoCommitInterval, buf);
sprintf(varDataVal(parasStr), "tbname:%d,commit:%d,interval:%dms,reset:%s", pConsumer->withTbName,
pConsumer->autoCommit, pConsumer->autoCommitInterval, buf);
varDataSetLen(parasStr, strlen(varDataVal(parasStr)));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);

5
source/dnode/mnode/impl/src/mndDef.c
View File

@ -85,6 +85,7 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) {
// 3.0.50 ver = 3
if (tEncodeI64(pEncoder, pObj->checkpointId) < 0) return -1;
if (tEncodeI8(pEncoder, pObj->subTableWithoutMd5) < 0) return -1;
if (tEncodeCStrWithLen(pEncoder, pObj->reserve, sizeof(pObj->reserve) - 1) < 0) return -1;
@ -168,6 +169,10 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj, int32_t sver) {
if (sver >= 3) {
if (tDecodeI64(pDecoder, &pObj->checkpointId) < 0) return -1;
}
if (sver >= 5) {
if (tDecodeI8(pDecoder, &pObj->subTableWithoutMd5) < 0) return -1;
}
if (tDecodeCStrTo(pDecoder, pObj->reserve) < 0) return -1;
tEndDecode(pDecoder);

8
source/dnode/mnode/impl/src/mndScheduler.c
View File

@ -221,8 +221,8 @@ static int32_t doAddSinkTask(SStreamObj* pStream, SMnode* pMnode, SVgObj* pVgrou
int64_t uid = (isFillhistory) ? pStream->hTaskUid : pStream->uid;
SArray** pTaskList = (isFillhistory) ? taosArrayGetLast(pStream->pHTasksList) : taosArrayGetLast(pStream->tasks);
SStreamTask* pTask =
tNewStreamTask(uid, TASK_LEVEL__SINK, pEpset, isFillhistory, 0, *pTaskList, pStream->conf.fillHistory);
SStreamTask* pTask = tNewStreamTask(uid, TASK_LEVEL__SINK, pEpset, isFillhistory, 0, *pTaskList,
pStream->conf.fillHistory, pStream->subTableWithoutMd5);
if (pTask == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return terrno;
@ -326,7 +326,7 @@ static SStreamTask* buildSourceTask(SStreamObj* pStream, SEpSet* pEpset, bool is
SStreamTask* pTask =
tNewStreamTask(uid, TASK_LEVEL__SOURCE, pEpset, isFillhistory, useTriggerParam ? pStream->conf.triggerParam : 0,
*pTaskList, pStream->conf.fillHistory);
*pTaskList, pStream->conf.fillHistory, pStream->subTableWithoutMd5);
if (pTask == NULL) {
return NULL;
}
@ -463,7 +463,7 @@ static SStreamTask* buildAggTask(SStreamObj* pStream, SEpSet* pEpset, bool isFil
SStreamTask* pAggTask =
tNewStreamTask(uid, TASK_LEVEL__AGG, pEpset, isFillhistory, useTriggerParam ? pStream->conf.triggerParam : 0,
*pTaskList, pStream->conf.fillHistory);
*pTaskList, pStream->conf.fillHistory, pStream->subTableWithoutMd5);
if (pAggTask == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;

1
source/dnode/mnode/impl/src/mndSma.c
View File

@ -567,6 +567,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea
streamObj.conf.triggerParam = pCreate->maxDelay;
streamObj.ast = taosStrdup(smaObj.ast);
streamObj.indexForMultiAggBalance = -1;
streamObj.subTableWithoutMd5 = 1;
// check the maxDelay
if (streamObj.conf.triggerParam < TSDB_MIN_ROLLUP_MAX_DELAY) {

16
source/dnode/vnode/src/tq/tq.c
View File

@ -14,8 +14,8 @@
*/
#include "tq.h"
#include "vnd.h"
#include "tqCommon.h"
#include "vnd.h"
// 0: not init
// 1: already inited
@ -862,9 +862,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) {
return 0;
}
int32_t tqProcessTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) {
return tqStreamTaskProcessCheckReq(pTq->pStreamMeta, pMsg);
}
int32_t tqProcessTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) { return tqStreamTaskProcessCheckReq(pTq->pStreamMeta, pMsg); }
int32_t tqProcessTaskCheckRsp(STQ* pTq, SRpcMsg* pMsg) {
return tqStreamTaskProcessCheckRsp(pTq->pStreamMeta, pMsg, vnodeIsRoleLeader(pTq->pVnode));
@ -988,7 +986,7 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
streamReExecScanHistoryFuture(pTask, retInfo.idleTime);
} else {
SStreamTaskState* p = streamTaskGetStatus(pTask);
ETaskStatus s = p->state;
ETaskStatus s = p->state;
if (s == TASK_STATUS__PAUSE) {
tqDebug("s-task:%s is paused in the step1, elapsed time:%.2fs total:%.2fs, sched-status:%d", pTask->id.idStr,
@ -1062,7 +1060,7 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) {
}
int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg) {
return tqStreamTaskProcessDispatchReq(pTq->pStreamMeta, pMsg);
return tqStreamTaskProcessDispatchReq(pTq->pStreamMeta, pMsg);
}
int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) {
@ -1101,7 +1099,7 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
pRsp->info.handle = NULL;
SStreamCheckpointSourceReq req = {0};
SDecoder decoder;
SDecoder decoder;
tDecoderInit(&decoder, (uint8_t*)msg, len);
if (tDecodeStreamCheckpointSourceReq(&decoder, &req) < 0) {
code = TSDB_CODE_MSG_DECODE_ERROR;
@ -1192,8 +1190,8 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
streamProcessCheckpointSourceReq(pTask, &req);
taosThreadMutexUnlock(&pTask->lock);
qInfo("s-task:%s (vgId:%d) level:%d receive checkpoint-source msg chkpt:%" PRId64 ", transId:%d",
pTask->id.idStr, vgId, pTask->info.taskLevel, req.checkpointId, req.transId);
qInfo("s-task:%s (vgId:%d) level:%d receive checkpoint-source msg chkpt:%" PRId64 ", transId:%d", pTask->id.idStr,
vgId, pTask->info.taskLevel, req.checkpointId, req.transId);
code = streamAddCheckpointSourceRspMsg(&req, &pMsg->info, pTask, 1);
if (code != TSDB_CODE_SUCCESS) {

10
source/dnode/vnode/src/tq/tqHandleSnapshot.c
View File

@ -74,11 +74,11 @@ int32_t tqSnapReaderClose(STqSnapReader** ppReader) {
}
int32_t tqSnapRead(STqSnapReader* pReader, uint8_t** ppData) {
int32_t code = 0;
void* pKey = NULL;
void* pVal = NULL;
int32_t kLen = 0;
int32_t vLen = 0;
int32_t code = 0;
void* pKey = NULL;
void* pVal = NULL;
int32_t kLen = 0;
int32_t vLen = 0;
if (tdbTbcNext(pReader->pCur, &pKey, &kLen, &pVal, &vLen)) {
goto _exit;

63
source/dnode/vnode/src/tq/tqRead.c
View File

@ -207,8 +207,8 @@ int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, uint64_t
goto END;
}
tqDebug("vgId:%d, consumer:0x%" PRIx64 " taosx get msg ver %" PRId64 ", type: %s, reqId:0x%" PRIx64" 0x%"PRIx64, vgId,
pHandle->consumerId, offset, TMSG_INFO(pHandle->pWalReader->pHead->head.msgType), reqId, id);
tqDebug("vgId:%d, consumer:0x%" PRIx64 " taosx get msg ver %" PRId64 ", type: %s, reqId:0x%" PRIx64 " 0x%" PRIx64,
vgId, pHandle->consumerId, offset, TMSG_INFO(pHandle->pWalReader->pHead->head.msgType), reqId, id);
if (pHandle->pWalReader->pHead->head.msgType == TDMT_VND_SUBMIT) {
code = walFetchBody(pHandle->pWalReader);
@ -303,7 +303,7 @@ int32_t tqReaderSeek(STqReader* pReader, int64_t ver, const char* id) {
int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, int64_t maxVer, const char* id) {
int32_t code = 0;
while(1) {
while (1) {
code = walNextValidMsg(pReader);
if (code != TSDB_CODE_SUCCESS) {
return code;
@ -322,7 +322,8 @@ int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, int64_t maxVer, con
void* data = taosMemoryMalloc(len);
if (data == NULL) {
// todo: for all stream in this vnode, keep this offset in the offset files, and wait for a moment, and then retry
// todo: for all stream in this vnode, keep this offset in the offset files, and wait for a moment, and then
// retry
code = TSDB_CODE_OUT_OF_MEMORY;
terrno = code;
@ -369,7 +370,7 @@ int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, int64_t maxVer, con
// todo ignore the error in wal?
bool tqNextBlockInWal(STqReader* pReader, const char* id, int sourceExcluded) {
SWalReader* pWalReader = pReader->pWalReader;
SWalReader* pWalReader = pReader->pWalReader;
SSDataBlock* pDataBlock = NULL;
uint64_t st = taosGetTimestampMs();
@ -387,22 +388,22 @@ bool tqNextBlockInWal(STqReader* pReader, const char* id, int sourceExcluded) {
pReader->nextBlk = 0;
int32_t numOfBlocks = taosArrayGetSize(pReader->submit.aSubmitTbData);
while (pReader->nextBlk < numOfBlocks) {
tqTrace("tq reader next data block %d/%d, len:%d %" PRId64, pReader->nextBlk,
numOfBlocks, pReader->msg.msgLen, pReader->msg.ver);
tqTrace("tq reader next data block %d/%d, len:%d %" PRId64, pReader->nextBlk, numOfBlocks, pReader->msg.msgLen,
pReader->msg.ver);
SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk);
if ((pSubmitTbData->source & sourceExcluded) != 0){
if ((pSubmitTbData->source & sourceExcluded) != 0) {
pReader->nextBlk += 1;
continue;
}
if (pReader->tbIdHash == NULL || taosHashGet(pReader->tbIdHash, &pSubmitTbData->uid, sizeof(int64_t)) != NULL) {
tqTrace("tq reader return submit block, uid:%" PRId64, pSubmitTbData->uid);
SSDataBlock* pRes = NULL;
int32_t code = tqRetrieveDataBlock(pReader, &pRes, NULL);
int32_t code = tqRetrieveDataBlock(pReader, &pRes, NULL);
if (code == TSDB_CODE_SUCCESS && pRes->info.rows > 0) {
if(pDataBlock == NULL){
if (pDataBlock == NULL) {
pDataBlock = createOneDataBlock(pRes, true);
}else{
} else {
blockDataMerge(pDataBlock, pRes);
}
}
@ -414,16 +415,16 @@ bool tqNextBlockInWal(STqReader* pReader, const char* id, int sourceExcluded) {
tDestroySubmitReq(&pReader->submit, TSDB_MSG_FLG_DECODE);
pReader->msg.msgStr = NULL;
if(pDataBlock != NULL){
if (pDataBlock != NULL) {
blockDataCleanup(pReader->pResBlock);
copyDataBlock(pReader->pResBlock, pDataBlock);
blockDataDestroy(pDataBlock);
return true;
}else{
} else {
qTrace("stream scan return empty, all %d submit blocks consumed, %s", numOfBlocks, id);
}
if(taosGetTimestampMs() - st > 1000){
if (taosGetTimestampMs() - st > 1000) {
return false;
}
}
@ -448,17 +449,11 @@ int32_t tqReaderSetSubmitMsg(STqReader* pReader, void* msgStr, int32_t msgLen, i
return 0;
}
SWalReader* tqGetWalReader(STqReader* pReader) {
return pReader->pWalReader;
}
SWalReader* tqGetWalReader(STqReader* pReader) { return pReader->pWalReader; }
SSDataBlock* tqGetResultBlock (STqReader* pReader) {
return pReader->pResBlock;
}
SSDataBlock* tqGetResultBlock(STqReader* pReader) { return pReader->pResBlock; }
int64_t tqGetResultBlockTime(STqReader *pReader){
return pReader->lastTs;
}
int64_t tqGetResultBlockTime(STqReader* pReader) { return pReader->lastTs; }
bool tqNextBlockImpl(STqReader* pReader, const char* idstr) {
if (pReader->msg.msgStr == NULL) {
@ -599,7 +594,7 @@ static int32_t doSetVal(SColumnInfoData* pColumnInfoData, int32_t rowIndex, SCol
if (IS_STR_DATA_TYPE(pColVal->type)) {
char val[65535 + 2] = {0};
if(COL_VAL_IS_VALUE(pColVal)){
if (COL_VAL_IS_VALUE(pColVal)) {
if (pColVal->value.pData != NULL) {
memcpy(varDataVal(val), pColVal->value.pData, pColVal->value.nData);
}
@ -874,7 +869,7 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas
sourceIdx++;
} else if (pCol->cid == pColData->info.colId) {
tColDataGetValue(pCol, i, &colVal);
if(doSetVal(pColData, curRow - lastRow, &colVal) != TDB_CODE_SUCCESS){
if (doSetVal(pColData, curRow - lastRow, &colVal) != TDB_CODE_SUCCESS) {
goto FAIL;
}
sourceIdx++;
@ -958,10 +953,10 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas
if (colVal.cid < pColData->info.colId) {
sourceIdx++;
} else if (colVal.cid == pColData->info.colId) {
if(doSetVal(pColData, curRow - lastRow, &colVal) != TDB_CODE_SUCCESS){
if (doSetVal(pColData, curRow - lastRow, &colVal) != TDB_CODE_SUCCESS) {
goto FAIL;
}
sourceIdx++;
sourceIdx++;
targetIdx++;
}
}
@ -1001,7 +996,7 @@ int tqReaderSetTbUidList(STqReader* pReader, const SArray* tbUidList, const char
taosHashPut(pReader->tbIdHash, pKey, sizeof(int64_t), NULL, 0);
}
tqDebug("s-task:%s %d tables are set to be queried target table", id, (int32_t) taosArrayGetSize(tbUidList));
tqDebug("s-task:%s %d tables are set to be queried target table", id, (int32_t)taosArrayGetSize(tbUidList));
return 0;
}
@ -1027,9 +1022,7 @@ bool tqReaderIsQueriedTable(STqReader* pReader, uint64_t uid) {
return taosHashGet(pReader->tbIdHash, &uid, sizeof(uint64_t));
}
bool tqCurrentBlockConsumed(const STqReader* pReader) {
return pReader->msg.msgStr == NULL;
}
bool tqCurrentBlockConsumed(const STqReader* pReader) { return pReader->msg.msgStr == NULL; }
int tqReaderRemoveTbUidList(STqReader* pReader, const SArray* tbUidList) {
for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) {
@ -1071,9 +1064,11 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
} else if (pTqHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) {
if (isAdd) {
SArray* list = NULL;
int ret = qGetTableList(pTqHandle->execHandle.execTb.suid, pTq->pVnode, pTqHandle->execHandle.execTb.node, &list, pTqHandle->execHandle.task);
if(ret != TDB_CODE_SUCCESS) {
tqError("qGetTableList in tqUpdateTbUidList error:%d handle %s consumer:0x%" PRIx64, ret, pTqHandle->subKey, pTqHandle->consumerId);
int ret = qGetTableList(pTqHandle->execHandle.execTb.suid, pTq->pVnode, pTqHandle->execHandle.execTb.node,
&list, pTqHandle->execHandle.task);
if (ret != TDB_CODE_SUCCESS) {
tqError("qGetTableList in tqUpdateTbUidList error:%d handle %s consumer:0x%" PRIx64, ret, pTqHandle->subKey,
pTqHandle->consumerId);
taosArrayDestroy(list);
taosHashCancelIterate(pTq->pHandle, pIter);
taosWUnLockLatch(&pTq->lock);

33
source/dnode/vnode/src/tq/tqScan.c
View File

@ -63,8 +63,8 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, STaosxRsp* pRsp, in
return 0;
}
int32_t getDataBlock(qTaskInfo_t task, const STqHandle* pHandle, int32_t vgId, SSDataBlock** res){
uint64_t ts = 0;
int32_t getDataBlock(qTaskInfo_t task, const STqHandle* pHandle, int32_t vgId, SSDataBlock** res) {
uint64_t ts = 0;
qStreamSetOpen(task);
tqDebug("consumer:0x%" PRIx64 " vgId:%d, tmq one task start execute", pHandle->consumerId, vgId);
@ -97,27 +97,27 @@ int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal*
while (1) {
SSDataBlock* pDataBlock = NULL;
code = getDataBlock(task, pHandle, vgId, &pDataBlock);
if (code != 0){
if (code != 0) {
return code;
}
if(pRequest->enableReplay){
if(IS_OFFSET_RESET_TYPE(pRequest->reqOffset.type) && pHandle->block != NULL){
if (pRequest->enableReplay) {
if (IS_OFFSET_RESET_TYPE(pRequest->reqOffset.type) && pHandle->block != NULL) {
blockDataDestroy(pHandle->block);
pHandle->block = NULL;
}
if(pHandle->block == NULL){
if (pHandle->block == NULL) {
if (pDataBlock == NULL) {
break;
}
STqOffsetVal offset = {0};
qStreamExtractOffset(task, &offset);
pHandle->block = createOneDataBlock(pDataBlock, true);
// pHandle->block = createDataBlock();
// copyDataBlock(pHandle->block, pDataBlock);
// pHandle->block = createDataBlock();
// copyDataBlock(pHandle->block, pDataBlock);
pHandle->blockTime = offset.ts;
code = getDataBlock(task, pHandle, vgId, &pDataBlock);
if (code != 0){
if (code != 0) {
return code;
}
}
@ -132,7 +132,7 @@ int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal*
if (pDataBlock == NULL) {
blockDataDestroy(pHandle->block);
pHandle->block = NULL;
}else{
} else {
copyDataBlock(pHandle->block, pDataBlock);
STqOffsetVal offset = {0};
@ -141,7 +141,7 @@ int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal*
pHandle->blockTime = offset.ts;
}
break;
}else{
} else {
if (pDataBlock == NULL) {
break;
}
@ -250,7 +250,8 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta
return 0;
}
int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp, int32_t* totalRows, int8_t sourceExcluded) {
int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp, int32_t* totalRows,
int8_t sourceExcluded) {
STqExecHandle* pExec = &pHandle->execHandle;
SArray* pBlocks = taosArrayInit(0, sizeof(SSDataBlock));
SArray* pSchemas = taosArrayInit(0, sizeof(void*));
@ -266,7 +267,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR
if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) goto loop_table;
}
if ((pSubmitTbDataRet->source & sourceExcluded) != 0){
if ((pSubmitTbDataRet->source & sourceExcluded) != 0) {
goto loop_table;
}
if (pRsp->withTbName) {
@ -303,7 +304,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR
tEncoderClear(&encoder);
}
if (pHandle->fetchMeta == ONLY_META && pSubmitTbDataRet->pCreateTbReq == NULL){
if (pHandle->fetchMeta == ONLY_META && pSubmitTbDataRet->pCreateTbReq == NULL) {
goto loop_table;
}
for (int32_t i = 0; i < taosArrayGetSize(pBlocks); i++) {
@ -334,7 +335,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR
if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) goto loop_db;
}
if ((pSubmitTbDataRet->source & sourceExcluded) != 0){
if ((pSubmitTbDataRet->source & sourceExcluded) != 0) {
goto loop_db;
}
if (pRsp->withTbName) {
@ -371,7 +372,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR
tEncoderClear(&encoder);
}
if (pHandle->fetchMeta == ONLY_META && pSubmitTbDataRet->pCreateTbReq == NULL){
if (pHandle->fetchMeta == ONLY_META && pSubmitTbDataRet->pCreateTbReq == NULL) {
goto loop_db;
}
for (int32_t i = 0; i < taosArrayGetSize(pBlocks); i++) {

13
source/dnode/vnode/src/tq/tqSink.c
View File

@ -262,7 +262,7 @@ static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, S
}
setCreateTableMsgTableName(pCreateTbReq, pDataBlock, stbFullName, gid,
pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER);
pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1);
taosArrayPush(reqs.pArray, pCreateTbReq);
tqDebug("s-task:%s build create table:%s msg complete", pTask->id.idStr, pCreateTbReq->name);
@ -374,7 +374,7 @@ int32_t doBuildAndSendDeleteMsg(SVnode* pVnode, char* stbFullName, SSDataBlock*
SBatchDeleteReq deleteReq = {.suid = suid, .deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq))};
int32_t code = tqBuildDeleteReq(pVnode->pTq, stbFullName, pDataBlock, &deleteReq, pTask->id.idStr,
pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER);
pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
@ -668,8 +668,8 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
memset(dstTableName, 0, TSDB_TABLE_NAME_LEN);
buildCtbNameByGroupIdImpl(stbFullName, groupId, dstTableName);
} else {
if (pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && !isAutoTableName(dstTableName) &&
!alreadyAddGroupId(dstTableName) && groupId != 0) {
if (pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1 &&
!isAutoTableName(dstTableName) && !alreadyAddGroupId(dstTableName) && groupId != 0) {
buildCtbNameAddGruopId(dstTableName, groupId);
}
}
@ -712,8 +712,9 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
SArray* pTagArray = taosArrayInit(pTSchema->numOfCols + 1, sizeof(STagVal));
pTableData->flags = SUBMIT_REQ_AUTO_CREATE_TABLE;
pTableData->pCreateTbReq = buildAutoCreateTableReq(stbFullName, suid, pTSchema->numOfCols + 1, pDataBlock,
pTagArray, pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER);
pTableData->pCreateTbReq =
buildAutoCreateTableReq(stbFullName, suid, pTSchema->numOfCols + 1, pDataBlock, pTagArray,
pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1);
taosArrayDestroy(pTagArray);
if (pTableData->pCreateTbReq == NULL) {

2
source/libs/parser/inc/sql.y
View File

@ -516,7 +516,7 @@ cmd ::= SHOW VNODES.
// show alive
cmd ::= SHOW db_name_cond_opt(A) ALIVE. { pCxt->pRootNode = createShowAliveStmt(pCxt, A, QUERY_NODE_SHOW_DB_ALIVE_STMT); }
cmd ::= SHOW CLUSTER ALIVE. { pCxt->pRootNode = createShowAliveStmt(pCxt, NULL, QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT); }
cmd ::= SHOW db_name_cond_opt(A) VIEWS. { pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_VIEWS_STMT, A, NULL, OP_TYPE_LIKE); }
cmd ::= SHOW db_name_cond_opt(A) VIEWS like_pattern_opt(B). { pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_VIEWS_STMT, A, B, OP_TYPE_LIKE); }
cmd ::= SHOW CREATE VIEW full_table_name(A). { pCxt->pRootNode = createShowCreateViewStmt(pCxt, QUERY_NODE_SHOW_CREATE_VIEW_STMT, A); }
cmd ::= SHOW COMPACTS. { pCxt->pRootNode = createShowCompactsStmt(pCxt, QUERY_NODE_SHOW_COMPACTS_STMT); }
cmd ::= SHOW COMPACT NK_INTEGER(A). { pCxt->pRootNode = createShowCompactDetailsStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &A)); }

15
source/libs/parser/src/parTranslater.c
View File

@ -9399,7 +9399,20 @@ static int32_t createOperatorNode(EOperatorType opType, const char* pColName, SN
}
static const char* getTbNameColName(ENodeType type) {
return (QUERY_NODE_SHOW_STABLES_STMT == type ? "stable_name" : "table_name");
const char* colName;
switch (type)
{
case QUERY_NODE_SHOW_VIEWS_STMT:
colName = "view_name";
break;
case QUERY_NODE_SHOW_STABLES_STMT:
colName = "stable_name";
break;
default:
colName = "table_name";
break;
}
return colName;
}
static int32_t createLogicCondNode(SNode* pCond1, SNode* pCond2, SNode** pCond, ELogicConditionType logicCondType) {

2521
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

2
source/libs/scheduler/src/schJob.c
View File

@ -66,7 +66,7 @@ FORCE_INLINE bool schJobNeedToStop(SSchJob *pJob, int8_t *pStatus) {
return true;
}
if ((*pJob->chkKillFp)(pJob->chkKillParam)) {
if (pJob->chkKillFp && (*pJob->chkKillFp)(pJob->chkKillParam)) {
schUpdateJobErrCode(pJob, TSDB_CODE_TSC_QUERY_KILLED);
return true;
}

469
source/libs/scheduler/test/schedulerTests.cpp
View File

@ -54,9 +54,8 @@
namespace {
extern "C" int32_t schHandleResponseMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *msg, int32_t msgSize,
int32_t rspCode);
extern "C" int32_t schHandleCallback(void *param, const SDataBuf *pMsg, int32_t msgType, int32_t rspCode);
extern "C" int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDataBuf *pMsg, int32_t rspCode);
extern "C" int32_t schHandleCallback(void *param, const SDataBuf *pMsg, int32_t rspCode);
int64_t insertJobRefId = 0;
int64_t queryJobRefId = 0;
@ -67,7 +66,7 @@ uint64_t schtQueryId = 1;
bool schtTestStop = false;
bool schtTestDeadLoop = false;
int32_t schtTestMTRunSec = 10;
int32_t schtTestMTRunSec = 1;
int32_t schtTestPrintNum = 1000;
int32_t schtStartFetch = 0;
@ -85,10 +84,69 @@ void schtInitLogFile() {
}
void schtQueryCb(SExecResult *pResult, void *param, int32_t code) {
assert(TSDB_CODE_SUCCESS == code);
*(int32_t *)param = 1;
}
int32_t schtBuildQueryRspMsg(uint32_t *msize, void** rspMsg) {
SQueryTableRsp rsp = {0};
rsp.code = 0;
rsp.affectedRows = 0;
rsp.tbVerInfo = NULL;
int32_t msgSize = tSerializeSQueryTableRsp(NULL, 0, &rsp);
if (msgSize < 0) {
qError("tSerializeSQueryTableRsp failed");
return TSDB_CODE_OUT_OF_MEMORY;
}
void *pRsp = taosMemoryCalloc(msgSize, 1);
if (NULL == pRsp) {
qError("rpcMallocCont %d failed", msgSize);
return TSDB_CODE_OUT_OF_MEMORY;
}
if (tSerializeSQueryTableRsp(pRsp, msgSize, &rsp) < 0) {
qError("tSerializeSQueryTableRsp %d failed", msgSize);
return TSDB_CODE_OUT_OF_MEMORY;
}
*rspMsg = pRsp;
*msize = msgSize;
return TSDB_CODE_SUCCESS;
}
int32_t schtBuildFetchRspMsg(uint32_t *msize, void** rspMsg) {
SRetrieveTableRsp* rsp = (SRetrieveTableRsp*)taosMemoryCalloc(sizeof(SRetrieveTableRsp), 1);
rsp->completed = 1;
rsp->numOfRows = 10;
rsp->compLen = 0;
*rspMsg = rsp;
*msize = sizeof(SRetrieveTableRsp);
return TSDB_CODE_SUCCESS;
}
int32_t schtBuildSubmitRspMsg(uint32_t *msize, void** rspMsg) {
SSubmitRsp2 submitRsp = {0};
int32_t msgSize = 0, ret = 0;
SEncoder ec = {0};
tEncodeSize(tEncodeSSubmitRsp2, &submitRsp, msgSize, ret);
void* msg = taosMemoryCalloc(1, msgSize);
tEncoderInit(&ec, (uint8_t*)msg, msgSize);
tEncodeSSubmitRsp2(&ec, &submitRsp);
tEncoderClear(&ec);
*rspMsg = msg;
*msize = msgSize;
return TSDB_CODE_SUCCESS;
}
void schtBuildQueryDag(SQueryPlan *dag) {
uint64_t qId = schtQueryId;
@ -98,8 +156,8 @@ void schtBuildQueryDag(SQueryPlan *dag) {
SNodeListNode *scan = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST);
SNodeListNode *merge = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST);
SSubplan *scanPlan = (SSubplan *)taosMemoryCalloc(1, sizeof(SSubplan));
SSubplan *mergePlan = (SSubplan *)taosMemoryCalloc(1, sizeof(SSubplan));
SSubplan *scanPlan = (SSubplan*)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN);
SSubplan *mergePlan = (SSubplan*)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN);
scanPlan->id.queryId = qId;
scanPlan->id.groupId = 0x0000000000000002;
@ -113,7 +171,7 @@ void schtBuildQueryDag(SQueryPlan *dag) {
scanPlan->pChildren = NULL;
scanPlan->level = 1;
scanPlan->pParents = nodesMakeList();
scanPlan->pNode = (SPhysiNode *)taosMemoryCalloc(1, sizeof(SPhysiNode));
scanPlan->pNode = (SPhysiNode *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN);
scanPlan->msgType = TDMT_SCH_QUERY;
mergePlan->id.queryId = qId;
@ -125,7 +183,7 @@ void schtBuildQueryDag(SQueryPlan *dag) {
mergePlan->pChildren = nodesMakeList();
mergePlan->pParents = NULL;
mergePlan->pNode = (SPhysiNode *)taosMemoryCalloc(1, sizeof(SPhysiNode));
mergePlan->pNode = (SPhysiNode *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_MERGE);
mergePlan->msgType = TDMT_SCH_QUERY;
merge->pNodeList = nodesMakeList();
@ -151,8 +209,7 @@ void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) {
SNodeListNode *scan = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST);
SNodeListNode *merge = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST);
SSubplan *scanPlan = (SSubplan *)taosMemoryCalloc(scanPlanNum, sizeof(SSubplan));
SSubplan *mergePlan = (SSubplan *)taosMemoryCalloc(1, sizeof(SSubplan));
SSubplan *mergePlan = (SSubplan*)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN);
merge->pNodeList = nodesMakeList();
scan->pNodeList = nodesMakeList();
@ -160,29 +217,30 @@ void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) {
mergePlan->pChildren = nodesMakeList();
for (int32_t i = 0; i < scanPlanNum; ++i) {
scanPlan[i].id.queryId = qId;
scanPlan[i].id.groupId = 0x0000000000000002;
scanPlan[i].id.subplanId = 0x0000000000000003 + i;
scanPlan[i].subplanType = SUBPLAN_TYPE_SCAN;
SSubplan *scanPlan = (SSubplan*)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN);
scanPlan->id.queryId = qId;
scanPlan->id.groupId = 0x0000000000000002;
scanPlan->id.subplanId = 0x0000000000000003 + i;
scanPlan->subplanType = SUBPLAN_TYPE_SCAN;
scanPlan[i].execNode.nodeId = 1 + i;
scanPlan[i].execNode.epSet.inUse = 0;
scanPlan[i].execNodeStat.tableNum = taosRand() % 30;
addEpIntoEpSet(&scanPlan[i].execNode.epSet, "ep0", 6030);
addEpIntoEpSet(&scanPlan[i].execNode.epSet, "ep1", 6030);
addEpIntoEpSet(&scanPlan[i].execNode.epSet, "ep2", 6030);
scanPlan[i].execNode.epSet.inUse = taosRand() % 3;
scanPlan->execNode.nodeId = 1 + i;
scanPlan->execNode.epSet.inUse = 0;
scanPlan->execNodeStat.tableNum = taosRand() % 30;
addEpIntoEpSet(&scanPlan->execNode.epSet, "ep0", 6030);
addEpIntoEpSet(&scanPlan->execNode.epSet, "ep1", 6030);
addEpIntoEpSet(&scanPlan->execNode.epSet, "ep2", 6030);
scanPlan->execNode.epSet.inUse = taosRand() % 3;
scanPlan[i].pChildren = NULL;
scanPlan[i].level = 1;
scanPlan[i].pParents = nodesMakeList();
scanPlan[i].pNode = (SPhysiNode *)taosMemoryCalloc(1, sizeof(SPhysiNode));
scanPlan[i].msgType = TDMT_SCH_QUERY;
scanPlan->pChildren = NULL;
scanPlan->level = 1;
scanPlan->pParents = nodesMakeList();
scanPlan->pNode = (SPhysiNode *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN);
scanPlan->msgType = TDMT_SCH_QUERY;
nodesListAppend(scanPlan[i].pParents, (SNode *)mergePlan);
nodesListAppend(mergePlan->pChildren, (SNode *)(scanPlan + i));
nodesListAppend(scanPlan->pParents, (SNode *)mergePlan);
nodesListAppend(mergePlan->pChildren, (SNode *)scanPlan);
nodesListAppend(scan->pNodeList, (SNode *)(scanPlan + i));
nodesListAppend(scan->pNodeList, (SNode *)scanPlan);
}
mergePlan->id.queryId = qId;
@ -193,7 +251,7 @@ void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) {
mergePlan->execNode.epSet.numOfEps = 0;
mergePlan->pParents = NULL;
mergePlan->pNode = (SPhysiNode *)taosMemoryCalloc(1, sizeof(SPhysiNode));
mergePlan->pNode = (SPhysiNode *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_MERGE);
mergePlan->msgType = TDMT_SCH_QUERY;
nodesListAppend(merge->pNodeList, (SNode *)mergePlan);
@ -211,45 +269,50 @@ void schtBuildInsertDag(SQueryPlan *dag) {
dag->numOfSubplans = 2;
dag->pSubplans = nodesMakeList();
SNodeListNode *inserta = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST);
SSubplan *insertPlan = (SSubplan *)taosMemoryCalloc(2, sizeof(SSubplan));
insertPlan[0].id.queryId = qId;
insertPlan[0].id.groupId = 0x0000000000000003;
insertPlan[0].id.subplanId = 0x0000000000000004;
insertPlan[0].subplanType = SUBPLAN_TYPE_MODIFY;
insertPlan[0].level = 0;
insertPlan[0].execNode.nodeId = 1;
insertPlan[0].execNode.epSet.inUse = 0;
addEpIntoEpSet(&insertPlan[0].execNode.epSet, "ep0", 6030);
insertPlan[0].pChildren = NULL;
insertPlan[0].pParents = NULL;
insertPlan[0].pNode = NULL;
insertPlan[0].pDataSink = (SDataSinkNode *)taosMemoryCalloc(1, sizeof(SDataSinkNode));
insertPlan[0].msgType = TDMT_VND_SUBMIT;
insertPlan[1].id.queryId = qId;
insertPlan[1].id.groupId = 0x0000000000000003;
insertPlan[1].id.subplanId = 0x0000000000000005;
insertPlan[1].subplanType = SUBPLAN_TYPE_MODIFY;
insertPlan[1].level = 0;
insertPlan[1].execNode.nodeId = 1;
insertPlan[1].execNode.epSet.inUse = 0;
addEpIntoEpSet(&insertPlan[1].execNode.epSet, "ep0", 6030);
insertPlan[1].pChildren = NULL;
insertPlan[1].pParents = NULL;
insertPlan[1].pNode = NULL;
insertPlan[1].pDataSink = (SDataSinkNode *)taosMemoryCalloc(1, sizeof(SDataSinkNode));
insertPlan[1].msgType = TDMT_VND_SUBMIT;
inserta->pNodeList = nodesMakeList();
SSubplan *insertPlan = (SSubplan*)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN);
insertPlan->id.queryId = qId;
insertPlan->id.groupId = 0x0000000000000003;
insertPlan->id.subplanId = 0x0000000000000004;
insertPlan->subplanType = SUBPLAN_TYPE_MODIFY;
insertPlan->level = 0;
insertPlan->execNode.nodeId = 1;
insertPlan->execNode.epSet.inUse = 0;
addEpIntoEpSet(&insertPlan->execNode.epSet, "ep0", 6030);
insertPlan->pChildren = NULL;
insertPlan->pParents = NULL;
insertPlan->pNode = NULL;
insertPlan->pDataSink = (SDataSinkNode*)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_INSERT);
((SDataInserterNode*)insertPlan->pDataSink)->size = 1;
((SDataInserterNode*)insertPlan->pDataSink)->pData = taosMemoryCalloc(1, 1);
insertPlan->msgType = TDMT_VND_SUBMIT;
nodesListAppend(inserta->pNodeList, (SNode *)insertPlan);
insertPlan += 1;
insertPlan = (SSubplan*)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN);
insertPlan->id.queryId = qId;
insertPlan->id.groupId = 0x0000000000000003;
insertPlan->id.subplanId = 0x0000000000000005;
insertPlan->subplanType = SUBPLAN_TYPE_MODIFY;
insertPlan->level = 0;
insertPlan->execNode.nodeId = 1;
insertPlan->execNode.epSet.inUse = 0;
addEpIntoEpSet(&insertPlan->execNode.epSet, "ep0", 6030);
insertPlan->pChildren = NULL;
insertPlan->pParents = NULL;
insertPlan->pNode = NULL;
insertPlan->pDataSink = (SDataSinkNode*)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_INSERT);
((SDataInserterNode*)insertPlan->pDataSink)->size = 1;
((SDataInserterNode*)insertPlan->pDataSink)->pData = taosMemoryCalloc(1, 1);
insertPlan->msgType = TDMT_VND_SUBMIT;
nodesListAppend(inserta->pNodeList, (SNode *)insertPlan);
nodesListAppend(dag->pSubplans, (SNode *)inserta);
@ -325,7 +388,7 @@ void schtSetRpcSendRequest() {
}
}
int32_t schtAsyncSendMsgToServer(void *pTransporter, SEpSet *epSet, int64_t *pTransporterId, SMsgSendInfo *pInfo) {
int32_t schtAsyncSendMsgToServer(void *pTransporter, SEpSet *epSet, int64_t *pTransporterId, SMsgSendInfo *pInfo, bool persistHandle, void* rpcCtx) {
if (pInfo) {
taosMemoryFreeClear(pInfo->param);
taosMemoryFreeClear(pInfo->msgInfo.pData);
@ -336,17 +399,17 @@ int32_t schtAsyncSendMsgToServer(void *pTransporter, SEpSet *epSet, int64_t *pTr
void schtSetAsyncSendMsgToServer() {
static Stub stub;
stub.set(asyncSendMsgToServer, schtAsyncSendMsgToServer);
stub.set(asyncSendMsgToServerExt, schtAsyncSendMsgToServer);
{
#ifdef WINDOWS
AddrAny any;
std::map<std::string, void *> result;
any.get_func_addr("asyncSendMsgToServer", result);
any.get_func_addr("asyncSendMsgToServerExt", result);
#endif
#ifdef LINUX
AddrAny any("libtransport.so");
std::map<std::string, void *> result;
any.get_global_func_addr_dynsym("^asyncSendMsgToServer$", result);
any.get_global_func_addr_dynsym("^asyncSendMsgToServerExt$", result);
#endif
for (const auto &f : result) {
stub.set(f.second, schtAsyncSendMsgToServer);
@ -374,9 +437,13 @@ void *schtSendRsp(void *param) {
while (pIter) {
SSchTask *task = *(SSchTask **)pIter;
SSubmitRsp rsp = {0};
rsp.affectedRows = 10;
schHandleResponseMsg(pJob, task, TDMT_VND_SUBMIT_RSP, (char *)&rsp, sizeof(rsp), 0);
SDataBuf msg = {0};
void* rmsg = NULL;
schtBuildSubmitRspMsg(&msg.len, &rmsg);
msg.msgType = TDMT_VND_SUBMIT_RSP;
msg.pData = rmsg;
schHandleResponseMsg(pJob, task, task->execId, &msg, 0);
pIter = taosHashIterate(pJob->execTasks, pIter);
}
@ -393,11 +460,13 @@ void *schtCreateFetchRspThread(void *param) {
taosSsleep(1);
int32_t code = 0;
SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, sizeof(SRetrieveTableRsp));
rsp->completed = 1;
rsp->numOfRows = 10;
SDataBuf msg = {0};
void* rmsg = NULL;
schtBuildFetchRspMsg(&msg.len, &rmsg);
msg.msgType = TDMT_SCH_MERGE_FETCH_RSP;
msg.pData = rmsg;
code = schHandleResponseMsg(pJob, pJob->fetchTask, TDMT_SCH_FETCH_RSP, (char *)rsp, sizeof(*rsp), 0);
code = schHandleResponseMsg(pJob, pJob->fetchTask, pJob->fetchTask->execId, &msg, 0);
schReleaseJob(job);
@ -414,7 +483,7 @@ void *schtFetchRspThread(void *aa) {
continue;
}
taosUsleep(1);
taosUsleep(100);
param = (SSchTaskCallbackParam *)taosMemoryCalloc(1, sizeof(*param));
@ -426,10 +495,11 @@ void *schtFetchRspThread(void *aa) {
rsp->completed = 1;
rsp->numOfRows = 10;
dataBuf.msgType = TDMT_SCH_FETCH_RSP;
dataBuf.pData = rsp;
dataBuf.len = sizeof(*rsp);
code = schHandleCallback(param, &dataBuf, TDMT_SCH_FETCH_RSP, 0);
code = schHandleCallback(param, &dataBuf, 0);
assert(code == 0 || code);
}
@ -456,7 +526,7 @@ void *schtRunJobThread(void *aa) {
char *dbname = "1.db1";
char *tablename = "table1";
SVgroupInfo vgInfo = {0};
SQueryPlan dag;
SQueryPlan* dag = (SQueryPlan*)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN);
schtInitLogFile();
@ -470,19 +540,19 @@ void *schtRunJobThread(void *aa) {
SSchJob *pJob = NULL;
SSchTaskCallbackParam *param = NULL;
SHashObj *execTasks = NULL;
SDataBuf dataBuf = {0};
uint32_t jobFinished = 0;
int32_t queryDone = 0;
while (!schtTestStop) {
schtBuildQueryDag(&dag);
schtBuildQueryDag(dag);
SArray *qnodeList = taosArrayInit(1, sizeof(SEp));
SArray *qnodeList = taosArrayInit(1, sizeof(SQueryNodeLoad));
SEp qnodeAddr = {0};
strcpy(qnodeAddr.fqdn, "qnode0.ep");
qnodeAddr.port = 6031;
taosArrayPush(qnodeList, &qnodeAddr);
SQueryNodeLoad load = {0};
load.addr.epSet.numOfEps = 1;
strcpy(load.addr.epSet.eps[0].fqdn, "qnode0.ep");
load.addr.epSet.eps[0].port = 6031;
taosArrayPush(qnodeList, &load);
queryDone = 0;
@ -492,7 +562,7 @@ void *schtRunJobThread(void *aa) {
req.syncReq = false;
req.pConn = &conn;
req.pNodeList = qnodeList;
req.pDag = &dag;
req.pDag = dag;
req.sql = "select * from tb";
req.execFp = schtQueryCb;
req.cbParam = &queryDone;
@ -503,7 +573,7 @@ void *schtRunJobThread(void *aa) {
pJob = schAcquireJob(queryJobRefId);
if (NULL == pJob) {
taosArrayDestroy(qnodeList);
schtFreeQueryDag(&dag);
schtFreeQueryDag(dag);
continue;
}
@ -526,11 +596,14 @@ void *schtRunJobThread(void *aa) {
SSchTask *task = (SSchTask *)pIter;
param->taskId = task->taskId;
SQueryTableRsp rsp = {0};
dataBuf.pData = &rsp;
dataBuf.len = sizeof(rsp);
code = schHandleCallback(param, &dataBuf, TDMT_SCH_QUERY_RSP, 0);
SDataBuf msg = {0};
void* rmsg = NULL;
schtBuildQueryRspMsg(&msg.len, &rmsg);
msg.msgType = TDMT_SCH_QUERY_RSP;
msg.pData = rmsg;
code = schHandleCallback(param, &msg, 0);
assert(code == 0 || code);
pIter = taosHashIterate(execTasks, pIter);
@ -545,11 +618,13 @@ void *schtRunJobThread(void *aa) {
SSchTask *task = (SSchTask *)pIter;
param->taskId = task->taskId - 1;
SQueryTableRsp rsp = {0};
dataBuf.pData = &rsp;
dataBuf.len = sizeof(rsp);
SDataBuf msg = {0};
void* rmsg = NULL;
schtBuildQueryRspMsg(&msg.len, &rmsg);
msg.msgType = TDMT_SCH_QUERY_RSP;
msg.pData = rmsg;
code = schHandleCallback(param, &dataBuf, TDMT_SCH_QUERY_RSP, 0);
code = schHandleCallback(param, &msg, 0);
assert(code == 0 || code);
pIter = taosHashIterate(execTasks, pIter);
@ -575,7 +650,6 @@ void *schtRunJobThread(void *aa) {
if (0 == code) {
SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)data;
assert(pRsp->completed == 1);
assert(pRsp->numOfRows == 10);
}
data = NULL;
@ -587,7 +661,7 @@ void *schtRunJobThread(void *aa) {
taosHashCleanup(execTasks);
taosArrayDestroy(qnodeList);
schtFreeQueryDag(&dag);
schtFreeQueryDag(dag);
if (++jobFinished % schtTestPrintNum == 0) {
printf("jobFinished:%d\n", jobFinished);
@ -609,6 +683,7 @@ void *schtFreeJobThread(void *aa) {
return NULL;
}
} // namespace
TEST(queryTest, normalCase) {
@ -618,21 +693,20 @@ TEST(queryTest, normalCase) {
char *tablename = "table1";
SVgroupInfo vgInfo = {0};
int64_t job = 0;
SQueryPlan dag;
SQueryPlan* dag = (SQueryPlan*)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN);
memset(&dag, 0, sizeof(dag));
SArray *qnodeList = taosArrayInit(1, sizeof(SQueryNodeLoad));
SArray *qnodeList = taosArrayInit(1, sizeof(SEp));
SEp qnodeAddr = {0};
strcpy(qnodeAddr.fqdn, "qnode0.ep");
qnodeAddr.port = 6031;
taosArrayPush(qnodeList, &qnodeAddr);
SQueryNodeLoad load = {0};
load.addr.epSet.numOfEps = 1;
strcpy(load.addr.epSet.eps[0].fqdn, "qnode0.ep");
load.addr.epSet.eps[0].port = 6031;
taosArrayPush(qnodeList, &load);
int32_t code = schedulerInit();
ASSERT_EQ(code, 0);
schtBuildQueryDag(&dag);
schtBuildQueryDag(dag);
schtSetPlanToString();
schtSetExecNode();
@ -645,7 +719,7 @@ TEST(queryTest, normalCase) {
SSchedulerReq req = {0};
req.pConn = &conn;
req.pNodeList = qnodeList;
req.pDag = &dag;
req.pDag = dag;
req.sql = "select * from tb";
req.execFp = schtQueryCb;
req.cbParam = &queryDone;
@ -659,8 +733,13 @@ TEST(queryTest, normalCase) {
while (pIter) {
SSchTask *task = *(SSchTask **)pIter;
SQueryTableRsp rsp = {0};
code = schHandleResponseMsg(pJob, task, TDMT_SCH_QUERY_RSP, (char *)&rsp, sizeof(rsp), 0);
SDataBuf msg = {0};
void* rmsg = NULL;
schtBuildQueryRspMsg(&msg.len, &rmsg);
msg.msgType = TDMT_SCH_QUERY_RSP;
msg.pData = rmsg;
code = schHandleResponseMsg(pJob, task, task->execId, &msg, 0);
ASSERT_EQ(code, 0);
pIter = taosHashIterate(pJob->execTasks, pIter);
@ -669,11 +748,18 @@ TEST(queryTest, normalCase) {
pIter = taosHashIterate(pJob->execTasks, NULL);
while (pIter) {
SSchTask *task = *(SSchTask **)pIter;
if (JOB_TASK_STATUS_EXEC == task->status) {
SDataBuf msg = {0};
void* rmsg = NULL;
schtBuildQueryRspMsg(&msg.len, &rmsg);
msg.msgType = TDMT_SCH_QUERY_RSP;
msg.pData = rmsg;
SQueryTableRsp rsp = {0};
code = schHandleResponseMsg(pJob, task, TDMT_SCH_QUERY_RSP, (char *)&rsp, sizeof(rsp), 0);
code = schHandleResponseMsg(pJob, task, task->execId, &msg, 0);
ASSERT_EQ(code, 0);
}
ASSERT_EQ(code, 0);
pIter = taosHashIterate(pJob->execTasks, pIter);
}
@ -703,18 +789,12 @@ TEST(queryTest, normalCase) {
ASSERT_EQ(pRsp->numOfRows, 10);
taosMemoryFreeClear(data);
data = NULL;
code = schedulerFetchRows(job, &req);
ASSERT_EQ(code, 0);
ASSERT_TRUE(data == NULL);
schReleaseJob(job);
schedulerDestroy();
schedulerFreeJob(&job, 0);
schtFreeQueryDag(&dag);
schedulerDestroy();
}
TEST(queryTest, readyFirstCase) {
@ -724,21 +804,20 @@ TEST(queryTest, readyFirstCase) {
char *tablename = "table1";
SVgroupInfo vgInfo = {0};
int64_t job = 0;
SQueryPlan dag;
SQueryPlan* dag = (SQueryPlan*)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN);
memset(&dag, 0, sizeof(dag));
SArray *qnodeList = taosArrayInit(1, sizeof(SQueryNodeLoad));
SArray *qnodeList = taosArrayInit(1, sizeof(SEp));
SEp qnodeAddr = {0};
strcpy(qnodeAddr.fqdn, "qnode0.ep");
qnodeAddr.port = 6031;
taosArrayPush(qnodeList, &qnodeAddr);
SQueryNodeLoad load = {0};
load.addr.epSet.numOfEps = 1;
strcpy(load.addr.epSet.eps[0].fqdn, "qnode0.ep");
load.addr.epSet.eps[0].port = 6031;
taosArrayPush(qnodeList, &load);
int32_t code = schedulerInit();
ASSERT_EQ(code, 0);
schtBuildQueryDag(&dag);
schtBuildQueryDag(dag);
schtSetPlanToString();
schtSetExecNode();
@ -751,7 +830,7 @@ TEST(queryTest, readyFirstCase) {
SSchedulerReq req = {0};
req.pConn = &conn;
req.pNodeList = qnodeList;
req.pDag = &dag;
req.pDag = dag;
req.sql = "select * from tb";
req.execFp = schtQueryCb;
req.cbParam = &queryDone;
@ -764,8 +843,13 @@ TEST(queryTest, readyFirstCase) {
while (pIter) {
SSchTask *task = *(SSchTask **)pIter;
SQueryTableRsp rsp = {0};
code = schHandleResponseMsg(pJob, task, TDMT_SCH_QUERY_RSP, (char *)&rsp, sizeof(rsp), 0);
SDataBuf msg = {0};
void* rmsg = NULL;
schtBuildQueryRspMsg(&msg.len, &rmsg);
msg.msgType = TDMT_SCH_QUERY_RSP;
msg.pData = rmsg;
code = schHandleResponseMsg(pJob, task, task->execId, &msg, 0);
ASSERT_EQ(code, 0);
pIter = taosHashIterate(pJob->execTasks, pIter);
@ -775,10 +859,18 @@ TEST(queryTest, readyFirstCase) {
while (pIter) {
SSchTask *task = *(SSchTask **)pIter;
SQueryTableRsp rsp = {0};
code = schHandleResponseMsg(pJob, task, TDMT_SCH_QUERY_RSP, (char *)&rsp, sizeof(rsp), 0);
if (JOB_TASK_STATUS_EXEC == task->status) {
SDataBuf msg = {0};
void* rmsg = NULL;
schtBuildQueryRspMsg(&msg.len, &rmsg);
msg.msgType = TDMT_SCH_QUERY_RSP;
msg.pData = rmsg;
code = schHandleResponseMsg(pJob, task, task->execId, &msg, 0);
ASSERT_EQ(code, 0);
}
ASSERT_EQ(code, 0);
pIter = taosHashIterate(pJob->execTasks, pIter);
}
@ -807,18 +899,11 @@ TEST(queryTest, readyFirstCase) {
ASSERT_EQ(pRsp->numOfRows, 10);
taosMemoryFreeClear(data);
data = NULL;
code = schedulerFetchRows(job, &req);
ASSERT_EQ(code, 0);
ASSERT_TRUE(data == NULL);
schReleaseJob(job);
schedulerFreeJob(&job, 0);
schtFreeQueryDag(&dag);
schedulerDestroy();
schedulerFreeJob(&job, 0);
}
TEST(queryTest, flowCtrlCase) {
@ -828,35 +913,39 @@ TEST(queryTest, flowCtrlCase) {
char *tablename = "table1";
SVgroupInfo vgInfo = {0};
int64_t job = 0;
SQueryPlan dag;
SQueryPlan* dag = (SQueryPlan*)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN);
schtInitLogFile();
taosSeedRand(taosGetTimestampSec());
SArray *qnodeList = taosArrayInit(1, sizeof(SEp));
SArray *qnodeList = taosArrayInit(1, sizeof(SQueryNodeLoad));
SQueryNodeLoad load = {0};
load.addr.epSet.numOfEps = 1;
strcpy(load.addr.epSet.eps[0].fqdn, "qnode0.ep");
load.addr.epSet.eps[0].port = 6031;
taosArrayPush(qnodeList, &load);
SEp qnodeAddr = {0};
strcpy(qnodeAddr.fqdn, "qnode0.ep");
qnodeAddr.port = 6031;
taosArrayPush(qnodeList, &qnodeAddr);
int32_t code = schedulerInit();
ASSERT_EQ(code, 0);
schtBuildQueryFlowCtrlDag(&dag);
schtBuildQueryFlowCtrlDag(dag);
schtSetPlanToString();
schtSetExecNode();
schtSetAsyncSendMsgToServer();
initTaskQueue();
int32_t queryDone = 0;
SRequestConnInfo conn = {0};
conn.pTrans = mockPointer;
SSchedulerReq req = {0};
req.pConn = &conn;
req.pNodeList = qnodeList;
req.pDag = &dag;
req.pDag = dag;
req.sql = "select * from tb";
req.execFp = schtQueryCb;
req.cbParam = &queryDone;
@ -866,41 +955,27 @@ TEST(queryTest, flowCtrlCase) {
SSchJob *pJob = schAcquireJob(job);
bool qDone = false;
while (!qDone) {
while (!queryDone) {
void *pIter = taosHashIterate(pJob->execTasks, NULL);
if (NULL == pIter) {
break;
}
while (pIter) {
SSchTask *task = *(SSchTask **)pIter;
taosHashCancelIterate(pJob->execTasks, pIter);
if (JOB_TASK_STATUS_EXEC == task->status && 0 != task->lastMsgType) {
SDataBuf msg = {0};
void* rmsg = NULL;
schtBuildQueryRspMsg(&msg.len, &rmsg);
msg.msgType = TDMT_SCH_QUERY_RSP;
msg.pData = rmsg;
if (task->lastMsgType == TDMT_SCH_QUERY) {
SQueryTableRsp rsp = {0};
code = schHandleResponseMsg(pJob, task, TDMT_SCH_QUERY_RSP, (char *)&rsp, sizeof(rsp), 0);
code = schHandleResponseMsg(pJob, task, task->execId, &msg, 0);
ASSERT_EQ(code, 0);
} else {
qDone = true;
break;
}
pIter = NULL;
pIter = taosHashIterate(pJob->execTasks, pIter);
}
}
while (true) {
if (queryDone) {
break;
}
taosUsleep(10000);
}
TdThreadAttr thattr;
taosThreadAttrInit(&thattr);
@ -918,18 +993,11 @@ TEST(queryTest, flowCtrlCase) {
ASSERT_EQ(pRsp->numOfRows, 10);
taosMemoryFreeClear(data);
data = NULL;
code = schedulerFetchRows(job, &req);
ASSERT_EQ(code, 0);
ASSERT_TRUE(data == NULL);
schReleaseJob(job);
schedulerFreeJob(&job, 0);
schtFreeQueryDag(&dag);
schedulerDestroy();
schedulerFreeJob(&job, 0);
}
TEST(insertTest, normalCase) {
@ -938,20 +1006,21 @@ TEST(insertTest, normalCase) {
char *dbname = "1.db1";
char *tablename = "table1";
SVgroupInfo vgInfo = {0};
SQueryPlan dag;
SQueryPlan* dag = (SQueryPlan*)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN);
uint64_t numOfRows = 0;
SArray *qnodeList = taosArrayInit(1, sizeof(SEp));
SArray *qnodeList = taosArrayInit(1, sizeof(SQueryNodeLoad));
SEp qnodeAddr = {0};
strcpy(qnodeAddr.fqdn, "qnode0.ep");
qnodeAddr.port = 6031;
taosArrayPush(qnodeList, &qnodeAddr);
SQueryNodeLoad load = {0};
load.addr.epSet.numOfEps = 1;
strcpy(load.addr.epSet.eps[0].fqdn, "qnode0.ep");
load.addr.epSet.eps[0].port = 6031;
taosArrayPush(qnodeList, &load);
int32_t code = schedulerInit();
ASSERT_EQ(code, 0);
schtBuildInsertDag(&dag);
schtBuildInsertDag(dag);
schtSetPlanToString();
schtSetAsyncSendMsgToServer();
@ -962,21 +1031,19 @@ TEST(insertTest, normalCase) {
TdThread thread1;
taosThreadCreate(&(thread1), &thattr, schtSendRsp, &insertJobRefId);
SExecResult res = {0};
int32_t queryDone = 0;
SRequestConnInfo conn = {0};
conn.pTrans = mockPointer;
SSchedulerReq req = {0};
req.pConn = &conn;
req.pNodeList = qnodeList;
req.pDag = &dag;
req.pDag = dag;
req.sql = "insert into tb values(now,1)";
req.execFp = schtQueryCb;
req.cbParam = NULL;
req.cbParam = &queryDone;
code = schedulerExecJob(&req, &insertJobRefId);
ASSERT_EQ(code, 0);
ASSERT_EQ(res.numOfRows, 20);
schedulerFreeJob(&insertJobRefId, 0);
@ -989,7 +1056,7 @@ TEST(multiThread, forceFree) {
TdThread thread1, thread2, thread3;
taosThreadCreate(&(thread1), &thattr, schtRunJobThread, NULL);
taosThreadCreate(&(thread2), &thattr, schtFreeJobThread, NULL);
// taosThreadCreate(&(thread2), &thattr, schtFreeJobThread, NULL);
taosThreadCreate(&(thread3), &thattr, schtFetchRspThread, NULL);
while (true) {
@ -1002,7 +1069,7 @@ TEST(multiThread, forceFree) {
}
schtTestStop = true;
taosSsleep(3);
//taosSsleep(3);
}
int main(int argc, char **argv) {

1
source/libs/stream/src/streamDispatch.c
View File

@ -569,6 +569,7 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S
char ctbName[TSDB_TABLE_FNAME_LEN] = {0};
if (pDataBlock->info.parTbName[0]) {
if(pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER &&
pTask->subtableWithoutMd5 != 1 &&
!isAutoTableName(pDataBlock->info.parTbName) &&
!alreadyAddGroupId(pDataBlock->info.parTbName) &&
groupId != 0){

5
source/libs/stream/src/streamTask.c
View File

@ -80,7 +80,7 @@ static SStreamChildEpInfo* createStreamTaskEpInfo(const SStreamTask* pTask) {
}
SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, bool fillHistory, int64_t triggerParam,
SArray* pTaskList, bool hasFillhistory) {
SArray* pTaskList, bool hasFillhistory, int8_t subtableWithoutMd5) {
SStreamTask* pTask = (SStreamTask*)taosMemoryCalloc(1, sizeof(SStreamTask));
if (pTask == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -96,6 +96,7 @@ SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset,
pTask->info.taskLevel = taskLevel;
pTask->info.fillHistory = fillHistory;
pTask->info.triggerParam = triggerParam;
pTask->subtableWithoutMd5 = subtableWithoutMd5;
pTask->status.pSM = streamCreateStateMachine(pTask);
if (pTask->status.pSM == NULL) {
@ -205,6 +206,7 @@ int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) {
if (tEncodeCStr(pEncoder, pTask->outputInfo.shuffleDispatcher.stbFullName) < 0) return -1;
}
if (tEncodeI64(pEncoder, pTask->info.triggerParam) < 0) return -1;
if (tEncodeI8(pEncoder, pTask->subtableWithoutMd5) < 0) return -1;
if (tEncodeCStrWithLen(pEncoder, pTask->reserve, sizeof(pTask->reserve) - 1) < 0) return -1;
tEndEncode(pEncoder);
@ -287,6 +289,7 @@ int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask) {
if (tDecodeCStrTo(pDecoder, pTask->outputInfo.shuffleDispatcher.stbFullName) < 0) return -1;
}
if (tDecodeI64(pDecoder, &pTask->info.triggerParam) < 0) return -1;
if (tDecodeI8(pDecoder, &pTask->subtableWithoutMd5) < 0) return -1;
if (tDecodeCStrTo(pDecoder, pTask->reserve) < 0) return -1;
tEndDecode(pDecoder);

1
source/libs/stream/src/streamUpdate.c
View File

@ -261,6 +261,7 @@ void updateInfoDestroy(SUpdateInfo *pInfo) {
taosArrayDestroy(pInfo->pTsSBFs);
taosHashCleanup(pInfo->pMap);
updateInfoDestoryColseWinSBF(pInfo);
taosMemoryFree(pInfo);
}

6
source/libs/transport/src/trans.c
View File

@ -140,11 +140,7 @@ void* rpcMallocCont(int64_t contLen) {
return start + sizeof(STransMsgHead);
}
void rpcFreeCont(void* cont) {
if (cont == NULL) return;
taosMemoryFree((char*)cont - TRANS_MSG_OVERHEAD);
tTrace("rpc free cont:%p", (char*)cont - TRANS_MSG_OVERHEAD);
}
void rpcFreeCont(void* cont) { transFreeMsg(cont); }
void* rpcReallocCont(void* ptr, int64_t contLen) {
if (ptr == NULL) return rpcMallocCont(contLen);

13
source/libs/transport/src/transCli.c
View File

@ -218,7 +218,6 @@ static void (*cliAsyncHandle[])(SCliMsg* pMsg, SCliThrd* pThrd) = {cliHandleReq,
/// static void (*cliAsyncHandle[])(SCliMsg* pMsg, SCliThrd* pThrd) = {cliHandleReq, cliHandleQuit, cliHandleRelease,
/// NULL,cliHandleUpdate};
static FORCE_INLINE void destroyUserdata(STransMsg* userdata);
static FORCE_INLINE void destroyCmsg(void* cmsg);
static FORCE_INLINE void destroyCmsgAndAhandle(void* cmsg);
static FORCE_INLINE int cliRBChoseIdx(STrans* pTransInst);
@ -1950,14 +1949,6 @@ _err:
return NULL;
}
static FORCE_INLINE void destroyUserdata(STransMsg* userdata) {
if (userdata->pCont == NULL) {
return;
}
transFreeMsg(userdata->pCont);
userdata->pCont = NULL;
}
static FORCE_INLINE void destroyCmsg(void* arg) {
SCliMsg* pMsg = arg;
if (pMsg == NULL) {
@ -1965,7 +1956,7 @@ static FORCE_INLINE void destroyCmsg(void* arg) {
}
transDestroyConnCtx(pMsg->ctx);
destroyUserdata(&pMsg->msg);
transFreeMsg(pMsg->msg.pCont);
taosMemoryFree(pMsg);
}
@ -1984,7 +1975,7 @@ static FORCE_INLINE void destroyCmsgAndAhandle(void* param) {
tDebug("destroy Ahandle C");
transDestroyConnCtx(pMsg->ctx);
destroyUserdata(&pMsg->msg);
transFreeMsg(pMsg->msg.pCont);
taosMemoryFree(pMsg);
}

1
source/libs/transport/src/transComm.c
View File

@ -87,6 +87,7 @@ void transFreeMsg(void* msg) {
if (msg == NULL) {
return;
}
tTrace("rpc free cont:%p", (char*)msg - TRANS_MSG_OVERHEAD);
taosMemoryFree((char*)msg - sizeof(STransMsgHead));
}
int transSockInfo2Str(struct sockaddr* sockname, char* dst) {

7
source/os/src/osFile.c
View File

@ -632,6 +632,11 @@ int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, in
return writeLen;
}
bool lastErrorIsFileNotExist() {
DWORD dwError = GetLastError();
return dwError == ERROR_FILE_NOT_FOUND;
}
#else
int taosOpenFileNotStream(const char *path, int32_t tdFileOptions) {
int access = O_BINARY;
@ -1028,6 +1033,8 @@ int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, in
#endif
}
bool lastErrorIsFileNotExist() { return errno == ENOENT; }
#endif // WINDOWS
TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) {

2
source/util/src/tlog.c
View File

@ -350,7 +350,7 @@ void taosResetLog() {
static bool taosCheckFileIsOpen(char *logFileName) {
TdFilePtr pFile = taosOpenFile(logFileName, TD_FILE_WRITE);
if (pFile == NULL) {
if (errno == ENOENT) {
if (lastErrorIsFileNotExist()) {
return false;
} else {
printf("\nfailed to open log file:%s, reason:%s\n", logFileName, strerror(errno));

1
tests/parallel_test/cases.task
View File

@ -575,6 +575,7 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ts_3405_3398_3423.py -N 3 -n 3
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ts-4348-td-27939.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/backslash_g.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/test_ts4467.py
,,n,system-test,python3 ./test.py -f 2-query/queryQnode.py
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode1mnode.py

16
tests/script/tsim/parser/like.sim
View File

@ -51,6 +51,22 @@ if $rows != 1 then
return -1
endi
$view1 = view1_name
$view2 = view2_name
sql CREATE VIEW $view1 as select * from $table1
sql CREATE VIEW $view2 AS select * from $table2
sql show views like 'view%'
if $rows != 2 then
return -1
endi
sql show views like 'view1%'
if $rows != 1 then
return -1
endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT

63
tests/system-test/2-query/test_ts4467.py Normal file
View File

@ -0,0 +1,63 @@
import random
import itertools
from util.log import *
from util.cases import *
from util.sql import *
from util.sqlset import *
from util import constant
from util.common import *
class TDTestCase:
"""Verify the jira TS-4467
"""
def init(self, conn, logSql, replicaVar=1):
self.replicaVar = int(replicaVar)
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor())
def prepareData(self):
# db
tdSql.execute("create database if not exists db")
tdSql.execute("use db")
# table
tdSql.execute("create table t (ts timestamp, c1 varchar(16));")
# insert data
sql = "insert into t values"
for i in range(6):
sql += f"(now+{str(i+1)}s, '{'name' + str(i+1)}')"
sql += ";"
tdSql.execute(sql)
tdLog.debug("insert data successfully")
def run(self):
self.prepareData()
# join query with order by
sql = "select * from t t1, (select * from t order by ts limit 5) t2 where t1.ts = t2.ts;"
tdSql.query(sql)
tdSql.checkRows(5)
sql = "select * from t t1, (select * from t order by ts desc limit 5) t2 where t1.ts = t2.ts;"
tdSql.query(sql)
tdSql.checkRows(5)
sql = "select * from t t1, (select * from t order by ts limit 5) t2 where t1.ts = t2.ts order by t1.ts;"
tdSql.query(sql)
res1 = tdSql.queryResult
tdLog.debug("res1: %s" % str(res1))
sql = "select * from t t1, (select * from t order by ts limit 5) t2 where t1.ts = t2.ts order by t1.ts desc;"
tdSql.query(sql)
res2 = tdSql.queryResult
tdLog.debug("res2: %s" % str(res2))
assert(len(res1) == len(res2) and res1[0][0] == res2[4][0])
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())