xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into enh/TD-28945

This commit is contained in:
Minglei Jin 2024-04-08 10:55:47 +08:00
parent 26d66e8b32 fff0def685
commit a407f76165
252 changed files with 21233 additions and 9265 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/en/08-client-libraries/50-odbc.mdx
View File

@ -49,7 +49,7 @@ TDengine ODBC driver supports two kinds of connections to TDengine cluster, nati
4.2 [Connection Type]: required field, we choose "WebSocket"
4.3 [URL]: required field, the URL for the ODBC data source, for example, `http://localhost:6041` is the URL for a local TDengine cluster, `https://gw.cloud.taosdata.com?token=your_token` is the URL for a TDengine cloud service.
4.3 [URL]: required field, the URL for the ODBC data source, for example, `http://localhost:6041` is the URL for a local TDengine cluster.
4.4 [Database]: optional field, the default database to access

6
docs/en/10-deployment/02-docker.md
View File

@ -128,7 +128,7 @@ If you want to start your application in a container, you need to add the corres
FROM ubuntu:20.04
RUN apt-get update && apt-get install -y wget
ENV TDENGINE_VERSION=3.0.0.0
RUN wget -c https://www.taosdata.com/assets-download/3.0/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \
RUN wget -c https://tdengine.com/assets-download/3.0/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \
&& tar xvf TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \
&& cd TDengine-client-${TDENGINE_VERSION} \
&& ./install_client.sh \
@ -229,7 +229,7 @@ Here is the full Dockerfile:
```docker
FROM golang:1.17.6-buster as builder
ENV TDENGINE_VERSION=3.0.0.0
RUN wget -c https://www.taosdata.com/assets-download/3.0/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \
RUN wget -c https://tdengine.com/assets-download/3.0/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \
&& tar xvf TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \
&& cd TDengine-client-${TDENGINE_VERSION} \
&& ./install_client.sh \
@ -245,7 +245,7 @@ RUN go build
FROM ubuntu:20.04
RUN apt-get update && apt-get install -y wget
ENV TDENGINE_VERSION=3.0.0.0
RUN wget -c https://www.taosdata.com/assets-download/3.0/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \
RUN wget -c https://tdengine.com/assets-download/3.0/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \
&& tar xvf TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \
&& cd TDengine-client-${TDENGINE_VERSION} \
&& ./install_client.sh \

2
docs/en/10-deployment/03-k8s.md
View File

@ -8,7 +8,7 @@ description: This document describes how to deploy TDengine on Kubernetes.
As a time series database for Cloud Native architecture design, TDengine supports Kubernetes deployment. Firstly we introduce how to use YAML files to create a highly available TDengine cluster from scratch step by step for production usage, and highlight the common operations of TDengine in Kubernetes environment.
To meet [high availability ](https://docs.taosdata.com/tdinternal/high-availability/)requirements, clusters need to meet the following requirements:
To meet [high availability](../../tdinternal/high-availability/) requirements, clusters need to meet the following requirements:
- 3 or more dnodes: multiple vnodes in the same vgroup of TDengine are not allowed to be distributed in one dnode at the same time, so if you create a database with 3 replicas, the number of dnodes is greater than or equal to 3
- 3 mnodes: mnode is responsible for the management of the entire TDengine cluster. The default number of mnode in TDengine cluster is only one. If the dnode where the mnode located is dropped, the entire cluster is unavailable.

2
docs/en/12-taos-sql/05-insert.md
View File

@ -1,7 +1,7 @@
---
title: Insert
sidebar_label: Insert
description: This document describes how to insert data into TDengine.
description: This document describes the SQL commands and syntax for inserting data into TDengine.
---
## Syntax

4
docs/en/12-taos-sql/06-select.md
View File

@ -24,7 +24,7 @@ SELECT [hints] [DISTINCT] [TAGS] select_list
hints: /*+ [hint([hint_param_list])] [hint([hint_param_list])] */
hint:
BATCH_SCAN | NO_BATCH_SCAN | SORT_FOR_GROUP | PARA_TABLES_SORT
BATCH_SCAN | NO_BATCH_SCAN | SORT_FOR_GROUP | PARA_TABLES_SORT | PARTITION_FIRST | SMALLDATA_TS_SORT
select_list:
select_expr [, select_expr] ...
@ -94,6 +94,7 @@ The list of currently supported Hints is as follows:
| SORT_FOR_GROUP| None | Use sort for partition, conflict with PARTITION_FIRST | With normal column in partition by list |
| PARTITION_FIRST| None | Use Partition before aggregate, conflict with SORT_FOR_GROUP | With normal column in partition by list |
| PARA_TABLES_SORT| None | When sorting the supertable rows by timestamp, No temporary disk space is used. When there are numerous tables, each with long rows, the corresponding algorithm associated with this prompt may consume a substantial amount of memory, potentially leading to an Out Of Memory (OOM) situation. | Sorting the supertable rows by timestamp |
| SMALLDATA_TS_SORT| None | When sorting the supertable rows by timestamp, if the length of query columns >= 256, and there are relatively few rows, this hint can improve performance. | Sorting the supertable rows by timestamp |
For example:
@ -102,6 +103,7 @@ SELECT /*+ BATCH_SCAN() */ a.ts FROM stable1 a, stable2 b where a.tag0 = b.tag0
SELECT /*+ SORT_FOR_GROUP() */ count(*), c1 FROM stable1 PARTITION BY c1;
SELECT /*+ PARTITION_FIRST() */ count(*), c1 FROM stable1 PARTITION BY c1;
SELECT /*+ PARA_TABLES_SORT() */ * from stable1 order by ts;
SELECT /*+ SMALLDATA_TS_SORT() */ * from stable1 order by ts;
```
## Lists

2
docs/en/12-taos-sql/13-tmq.md
View File

@ -1,5 +1,5 @@
---
title: Data Subscription
title: Data Subscription SQL Reference
sidebar_label: Data Subscription
description: This document describes the SQL statements related to the data subscription component of TDengine.
---

12
docs/en/12-taos-sql/14-stream.md
View File

@ -1,5 +1,5 @@
---
title: Stream Processing
title: Stream Processing SQL Reference
sidebar_label: Stream Processing
description: This document describes the SQL statements related to the stream processing component of TDengine.
---
@ -148,7 +148,7 @@ T = latest event time - watermark
The window closing time for each batch of data that arrives at the system is updated using the preceding formula, and all windows are closed whose closing time is less than T. If the triggering method is WINDOW_CLOSE or MAX_DELAY, the aggregate result for the window is pushed.
Stream processing strategy for expired data
## Stream processing strategy for expired data
The data in expired windows is tagged as expired. TDengine stream processing provides two methods for handling such data:
1. Drop the data. This is the default and often only handling method for most stream processing engines.
@ -157,6 +157,14 @@ The data in expired windows is tagged as expired. TDengine stream processing pro
In both of these methods, configuring the watermark is essential for obtaining accurate results (if expired data is dropped) and avoiding repeated triggers that affect system performance (if expired data is recalculated).
## Stream processing strategy for modifying data
TDengine provides two ways to handle modified data, which are specified by the IGNORE UPDATE option:
1. Check whether the data has been modified, i.e. IGNORE UPDATE 0, and recalculate the corresponding window if the data has been modified.
2. Do not check whether the data has been modified, and calculate all the data as incremental data, i.e. IGNORE UPDATE 1, the default configuration.
## Supported functions
All [scalar functions](../function/#scalar-functions) are available in stream processing. All [Aggregate functions](../function/#aggregate-functions) and [Selection functions](../function/#selection-functions) are available in stream processing, except the followings:

2
docs/en/12-taos-sql/26-udf.md
View File

@ -1,5 +1,5 @@
---
title: User-Defined Functions (UDF)
title: User-Defined Functions (UDF) SQL Reference
sidebar_label: User-Defined Functions
description: This document describes the SQL statements related to user-defined functions (UDF) in TDengine.
---

2
docs/en/14-reference/06-taosdump.md
View File

@ -19,7 +19,7 @@ Users should not use taosdump to back up raw data, environment settings, hardwar
There are two ways to install taosdump:
- Install the taosTools official installer. Please find taosTools from [Release History](https://docs.taosdata.com/releases/tools/) page and download and install it.
- Install the taosTools official installer. Please find taosTools from [Release History](../../releases/tools/) page and download and install it.
- Compile taos-tools separately and install it. Please refer to the [taos-tools](https://github.com/taosdata/taos-tools) repository for details.

2
docs/en/14-reference/07-tdinsight/index.md
View File

@ -1,5 +1,5 @@
---
title: TDinsight - Grafana-based Zero-Dependency Monitoring Solution for TDengine
title: TDinsight
sidebar_label: TDinsight
description: This document describes TDinsight, a monitoring solution for TDengine.
---

4
docs/en/20-third-party/75-powerbi.md vendored
View File

@ -19,7 +19,7 @@ With TDengine ODBC driver, PowerBI can access time series data stored in TDengin
## Install Driver
Depending on your TDengine server version, download appropriate version of TDengine client package from TDengine website [Download Link](https://docs.taosdata.com/get-started/package/), or TDengine explorer if you are using a local TDengine cluster. Install the TDengine client package on same Windows machine where PowerBI is running.
Depending on your TDengine server version, download appropriate version of TDengine client package from TDengine website [Download Link](../../get-started/package/), or TDengine explorer if you are using a local TDengine cluster. Install the TDengine client package on same Windows machine where PowerBI is running.
### Configure Data Source
@ -38,7 +38,7 @@ To better use Power BI to analyze the data stored in TDengine, you need to under
1. Dimention: it's normally category (text) data to describe such information as device, collection point, model. In the supertable template of TDengine, we use tag columns to store the dimention information. You can use SQL like `select distinct tbname, tag1, tag2 from supertable` to get dimentions.
2. Metric: quantitive (numeric) fileds that can be calculated, like SUM, AVERAGE, MINIMUM. If the collecting frequency is 1 second, then there are 31,536,000 records in one year, it will be too low efficient to import so big data into Power BI. In TDengine, you can use data partition query, window partition query, in combination with pseudo columns related to window, to import downsampled data into Power BI. For more details, please refer to [TDengine Specialized Queries](https://docs.taosdata.com/taos-sql/distinguished/)。
2. Metric: quantitive (numeric) fileds that can be calculated, like SUM, AVERAGE, MINIMUM. If the collecting frequency is 1 second, then there are 31,536,000 records in one year, it will be too low efficient to import so big data into Power BI. In TDengine, you can use data partition query, window partition query, in combination with pseudo columns related to window, to import downsampled data into Power BI. For more details, please refer to [TDengine Specialized Queries](../../taos-sql/distinguished/)。
- Window partition query: for example, thermal meters collect one data per second, but you need to query the average temperature every 10 minutes, you can use window subclause to get the downsampling data you need. The corresponding SQL is like `select tbname, _wstart date，avg(temperature) temp from table interval(10m)`, in which _wstart is a pseudo column indicting the start time of a widow, 10m is the duration of the window, `avg(temperature)` indicates the aggregate value inside a window.

2
docs/en/25-application/01-telegraf.md
View File

@ -1,5 +1,5 @@
---
title: Quickly Build IT DevOps Visualization System with TDengine + Telegraf + Grafana
title: IT Visualization with TDengine + Telegraf + Grafana
sidebar_label: TDengine + Telegraf + Grafana
description: This document describes how to create an IT visualization system by integrating TDengine with Telegraf and Grafana.
---

2
docs/en/25-application/_03-immigrate.md
View File

@ -184,8 +184,6 @@ TDengine supports the standard JDBC 3.0 interface for manipulating databases, bu
To facilitate historical data migration, we provide a plug-in for the data synchronization tool DataX, which can automatically write data into TDengine.The automatic data migration of DataX can only support the data migration process of a single value model.
For the specific usage of DataX and how to use DataX to write data to TDengine, please refer to [DataX-based TDengine Data Migration Tool](https://www.taosdata.com/engineering/16401.html).
After migrating via DataX, we found that we can significantly improve the efficiency of migrating historical data by starting multiple processes and migrating numerous metrics simultaneously. The following are some records of the migration process. We provide these as a reference for application migration.
| Number of datax instances (number of concurrent processes) | Migration record speed (pieces/second) |

2
docs/en/28-releases/02-tools.md
View File

@ -6,7 +6,7 @@ description: This document provides download links for all released versions of
taosTools installation packages can be downloaded at the following links:
For other historical version installers, please visit [here](https://www.taosdata.com/all-downloads).
For other historical version installers, please visit [here](https://tdengine.com/downloads/historical/).
import Release from "/components/ReleaseV3";

66
docs/examples/rust/nativeexample/examples/query.rs Normal file
View File

@ -0,0 +1,66 @@
use taos::*;
#[tokio::main]
async fn main() -> anyhow::Result<()> {
let dsn = "taos://localhost:6030";
let builder = TaosBuilder::from_dsn(dsn)?;
let taos = builder.build()?;
// ANCHOR: create_db_and_table
let db = "power";
// create database
taos.exec_many([
format!("DROP DATABASE IF EXISTS `{db}`"),
format!("CREATE DATABASE `{db}`"),
format!("USE `{db}`"),
])
.await?;
// create table
taos.exec_many([
// create super table
"CREATE TABLE `meters` (`ts` TIMESTAMP, `current` FLOAT, `voltage` INT, `phase` FLOAT) \
TAGS (`groupid` INT, `location` BINARY(24))",
]).await?;
// ANCHOR_END: create_db_and_table
// ANCHOR: insert_data
let inserted = taos.exec("INSERT INTO " +
"power.d1001 USING power.meters TAGS(2,'California.SanFrancisco') " +
"VALUES " +
"(NOW + 1a, 10.30000, 219, 0.31000) " +
"(NOW + 2a, 12.60000, 218, 0.33000) " +
"(NOW + 3a, 12.30000, 221, 0.31000) " +
"power.d1002 USING power.meters TAGS(3, 'California.SanFrancisco') " +
"VALUES " +
"(NOW + 1a, 10.30000, 218, 0.25000) ").await?;
println!("inserted: {} rows", inserted);
// ANCHOR_END: insert_data
// ANCHOR: query_data
let mut result = taos.query("SELECT * FROM power.meters").await?;
for field in result.fields() {
println!("got field: {}", field.name());
}
let mut rows = result.rows();
let mut nrows = 0;
while let Some(row) = rows.try_next().await? {
for (col, (name, value)) in row.enumerate() {
println!(
"[{}] got value in col {} (named `{:>8}`): {}",
nrows, col, name, value
);
}
nrows += 1;
}
// ANCHOR_END: query_data
// ANCHOR: query_with_req_id
let result = taos.query_with_req_id("SELECT * FROM power.meters", 0).await?;
// ANCHOR_END: query_with_req_id
}

80
docs/examples/rust/nativeexample/examples/schemaless.rs Normal file
View File

@ -0,0 +1,80 @@
use taos_query::common::SchemalessPrecision;
use taos_query::common::SchemalessProtocol;
use taos_query::common::SmlDataBuilder;
use crate::AsyncQueryable;
use crate::AsyncTBuilder;
use crate::TaosBuilder;
async fn put() -> anyhow::Result<()> {
std::env::set_var("RUST_LOG", "taos=debug");
pretty_env_logger::init();
let dsn =
std::env::var("TDENGINE_ClOUD_DSN").unwrap_or("http://localhost:6041".to_string());
log::debug!("dsn: {:?}", &dsn);
let client = TaosBuilder::from_dsn(dsn)?.build().await?;
let db = "power";
client.exec(format!("drop database if exists {db}")).await?;
client
.exec(format!("create database if not exists {db}"))
.await?;
// should specify database before insert
client.exec(format!("use {db}")).await?;
// SchemalessProtocol::Line
let data = [
"meters,groupid=2,location=California.SanFrancisco current=10.3000002f64,voltage=219i32,phase=0.31f64 1626006833639000000",
]
.map(String::from)
.to_vec();
let sml_data = SmlDataBuilder::default()
.protocol(SchemalessProtocol::Line)
.precision(SchemalessPrecision::Millisecond)
.data(data.clone())
.ttl(1000)
.req_id(100u64)
.build()?;
assert_eq!(client.put(&sml_data).await?, ());
// SchemalessProtocol::Telnet
let data = [
"meters.current 1648432611249 10.3 location=California.SanFrancisco group=2",
]
.map(String::from)
.to_vec();
let sml_data = SmlDataBuilder::default()
.protocol(SchemalessProtocol::Telnet)
.precision(SchemalessPrecision::Millisecond)
.data(data.clone())
.ttl(1000)
.req_id(200u64)
.build()?;
assert_eq!(client.put(&sml_data).await?, ());
// SchemalessProtocol::Json
let data = [
r#"[{"metric": "meters.current", "timestamp": 1681345954000, "value": 10.3, "tags": {"location": "California.SanFrancisco", "groupid": 2}}, {"metric": "meters.voltage", "timestamp": 1648432611249, "value": 219, "tags": {"location": "California.LosAngeles", "groupid": 1}}, {"metric": "meters.current", "timestamp": 1648432611250, "value": 12.6, "tags": {"location": "California.SanFrancisco", "groupid": 2}}, {"metric": "meters.voltage", "timestamp": 1648432611250, "value": 221, "tags": {"location": "California.LosAngeles", "groupid": 1}}]"#
]
.map(String::from)
.to_vec();
let sml_data = SmlDataBuilder::default()
.protocol(SchemalessProtocol::Json)
.precision(SchemalessPrecision::Millisecond)
.data(data.clone())
.ttl(1000)
.req_id(300u64)
.build()?;
assert_eq!(client.put(&sml_data).await?, ());
client.exec(format!("drop database if exists {db}")).await?;
Ok(())
}

37
docs/examples/rust/nativeexample/examples/stmt.rs Normal file
View File

@ -0,0 +1,37 @@
use taos::*;
#[tokio::main]
async fn main() -> anyhow::Result<()> {
let taos = TaosBuilder::from_dsn("taos://")?.build().await?;
taos.exec("DROP DATABASE IF EXISTS power").await?;
taos.create_database("power").await?;
taos.use_database("power").await?;
taos.exec("CREATE STABLE IF NOT EXISTS meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)").await?;
let mut stmt = Stmt::init(&taos).await?;
stmt.prepare("INSERT INTO ? USING meters TAGS(?, ?) VALUES(?, ?, ?, ?)").await?;
const NUM_TABLES: usize = 10;
const NUM_ROWS: usize = 10;
for i in 0..NUM_TABLES {
let table_name = format!("d{}", i);
let tags = vec![Value::VarChar("California.SanFransico".into()), Value::Int(2)];
stmt.set_tbname_tags(&table_name, &tags).await?;
for j in 0..NUM_ROWS {
let values = vec![
ColumnView::from_millis_timestamp(vec![1648432611249 + j as i64]),
ColumnView::from_floats(vec![10.3 + j as f32]),
ColumnView::from_ints(vec![219 + j as i32]),
ColumnView::from_floats(vec![0.31 + j as f32]),
];
stmt.bind(&values).await?;
}
stmt.add_batch().await?;
}
// execute.
let rows = stmt.execute().await?;
assert_eq!(rows, NUM_TABLES * NUM_ROWS);
Ok(())
}

166
docs/examples/rust/nativeexample/examples/tmq.rs Normal file
View File

@ -0,0 +1,166 @@
use std::time::Duration;
use std::str::FromStr;
use taos::*;
#[tokio::main]
async fn main() -> anyhow::Result<()> {
pretty_env_logger::formatted_timed_builder()
.filter_level(log::LevelFilter::Info)
.init();
use taos_query::prelude::*;
let dsn = "taos://localhost:6030".to_string();
log::info!("dsn: {}", dsn);
let mut dsn = Dsn::from_str(&dsn)?;
let taos = TaosBuilder::from_dsn(&dsn)?.build().await?;
// prepare database and table
taos.exec_many([
"drop topic if exists topic_meters",
"drop database if exists power",
"create database if not exists power WAL_RETENTION_PERIOD 86400",
"use power",
"CREATE STABLE IF NOT EXISTS power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (groupId INT, location BINARY(24))",
"create table if not exists power.d001 using power.meters tags(1,'location')",
])
.await?;
taos.exec_many([
"drop database if exists db2",
"create database if not exists db2 wal_retention_period 3600",
"use db2",
])
.await?;
// ANCHOR: create_topic
taos.exec_many([
"CREATE TOPIC IF NOT EXISTS topic_meters AS SELECT ts, current, voltage, phase, groupid, location FROM power.meters",
])
.await?;
// ANCHOR_END: create_topic
// ANCHOR: create_consumer
dsn.params.insert("group.id".to_string(), "abc".to_string());
dsn.params.insert("auto.offset.reset".to_string(), "earliest".to_string());
let builder = TmqBuilder::from_dsn(&dsn)?;
let mut consumer = builder.build().await?;
// ANCHOR_END: create_consumer
// ANCHOR: subscribe
consumer.subscribe(["topic_meters"]).await?;
// ANCHOR_END: subscribe
// ANCHOR: consume
{
let mut stream = consumer.stream_with_timeout(Timeout::from_secs(1));
while let Some((offset, message)) = stream.try_next().await? {
let topic: &str = offset.topic();
let database = offset.database();
let vgroup_id = offset.vgroup_id();
log::debug!(
"topic: {}, database: {}, vgroup_id: {}",
topic,
database,
vgroup_id
);
match message {
MessageSet::Meta(meta) => {
log::info!("Meta");
let raw = meta.as_raw_meta().await?;
taos.write_raw_meta(&raw).await?;
let json = meta.as_json_meta().await?;
let sql = json.to_string();
if let Err(err) = taos.exec(sql).await {
println!("maybe error: {}", err);
}
}
MessageSet::Data(data) => {
log::info!("Data");
while let Some(data) = data.fetch_raw_block().await? {
log::debug!("data: {:?}", data);
}
}
MessageSet::MetaData(meta, data) => {
log::info!("MetaData");
let raw = meta.as_raw_meta().await?;
taos.write_raw_meta(&raw).await?;
let json = meta.as_json_meta().await?;
let sql = json.to_string();
if let Err(err) = taos.exec(sql).await {
println!("maybe error: {}", err);
}
while let Some(data) = data.fetch_raw_block().await? {
log::debug!("data: {:?}", data);
}
}
}
consumer.commit(offset).await?;
}
}
// ANCHOR_END: consume
// ANCHOR: assignments
let assignments = consumer.assignments().await.unwrap();
log::info!("assignments: {:?}", assignments);
// ANCHOR_END: assignments
// seek offset
for topic_vec_assignment in assignments {
let topic = &topic_vec_assignment.0;
let vec_assignment = topic_vec_assignment.1;
for assignment in vec_assignment {
let vgroup_id = assignment.vgroup_id();
let current = assignment.current_offset();
let begin = assignment.begin();
let end = assignment.end();
log::debug!(
"topic: {}, vgroup_id: {}, current offset: {} begin {}, end: {}",
topic,
vgroup_id,
current,
begin,
end
);
// ANCHOR: seek_offset
let res = consumer.offset_seek(topic, vgroup_id, end).await;
if res.is_err() {
log::error!("seek offset error: {:?}", res);
let a = consumer.assignments().await.unwrap();
log::error!("assignments: {:?}", a);
}
// ANCHOR_END: seek_offset
}
let topic_assignment = consumer.topic_assignment(topic).await;
log::debug!("topic assignment: {:?}", topic_assignment);
}
// after seek offset
let assignments = consumer.assignments().await.unwrap();
log::info!("after seek offset assignments: {:?}", assignments);
// ANCHOR: unsubscribe
consumer.unsubscribe().await;
// ANCHOR_END: unsubscribe
tokio::time::sleep(Duration::from_secs(1)).await;
taos.exec_many([
"drop database db2",
"drop topic topic_meters",
"drop database power",
])
.await?;
Ok(())
}

2
docs/zh/07-develop/06-stream.md
View File

@ -8,7 +8,7 @@ title: 流式计算
TDengine 3.0 的流式计算引擎提供了实时处理写入的数据流的能力，使用 SQL 定义实时流变换，当数据被写入流的源表后，数据会被以定义的方式自动处理，并根据定义的触发模式向目的表推送结果。它提供了替代复杂流处理系统的轻量级解决方案，并能够在高吞吐的数据写入的情况下，提供毫秒级的计算结果延迟。
流式计算可以包含数据过滤，标量函数计算（含UDF），以及窗口聚合（支持滑动窗口、会话窗口与状态窗口），可以以超级表、子表、普通表为源表，写入到目的超级表。在创建流时，目的超级表将被自动创建，随后新插入的数据会被流定义的方式处理并写入其中，通过 partition by 子句，可以以表名或标签划分 partition，不同的 partition 将写入到目的超级表的不同子表。
流式计算可以包含数据过滤，标量函数计算（含UDF），以及窗口聚合（支持滑动窗口、会话窗口、状态窗口、事件窗口与计数窗口），可以以超级表、子表、普通表为源表，写入到目的超级表。在创建流时，目的超级表将被自动创建，随后新插入的数据会被流定义的方式处理并写入其中，通过 partition by 子句，可以以表名或标签划分 partition，不同的 partition 将写入到目的超级表的不同子表。
TDengine 的流式计算能够支持分布在多个 vnode 中的超级表聚合；还能够处理乱序数据的写入：它提供了 watermark 机制以度量容忍数据乱序的程度，并提供了 ignore expired 配置项以决定乱序数据的处理策略——丢弃或者重新计算。

5
docs/zh/12-taos-sql/06-select.md
View File

@ -24,7 +24,7 @@ SELECT [hints] [DISTINCT] [TAGS] select_list
hints: /*+ [hint([hint_param_list])] [hint([hint_param_list])] */
hint:
BATCH_SCAN | NO_BATCH_SCAN | SORT_FOR_GROUP | PARA_TABLES_SORT
BATCH_SCAN | NO_BATCH_SCAN | SORT_FOR_GROUP | PARTITION_FIRST | PARA_TABLES_SORT | SMALLDATA_TS_SORT
select_list:
select_expr [, select_expr] ...
@ -94,6 +94,8 @@ Hints 是用户控制单个语句查询优化的一种手段，当 Hint 不适
| SORT_FOR_GROUP| 无 | 采用sort方式进行分组, 与PARTITION_FIRST冲突 | partition by 列表有普通列时 |
| PARTITION_FIRST| 无 | 在聚合之前使用PARTITION计算分组, 与SORT_FOR_GROUP冲突 | partition by 列表有普通列时 |
| PARA_TABLES_SORT| 无 | 超级表的数据按时间戳排序时, 不使用临时磁盘空间, 只使用内存。当子表数量多, 行长比较大时候, 会使用大量内存, 可能发生OOM | 超级表的数据按时间戳排序时 |
| SMALLDATA_TS_SORT| 无 | 超级表的数据按时间戳排序时, 查询列长度大于等于256, 但是行数不多, 使用这个提示, 可以提高性能 | 超级表的数据按时间戳排序时 |
举例：
```sql
@ -101,6 +103,7 @@ SELECT /*+ BATCH_SCAN() */ a.ts FROM stable1 a, stable2 b where a.tag0 = b.tag0
SELECT /*+ SORT_FOR_GROUP() */ count(*), c1 FROM stable1 PARTITION BY c1;
SELECT /*+ PARTITION_FIRST() */ count(*), c1 FROM stable1 PARTITION BY c1;
SELECT /*+ PARA_TABLES_SORT() */ * from stable1 order by ts;
SELECT /*+ SMALLDATA_TS_SORT() */ * from stable1 order by ts;
```
## 列表

14
docs/zh/12-taos-sql/12-distinguished.md
View File

@ -49,6 +49,7 @@ window_clause: {
| STATE_WINDOW(col)
| INTERVAL(interval_val [, interval_offset]) [SLIDING (sliding_val)] [FILL(fill_mod_and_val)]
| EVENT_WINDOW START WITH start_trigger_condition END WITH end_trigger_condition
| COUNT_WINDOW(count_val[, sliding_val])
}
```
@ -180,6 +181,19 @@ select _wstart, _wend, count(*) from t event_window start with c1 > 0 end with c
![TDengine Database 事件窗口示意图](./event_window.webp)
### 计数窗口
计数窗口按固定的数据行数来划分窗口。默认将数据按时间戳排序，再按照count_val的值，将数据划分为多个窗口，然后做聚合计算。count_val表示每个count window包含的最大数据行数，总数据行数不能整除count_val时，最后一个窗口的行数会小于count_val。sliding_val是常量，表示窗口滑动的数量，类似于 interval的SLIDING。
以下面的 SQL 语句为例，计数窗口切分如图所示：
```sql
select _wstart, _wend, count(*) from t count_window(4);
```
![count_window](https://github.com/taosdata/TDengine/assets/38781207/ff86278e-cbff-477f-a866-6e5cc150b712)
### 时间戳伪列
窗口聚合查询结果中，如果 SQL 语句中没有指定输出查询结果中的时间戳列，那么最终结果中不会自动包含窗口的时间列信息。如果需要在结果中输出聚合结果所对应的时间窗口信息，需要在 SELECT 子句中使用时间戳相关的伪列: 时间窗口起始时间 (\_WSTART), 时间窗口结束时间 (\_WEND), 时间窗口持续时间 (\_WDURATION), 以及查询整体窗口相关的伪列: 查询窗口起始时间(\_QSTART) 和查询窗口结束时间(\_QEND)。需要注意的是时间窗口起始时间和结束时间均是闭区间，时间窗口持续时间是数据当前时间分辨率下的数值。例如，如果当前数据库的时间分辨率是毫秒，那么结果中 500 就表示当前时间窗口的持续时间是 500毫秒 (500 ms)。

12
docs/zh/12-taos-sql/14-stream.md
View File

@ -30,7 +30,7 @@ subquery: SELECT select_list
[window_clause]
```
支持会话窗口、状态窗口与滑动窗口，其中，会话窗口与状态窗口搭配超级表时必须与partition by tbname一起使用
支持会话窗口、状态窗口、滑动窗口、事件窗口和计数窗口，其中，状态窗口、事件窗口和计数窗口搭配超级表时必须与partition by tbname一起使用
stb_name 是保存计算结果的超级表的表名，如果该超级表不存在，会自动创建；如果已存在，则检查列的schema信息。详见 写入已存在的超级表
@ -60,7 +60,11 @@ COUNT_WINDOW 是计数窗口,按固定的数据行数来划分窗口。 count_va
窗口的定义与时序数据特色查询中的定义完全相同，详见 [TDengine 特色查询](../distinguished)
例如，如下语句创建流式计算，同时自动创建名为 avg_vol 的超级表，此流计算以一分钟为时间窗口、30 秒为前向增量统计这些电表的平均电压，并将来自 meters 表的数据的计算结果写入 avg_vol 表，不同 partition 的数据会分别创建子表并写入不同子表。
例如，如下语句创建流式计算。第一个流计算，自动创建名为 avg_vol 的超级表，以一分钟为时间窗口、30 秒为前向增量统计这些电表的平均电压，并将来自 meters 表的数据的计算结果写入 avg_vol 表，不同 partition 的数据会分别创建子表并写入不同子表。
第二个流计算，自动创建名为 streamt0 的超级表，将数据按时间戳的顺序，以 voltage < 0 作为窗口的开始条件，voltage > 9作为窗口的结束条件，划分窗口做聚合运算，并将来自 meters 表的数据的计算结果写入 streamt0 表，不同 partition 的数据会分别创建子表并写入不同子表。
第三个流计算，自动创建名为 streamt1 的超级表，将数据按时间戳的顺序，以10条数据为一组，划分窗口做聚合运算，并将来自 meters 表的数据的计算结果写入 streamt1 表，不同 partition 的数据会分别创建子表并写入不同子表。
```sql
CREATE STREAM avg_vol_s INTO avg_vol AS
@ -197,9 +201,9 @@ TDengine 对于过期数据提供两种处理方式，由 IGNORE EXPIRED 选项
TDengine 对于修改数据提供两种处理方式，由 IGNORE UPDATE 选项指定：
1. 检查数据是否被修改，即 IGNORE UPDATE 0：默认配置，如果被修改，则重新计算对应窗口。
1. 检查数据是否被修改，即 IGNORE UPDATE 0，如果数据被修改，则重新计算对应窗口。
2. 不检查数据是否被修改，全部按增量数据计算，即 IGNORE UPDATE 1。
2. 不检查数据是否被修改，全部按增量数据计算，即 IGNORE UPDATE 1，默认配置。
## 写入已存在的超级表

1
include/common/systable.h
View File

@ -29,6 +29,7 @@ extern "C" {
#define TSDB_INS_TABLE_QNODES "ins_qnodes"
#define TSDB_INS_TABLE_BNODES "ins_bnodes" // no longer used
#define TSDB_INS_TABLE_SNODES "ins_snodes"
#define TSDB_INS_TABLE_ARBGROUPS "ins_arbgroups"
#define TSDB_INS_TABLE_CLUSTER "ins_cluster"
#define TSDB_INS_TABLE_DATABASES "ins_databases"
#define TSDB_INS_TABLE_FUNCTIONS "ins_functions"

2
include/common/tdatablock.h
View File

@ -267,7 +267,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq2** pReq, const SSDataBlock* pData
bool alreadyAddGroupId(char* ctbName);
bool isAutoTableName(char* ctbName);
void buildCtbNameAddGroupId(char* ctbName, uint64_t groupId);
void buildCtbNameAddGroupId(const char* stbName, char* ctbName, uint64_t groupId);
char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId);
int32_t buildCtbNameByGroupIdImpl(const char* stbName, uint64_t groupId, char* pBuf);

5
include/common/tglobal.h
View File

@ -76,6 +76,11 @@ extern int32_t tsHeartbeatInterval;
extern int32_t tsHeartbeatTimeout;
extern int32_t tsSnapReplMaxWaitN;
// arbitrator
extern int32_t tsArbHeartBeatIntervalSec;
extern int32_t tsArbCheckSyncIntervalSec;
extern int32_t tsArbSetAssignedTimeoutSec;
// vnode
extern int64_t tsVndCommitMaxIntervalMs;

7
include/common/tgrant.h
View File

@ -61,9 +61,6 @@ typedef enum {
int32_t grantCheck(EGrantType grant);
int32_t grantCheckExpire(EGrantType grant);
char* tGetMachineId();
#ifdef TD_UNIQ_GRANT
int32_t grantCheckLE(EGrantType grant);
#endif
// #ifndef GRANTS_CFG
#ifdef TD_ENTERPRISE
@ -76,7 +73,7 @@ int32_t grantCheckLE(EGrantType grant);
{.name = "state", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \
{.name = "timeseries", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \
{.name = "dnodes", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \
{.name = "cpu_cores", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \
{.name = "cpu_cores", .bytes = 13 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \
}
#else
#define GRANTS_SCHEMA \
@ -88,7 +85,7 @@ int32_t grantCheckLE(EGrantType grant);
{.name = "state", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \
{.name = "timeseries", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \
{.name = "dnodes", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \
{.name = "cpu_cores", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \
{.name = "cpu_cores", .bytes = 13 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \
}
#endif
// #define GRANT_CFG_ADD

106
include/common/tmsg.h
View File

@ -150,6 +150,7 @@ typedef enum _mgmt_table {
TSDB_MGMT_TABLE_GRANTS_FULL,
TSDB_MGMT_TABLE_GRANTS_LOGS,
TSDB_MGMT_TABLE_MACHINES,
TSDB_MGMT_TABLE_ARBGROUP,
TSDB_MGMT_TABLE_MAX,
} EShowType;
@ -337,6 +338,7 @@ typedef enum ENodeType {
QUERY_NODE_SHOW_QNODES_STMT,
QUERY_NODE_SHOW_SNODES_STMT,
QUERY_NODE_SHOW_BNODES_STMT,
QUERY_NODE_SHOW_ARBGROUPS_STMT,
QUERY_NODE_SHOW_CLUSTER_STMT,
QUERY_NODE_SHOW_DATABASES_STMT,
QUERY_NODE_SHOW_FUNCTIONS_STMT,
@ -1156,6 +1158,7 @@ typedef struct {
int32_t tsdbPageSize;
int32_t sqlLen;
char* sql;
int8_t withArbitrator;
} SCreateDbReq;
int32_t tSerializeSCreateDbReq(void* buf, int32_t bufLen, SCreateDbReq* pReq);
@ -1186,6 +1189,7 @@ typedef struct {
int8_t s3Compact;
int32_t sqlLen;
char* sql;
int8_t withArbitrator;
} SAlterDbReq;
int32_t tSerializeSAlterDbReq(void* buf, int32_t bufLen, SAlterDbReq* pReq);
@ -1325,6 +1329,7 @@ typedef struct {
SArray* pRetensions;
int8_t schemaless;
int16_t sstTrigger;
int8_t withArbitrator;
} SDbCfgRsp;
typedef SDbCfgRsp SDbCfgInfo;
@ -2173,6 +2178,99 @@ typedef struct {
int32_t tSerializeSDCreateMnodeReq(void* buf, int32_t bufLen, SDCreateMnodeReq* pReq);
int32_t tDeserializeSDCreateMnodeReq(void* buf, int32_t bufLen, SDCreateMnodeReq* pReq);
typedef struct {
int32_t vgId;
int32_t hbSeq;
} SVArbHbReqMember;
typedef struct {
int32_t dnodeId;
char* arbToken;
int64_t arbTerm;
SArray* hbMembers; // SVArbHbReqMember
} SVArbHeartBeatReq;
int32_t tSerializeSVArbHeartBeatReq(void* buf, int32_t bufLen, SVArbHeartBeatReq* pReq);
int32_t tDeserializeSVArbHeartBeatReq(void* buf, int32_t bufLen, SVArbHeartBeatReq* pReq);
void tFreeSVArbHeartBeatReq(SVArbHeartBeatReq* pReq);
typedef struct {
int32_t vgId;
char memberToken[TSDB_ARB_TOKEN_SIZE];
int32_t hbSeq;
} SVArbHbRspMember;
typedef struct {
char arbToken[TSDB_ARB_TOKEN_SIZE];
int32_t dnodeId;
SArray* hbMembers; // SVArbHbRspMember
} SVArbHeartBeatRsp;
int32_t tSerializeSVArbHeartBeatRsp(void* buf, int32_t bufLen, SVArbHeartBeatRsp* pRsp);
int32_t tDeserializeSVArbHeartBeatRsp(void* buf, int32_t bufLen, SVArbHeartBeatRsp* pRsp);
void tFreeSVArbHeartBeatRsp(SVArbHeartBeatRsp* pRsp);
typedef struct {
char* arbToken;
int64_t arbTerm;
char* member0Token;
char* member1Token;
} SVArbCheckSyncReq;
int32_t tSerializeSVArbCheckSyncReq(void* buf, int32_t bufLen, SVArbCheckSyncReq* pReq);
int32_t tDeserializeSVArbCheckSyncReq(void* buf, int32_t bufLen, SVArbCheckSyncReq* pReq);
void tFreeSVArbCheckSyncReq(SVArbCheckSyncReq* pRsp);
typedef struct {
char* arbToken;
char* member0Token;
char* member1Token;
int32_t vgId;
int32_t errCode;
} SVArbCheckSyncRsp;
int32_t tSerializeSVArbCheckSyncRsp(void* buf, int32_t bufLen, SVArbCheckSyncRsp* pRsp);
int32_t tDeserializeSVArbCheckSyncRsp(void* buf, int32_t bufLen, SVArbCheckSyncRsp* pRsp);
void tFreeSVArbCheckSyncRsp(SVArbCheckSyncRsp* pRsp);
typedef struct {
char* arbToken;
int64_t arbTerm;
char* memberToken;
} SVArbSetAssignedLeaderReq;
int32_t tSerializeSVArbSetAssignedLeaderReq(void* buf, int32_t bufLen, SVArbSetAssignedLeaderReq* pReq);
int32_t tDeserializeSVArbSetAssignedLeaderReq(void* buf, int32_t bufLen, SVArbSetAssignedLeaderReq* pReq);
void tFreeSVArbSetAssignedLeaderReq(SVArbSetAssignedLeaderReq* pReq);
typedef struct {
char* arbToken;
char* memberToken;
int32_t vgId;
} SVArbSetAssignedLeaderRsp;
int32_t tSerializeSVArbSetAssignedLeaderRsp(void* buf, int32_t bufLen, SVArbSetAssignedLeaderRsp* pRsp);
int32_t tDeserializeSVArbSetAssignedLeaderRsp(void* buf, int32_t bufLen, SVArbSetAssignedLeaderRsp* pRsp);
void tFreeSVArbSetAssignedLeaderRsp(SVArbSetAssignedLeaderRsp* pRsp);
typedef struct {
int32_t dnodeId;
char* token;
} SMArbUpdateGroupReqMember;
typedef struct {
int32_t vgId;
int64_t dbUid;
SMArbUpdateGroupReqMember members[2];
int8_t isSync;
SMArbUpdateGroupReqMember assignedLeader;
int64_t version;
} SMArbUpdateGroupReq;
int32_t tSerializeSMArbUpdateGroupReq(void* buf, int32_t bufLen, SMArbUpdateGroupReq* pReq);
int32_t tDeserializeSMArbUpdateGroupReq(void* buf, int32_t bufLen, SMArbUpdateGroupReq* pReq);
void tFreeSMArbUpdateGroupReq(SMArbUpdateGroupReq* pReq);
typedef struct {
char queryStrId[TSDB_QUERY_ID_LEN];
} SKillQueryReq;
@ -2848,8 +2946,10 @@ typedef struct {
char* tagName;
int8_t isNull;
int8_t tagType;
int8_t tagFree;
uint32_t nTagVal;
uint8_t* pTagVal;
SArray* pTagArray;
// TSDB_ALTER_TABLE_UPDATE_OPTIONS
int8_t updateTTL;
int32_t newTTL;
@ -3848,6 +3948,7 @@ typedef struct {
uint32_t phyLen;
char* sql;
char* msg;
int8_t source;
} SVDeleteReq;
int32_t tSerializeSVDeleteReq(void* buf, int32_t bufLen, SVDeleteReq* pReq);
@ -3869,6 +3970,7 @@ typedef struct SDeleteRes {
char tableFName[TSDB_TABLE_NAME_LEN];
char tsColName[TSDB_COL_NAME_LEN];
int64_t ctimeMs; // fill by vnode
int8_t source;
} SDeleteRes;
int32_t tEncodeDeleteRes(SEncoder* pCoder, const SDeleteRes* pRes);
@ -3945,11 +4047,11 @@ int32_t tSerializeSMqAskEpReq(void* buf, int32_t bufLen, SMqAskEpReq* pReq);
int32_t tDeserializeSMqAskEpReq(void* buf, int32_t bufLen, SMqAskEpReq* pReq);
int32_t tSerializeSMqHbReq(void* buf, int32_t bufLen, SMqHbReq* pReq);
int32_t tDeserializeSMqHbReq(void* buf, int32_t bufLen, SMqHbReq* pReq);
int32_t tDeatroySMqHbReq(SMqHbReq* pReq);
void tDestroySMqHbReq(SMqHbReq* pReq);
int32_t tSerializeSMqHbRsp(void* buf, int32_t bufLen, SMqHbRsp* pRsp);
int32_t tDeserializeSMqHbRsp(void* buf, int32_t bufLen, SMqHbRsp* pRsp);
int32_t tDeatroySMqHbRsp(SMqHbRsp* pRsp);
void tDestroySMqHbRsp(SMqHbRsp* pRsp);
int32_t tSerializeSMqSeekReq(void* buf, int32_t bufLen, SMqSeekReq* pReq);
int32_t tDeserializeSMqSeekReq(void* buf, int32_t bufLen, SMqSeekReq* pReq);

3
include/common/tmsgcb.h
View File

@ -36,9 +36,11 @@ typedef enum {
SYNC_QUEUE,
SYNC_RD_QUEUE,
STREAM_QUEUE,
ARB_QUEUE,
QUEUE_MAX,
} EQueueType;
typedef void (*GetDnodeEp)(void* pData, int32_t dnodeId, char* pEp, char* pFqdn, uint16_t* pPort);
typedef bool (*UpdateDnodeInfoFp)(void* pData, int32_t* dnodeId, int64_t* clusterId, char* fqdn, uint16_t* port);
typedef int32_t (*PutToQueueFp)(void* pMgmt, EQueueType qtype, SRpcMsg* pMsg);
typedef int32_t (*GetQueueSizeFp)(void* pMgmt, int32_t vgId, EQueueType qtype);
@ -64,6 +66,7 @@ typedef struct {
ReleaseHandleFp releaseHandleFp;
ReportStartup reportStartupFp;
UpdateDnodeInfoFp updateDnodeInfoFp;
GetDnodeEp getDnodeEpFp;
} SMsgCb;
void tmsgSetDefault(const SMsgCb* msgcb);

10
include/common/tmsgdef.h
View File

@ -275,6 +275,8 @@
TD_DEF_MSG_TYPE(TDMT_VND_QUERY_COMPACT_PROGRESS, "vnode-query-compact-progress", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_KILL_COMPACT, "kill-compact", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_S3MIGRATE, "vnode-s3migrate", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_ARB_HEARTBEAT, "vnode-arb-hb", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_ARB_CHECK_SYNC, "vnode-arb-check-sync", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_MAX_MSG, "vnd-max", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_END_VND_MSG)
@ -344,6 +346,7 @@
TD_DEF_MSG_TYPE(TDMT_SYNC_PREP_SNAPSHOT_REPLY, "sync-prep-snapshot-reply", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_SYNC_UNUSED_CODE, "sync-unused", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_SYNC_FORCE_FOLLOWER, "sync-force-become-follower", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_SYNC_SET_ASSIGNED_LEADER, "sync-set-assigned-leader", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_SYNC_MAX_MSG, "sync-max", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_END_SYNC_MSG)
@ -370,6 +373,13 @@
TD_DEF_MSG_TYPE(TDMT_VND_TMQ_MAX_MSG, "vnd-tmq-max", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_END_TMQ_MSG)
TD_NEW_MSG_SEG(TDMT_MND_ARB_MSG) //9 << 8
TD_DEF_MSG_TYPE(TDMT_MND_ARB_HEARTBEAT_TIMER, "mnd-arb-hb-tmr", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_ARB_CHECK_SYNC_TIMER, "mnd-arb-check-sync-tmr", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_ARB_UPDATE_GROUP, "mnd-arb-update-group", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_ARB_MAX_MSG, "mnd-arb-max", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_END_ARB_MSG)
TD_NEW_MSG_SEG(TDMT_MAX_MSG) // msg end mark
TD_CLOSE_MSG_SEG(TDMT_END_MAX_MSG)

408
include/common/ttokendef.h
View File

@ -171,218 +171,222 @@
#define TK_STABLES 153
#define TK_MNODES 154
#define TK_QNODES 155
#define TK_FUNCTIONS 156
#define TK_INDEXES 157
#define TK_ACCOUNTS 158
#define TK_APPS 159
#define TK_CONNECTIONS 160
#define TK_LICENCES 161
#define TK_GRANTS 162
#define TK_FULL 163
#define TK_LOGS 164
#define TK_MACHINES 165
#define TK_QUERIES 166
#define TK_SCORES 167
#define TK_TOPICS 168
#define TK_VARIABLES 169
#define TK_BNODES 170
#define TK_SNODES 171
#define TK_TRANSACTIONS 172
#define TK_DISTRIBUTED 173
#define TK_CONSUMERS 174
#define TK_SUBSCRIPTIONS 175
#define TK_VNODES 176
#define TK_ALIVE 177
#define TK_VIEWS 178
#define TK_VIEW 179
#define TK_COMPACTS 180
#define TK_NORMAL 181
#define TK_CHILD 182
#define TK_LIKE 183
#define TK_TBNAME 184
#define TK_QTAGS 185
#define TK_AS 186
#define TK_SYSTEM 187
#define TK_INDEX 188
#define TK_FUNCTION 189
#define TK_INTERVAL 190
#define TK_COUNT 191
#define TK_LAST_ROW 192
#define TK_META 193
#define TK_ONLY 194
#define TK_TOPIC 195
#define TK_CONSUMER 196
#define TK_GROUP 197
#define TK_DESC 198
#define TK_DESCRIBE 199
#define TK_RESET 200
#define TK_QUERY 201
#define TK_CACHE 202
#define TK_EXPLAIN 203
#define TK_ANALYZE 204
#define TK_VERBOSE 205
#define TK_NK_BOOL 206
#define TK_RATIO 207
#define TK_NK_FLOAT 208
#define TK_OUTPUTTYPE 209
#define TK_AGGREGATE 210
#define TK_BUFSIZE 211
#define TK_LANGUAGE 212
#define TK_REPLACE 213
#define TK_STREAM 214
#define TK_INTO 215
#define TK_PAUSE 216
#define TK_RESUME 217
#define TK_TRIGGER 218
#define TK_AT_ONCE 219
#define TK_WINDOW_CLOSE 220
#define TK_IGNORE 221
#define TK_EXPIRED 222
#define TK_FILL_HISTORY 223
#define TK_UPDATE 224
#define TK_SUBTABLE 225
#define TK_UNTREATED 226
#define TK_KILL 227
#define TK_CONNECTION 228
#define TK_TRANSACTION 229
#define TK_BALANCE 230
#define TK_VGROUP 231
#define TK_LEADER 232
#define TK_MERGE 233
#define TK_REDISTRIBUTE 234
#define TK_SPLIT 235
#define TK_DELETE 236
#define TK_INSERT 237
#define TK_NULL 238
#define TK_NK_QUESTION 239
#define TK_NK_ALIAS 240
#define TK_NK_ARROW 241
#define TK_ROWTS 242
#define TK_QSTART 243
#define TK_QEND 244
#define TK_QDURATION 245
#define TK_WSTART 246
#define TK_WEND 247
#define TK_WDURATION 248
#define TK_IROWTS 249
#define TK_ISFILLED 250
#define TK_CAST 251
#define TK_NOW 252
#define TK_TODAY 253
#define TK_TIMEZONE 254
#define TK_CLIENT_VERSION 255
#define TK_SERVER_VERSION 256
#define TK_SERVER_STATUS 257
#define TK_CURRENT_USER 258
#define TK_CASE 259
#define TK_WHEN 260
#define TK_THEN 261
#define TK_ELSE 262
#define TK_BETWEEN 263
#define TK_IS 264
#define TK_NK_LT 265
#define TK_NK_GT 266
#define TK_NK_LE 267
#define TK_NK_GE 268
#define TK_NK_NE 269
#define TK_MATCH 270
#define TK_NMATCH 271
#define TK_CONTAINS 272
#define TK_IN 273
#define TK_JOIN 274
#define TK_INNER 275
#define TK_SELECT 276
#define TK_NK_HINT 277
#define TK_DISTINCT 278
#define TK_WHERE 279
#define TK_PARTITION 280
#define TK_BY 281
#define TK_SESSION 282
#define TK_STATE_WINDOW 283
#define TK_EVENT_WINDOW 284
#define TK_COUNT_WINDOW 285
#define TK_SLIDING 286
#define TK_FILL 287
#define TK_VALUE 288
#define TK_VALUE_F 289
#define TK_NONE 290
#define TK_PREV 291
#define TK_NULL_F 292
#define TK_LINEAR 293
#define TK_NEXT 294
#define TK_HAVING 295
#define TK_RANGE 296
#define TK_EVERY 297
#define TK_ORDER 298
#define TK_SLIMIT 299
#define TK_SOFFSET 300
#define TK_LIMIT 301
#define TK_OFFSET 302
#define TK_ASC 303
#define TK_NULLS 304
#define TK_ABORT 305
#define TK_AFTER 306
#define TK_ATTACH 307
#define TK_BEFORE 308
#define TK_BEGIN 309
#define TK_BITAND 310
#define TK_BITNOT 311
#define TK_BITOR 312
#define TK_BLOCKS 313
#define TK_CHANGE 314
#define TK_COMMA 315
#define TK_CONCAT 316
#define TK_CONFLICT 317
#define TK_COPY 318
#define TK_DEFERRED 319
#define TK_DELIMITERS 320
#define TK_DETACH 321
#define TK_DIVIDE 322
#define TK_DOT 323
#define TK_EACH 324
#define TK_FAIL 325
#define TK_FILE 326
#define TK_FOR 327
#define TK_GLOB 328
#define TK_ID 329
#define TK_IMMEDIATE 330
#define TK_IMPORT 331
#define TK_INITIALLY 332
#define TK_INSTEAD 333
#define TK_ISNULL 334
#define TK_KEY 335
#define TK_MODULES 336
#define TK_NK_BITNOT 337
#define TK_NK_SEMI 338
#define TK_NOTNULL 339
#define TK_OF 340
#define TK_PLUS 341
#define TK_PRIVILEGE 342
#define TK_RAISE 343
#define TK_RESTRICT 344
#define TK_ROW 345
#define TK_SEMI 346
#define TK_STAR 347
#define TK_STATEMENT 348
#define TK_STRICT 349
#define TK_STRING 350
#define TK_TIMES 351
#define TK_VALUES 352
#define TK_VARIABLE 353
#define TK_WAL 354
#define TK_ARBGROUPS 156
#define TK_FUNCTIONS 157
#define TK_INDEXES 158
#define TK_ACCOUNTS 159
#define TK_APPS 160
#define TK_CONNECTIONS 161
#define TK_LICENCES 162
#define TK_GRANTS 163
#define TK_FULL 164
#define TK_LOGS 165
#define TK_MACHINES 166
#define TK_QUERIES 167
#define TK_SCORES 168
#define TK_TOPICS 169
#define TK_VARIABLES 170
#define TK_BNODES 171
#define TK_SNODES 172
#define TK_TRANSACTIONS 173
#define TK_DISTRIBUTED 174
#define TK_CONSUMERS 175
#define TK_SUBSCRIPTIONS 176
#define TK_VNODES 177
#define TK_ALIVE 178
#define TK_VIEWS 179
#define TK_VIEW 180
#define TK_COMPACTS 181
#define TK_NORMAL 182
#define TK_CHILD 183
#define TK_LIKE 184
#define TK_TBNAME 185
#define TK_QTAGS 186
#define TK_AS 187
#define TK_SYSTEM 188
#define TK_INDEX 189
#define TK_FUNCTION 190
#define TK_INTERVAL 191
#define TK_COUNT 192
#define TK_LAST_ROW 193
#define TK_META 194
#define TK_ONLY 195
#define TK_TOPIC 196
#define TK_CONSUMER 197
#define TK_GROUP 198
#define TK_DESC 199
#define TK_DESCRIBE 200
#define TK_RESET 201
#define TK_QUERY 202
#define TK_CACHE 203
#define TK_EXPLAIN 204
#define TK_ANALYZE 205
#define TK_VERBOSE 206
#define TK_NK_BOOL 207
#define TK_RATIO 208
#define TK_NK_FLOAT 209
#define TK_OUTPUTTYPE 210
#define TK_AGGREGATE 211
#define TK_BUFSIZE 212
#define TK_LANGUAGE 213
#define TK_REPLACE 214
#define TK_STREAM 215
#define TK_INTO 216
#define TK_PAUSE 217
#define TK_RESUME 218
#define TK_TRIGGER 219
#define TK_AT_ONCE 220
#define TK_WINDOW_CLOSE 221
#define TK_IGNORE 222
#define TK_EXPIRED 223
#define TK_FILL_HISTORY 224
#define TK_UPDATE 225
#define TK_SUBTABLE 226
#define TK_UNTREATED 227
#define TK_KILL 228
#define TK_CONNECTION 229
#define TK_TRANSACTION 230
#define TK_BALANCE 231
#define TK_VGROUP 232
#define TK_LEADER 233
#define TK_MERGE 234
#define TK_REDISTRIBUTE 235
#define TK_SPLIT 236
#define TK_DELETE 237
#define TK_INSERT 238
#define TK_NK_BIN 239
#define TK_NK_HEX 240
#define TK_NULL 241
#define TK_NK_QUESTION 242
#define TK_NK_ALIAS 243
#define TK_NK_ARROW 244
#define TK_ROWTS 245
#define TK_QSTART 246
#define TK_QEND 247
#define TK_QDURATION 248
#define TK_WSTART 249
#define TK_WEND 250
#define TK_WDURATION 251
#define TK_IROWTS 252
#define TK_ISFILLED 253
#define TK_CAST 254
#define TK_NOW 255
#define TK_TODAY 256
#define TK_TIMEZONE 257
#define TK_CLIENT_VERSION 258
#define TK_SERVER_VERSION 259
#define TK_SERVER_STATUS 260
#define TK_CURRENT_USER 261
#define TK_CASE 262
#define TK_WHEN 263
#define TK_THEN 264
#define TK_ELSE 265
#define TK_BETWEEN 266
#define TK_IS 267
#define TK_NK_LT 268
#define TK_NK_GT 269
#define TK_NK_LE 270
#define TK_NK_GE 271
#define TK_NK_NE 272
#define TK_MATCH 273
#define TK_NMATCH 274
#define TK_CONTAINS 275
#define TK_IN 276
#define TK_JOIN 277
#define TK_INNER 278
#define TK_SELECT 279
#define TK_NK_HINT 280
#define TK_DISTINCT 281
#define TK_WHERE 282
#define TK_PARTITION 283
#define TK_BY 284
#define TK_SESSION 285
#define TK_STATE_WINDOW 286
#define TK_EVENT_WINDOW 287
#define TK_COUNT_WINDOW 288
#define TK_SLIDING 289
#define TK_FILL 290
#define TK_VALUE 291
#define TK_VALUE_F 292
#define TK_NONE 293
#define TK_PREV 294
#define TK_NULL_F 295
#define TK_LINEAR 296
#define TK_NEXT 297
#define TK_HAVING 298
#define TK_RANGE 299
#define TK_EVERY 300
#define TK_ORDER 301
#define TK_SLIMIT 302
#define TK_SOFFSET 303
#define TK_LIMIT 304
#define TK_OFFSET 305
#define TK_ASC 306
#define TK_NULLS 307
#define TK_ABORT 308
#define TK_AFTER 309
#define TK_ATTACH 310
#define TK_BEFORE 311
#define TK_BEGIN 312
#define TK_BITAND 313
#define TK_BITNOT 314
#define TK_BITOR 315
#define TK_BLOCKS 316
#define TK_CHANGE 317
#define TK_COMMA 318
#define TK_CONCAT 319
#define TK_CONFLICT 320
#define TK_COPY 321
#define TK_DEFERRED 322
#define TK_DELIMITERS 323
#define TK_DETACH 324
#define TK_DIVIDE 325
#define TK_DOT 326
#define TK_EACH 327
#define TK_FAIL 328
#define TK_FILE 329
#define TK_FOR 330
#define TK_GLOB 331
#define TK_ID 332
#define TK_IMMEDIATE 333
#define TK_IMPORT 334
#define TK_INITIALLY 335
#define TK_INSTEAD 336
#define TK_ISNULL 337
#define TK_KEY 338
#define TK_MODULES 339
#define TK_NK_BITNOT 340
#define TK_NK_SEMI 341
#define TK_NOTNULL 342
#define TK_OF 343
#define TK_PLUS 344
#define TK_PRIVILEGE 345
#define TK_RAISE 346
#define TK_RESTRICT 347
#define TK_ROW 348
#define TK_SEMI 349
#define TK_STAR 350
#define TK_STATEMENT 351
#define TK_STRICT 352
#define TK_STRING 353
#define TK_TIMES 354
#define TK_VALUES 355
#define TK_VARIABLE 356
#define TK_WAL 357
#define TK_NK_SPACE 600
#define TK_NK_COMMENT 601
#define TK_NK_ILLEGAL 602
#define TK_NK_HEX 603 // hex number 0x123
// #define TK_NK_HEX 603 // hex number 0x123
#define TK_NK_OCT 604 // oct number
#define TK_NK_BIN 605 // bin format data 0b111
// #define TK_NK_BIN 605 // bin format data 0b111
#define TK_BATCH_SCAN 606
#define TK_NO_BATCH_SCAN 607
#define TK_SORT_FOR_GROUP 608
#define TK_PARTITION_FIRST 609
#define TK_PARA_TABLES_SORT 610
#define TK_PARA_TABLES_SORT 610
#define TK_SMALLDATA_TS_SORT 611
#define TK_NK_NIL 65535

5
include/common/tvariant.h
View File

@ -44,6 +44,11 @@ int32_t toDoubleEx(const char *z, int32_t n, uint32_t type, double *value);
int32_t toInteger(const char *z, int32_t n, int32_t base, int64_t *value);
int32_t toUInteger(const char *z, int32_t n, int32_t base, uint64_t *value);
/**
* non floating point integers
*/
int32_t toIntegerPure(const char *z, int32_t n, int32_t base, int64_t *value);
void taosVariantCreateFromBinary(SVariant *pVar, const char *pz, size_t len, uint32_t type);
void taosVariantDestroy(SVariant *pV);

3
include/dnode/mnode/mnode.h
View File

@ -75,6 +75,9 @@ void mndStop(SMnode *pMnode);
int32_t mndIsCatchUp(SMnode *pMnode);
ESyncRole mndGetRole(SMnode *pMnode);
int64_t mndGetTerm(SMnode *pMnode);
int32_t mndGetArbToken(SMnode *pMnode, char *outToken);
/**
* @brief Get mnode monitor info.

17
include/libs/executor/storageapi.h
View File

@ -36,6 +36,7 @@ extern "C" {
#define CACHESCAN_RETRIEVE_LAST_ROW 0x4
#define CACHESCAN_RETRIEVE_LAST 0x8
#define META_READER_LOCK 0x0
#define META_READER_NOLOCK 0x1
#define STREAM_STATE_BUFF_HASH 1
@ -223,10 +224,10 @@ typedef struct SStoreTqReader {
bool (*tqReaderCurrentBlockConsumed)();
struct SWalReader* (*tqReaderGetWalReader)(); // todo remove it
int32_t (*tqReaderRetrieveTaosXBlock)(); // todo remove it
// int32_t (*tqReaderRetrieveTaosXBlock)(); // todo remove it
int32_t (*tqReaderSetSubmitMsg)(); // todo remove it
bool (*tqReaderNextBlockFilterOut)();
// bool (*tqReaderNextBlockFilterOut)();
} SStoreTqReader;
typedef struct SStoreSnapshotFn {
@ -237,12 +238,12 @@ typedef struct SStoreSnapshotFn {
} SStoreSnapshotFn;
typedef struct SStoreMeta {
SMTbCursor* (*openTableMetaCursor)(void* pVnode); // metaOpenTbCursor
void (*closeTableMetaCursor)(SMTbCursor* pTbCur); // metaCloseTbCursor
void (*pauseTableMetaCursor)(SMTbCursor* pTbCur); // metaPauseTbCursor
void (*resumeTableMetaCursor)(SMTbCursor* pTbCur, int8_t first); // metaResumeTbCursor
int32_t (*cursorNext)(SMTbCursor* pTbCur, ETableType jumpTableType); // metaTbCursorNext
int32_t (*cursorPrev)(SMTbCursor* pTbCur, ETableType jumpTableType); // metaTbCursorPrev
SMTbCursor* (*openTableMetaCursor)(void* pVnode); // metaOpenTbCursor
void (*closeTableMetaCursor)(SMTbCursor* pTbCur); // metaCloseTbCursor
void (*pauseTableMetaCursor)(SMTbCursor* pTbCur); // metaPauseTbCursor
void (*resumeTableMetaCursor)(SMTbCursor* pTbCur, int8_t first, int8_t move); // metaResumeTbCursor
int32_t (*cursorNext)(SMTbCursor* pTbCur, ETableType jumpTableType); // metaTbCursorNext
int32_t (*cursorPrev)(SMTbCursor* pTbCur, ETableType jumpTableType); // metaTbCursorPrev
int32_t (*getTableTags)(void* pVnode, uint64_t suid, SArray* uidList);
int32_t (*getTableTagsByUid)(void* pVnode, int64_t suid, SArray* uidList);

1
include/libs/nodes/cmdnodes.h
View File

@ -104,6 +104,7 @@ typedef struct SDatabaseOptions {
int32_t s3KeepLocal;
SValueNode* s3KeepLocalStr;
int8_t s3Compact;
int8_t withArbitrator;
} SDatabaseOptions;
typedef struct SCreateDatabaseStmt {

2
include/libs/nodes/plannodes.h
View File

@ -122,6 +122,7 @@ typedef struct SScanLogicNode {
bool isCountByTag; // true if selectstmt hasCountFunc & part by tag/tbname
SArray* pFuncTypes; // for last, last_row
bool paraTablesSort; // for table merge scan
bool smallDataTsSort; // disable row id sort for table merge scan
} SScanLogicNode;
typedef struct SJoinLogicNode {
@ -445,6 +446,7 @@ typedef struct STableScanPhysiNode {
bool filesetDelimited;
bool needCountEmptyTable;
bool paraTablesSort;
bool smallDataTsSort;
} STableScanPhysiNode;
typedef STableScanPhysiNode STableSeqScanPhysiNode;

3
include/libs/nodes/querynodes.h
View File

@ -128,7 +128,8 @@ typedef enum EHintOption {
HINT_BATCH_SCAN,
HINT_SORT_FOR_GROUP,
HINT_PARTITION_FIRST,
HINT_PARA_TABLES_SORT
HINT_PARA_TABLES_SORT,
HINT_SMALLDATA_TS_SORT,
} EHintOption;
typedef struct SHintNode {

1
include/libs/scheduler/scheduler.h
View File

@ -78,6 +78,7 @@ typedef struct SSchedulerReq {
void* chkKillParam;
SExecResult* pExecRes;
void** pFetchRes;
int8_t source;
} SSchedulerReq;
int32_t schedulerInit(void);

4
include/libs/stream/tstream.h
View File

@ -61,7 +61,7 @@ typedef struct SStreamTask SStreamTask;
typedef struct SStreamQueue SStreamQueue;
typedef struct SStreamTaskSM SStreamTaskSM;
#define SSTREAM_TASK_VER 3
#define SSTREAM_TASK_VER 4
#define SSTREAM_TASK_INCOMPATIBLE_VER 1
#define SSTREAM_TASK_NEED_CONVERT_VER 2
#define SSTREAM_TASK_SUBTABLE_CHANGED_VER 3
@ -355,6 +355,8 @@ typedef struct SMetaHbInfo SMetaHbInfo;
typedef struct SDispatchMsgInfo {
SStreamDispatchReq* pData; // current dispatch data
int8_t dispatchMsgType;
int64_t checkpointId;// checkpoint id msg
int32_t transId; // transId for current checkpoint
int16_t msgType; // dispatch msg type
int32_t retryCount; // retry send data count
int64_t startTs; // dispatch start time, record total elapsed time for dispatch

8
include/libs/sync/sync.h
View File

@ -80,6 +80,7 @@ typedef enum {
TAOS_SYNC_STATE_LEADER = 102,
TAOS_SYNC_STATE_ERROR = 103,
TAOS_SYNC_STATE_LEARNER = 104,
TAOS_SYNC_STATE_ASSIGNED_LEADER = 105,
} ESyncState;
typedef enum {
@ -184,6 +185,7 @@ typedef struct SSyncFSM {
void (*FpBecomeLeaderCb)(const struct SSyncFSM* pFsm);
void (*FpBecomeFollowerCb)(const struct SSyncFSM* pFsm);
void (*FpBecomeLearnerCb)(const struct SSyncFSM* pFsm);
void (*FpBecomeAssignedLeaderCb)(const struct SSyncFSM* pFsm);
int32_t (*FpGetSnapshot)(const struct SSyncFSM* pFsm, SSnapshot* pSnapshot, void* pReaderParam, void** ppReader);
int32_t (*FpGetSnapshotInfo)(const struct SSyncFSM* pFsm, SSnapshot* pSnapshot);
@ -273,6 +275,7 @@ int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak, int64_t* seq);
int32_t syncCheckMember(int64_t rid);
int32_t syncIsCatchUp(int64_t rid);
ESyncRole syncGetRole(int64_t rid);
int64_t syncGetTerm(int64_t rid);
int32_t syncProcessMsg(int64_t rid, SRpcMsg* pMsg);
int32_t syncReconfig(int64_t rid, SSyncCfg* pCfg);
int32_t syncBeginSnapshot(int64_t rid, int64_t lastApplyIndex);
@ -284,8 +287,13 @@ bool syncSnapshotSending(int64_t rid);
bool syncSnapshotRecving(int64_t rid);
int32_t syncSendTimeoutRsp(int64_t rid, int64_t seq);
int32_t syncForceBecomeFollower(SSyncNode* ths, const SRpcMsg* pRpcMsg);
int32_t syncBecomeAssignedLeader(SSyncNode* ths, SRpcMsg* pRpcMsg);
int32_t syncUpdateArbTerm(int64_t rid, SyncTerm arbTerm);
SSyncState syncGetState(int64_t rid);
int32_t syncGetArbToken(int64_t rid, char* outToken);
int32_t syncGetAssignedLogSynced(int64_t rid);
void syncGetRetryEpSet(int64_t rid, SEpSet* pEpSet);
const char* syncStr(ESyncState state);

1
include/libs/transport/trpc.h
View File

@ -62,6 +62,7 @@ typedef struct SRpcHandleInfo {
SRpcConnInfo conn;
int8_t forbiddenIp;
int8_t notFreeAhandle;
} SRpcHandleInfo;

7
include/os/osFile.h
View File

@ -119,6 +119,13 @@ int32_t taosSetFileHandlesLimit();
int32_t taosLinkFile(char *src, char *dst);
FILE* taosOpenCFile(const char* filename, const char* mode);
int taosSeekCFile(FILE* file, int64_t offset, int whence);
size_t taosReadFromCFile(void *buffer, size_t size, size_t count, FILE *stream );
size_t taosWriteToCFile(const void* ptr, size_t size, size_t nitems, FILE* stream);
int taosCloseCFile(FILE *);
int taosSetAutoDelFile(char* path);
bool lastErrorIsFileNotExist();
#ifdef __cplusplus

4
include/util/taoserror.h
View File

@ -323,6 +323,9 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_BNODE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03A7)
#define TSDB_CODE_MND_TOO_FEW_MNODES TAOS_DEF_ERROR_CODE(0, 0x03A8)
#define TSDB_CODE_MND_TOO_MANY_MNODES TAOS_DEF_ERROR_CODE(0, 0x03A9)
#define TSDB_CODE_MND_ARBGROUP_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03AA)
#define TSDB_CODE_MND_ARBGROUP_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03AB)
#define TSDB_CODE_MND_ARB_TOKEN_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x03AC)
// mnode-dnode-part2
#define TSDB_CODE_MND_TOO_MANY_DNODES TAOS_DEF_ERROR_CODE(0, 0x03B0)
@ -472,6 +475,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_VND_ALREADY_IS_VOTER TAOS_DEF_ERROR_CODE(0, 0x0533) // internal
#define TSDB_CODE_VND_DIR_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0534)
#define TSDB_CODE_VND_META_DATA_UNSAFE_DELETE TAOS_DEF_ERROR_CODE(0, 0x0535)
#define TSDB_CODE_VND_ARB_NOT_SYNCED TAOS_DEF_ERROR_CODE(0, 0x0536) // internal
// tsdb
#define TSDB_CODE_TDB_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0600)

23
include/util/tdef.h
View File

@ -279,6 +279,9 @@ typedef enum ELogicConditionType {
#define TSDB_SHOW_SUBQUERY_LEN 1000
#define TSDB_LOG_VAR_LEN 32
#define TSDB_ARB_GROUP_MEMBER_NUM 2
#define TSDB_ARB_TOKEN_SIZE 32
#define TSDB_TRANS_STAGE_LEN 12
#define TSDB_TRANS_TYPE_LEN 16
#define TSDB_TRANS_ERROR_LEN 512
@ -435,6 +438,10 @@ typedef enum ELogicConditionType {
#define TSDB_DB_MIN_WAL_SEGMENT_SIZE 0
#define TSDB_DEFAULT_DB_WAL_SEGMENT_SIZE 0
#define TSDB_DEFAULT_DB_WITH_ARBITRATOR 0
#define TSDB_MIN_DB_WITH_ARBITRATOR 0
#define TSDB_MAX_DB_WITH_ARBITRATOR 1
#define TSDB_MIN_ROLLUP_MAX_DELAY 1 // unit millisecond
#define TSDB_MAX_ROLLUP_MAX_DELAY (15 * 60 * 1000)
#define TSDB_MIN_ROLLUP_WATERMARK 0 // unit millisecond
@ -533,12 +540,12 @@ enum {
SND_WORKER_TYPE__UNIQUE,
};
#define DEFAULT_HANDLE 0
#define MNODE_HANDLE 1
#define QNODE_HANDLE -1
#define SNODE_HANDLE -2
#define VNODE_HANDLE -3
#define CLIENT_HANDLE -5
#define DEFAULT_HANDLE 0
#define MNODE_HANDLE 1
#define QNODE_HANDLE -1
#define SNODE_HANDLE -2
#define VNODE_HANDLE -3
#define CLIENT_HANDLE -5
#define TSDB_CONFIG_OPTION_LEN 32
#define TSDB_CONFIG_VALUE_LEN 64
@ -558,8 +565,8 @@ enum {
// sort page size by default
#define DEFAULT_PAGESIZE 4096
#define VNODE_TIMEOUT_SEC 60
#define MNODE_TIMEOUT_SEC 60
#define VNODE_TIMEOUT_SEC 60
#define MNODE_TIMEOUT_SEC 60
#define MONITOR_TABLENAME_LEN 200
#define MONITOR_TAG_NAME_LEN 100

8
source/client/inc/clientInt.h
View File

@ -284,6 +284,7 @@ typedef struct SRequestObj {
void* pWrapper;
SMetaData parseMeta;
char* effectiveUser;
int8_t source;
} SRequestObj;
typedef struct SSyncQueryParam {
@ -297,8 +298,7 @@ void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4);
void doSetOneRowPtr(SReqResultInfo* pResultInfo);
void setResPrecision(SReqResultInfo* pResInfo, int32_t precision);
int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4,
bool freeAfterUse);
int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4);
int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows,
bool convertUcs4);
void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols);
@ -306,10 +306,10 @@ void doFreeReqResultInfo(SReqResultInfo* pResInfo);
int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName, SArray** pReq);
void syncCatalogFn(SMetaData* pResult, void* param, int32_t code);
TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly);
TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly, int8_t source);
TAOS_RES* taosQueryImplWithReqid(TAOS* taos, const char* sql, bool validateOnly, int64_t reqid);
void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly);
void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly, int8_t source);
void taosAsyncQueryImplWithReqid(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly,
int64_t reqid);
void taosAsyncFetchImpl(SRequestObj *pRequest, __taos_async_fn_t fp, void *param);

23
source/client/src/clientImpl.c
View File

@ -302,7 +302,7 @@ int32_t execLocalCmd(SRequestObj* pRequest, SQuery* pQuery) {
int8_t biMode = atomic_load_8(&pRequest->pTscObj->biMode);
int32_t code = qExecCommand(&pRequest->pTscObj->id, pRequest->pTscObj->sysInfo, pQuery->pRoot, &pRsp, biMode);
if (TSDB_CODE_SUCCESS == code && NULL != pRsp) {
code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, false, true);
code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, false);
}
return code;
@ -341,7 +341,7 @@ void asyncExecLocalCmd(SRequestObj* pRequest, SQuery* pQuery) {
int32_t code = qExecCommand(&pRequest->pTscObj->id, pRequest->pTscObj->sysInfo, pQuery->pRoot, &pRsp,
atomic_load_8(&pRequest->pTscObj->biMode));
if (TSDB_CODE_SUCCESS == code && NULL != pRsp) {
code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, false, true);
code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, false);
}
SReqResultInfo* pResultInfo = &pRequest->body.resInfo;
@ -743,6 +743,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList
.chkKillFp = chkRequestKilled,
.chkKillParam = (void*)pRequest->self,
.pExecRes = &res,
.source = pRequest->source,
};
int32_t code = schedulerExecJob(&req, &pRequest->body.queryJob);
@ -1212,6 +1213,7 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat
.chkKillFp = chkRequestKilled,
.chkKillParam = (void*)pRequest->self,
.pExecRes = NULL,
.source = pRequest->source,
};
code = schedulerExecJob(&req, &pRequest->body.queryJob);
taosArrayDestroy(pNodeList);
@ -1719,7 +1721,7 @@ void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4)
}
pRequest->code =
setQueryResultFromRsp(&pRequest->body.resInfo, (const SRetrieveTableRsp*)pResInfo->pData, convertUcs4, true);
setQueryResultFromRsp(&pRequest->body.resInfo, (const SRetrieveTableRsp*)pResInfo->pData, convertUcs4);
if (pRequest->code != TSDB_CODE_SUCCESS) {
pResultInfo->numOfRows = 0;
return NULL;
@ -2178,15 +2180,13 @@ void resetConnectDB(STscObj* pTscObj) {
taosThreadMutexUnlock(&pTscObj->mutex);
}
int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4,
bool freeAfterUse) {
int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4) {
if (pResultInfo == NULL || pRsp == NULL) {
tscError("setQueryResultFromRsp paras is null");
return TSDB_CODE_TSC_INTERNAL_ERROR;
}
if (freeAfterUse) taosMemoryFreeClear(pResultInfo->pRspMsg);
taosMemoryFreeClear(pResultInfo->pRspMsg);
pResultInfo->pRspMsg = (const char*)pRsp;
pResultInfo->pData = (void*)pRsp->data;
pResultInfo->numOfRows = htobe64(pRsp->numOfRows);
@ -2475,7 +2475,7 @@ void syncQueryFn(void* param, void* res, int32_t code) {
tsem_post(&pParam->sem);
}
void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly) {
void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly, int8_t source) {
if (sql == NULL || NULL == fp) {
terrno = TSDB_CODE_INVALID_PARA;
if (fp) {
@ -2501,6 +2501,7 @@ void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp,
return;
}
pRequest->source = source;
pRequest->body.queryFp = fp;
doAsyncQuery(pRequest, false);
}
@ -2535,7 +2536,7 @@ void taosAsyncQueryImplWithReqid(uint64_t connId, const char* sql, __taos_async_
doAsyncQuery(pRequest, false);
}
TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) {
TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly, int8_t source) {
if (NULL == taos) {
terrno = TSDB_CODE_TSC_DISCONNECTED;
return NULL;
@ -2550,7 +2551,7 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) {
}
tsem_init(&param->sem, 0, 0);
taosAsyncQueryImpl(*(int64_t*)taos, sql, syncQueryFn, param, validateOnly);
taosAsyncQueryImpl(*(int64_t*)taos, sql, syncQueryFn, param, validateOnly, source);
tsem_wait(&param->sem);
SRequestObj* pRequest = NULL;
@ -2615,7 +2616,7 @@ static void fetchCallback(void* pResult, void* param, int32_t code) {
}
pRequest->code =
setQueryResultFromRsp(pResultInfo, (const SRetrieveTableRsp*)pResultInfo->pData, pResultInfo->convertUcs4, true);
setQueryResultFromRsp(pResultInfo, (const SRetrieveTableRsp*)pResultInfo->pData, pResultInfo->convertUcs4);
if (pRequest->code != TSDB_CODE_SUCCESS) {
pResultInfo->numOfRows = 0;
pRequest->code = code;

8
source/client/src/clientMain.c
View File

@ -350,7 +350,6 @@ void taos_free_result(TAOS_RES *res) {
taosArrayDestroy(pRsp->rsp.createTableLen);
taosArrayDestroyP(pRsp->rsp.createTableReq, taosMemoryFree);
pRsp->resInfo.pRspMsg = NULL;
doFreeReqResultInfo(&pRsp->resInfo);
taosMemoryFree(pRsp);
} else if (TD_RES_TMQ(res)) {
@ -359,7 +358,6 @@ void taos_free_result(TAOS_RES *res) {
taosArrayDestroy(pRsp->rsp.blockDataLen);
taosArrayDestroyP(pRsp->rsp.blockTbName, taosMemoryFree);
taosArrayDestroyP(pRsp->rsp.blockSchema, (FDelete)tDeleteSchemaWrapper);
pRsp->resInfo.pRspMsg = NULL;
doFreeReqResultInfo(&pRsp->resInfo);
taosMemoryFree(pRsp);
} else if (TD_RES_TMQ_META(res)) {
@ -402,7 +400,7 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) {
return pResInfo->userFields;
}
TAOS_RES *taos_query(TAOS *taos, const char *sql) { return taosQueryImpl(taos, sql, false); }
TAOS_RES *taos_query(TAOS *taos, const char *sql) { return taosQueryImpl(taos, sql, false, TD_REQ_FROM_APP); }
TAOS_RES *taos_query_with_reqid(TAOS *taos, const char *sql, int64_t reqid) {
return taosQueryImplWithReqid(taos, sql, false, reqid);
}
@ -828,7 +826,7 @@ int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex) {
}
int taos_validate_sql(TAOS *taos, const char *sql) {
TAOS_RES *pObj = taosQueryImpl(taos, sql, true);
TAOS_RES *pObj = taosQueryImpl(taos, sql, true, TD_REQ_FROM_APP);
int code = taos_errno(pObj);
@ -1126,7 +1124,7 @@ void continueInsertFromCsv(SSqlCallbackWrapper *pWrapper, SRequestObj *pRequest)
void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param) {
int64_t connId = *(int64_t *)taos;
tscDebug("taos_query_a start with sql:%s", sql);
taosAsyncQueryImpl(connId, sql, fp, param, false);
taosAsyncQueryImpl(connId, sql, fp, param, false, TD_REQ_FROM_APP);
tscDebug("taos_query_a end with sql:%s", sql);
}

4
source/client/src/clientMsgHandler.c
View File

@ -538,7 +538,7 @@ int32_t processShowVariablesRsp(void* param, SDataBuf* pMsg, int32_t code) {
code = buildShowVariablesRsp(rsp.variables, &pRes);
}
if (TSDB_CODE_SUCCESS == code) {
code = setQueryResultFromRsp(&pRequest->body.resInfo, pRes, false, true);
code = setQueryResultFromRsp(&pRequest->body.resInfo, pRes, false);
}
if (code != 0) {
@ -651,7 +651,7 @@ int32_t processCompactDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
code = buildRetriveTableRspForCompactDb(&rsp, &pRes);
}
if (TSDB_CODE_SUCCESS == code) {
code = setQueryResultFromRsp(&pRequest->body.resInfo, pRes, false, true);
code = setQueryResultFromRsp(&pRequest->body.resInfo, pRes, false);
}
if (code != 0) {

2
source/client/src/clientRawBlockWrite.c
View File

@ -1256,7 +1256,7 @@ static int32_t taosDeleteData(TAOS* taos, void* meta, int32_t metaLen) {
snprintf(sql, sizeof(sql), "delete from `%s` where `%s` >= %" PRId64 " and `%s` <= %" PRId64, req.tableFName,
req.tsColName, req.skey, req.tsColName, req.ekey);
TAOS_RES* res = taos_query(taos, sql);
TAOS_RES* res = taosQueryImpl(taos, sql, false, TD_REQ_FROM_TAOX);
SRequestObj* pRequest = (SRequestObj*)res;
code = pRequest->code;
if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_PAR_GET_META_ERROR) {

48
source/client/src/clientTmq.c
View File

@ -779,7 +779,7 @@ int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) {
taosWUnLockLatch(&tmq->lock);
taosReleaseRef(tmqMgmt.rsetId, refId);
}
tDeatroySMqHbRsp(&rsp);
tDestroySMqHbRsp(&rsp);
taosMemoryFree(pMsg->pData);
taosMemoryFree(pMsg->pEpSet);
}
@ -861,7 +861,7 @@ void tmqSendHbReq(void* param, void* tmrId) {
asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo);
OVER:
tDeatroySMqHbReq(&req);
tDestroySMqHbReq(&req);
taosTmrReset(tmqSendHbReq, DEFAULT_HEARTBEAT_INTERVAL, param, tmqMgmt.timer, &tmq->hbLiveTimer);
taosReleaseRef(tmqMgmt.rsetId, refId);
}
@ -1626,6 +1626,22 @@ void changeByteEndian(char* pData){
}
}
static void tmqGetRawDataRowsPrecisionFromRes(void *pRetrieve, void** rawData, int64_t *rows, int32_t *precision){
if(*(int64_t*)pRetrieve == 0){
*rawData = ((SRetrieveTableRsp*)pRetrieve)->data;
*rows = htobe64(((SRetrieveTableRsp*)pRetrieve)->numOfRows);
if(precision != NULL){
*precision = ((SRetrieveTableRsp*)pRetrieve)->precision;
}
}else if(*(int64_t*)pRetrieve == 1){
*rawData = ((SRetrieveTableRspForTmq*)pRetrieve)->data;
*rows = htobe64(((SRetrieveTableRspForTmq*)pRetrieve)->numOfRows);
if(precision != NULL){
*precision = ((SRetrieveTableRspForTmq*)pRetrieve)->precision;
}
}
}
static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, SMqRspObj* pRspObj) {
(*numOfRows) = 0;
tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN);
@ -1648,13 +1664,7 @@ static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg
void* rawData = NULL;
int64_t rows = 0;
// deal with compatibility
if(*(int64_t*)pRetrieve == 0){
rawData = ((SRetrieveTableRsp*)pRetrieve)->data;
rows = htobe64(((SRetrieveTableRsp*)pRetrieve)->numOfRows);
}else if(*(int64_t*)pRetrieve == 1){
rawData = ((SRetrieveTableRspForTmq*)pRetrieve)->data;
rows = htobe64(((SRetrieveTableRspForTmq*)pRetrieve)->numOfRows);
}
tmqGetRawDataRowsPrecisionFromRes(pRetrieve, &rawData, &rows, NULL);
pVg->numOfRows += rows;
(*numOfRows) += rows;
@ -2625,22 +2635,22 @@ SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4) {
pRspObj->resIter++;
if (pRspObj->resIter < pRspObj->rsp.blockNum) {
SRetrieveTableRspForTmq* pRetrieveTmq =
(SRetrieveTableRspForTmq*)taosArrayGetP(pRspObj->rsp.blockData, pRspObj->resIter);
if (pRspObj->rsp.withSchema) {
doFreeReqResultInfo(&pRspObj->resInfo);
SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(pRspObj->rsp.blockSchema, pRspObj->resIter);
setResSchemaInfo(&pRspObj->resInfo, pSW->pSchema, pSW->nCols);
taosMemoryFreeClear(pRspObj->resInfo.row);
taosMemoryFreeClear(pRspObj->resInfo.pCol);
taosMemoryFreeClear(pRspObj->resInfo.length);
taosMemoryFreeClear(pRspObj->resInfo.convertBuf);
taosMemoryFreeClear(pRspObj->resInfo.convertJson);
}
pRspObj->resInfo.pData = (void*)pRetrieveTmq->data;
pRspObj->resInfo.numOfRows = htobe64(pRetrieveTmq->numOfRows);
void* pRetrieve = taosArrayGetP(pRspObj->rsp.blockData, pRspObj->resIter);
void* rawData = NULL;
int64_t rows = 0;
int32_t precision = 0;
tmqGetRawDataRowsPrecisionFromRes(pRetrieve, &rawData, &rows, &precision);
pRspObj->resInfo.pData = rawData;
pRspObj->resInfo.numOfRows = rows;
pRspObj->resInfo.current = 0;
pRspObj->resInfo.precision = pRetrieveTmq->precision;
pRspObj->resInfo.precision = precision;
// TODO handle the compressed case
pRspObj->resInfo.totalRows += pRspObj->resInfo.numOfRows;

27
source/common/src/systable.c
View File

@ -68,6 +68,16 @@ static const SSysDbTableSchema snodesSchema[] = {
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true},
};
static const SSysDbTableSchema arbGroupsSchema[] = {
{.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
{.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
{.name = "v1_dnode", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true},
{.name = "v2_dnode", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true},
{.name = "is_sync", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true},
{.name = "assigned_dnode", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true},
{.name = "assigned_token", .bytes = TSDB_ARB_TOKEN_SIZE + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
};
static const SSysDbTableSchema clusterSchema[] = {
{.name = "id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = true},
{.name = "name", .bytes = TSDB_CLUSTER_ID_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
@ -110,6 +120,7 @@ static const SSysDbTableSchema userDBSchema[] = {
{.name = "s3_chunksize", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
{.name = "s3_keeplocal", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
{.name = "s3_compact", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = false},
{.name = "with_arbitrator", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
};
static const SSysDbTableSchema userFuncSchema[] = {
@ -351,21 +362,24 @@ static const SSysDbTableSchema userCompactsDetailSchema[] = {
{.name = "start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
};
static const SSysDbTableSchema useGrantsFullSchema[] = {
static const SSysDbTableSchema userGrantsFullSchema[] = {
{.name = "grant_name", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
{.name = "display_name", .bytes = 256 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
{.name = "expire", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
{.name = "limits", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
};
static const SSysDbTableSchema useGrantsLogsSchema[] = {
static const SSysDbTableSchema userGrantsLogsSchema[] = {
{.name = "state", .bytes = 1536 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
{.name = "active", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
{.name = "machine", .bytes = TSDB_GRANT_LOG_COL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
};
static const SSysDbTableSchema useMachinesSchema[] = {
static const SSysDbTableSchema userMachinesSchema[] = {
{.name = "id", .bytes = TSDB_CLUSTER_ID_LEN + 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
#ifndef TD_UNIQ_GRANT
{.name = "dnode_num", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
#endif
{.name = "machine", .bytes = 7552 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
};
@ -398,9 +412,10 @@ static const SSysTableMeta infosMeta[] = {
{TSDB_INS_TABLE_VIEWS, userViewsSchema, tListLen(userViewsSchema), false},
{TSDB_INS_TABLE_COMPACTS, userCompactsSchema, tListLen(userCompactsSchema), false},
{TSDB_INS_TABLE_COMPACT_DETAILS, userCompactsDetailSchema, tListLen(userCompactsDetailSchema), false},
{TSDB_INS_TABLE_GRANTS_FULL, useGrantsFullSchema, tListLen(useGrantsFullSchema), true},
{TSDB_INS_TABLE_GRANTS_LOGS, useGrantsLogsSchema, tListLen(useGrantsLogsSchema), true},
{TSDB_INS_TABLE_MACHINES, useMachinesSchema, tListLen(useMachinesSchema), true},
{TSDB_INS_TABLE_GRANTS_FULL, userGrantsFullSchema, tListLen(userGrantsFullSchema), true},
{TSDB_INS_TABLE_GRANTS_LOGS, userGrantsLogsSchema, tListLen(userGrantsLogsSchema), true},
{TSDB_INS_TABLE_MACHINES, userMachinesSchema, tListLen(userMachinesSchema), true},
{TSDB_INS_TABLE_ARBGROUPS, arbGroupsSchema, tListLen(arbGroupsSchema), true},
};
static const SSysDbTableSchema connectionsSchema[] = {

11
source/common/src/tdatablock.c
View File

@ -2141,10 +2141,14 @@ _end:
return TSDB_CODE_SUCCESS;
}
void buildCtbNameAddGroupId(char* ctbName, uint64_t groupId){
void buildCtbNameAddGroupId(const char* stbName, char* ctbName, uint64_t groupId){
char tmp[TSDB_TABLE_NAME_LEN] = {0};
snprintf(tmp, TSDB_TABLE_NAME_LEN, "_%"PRIu64, groupId);
ctbName[TSDB_TABLE_NAME_LEN - strlen(tmp) - 1] = 0; // put groupId to the end
if (stbName == NULL){
snprintf(tmp, TSDB_TABLE_NAME_LEN, "_%"PRIu64, groupId);
}else{
snprintf(tmp, TSDB_TABLE_NAME_LEN, "_%s_%"PRIu64, stbName, groupId);
}
ctbName[TSDB_TABLE_NAME_LEN - strlen(tmp) - 1] = 0; // put stbname + groupId to the end
strcat(ctbName, tmp);
}
@ -2154,6 +2158,7 @@ bool isAutoTableName(char* ctbName) { return (strlen(ctbName) == 34 && ctbName[0
bool alreadyAddGroupId(char* ctbName) {
size_t len = strlen(ctbName);
if (len == 0) return false;
size_t _location = len - 1;
while (_location > 0) {
if (ctbName[_location] < '0' || ctbName[_location] > '9') {

55
source/common/src/tglobal.c
View File

@ -82,6 +82,11 @@ int32_t tsMndGrantMode = 0;
bool tsMndSkipGrant = false;
bool tsEnableWhiteList = false; // ip white list cfg
// arbitrator
int32_t tsArbHeartBeatIntervalSec = 5;
int32_t tsArbCheckSyncIntervalSec = 10;
int32_t tsArbSetAssignedTimeoutSec = 30;
// dnode
int64_t tsDndStart = 0;
int64_t tsDndStartOsUptime = 0;
@ -270,7 +275,7 @@ bool tsDisableStream = false;
int64_t tsStreamBufferSize = 128 * 1024 * 1024;
bool tsFilterScalarMode = false;
int tsResolveFQDNRetryTime = 100; // seconds
int tsStreamAggCnt = 1000;
int tsStreamAggCnt = 100000;
char tsS3Endpoint[TSDB_FQDN_LEN] = "<endpoint>";
char tsS3AccessKey[TSDB_FQDN_LEN] = "<accesskey>";
@ -506,7 +511,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0)
return -1;
if (cfgAddBool(pCfg, "useAdapter", tsUseAdapter, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1;
if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1;
if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1;
if (cfgAddInt64(pCfg, "queryMaxConcurrentTables", tsQueryMaxConcurrentTables, INT64_MIN, INT64_MAX, CFG_SCOPE_CLIENT,
CFG_DYN_NONE) != 0)
return -1;
@ -542,9 +547,8 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
return -1;
if (cfgAddBool(pCfg, "experimental", tsExperimental, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1;
if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
return -1;
if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1;
if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
return 0;
}
@ -591,7 +595,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
tsNumOfSupportVnodes = tsNumOfCores * 2;
tsNumOfSupportVnodes = TMAX(tsNumOfSupportVnodes, 2);
if (cfgAddInt32(pCfg, "supportVnodes", tsNumOfSupportVnodes, 0, 4096, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddInt32(pCfg, "supportVnodes", tsNumOfSupportVnodes, 0, 4096, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
if (cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0)
@ -605,18 +609,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
return -1;
if (cfgAddInt32(pCfg, "queryRspPolicy", tsQueryRspPolicy, 0, 1, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
tsNumOfRpcThreads = tsNumOfCores / 2;
tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 2, TSDB_MAX_RPC_THREADS);
if (cfgAddInt32(pCfg, "numOfRpcThreads", tsNumOfRpcThreads, 1, 1024, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
tsNumOfRpcSessions = TRANGE(tsNumOfRpcSessions, 100, 10000);
if (cfgAddInt32(pCfg, "numOfRpcSessions", tsNumOfRpcSessions, 1, 100000, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0)
return -1;
tsTimeToGetAvailableConn = TRANGE(tsTimeToGetAvailableConn, 20, 1000000);
if (cfgAddInt32(pCfg, "timeToGetAvailableConn", tsNumOfRpcSessions, 20, 1000000, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0)
return -1;
tsNumOfCommitThreads = tsNumOfCores / 2;
tsNumOfCommitThreads = TRANGE(tsNumOfCommitThreads, 2, 4);
if (cfgAddInt32(pCfg, "numOfCommitThreads", tsNumOfCommitThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
@ -689,6 +681,16 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
return -1;
if (cfgAddInt32(pCfg, "arbHeartBeatIntervalSec", tsArbHeartBeatIntervalSec, 1, 60 * 24 * 2, CFG_SCOPE_SERVER,
CFG_DYN_NONE) != 0)
return -1;
if (cfgAddInt32(pCfg, "arbCheckSyncIntervalSec", tsArbCheckSyncIntervalSec, 1, 60 * 24 * 2, CFG_SCOPE_SERVER,
CFG_DYN_NONE) != 0)
return -1;
if (cfgAddInt32(pCfg, "arbSetAssignedTimeoutSec", tsArbSetAssignedTimeoutSec, 1, 60 * 24 * 2, CFG_SCOPE_SERVER,
CFG_DYN_NONE) != 0)
return -1;
if (cfgAddInt64(pCfg, "mndSdbWriteDelta", tsMndSdbWriteDelta, 20, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0)
return -1;
if (cfgAddInt64(pCfg, "mndLogRetention", tsMndLogRetention, 500, 10000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
@ -696,9 +698,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "grantMode", tsMndGrantMode, 0, 10000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "skipGrant", tsMndSkipGrant, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
return -1;
if (cfgAddString(pCfg, "monitorFqdn", tsMonitorFqdn, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddInt32(pCfg, "monitorPort", tsMonitorPort, 1, 65056, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddInt32(pCfg, "monitorMaxLogs", tsMonitorMaxLogs, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
@ -713,7 +712,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddBool(pCfg, "auditCreateTable", tsEnableAuditCreateTable, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddInt32(pCfg, "auditInterval", tsAuditInterval, 500, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "telemetryReporting", tsEnableTelem, CFG_SCOPE_BOTH, CFG_DYN_ENT_SERVER) != 0) return -1;
if (cfgAddInt32(pCfg, "telemetryInterval", tsTelemInterval, 1, 200000, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
if (cfgAddString(pCfg, "telemetryServer", tsTelemServer, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1;
@ -825,8 +823,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
return -1;
if (cfgAddBool(pCfg, "enableWhiteList", tsEnableWhiteList, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
if (cfgAddBool(pCfg, "experimental", tsExperimental, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1;
// GRANT_CFG_ADD;
return 0;
}
@ -1216,6 +1212,10 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsHeartbeatTimeout = cfgGetItem(pCfg, "syncHeartbeatTimeout")->i32;
tsSnapReplMaxWaitN = cfgGetItem(pCfg, "syncSnapReplMaxWaitN")->i32;
tsArbHeartBeatIntervalSec = cfgGetItem(pCfg, "arbHeartBeatIntervalSec")->i32;
tsArbCheckSyncIntervalSec = cfgGetItem(pCfg, "arbCheckSyncIntervalSec")->i32;
tsArbSetAssignedTimeoutSec = cfgGetItem(pCfg, "arbSetAssignedTimeoutSec")->i32;
tsMndSdbWriteDelta = cfgGetItem(pCfg, "mndSdbWriteDelta")->i64;
tsMndLogRetention = cfgGetItem(pCfg, "mndLogRetention")->i64;
tsMndSkipGrant = cfgGetItem(pCfg, "skipGrant")->bval;
@ -1364,6 +1364,7 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile
if (taosAddClientLogCfg(tsCfg) != 0) return -1;
if (taosAddServerLogCfg(tsCfg) != 0) return -1;
}
taosAddSystemCfg(tsCfg);
if (taosLoadCfg(tsCfg, envCmd, cfgDir, envFile, apolloUrl) != 0) {
@ -1390,7 +1391,9 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile
if (taosSetTfsCfg(tsCfg) != 0) return -1;
if (taosSetS3Cfg(tsCfg) != 0) return -1;
}
taosSetSystemCfg(tsCfg);
if (taosSetFileHandlesLimit() != 0) return -1;
taosSetAllDebugFlag(tsCfg, cfgGetItem(tsCfg, "debugFlag")->i32);
@ -1639,6 +1642,10 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, char *name) {
tsLogSpace.reserved = (int64_t)(((double)pItem->fval) * 1024 * 1024 * 1024);
uInfo("%s set to %" PRId64, name, tsLogSpace.reserved);
matched = true;
} else if (strcasecmp("monitor", name) == 0) {
tsEnableMonitor = pItem->bval;
uInfo("%s set to %d", name, tsEnableMonitor);
matched = true;
}
break;
}

6
source/common/src/tgrant.c
View File

@ -21,11 +21,5 @@
int32_t grantCheck(EGrantType grant) { return TSDB_CODE_SUCCESS; }
int32_t grantCheckExpire(EGrantType grant) { return TSDB_CODE_SUCCESS; }
#ifdef TD_UNIQ_GRANT
int32_t grantCheckLE(EGrantType grant) { return TSDB_CODE_SUCCESS; }
#endif
#else
#ifdef TD_UNIQ_GRANT
int32_t grantCheckExpire(EGrantType grant) { return TSDB_CODE_SUCCESS; }
#endif
#endif

374
source/common/src/tmsg.c
View File

@ -51,8 +51,8 @@
#define ENCODESQL() \
do { \
if (tEncodeI32(&encoder, pReq->sqlLen) < 0) return -1; \
if (pReq->sqlLen > 0 && pReq->sql != NULL) { \
if (tEncodeI32(&encoder, pReq->sqlLen) < 0) return -1; \
if (tEncodeBinary(&encoder, pReq->sql, pReq->sqlLen) < 0) return -1; \
} \
} while (0)
@ -3025,6 +3025,9 @@ int32_t tSerializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) {
if (tEncodeI8(&encoder, pReq->s3Compact) < 0) return -1;
ENCODESQL();
if (tEncodeI8(&encoder, pReq->withArbitrator) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
@ -3103,8 +3106,12 @@ int32_t tDeserializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq)
DECODESQL();
tEndDecode(&decoder);
pReq->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR;
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI8(&decoder, &pReq->withArbitrator) < 0) return -1;
}
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
@ -3147,6 +3154,7 @@ int32_t tSerializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) {
if (tEncodeI8(&encoder, pReq->s3Compact) < 0) return -1;
ENCODESQL();
if (tEncodeI8(&encoder, pReq->withArbitrator) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
@ -3203,6 +3211,10 @@ int32_t tDeserializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) {
}
DECODESQL();
pReq->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR;
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI8(&decoder, &pReq->withArbitrator) < 0) return -1;
}
tEndDecode(&decoder);
tDecoderClear(&decoder);
@ -4044,6 +4056,7 @@ int32_t tSerializeSDbCfgRspImpl(SEncoder *encoder, const SDbCfgRsp *pRsp) {
if (tEncodeI32(encoder, pRsp->s3ChunkSize) < 0) return -1;
if (tEncodeI32(encoder, pRsp->s3KeepLocal) < 0) return -1;
if (tEncodeI8(encoder, pRsp->s3Compact) < 0) return -1;
if (tEncodeI8(encoder, pRsp->withArbitrator) < 0) return -1;
return 0;
}
@ -4116,6 +4129,10 @@ int32_t tDeserializeSDbCfgRspImpl(SDecoder *decoder, SDbCfgRsp *pRsp) {
if (!tDecodeIsEnd(decoder)) {
if (tDecodeI32(decoder, &pRsp->keepTimeOffset) < 0) return -1;
}
pRsp->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR;
if (!tDecodeIsEnd(decoder)) {
if (tDecodeI8(decoder, &pRsp->withArbitrator) < 0) return -1;
}
pRsp->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE;
pRsp->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
@ -6023,6 +6040,345 @@ int32_t tDeserializeSDCreateMnodeReq(void *buf, int32_t bufLen, SDCreateMnodeReq
return 0;
}
int32_t tSerializeSVArbHeartBeatReq(void *buf, int32_t bufLen, SVArbHeartBeatReq *pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI32(&encoder, pReq->dnodeId) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->arbToken) < 0) return -1;
if (tEncodeI64(&encoder, pReq->arbTerm) < 0) return -1;
int32_t size = taosArrayGetSize(pReq->hbMembers);
if (tEncodeI32(&encoder, size) < 0) return -1;
for (int32_t i = 0; i < size; i++) {
SVArbHbReqMember *pMember = taosArrayGet(pReq->hbMembers, i);
if (tEncodeI32(&encoder, pMember->vgId) < 0) return -1;
if (tEncodeI32(&encoder, pMember->hbSeq) < 0) return -1;
}
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSVArbHeartBeatReq(void *buf, int32_t bufLen, SVArbHeartBeatReq *pReq) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1;
pReq->arbToken = taosMemoryMalloc(TSDB_ARB_TOKEN_SIZE);
if (tDecodeCStrTo(&decoder, pReq->arbToken) < 0) return -1;
if (tDecodeI64(&decoder, &pReq->arbTerm) < 0) return -1;
pReq->hbMembers = taosArrayInit(16, sizeof(SVArbHbReqMember));
int32_t size = 0;
if (tDecodeI32(&decoder, &size) < 0) return -1;
for (int32_t i = 0; i < size; i++) {
SVArbHbReqMember member = {0};
if (tDecodeI32(&decoder, &member.vgId) < 0) return -1;
if (tDecodeI32(&decoder, &member.hbSeq) < 0) return -1;
taosArrayPush(pReq->hbMembers, &member);
}
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
void tFreeSVArbHeartBeatReq(SVArbHeartBeatReq *pReq) {
if (!pReq) return;
taosMemoryFree(pReq->arbToken);
taosArrayDestroy(pReq->hbMembers);
}
int32_t tSerializeSVArbHeartBeatRsp(void *buf, int32_t bufLen, SVArbHeartBeatRsp *pRsp) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeCStr(&encoder, pRsp->arbToken) < 0) return -1;
if (tEncodeI32(&encoder, pRsp->dnodeId) < 0) return -1;
int32_t sz = taosArrayGetSize(pRsp->hbMembers);
if (tEncodeI32(&encoder, sz) < 0) return -1;
for (int32_t i = 0; i < sz; i++) {
SVArbHbRspMember *pMember = taosArrayGet(pRsp->hbMembers, i);
if (tEncodeI32(&encoder, pMember->vgId) < 0) return -1;
if (tEncodeI32(&encoder, pMember->hbSeq) < 0) return -1;
if (tEncodeCStr(&encoder, pMember->memberToken) < 0) return -1;
}
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSVArbHeartBeatRsp(void *buf, int32_t bufLen, SVArbHeartBeatRsp *pRsp) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeCStrTo(&decoder, pRsp->arbToken) < 0) return -1;
if (tDecodeI32(&decoder, &pRsp->dnodeId) < 0) return -1;
int32_t sz = 0;
if (tDecodeI32(&decoder, &sz) < 0) return -1;
pRsp->hbMembers = taosArrayInit(sz, sizeof(SVArbHbRspMember));
for (int32_t i = 0; i < sz; i++) {
SVArbHbRspMember hbMember = {0};
if (tDecodeI32(&decoder, &hbMember.vgId) < 0) return -1;
if (tDecodeI32(&decoder, &hbMember.hbSeq) < 0) return -1;
if (tDecodeCStrTo(&decoder, hbMember.memberToken) < 0) return -1;
taosArrayPush(pRsp->hbMembers, &hbMember);
}
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
void tFreeSVArbHeartBeatRsp(SVArbHeartBeatRsp *pRsp) {
if (NULL == pRsp) {
return;
}
taosArrayDestroy(pRsp->hbMembers);
}
int32_t tSerializeSVArbCheckSyncReq(void *buf, int32_t bufLen, SVArbCheckSyncReq *pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->arbToken) < 0) return -1;
if (tEncodeI64(&encoder, pReq->arbTerm) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->member0Token) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->member1Token) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSVArbCheckSyncReq(void *buf, int32_t bufLen, SVArbCheckSyncReq *pReq) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
pReq->arbToken = taosMemoryMalloc(TSDB_ARB_TOKEN_SIZE);
if (tDecodeCStrTo(&decoder, pReq->arbToken) < 0) return -1;
if (tDecodeI64(&decoder, &pReq->arbTerm) < 0) return -1;
pReq->member0Token = taosMemoryMalloc(TSDB_ARB_TOKEN_SIZE);
if (tDecodeCStrTo(&decoder, pReq->member0Token) < 0) return -1;
pReq->member1Token = taosMemoryMalloc(TSDB_ARB_TOKEN_SIZE);
if (tDecodeCStrTo(&decoder, pReq->member1Token) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
void tFreeSVArbCheckSyncReq(SVArbCheckSyncReq *pRsp) {
if (NULL == pRsp) {
return;
}
taosMemoryFreeClear(pRsp->arbToken);
taosMemoryFreeClear(pRsp->member0Token);
taosMemoryFreeClear(pRsp->member1Token);
}
int32_t tSerializeSVArbCheckSyncRsp(void *buf, int32_t bufLen, SVArbCheckSyncRsp *pRsp) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeCStr(&encoder, pRsp->arbToken) < 0) return -1;
if (tEncodeCStr(&encoder, pRsp->member0Token) < 0) return -1;
if (tEncodeCStr(&encoder, pRsp->member1Token) < 0) return -1;
if (tEncodeI32(&encoder, pRsp->vgId) < 0) return -1;
if (tEncodeI32(&encoder, pRsp->errCode) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSVArbCheckSyncRsp(void *buf, int32_t bufLen, SVArbCheckSyncRsp *pRsp) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
pRsp->arbToken = taosMemoryMalloc(TSDB_ARB_TOKEN_SIZE);
if (tDecodeCStrTo(&decoder, pRsp->arbToken) < 0) return -1;
pRsp->member0Token = taosMemoryMalloc(TSDB_ARB_TOKEN_SIZE);
if (tDecodeCStrTo(&decoder, pRsp->member0Token) < 0) return -1;
pRsp->member1Token = taosMemoryMalloc(TSDB_ARB_TOKEN_SIZE);
if (tDecodeCStrTo(&decoder, pRsp->member1Token) < 0) return -1;
if (tDecodeI32(&decoder, &pRsp->vgId) < 0) return -1;
if (tDecodeI32(&decoder, &pRsp->errCode) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
void tFreeSVArbCheckSyncRsp(SVArbCheckSyncRsp *pRsp) {
if (NULL == pRsp) {
return;
}
taosMemoryFreeClear(pRsp->arbToken);
taosMemoryFreeClear(pRsp->member0Token);
taosMemoryFreeClear(pRsp->member1Token);
}
int32_t tSerializeSVArbSetAssignedLeaderReq(void *buf, int32_t bufLen, SVArbSetAssignedLeaderReq *pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->arbToken) < 0) return -1;
if (tEncodeI64(&encoder, pReq->arbTerm) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->memberToken) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSVArbSetAssignedLeaderReq(void *buf, int32_t bufLen, SVArbSetAssignedLeaderReq *pReq) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
pReq->arbToken = taosMemoryMalloc(TSDB_ARB_TOKEN_SIZE);
if (tDecodeCStrTo(&decoder, pReq->arbToken) < 0) return -1;
if (tDecodeI64(&decoder, &pReq->arbTerm) < 0) return -1;
pReq->memberToken = taosMemoryMalloc(TSDB_ARB_TOKEN_SIZE);
if (tDecodeCStrTo(&decoder, pReq->memberToken) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
void tFreeSVArbSetAssignedLeaderReq(SVArbSetAssignedLeaderReq *pReq) {
if (NULL == pReq) {
return;
}
taosMemoryFreeClear(pReq->arbToken);
taosMemoryFreeClear(pReq->memberToken);
}
int32_t tSerializeSVArbSetAssignedLeaderRsp(void *buf, int32_t bufLen, SVArbSetAssignedLeaderRsp *pRsp) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeCStr(&encoder, pRsp->arbToken) < 0) return -1;
if (tEncodeCStr(&encoder, pRsp->memberToken) < 0) return -1;
if (tEncodeI32(&encoder, pRsp->vgId) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSVArbSetAssignedLeaderRsp(void *buf, int32_t bufLen, SVArbSetAssignedLeaderRsp *pRsp) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
pRsp->arbToken = taosMemoryMalloc(TSDB_ARB_TOKEN_SIZE);
if (tDecodeCStrTo(&decoder, pRsp->arbToken) < 0) return -1;
pRsp->memberToken = taosMemoryMalloc(TSDB_ARB_TOKEN_SIZE);
if (tDecodeCStrTo(&decoder, pRsp->memberToken) < 0) return -1;
if (tDecodeI32(&decoder, &pRsp->vgId) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
void tFreeSVArbSetAssignedLeaderRsp(SVArbSetAssignedLeaderRsp *pRsp) {
if (NULL == pRsp) {
return;
}
taosMemoryFreeClear(pRsp->arbToken);
taosMemoryFreeClear(pRsp->memberToken);
}
int32_t tSerializeSMArbUpdateGroupReq(void *buf, int32_t bufLen, SMArbUpdateGroupReq *pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1;
if (tEncodeI64(&encoder, pReq->dbUid) < 0) return -1;
for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) {
if (tEncodeI32(&encoder, pReq->members[i].dnodeId) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->members[i].token) < 0) return -1;
}
if (tEncodeI8(&encoder, pReq->isSync) < 0) return -1;
if (tEncodeI32(&encoder, pReq->assignedLeader.dnodeId) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->assignedLeader.token) < 0) return -1;
if (tEncodeI64(&encoder, pReq->version) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSMArbUpdateGroupReq(void *buf, int32_t bufLen, SMArbUpdateGroupReq *pReq) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1;
if (tDecodeI64(&decoder, &pReq->dbUid) < 0) return -1;
for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) {
if (tDecodeI32(&decoder, &pReq->members[i].dnodeId) < 0) return -1;
pReq->members[i].token = taosMemoryMalloc(TSDB_ARB_TOKEN_SIZE);
if (tDecodeCStrTo(&decoder, pReq->members[i].token) < 0) return -1;
}
if (tDecodeI8(&decoder, &pReq->isSync) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->assignedLeader.dnodeId) < 0) return -1;
pReq->assignedLeader.token = taosMemoryMalloc(TSDB_ARB_TOKEN_SIZE);
if (tDecodeCStrTo(&decoder, pReq->assignedLeader.token) < 0) return -1;
if (tDecodeI64(&decoder, &pReq->version) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
void tFreeSMArbUpdateGroupReq(SMArbUpdateGroupReq *pReq) {
if (NULL == pReq) {
return;
}
for (int i = 0; i < 2; i++) {
taosMemoryFreeClear(pReq->members[i].token);
}
taosMemoryFreeClear(pReq->assignedLeader.token);
}
// int32_t tSerializeSAuthReq(void *buf, int32_t bufLen, SAuthReq *pReq) {
// SEncoder encoder = {0};
// tEncoderInit(&encoder, buf, bufLen);
@ -6308,9 +6664,8 @@ int32_t tDeserializeSMqAskEpReq(void *buf, int32_t bufLen, SMqAskEpReq *pReq) {
return 0;
}
int32_t tDeatroySMqHbRsp(SMqHbRsp *pRsp) {
void tDestroySMqHbRsp(SMqHbRsp *pRsp) {
taosArrayDestroy(pRsp->topicPrivileges);
return 0;
}
int32_t tSerializeSMqHbRsp(void *buf, int32_t bufLen, SMqHbRsp *pRsp) {
@ -6357,13 +6712,12 @@ int32_t tDeserializeSMqHbRsp(void *buf, int32_t bufLen, SMqHbRsp *pRsp) {
return 0;
}
int32_t tDeatroySMqHbReq(SMqHbReq *pReq) {
void tDestroySMqHbReq(SMqHbReq *pReq) {
for (int i = 0; i < taosArrayGetSize(pReq->topics); i++) {
TopicOffsetRows *vgs = taosArrayGet(pReq->topics, i);
if (vgs) taosArrayDestroy(vgs->offsetRows);
}
taosArrayDestroy(pReq->topics);
return 0;
}
int32_t tSerializeSMqHbReq(void *buf, int32_t bufLen, SMqHbReq *pReq) {
@ -7299,6 +7653,7 @@ int32_t tSerializeSVDeleteReq(void *buf, int32_t bufLen, SVDeleteReq *pReq) {
if (tEncodeU32(&encoder, pReq->sqlLen) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->sql) < 0) return -1;
if (tEncodeBinary(&encoder, pReq->msg, pReq->phyLen) < 0) return -1;
if (tEncodeI8(&encoder, pReq->source) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
@ -7335,6 +7690,9 @@ int32_t tDeserializeSVDeleteReq(void *buf, int32_t bufLen, SVDeleteReq *pReq) {
if (tDecodeBinaryAlloc(&decoder, (void **)&pReq->msg, &msgLen) < 0) return -1;
pReq->phyLen = msgLen;
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI8(&decoder, &pReq->source) < 0) return -1;
}
tEndDecode(&decoder);
tDecoderClear(&decoder);
@ -8534,6 +8892,7 @@ int32_t tEncodeDeleteRes(SEncoder *pCoder, const SDeleteRes *pRes) {
if (tEncodeCStr(pCoder, pRes->tableFName) < 0) return -1;
if (tEncodeCStr(pCoder, pRes->tsColName) < 0) return -1;
if (tEncodeI64(pCoder, pRes->ctimeMs) < 0) return -1;
if (tEncodeI8(pCoder, pRes->source) < 0) return -1;
return 0;
}
@ -8558,6 +8917,9 @@ int32_t tDecodeDeleteRes(SDecoder *pCoder, SDeleteRes *pRes) {
if (!tDecodeIsEnd(pCoder)) {
if (tDecodeI64(pCoder, &pRes->ctimeMs) < 0) return -1;
}
if (!tDecodeIsEnd(pCoder)) {
if (tDecodeI8(pCoder, &pRes->source) < 0) return -1;
}
return 0;
}

2
source/common/src/ttypes.c
View File

@ -34,7 +34,7 @@ const int32_t TYPE_BYTES[21] = {
INT_BYTES, // TSDB_DATA_TYPE_UINT
sizeof(uint64_t), // TSDB_DATA_TYPE_UBIGINT
TSDB_MAX_JSON_TAG_LEN, // TSDB_DATA_TYPE_JSON
TSDB_MAX_TAGS_LEN, // TSDB_DATA_TYPE_VARBINARY: placeholder, not implemented
sizeof(VarDataOffsetT), // TSDB_DATA_TYPE_VARBINARY
TSDB_MAX_TAGS_LEN, // TSDB_DATA_TYPE_DECIMAL: placeholder, not implemented
TSDB_MAX_TAGS_LEN, // TSDB_DATA_TYPE_BLOB: placeholder, not implemented
TSDB_MAX_TAGS_LEN, // TSDB_DATA_TYPE_MEDIUMBLOB: placeholder, not implemented

122
source/common/src/tvariant.c
View File

@ -19,7 +19,7 @@
#include "ttokendef.h"
#include "tvariant.h"
int32_t parseBinaryUInteger(const char *z, int32_t n, uint64_t *value) {
static int32_t parseBinaryUInteger(const char *z, int32_t n, uint64_t *value) {
// skip head 0b
const char *p = z + 2;
int32_t l = n - 2;
@ -45,7 +45,7 @@ int32_t parseBinaryUInteger(const char *z, int32_t n, uint64_t *value) {
return TSDB_CODE_SUCCESS;
}
int32_t parseSignAndUInteger(const char *z, int32_t n, bool *is_neg, uint64_t *value) {
static int32_t parseSignAndUInteger(const char *z, int32_t n, bool *is_neg, uint64_t *value, bool parseFloat) {
// parse sign
bool has_sign = false;
if (z[0] == '-') {
@ -65,8 +65,7 @@ int32_t parseSignAndUInteger(const char *z, int32_t n, bool *is_neg, uint64_t *v
}
errno = 0;
char *endPtr = NULL;
bool parsed = false;
char *endPtr = NULL;
if (z[0] == '0' && n > 2) {
if (z[1] == 'b' || z[1] == 'B') {
// paring as binary
@ -76,44 +75,87 @@ int32_t parseSignAndUInteger(const char *z, int32_t n, bool *is_neg, uint64_t *v
if (z[1] == 'x' || z[1] == 'X') {
// parsing as hex
*value = taosStr2UInt64(z, &endPtr, 16);
parsed = true;
if (errno == ERANGE || errno == EINVAL || endPtr - z != n) {
return TSDB_CODE_FAILED;
}
return TSDB_CODE_SUCCESS;
}
}
if (!parsed) {
if (parseFloat) {
// parsing as double
double val = taosStr2Double(z, &endPtr);
if (errno == ERANGE || errno == EINVAL || endPtr - z != n) {
return TSDB_CODE_FAILED;
}
if (val > UINT64_MAX) {
errno = ERANGE;
return TSDB_CODE_FAILED;
}
*value = round(val);
}
if (errno == ERANGE || errno == EINVAL || endPtr - z != n) {
return TSDB_CODE_FAILED;
}
return TSDB_CODE_SUCCESS;
}
int32_t toDoubleEx(const char *z, int32_t n, uint32_t type, double* value) {
if (n == 0) {
*value = 0;
return TSDB_CODE_SUCCESS;
}
return TSDB_CODE_FAILED;
}
int32_t toDoubleEx(const char *z, int32_t n, uint32_t type, double *value) {
if (n == 0) {
errno = EINVAL;
return TSDB_CODE_FAILED;
}
errno = 0;
char* endPtr = NULL;
*value = taosStr2Double(z, &endPtr);
char *endPtr = NULL;
*value = taosStr2Double(z, &endPtr); // 0x already converted here
if (errno == ERANGE || errno == EINVAL) return TSDB_CODE_FAILED;
if (endPtr - z == n) return TSDB_CODE_SUCCESS;
if (errno == ERANGE || errno == EINVAL) {
return TSDB_CODE_FAILED;
if (type == TK_NK_BIN || type == TK_NK_STRING) {
bool is_neg = false;
uint64_t uv = 0;
if (TSDB_CODE_SUCCESS == parseSignAndUInteger(z, n, &is_neg, &uv, false)) {
*value = is_neg ? -(double)uv : uv;
return TSDB_CODE_SUCCESS;
}
}
if (endPtr - z != n) {
return TSDB_CODE_FAILED;
return TSDB_CODE_FAILED;
}
int32_t toIntegerPure(const char *z, int32_t n, int32_t base, int64_t *value) {
errno = 0;
char *endPtr = NULL;
*value = taosStr2Int64(z, &endPtr, base);
if (endPtr - z == n) {
if (errno == ERANGE || errno == EINVAL) {
return TSDB_CODE_FAILED;
}
return TSDB_CODE_SUCCESS;
}
return TSDB_CODE_SUCCESS;
bool is_neg = false;
uint64_t uv = 0;
int32_t code = parseSignAndUInteger(z, n, &is_neg, &uv, false); // support 0b/0x
if (code == TSDB_CODE_SUCCESS) {
if (is_neg) {
if (uv > 1ull + INT64_MAX) {
*value = INT64_MIN;
return TSDB_CODE_FAILED;
} else {
*value = -(int64_t)uv;
}
} else {
if (uv > INT64_MAX) {
*value = INT64_MAX;
return TSDB_CODE_FAILED;
}
*value = uv;
}
}
return code;
}
int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) {
@ -144,8 +186,8 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) {
}
if (n == 0) {
*value = 0;
return TSDB_CODE_SUCCESS;
errno = EINVAL;
return TSDB_CODE_FAILED;
}
// 1. try to parse as integer
@ -190,7 +232,7 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) {
// 2. parse as other
bool is_neg = false;
uint64_t uv = 0;
int32_t code = parseSignAndUInteger(z, n, &is_neg, &uv);
int32_t code = parseSignAndUInteger(z, n, &is_neg, &uv, true);
if (code == TSDB_CODE_SUCCESS) {
// truncate into int64
if (is_neg) {
@ -198,7 +240,7 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) {
*value = INT64_MIN;
return TSDB_CODE_FAILED;
} else {
*value = -uv;
*value = -(int64_t)uv;
}
} else {
if (uv > INT64_MAX) {
@ -216,9 +258,6 @@ int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value) {
errno = 0;
char *endPtr = NULL;
const char *p = z;
while (*p == ' ') {
p++;
}
switch (type) {
case TK_NK_INTEGER: {
*value = taosStr2UInt64(p, &endPtr, 10);
@ -246,24 +285,22 @@ int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value) {
}
if (n == 0) {
*value = 0;
return TSDB_CODE_SUCCESS;
errno = EINVAL;
return TSDB_CODE_FAILED;
}
// 1. parse as integer
*value = taosStr2UInt64(p, &endPtr, 10);
if (*p == '-' && *value) {
return TSDB_CODE_FAILED;
}
if (endPtr - z == n) {
if (errno == ERANGE || errno == EINVAL) {
if (errno == ERANGE || errno == EINVAL || (*p == '-' && *value)) {
return TSDB_CODE_FAILED;
}
return TSDB_CODE_SUCCESS;
} else if (errno == 0 && *endPtr == '.') {
const char *s = endPtr + 1;
const char *end = z + n;
bool pure = true;
bool pure = true;
while (s < end) {
if (*s < '0' || *s > '9') {
pure = false;
@ -275,13 +312,16 @@ int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value) {
if (endPtr + 1 < end && endPtr[1] > '4' && *value < UINT64_MAX) {
(*value)++;
}
if (*p == '-' && *value) {
return TSDB_CODE_FAILED;
}
return TSDB_CODE_SUCCESS;
}
}
// 2. parse as other
// 2. parse as other
bool is_neg = false;
int32_t code = parseSignAndUInteger(z, n, &is_neg, value);
int32_t code = parseSignAndUInteger(z, n, &is_neg, value, true);
if (is_neg) {
if (TSDB_CODE_SUCCESS == code && 0 == *value) {
return TSDB_CODE_SUCCESS;

10
source/dnode/mgmt/mgmt_dnode/src/dmHandle.c
View File

@ -90,16 +90,6 @@ static void dmProcessStatusRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) {
rpcFreeCont(pRsp->pCont);
}
void dmEpSetToStr(char *buf, int32_t len, SEpSet *epSet) {
int32_t n = 0;
n += snprintf(buf + n, len - n, "%s", "{");
for (int i = 0; i < epSet->numOfEps; i++) {
n += snprintf(buf + n, len - n, "%s:%d%s", epSet->eps[i].fqdn, epSet->eps[i].port,
(i + 1 < epSet->numOfEps ? ", " : ""));
}
n += snprintf(buf + n, len - n, "%s", "}");
}
void dmSendStatusReq(SDnodeMgmt *pMgmt) {
SStatusReq req = {0};

2
source/dnode/mgmt/mgmt_mnode/inc/mmInt.h
View File

@ -33,6 +33,7 @@ typedef struct SMnodeMgmt {
SSingleWorker fetchWorker;
SSingleWorker readWorker;
SSingleWorker writeWorker;
SSingleWorker arbWorker;
SSingleWorker syncWorker;
SSingleWorker syncRdWorker;
bool stopped;
@ -53,6 +54,7 @@ int32_t mmProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg);
int32_t mmStartWorker(SMnodeMgmt *pMgmt);
void mmStopWorker(SMnodeMgmt *pMgmt);
int32_t mmPutMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t mmPutMsgToArbQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t mmPutMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t mmPutMsgToSyncRdQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t mmPutMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg);

3
source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
View File

@ -237,6 +237,9 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_INDEX_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_INDEX_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_DISABLE_WRITE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_ARB_HEARTBEAT_RSP, mmPutMsgToArbQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_ARB_CHECK_SYNC_RSP, mmPutMsgToArbQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_SET_ASSIGNED_LEADER_RSP, mmPutMsgToArbQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT_ELECTION, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;

20
source/dnode/mgmt/mgmt_mnode/src/mmWorker.c
View File

@ -107,6 +107,10 @@ int32_t mmPutMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return mmPutMsgToWorker(pMgmt, &pMgmt->writeWorker, pMsg);
}
int32_t mmPutMsgToArbQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return mmPutMsgToWorker(pMgmt, &pMgmt->arbWorker, pMsg);
}
int32_t mmPutMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return mmPutMsgToWorker(pMgmt, &pMgmt->syncWorker, pMsg);
}
@ -153,6 +157,9 @@ int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
case READ_QUEUE:
pWorker = &pMgmt->readWorker;
break;
case ARB_QUEUE:
pWorker = &pMgmt->arbWorker;
break;
case SYNC_QUEUE:
pWorker = &pMgmt->syncWorker;
break;
@ -252,6 +259,18 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
return -1;
}
SSingleWorkerCfg arbCfg = {
.min = 1,
.max = 1,
.name = "mnode-arb",
.fp = (FItem)mmProcessRpcMsg,
.param = pMgmt,
};
if (tSingleWorkerInit(&pMgmt->arbWorker, &arbCfg) != 0) {
dError("failed to start mnode mnode-arb worker since %s", terrstr());
return -1;
}
dDebug("mnode workers are initialized");
return 0;
}
@ -263,6 +282,7 @@ void mmStopWorker(SMnodeMgmt *pMgmt) {
tSingleWorkerCleanup(&pMgmt->fetchWorker);
tSingleWorkerCleanup(&pMgmt->readWorker);
tSingleWorkerCleanup(&pMgmt->writeWorker);
tSingleWorkerCleanup(&pMgmt->arbWorker);
tSingleWorkerCleanup(&pMgmt->syncWorker);
tSingleWorkerCleanup(&pMgmt->syncRdWorker);
dDebug("mnode workers are closed");

1
source/dnode/mgmt/mgmt_vnode/inc/vmInt.h
View File

@ -101,6 +101,7 @@ int32_t vmProcessDisableVnodeWriteReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t vmProcessAlterHashRangeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t vmProcessAlterVnodeTypeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t vmProcessCheckLearnCatchupReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t vmProcessArbHeartBeatReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
// vmFile.c
int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes);

91
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -88,7 +88,9 @@ void vmGetMonitorInfo(SVnodeMgmt *pMgmt, SMonVmInfo *pInfo) {
numOfInsertSuccessReqs += pLoad->numOfInsertSuccessReqs;
numOfBatchInsertReqs += pLoad->numOfBatchInsertReqs;
numOfBatchInsertSuccessReqs += pLoad->numOfBatchInsertSuccessReqs;
if (pLoad->syncState == TAOS_SYNC_STATE_LEADER) masterNum++;
if (pLoad->syncState == TAOS_SYNC_STATE_LEADER || pLoad->syncState == TAOS_SYNC_STATE_ASSIGNED_LEADER) {
masterNum++;
}
totalVnodes++;
}
@ -775,6 +777,89 @@ int32_t vmProcessDropVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return 0;
}
int32_t vmProcessArbHeartBeatReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SVArbHeartBeatReq arbHbReq = {0};
SVArbHeartBeatRsp arbHbRsp = {0};
if (tDeserializeSVArbHeartBeatReq(pMsg->pCont, pMsg->contLen, &arbHbReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
if (arbHbReq.dnodeId != pMgmt->pData->dnodeId) {
terrno = TSDB_CODE_INVALID_MSG;
dError("dnodeId:%d not matched with local dnode", arbHbReq.dnodeId);
goto _OVER;
}
if (strlen(arbHbReq.arbToken) == 0) {
terrno = TSDB_CODE_INVALID_MSG;
dError("dnodeId:%d arbToken is empty", arbHbReq.dnodeId);
goto _OVER;
}
size_t size = taosArrayGetSize(arbHbReq.hbMembers);
arbHbRsp.dnodeId = pMgmt->pData->dnodeId;
strncpy(arbHbRsp.arbToken, arbHbReq.arbToken, TSDB_ARB_TOKEN_SIZE);
arbHbRsp.hbMembers = taosArrayInit(size, sizeof(SVArbHbRspMember));
for (int32_t i = 0; i < size; i++) {
SVArbHbReqMember *pReqMember = taosArrayGet(arbHbReq.hbMembers, i);
SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pReqMember->vgId);
if (pVnode == NULL) {
dError("dnodeId:%d vgId:%d not found failed to process arb hb req", arbHbReq.dnodeId, pReqMember->vgId);
continue;
}
SVArbHbRspMember rspMember = {0};
rspMember.vgId = pReqMember->vgId;
rspMember.hbSeq = pReqMember->hbSeq;
if (vnodeGetArbToken(pVnode->pImpl, rspMember.memberToken) != 0) {
dError("dnodeId:%d vgId:%d failed to get arb token", arbHbReq.dnodeId, pReqMember->vgId);
vmReleaseVnode(pMgmt, pVnode);
continue;
}
if (vnodeUpdateArbTerm(pVnode->pImpl, arbHbReq.arbTerm) != 0) {
dError("dnodeId:%d vgId:%d failed to update arb term", arbHbReq.dnodeId, pReqMember->vgId);
vmReleaseVnode(pMgmt, pVnode);
continue;
}
taosArrayPush(arbHbRsp.hbMembers, &rspMember);
vmReleaseVnode(pMgmt, pVnode);
}
SRpcMsg rspMsg = {.info = pMsg->info};
int32_t rspLen = tSerializeSVArbHeartBeatRsp(NULL, 0, &arbHbRsp);
if (rspLen < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
void *pRsp = rpcMallocCont(rspLen);
if (pRsp == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
if (tSerializeSVArbHeartBeatRsp(pRsp, rspLen, &arbHbRsp) <= 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
rpcFreeCont(pRsp);
goto _OVER;
}
pMsg->info.rsp = pRsp;
pMsg->info.rspLen = rspLen;
terrno = TSDB_CODE_SUCCESS;
_OVER:
tFreeSVArbHeartBeatReq(&arbHbReq);
tFreeSVArbHeartBeatRsp(&arbHbRsp);
return terrno == TSDB_CODE_SUCCESS ? 0 : -1;
}
SArray *vmGetMsgHandles() {
int32_t code = -1;
SArray *pArray = taosArrayInit(32, sizeof(SMgmtHandle));
@ -879,6 +964,10 @@ SArray *vmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_RSP, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_PREP_SNAPSHOT_REPLY, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_ARB_HEARTBEAT, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_ARB_CHECK_SYNC, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_SET_ASSIGNED_LEADER, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
code = 0;
_OVER:

3
source/dnode/mgmt/mgmt_vnode/src/vmWorker.c
View File

@ -56,6 +56,9 @@ static void vmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
case TDMT_DND_CHECK_VNODE_LEARNER_CATCHUP:
code = vmProcessCheckLearnCatchupReq(pMgmt, pMsg);
break;
case TDMT_VND_ARB_HEARTBEAT:
code = vmProcessArbHeartBeatReq(pMgmt, pMsg);
break;
default:
terrno = TSDB_CODE_MSG_NOT_PROCESSED;
dGError("msg:%p, not processed in vnode-mgmt queue", pMsg);

1
source/dnode/mgmt/node_mgmt/src/dmTransport.c
View File

@ -538,6 +538,7 @@ SMsgCb dmGetMsgcb(SDnode *pDnode) {
.releaseHandleFp = dmReleaseHandle,
.reportStartupFp = dmReportStartup,
.updateDnodeInfoFp = dmUpdateDnodeInfo,
.getDnodeEpFp = dmGetDnodeEp,
.data = &pDnode->data,
};
return msgCb;

2
source/dnode/mgmt/node_util/inc/dmUtil.h
View File

@ -184,11 +184,13 @@ int32_t dmReadEps(SDnodeData *pData);
int32_t dmWriteEps(SDnodeData *pData);
void dmUpdateEps(SDnodeData *pData, SArray *pDnodeEps);
void dmGetMnodeEpSet(SDnodeData *pData, SEpSet *pEpSet);
void dmEpSetToStr(char *buf, int32_t len, SEpSet *epSet);
void dmRotateMnodeEpSet(SDnodeData *pData);
void dmGetMnodeEpSetForRedirect(SDnodeData *pData, SRpcMsg *pMsg, SEpSet *pEpSet);
void dmSetMnodeEpSet(SDnodeData *pData, SEpSet *pEpSet);
bool dmUpdateDnodeInfo(void *pData, int32_t *dnodeId, int64_t *clusterId, char *fqdn, uint16_t *port);
void dmRemoveDnodePairs(SDnodeData *pData);
void dmGetDnodeEp(void *pData, int32_t dnodeId, char *pEp, char *pFqdn, uint16_t *pPort);
#ifdef __cplusplus
}

13
source/dnode/mgmt/node_util/src/dmEps.c
View File

@ -31,7 +31,8 @@ static bool dmIsEpChanged(SDnodeData *pData, int32_t dnodeId, const char *ep)
static void dmResetEps(SDnodeData *pData, SArray *dnodeEps);
static int32_t dmReadDnodePairs(SDnodeData *pData);
static void dmGetDnodeEp(SDnodeData *pData, int32_t dnodeId, char *pEp, char *pFqdn, uint16_t *pPort) {
void dmGetDnodeEp(void *data, int32_t dnodeId, char *pEp, char *pFqdn, uint16_t *pPort) {
SDnodeData *pData = data;
taosThreadRwlockRdlock(&pData->lock);
SDnodeEp *pDnodeEp = taosHashGet(pData->dnodeHash, &dnodeId, sizeof(int32_t));
@ -335,6 +336,16 @@ void dmGetMnodeEpSet(SDnodeData *pData, SEpSet *pEpSet) {
taosThreadRwlockUnlock(&pData->lock);
}
void dmEpSetToStr(char *buf, int32_t len, SEpSet *epSet) {
int32_t n = 0;
n += snprintf(buf + n, len - n, "%s", "{");
for (int i = 0; i < epSet->numOfEps; i++) {
n += snprintf(buf + n, len - n, "%s:%d%s", epSet->eps[i].fqdn, epSet->eps[i].port,
(i + 1 < epSet->numOfEps ? ", " : ""));
}
n += snprintf(buf + n, len - n, "%s", "}");
}
static FORCE_INLINE void dmSwapEps(SEp *epLhs, SEp *epRhs) {
SEp epTmp;

54
source/dnode/mnode/impl/inc/mndArbGroup.h Normal file
View File

@ -0,0 +1,54 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_MND_ARBGROUP_H_
#define _TD_MND_ARBGROUP_H_
#include "mndInt.h"
#ifdef __cplusplus
extern "C" {
#endif
int32_t mndInitArbGroup(SMnode *pMnode);
void mndCleanupArbGroup(SMnode *pMnode);
SArbGroup *mndAcquireArbGroup(SMnode *pMnode, int32_t vgId);
void mndReleaseArbGroup(SMnode *pMnode, SArbGroup *pObj);
SSdbRaw *mndArbGroupActionEncode(SArbGroup *pGroup);
SSdbRow *mndArbGroupActionDecode(SSdbRaw *pRaw);
void mndArbGroupInitFromVgObj(SVgObj *pVgObj, SArbGroup *outGroup);
int32_t mndSetCreateArbGroupRedoLogs(STrans *pTrans, SArbGroup *pGroup);
int32_t mndSetCreateArbGroupUndoLogs(STrans *pTrans, SArbGroup *pGroup);
int32_t mndSetCreateArbGroupCommitLogs(STrans *pTrans, SArbGroup *pGroup);
int32_t mndSetDropArbGroupPrepareLogs(STrans *pTrans, SArbGroup *pGroup);
int32_t mndSetDropArbGroupCommitLogs(STrans *pTrans, SArbGroup *pGroup);
bool mndUpdateArbGroupByHeartBeat(SArbGroup *pGroup, SVArbHbRspMember *pRspMember, int64_t nowMs, int32_t dnodeId,
SArbGroup *pNewGroup);
bool mndUpdateArbGroupByCheckSync(SArbGroup *pGroup, int32_t vgId, char *member0Token, char *member1Token,
bool newIsSync, SArbGroup *pNewGroup);
bool mndUpdateArbGroupBySetAssignedLeader(SArbGroup *pGroup, int32_t vgId, char *memberToken, int32_t errcode,
SArbGroup *pNewGroup);
int32_t mndGetArbGroupSize(SMnode *pMnode);
#ifdef __cplusplus
}
#endif
#endif /*_TD_MND_ARBGROUP_H_*/

13
source/dnode/mnode/impl/inc/mndConsumer.h
View File

@ -24,27 +24,22 @@ extern "C" {
enum {
MQ_CONSUMER_STATUS_REBALANCE = 1,
// MQ_CONSUMER_STATUS__MODIFY_IN_REB, // this value is not used anymore
MQ_CONSUMER_STATUS_READY,
MQ_CONSUMER_STATUS_LOST,
// MQ_CONSUMER_STATUS__LOST_IN_REB, // this value is not used anymore
// MQ_CONSUMER_STATUS__LOST_REBD,
};\
};
int32_t mndInitConsumer(SMnode *pMnode);
void mndCleanupConsumer(SMnode *pMnode);
void mndDropConsumerFromSdb(SMnode *pMnode, int64_t consumerId, SRpcHandleInfo* info);
void mndSendConsumerMsg(SMnode *pMnode, int64_t consumerId, uint16_t msgType, SRpcHandleInfo* info);
SMqConsumerObj *mndAcquireConsumer(SMnode *pMnode, int64_t consumerId);
void mndReleaseConsumer(SMnode *pMnode, SMqConsumerObj *pConsumer);
SMqConsumerObj *mndCreateConsumer(int64_t consumerId, const char *cgroup);
SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer);
SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw);
int32_t mndSetConsumerCommitLogs(SMnode *pMnode, STrans *pTrans, SMqConsumerObj *pConsumer);
int32_t mndSetConsumerDropLogs(SMnode *pMnode, STrans *pTrans, SMqConsumerObj *pConsumer);
int32_t mndSetConsumerCommitLogs(STrans *pTrans, SMqConsumerObj *pConsumer);
int32_t mndSetConsumerDropLogs(STrans *pTrans, SMqConsumerObj *pConsumer);
const char *mndConsumerStatusName(int status);

46
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -104,6 +104,7 @@ typedef enum {
TRN_CONFLICT_DB_INSIDE = 3,
TRN_CONFLICT_TOPIC = 4,
TRN_CONFLICT_TOPIC_INSIDE = 5,
TRN_CONFLICT_ARBGROUP = 6,
} ETrnConflct;
typedef enum {
@ -149,6 +150,7 @@ typedef enum {
CONSUMER_REMOVE_REB, // remove after rebalance
CONSUMER_UPDATE_REC, // update after recover
CONSUMER_UPDATE_SUB, // update after subscribe req
CONSUMER_INSERT_SUB,
} ECsmUpdateType;
typedef struct {
@ -158,6 +160,7 @@ typedef struct {
ETrnConflct conflict;
ETrnExec exec;
EOperType oper;
bool changeless;
int32_t code;
int32_t failedTimes;
void* rpcRsp;
@ -176,6 +179,7 @@ typedef struct {
tmsg_t originRpcType;
char dbname[TSDB_TABLE_FNAME_LEN];
char stbname[TSDB_TABLE_FNAME_LEN];
int32_t arbGroupId;
int32_t startFunc;
int32_t stopFunc;
int32_t paramLen;
@ -214,8 +218,6 @@ typedef struct {
uint16_t port;
char fqdn[TSDB_FQDN_LEN];
char ep[TSDB_EP_LEN];
char active[TSDB_ACTIVE_KEY_LEN];
char connActive[TSDB_CONN_ACTIVE_KEY_LEN];
char machineId[TSDB_MACHINE_ID_LEN + 1];
} SDnodeObj;
@ -247,6 +249,40 @@ typedef struct {
SDnodeObj* pDnode;
} SSnodeObj;
typedef struct {
int32_t dnodeId;
char token[TSDB_ARB_TOKEN_SIZE];
} SArbAssignedLeader;
typedef struct {
int32_t dnodeId;
} SArbMemberInfo;
typedef struct {
int32_t nextHbSeq;
int32_t responsedHbSeq;
char token[TSDB_ARB_TOKEN_SIZE];
int64_t lastHbMs;
} SArbMemberState;
typedef struct {
SArbMemberInfo info;
SArbMemberState state;
} SArbGroupMember;
typedef struct {
int32_t vgId;
int64_t dbUid;
SArbGroupMember members[TSDB_ARB_GROUP_MEMBER_NUM];
int8_t isSync;
SArbAssignedLeader assignedLeader;
int64_t version;
// following fields will not be duplicated
bool mutexInited;
TdThreadMutex mutex;
} SArbGroup;
typedef struct {
int32_t maxUsers;
int32_t maxDbs;
@ -346,6 +382,7 @@ typedef struct {
int32_t s3ChunkSize;
int32_t s3KeepLocal;
int8_t s3Compact;
int8_t withArbitrator;
} SDbCfg;
typedef struct {
@ -559,8 +596,9 @@ typedef struct {
int32_t resetOffsetCfg;
} SMqConsumerObj;
SMqConsumerObj* tNewSMqConsumerObj(int64_t consumerId, char cgroup[TSDB_CGROUP_LEN]);
void tDeleteSMqConsumerObj(SMqConsumerObj* pConsumer, bool isDeleted);
SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char *cgroup, int8_t updateType, char *topic, SCMSubscribeReq *subscribe);
void tClearSMqConsumerObj(SMqConsumerObj* pConsumer);
void tDeleteSMqConsumerObj(SMqConsumerObj* pConsumer);
int32_t tEncodeSMqConsumerObj(void** buf, const SMqConsumerObj* pConsumer);
void* tDecodeSMqConsumerObj(const void* buf, SMqConsumerObj* pConsumer, int8_t sver);

1
source/dnode/mnode/impl/inc/mndDnode.h
View File

@ -27,6 +27,7 @@ void mndCleanupDnode(SMnode *pMnode);
SDnodeObj *mndAcquireDnode(SMnode *pMnode, int32_t dnodeId);
void mndReleaseDnode(SMnode *pMnode, SDnodeObj *pDnode);
SEpSet mndGetDnodeEpset(SDnodeObj *pDnode);
SEpSet mndGetDnodeEpsetById(SMnode *pMnode, int32_t dnodeId);
int32_t mndGetDnodeSize(SMnode *pMnode);
bool mndIsDnodeOnline(SDnodeObj *pDnode, int64_t curMs);
void mndGetDnodeData(SMnode *pMnode, SArray *pDnodeInfo);

4
source/dnode/mnode/impl/inc/mndStream.h
View File

@ -107,11 +107,11 @@ SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady);
void mndKillTransImpl(SMnode *pMnode, int32_t transId, const char *pDbName);
int32_t setTransAction(STrans *pTrans, void *pCont, int32_t contLen, int32_t msgType, const SEpSet *pEpset,
int32_t retryCode);
STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, const char *name, const char *pMsg);
STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, ETrnConflct conflict, const char *name, const char *pMsg);
int32_t mndPersistTransLog(SStreamObj *pStream, STrans *pTrans, int32_t status);
SSdbRaw *mndStreamActionEncode(SStreamObj *pStream);
void killAllCheckpointTrans(SMnode *pMnode, SVgroupChangeInfo *pChangeInfo);
int32_t mndStreamSetUpdateEpsetAction(SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans);
int32_t mndStreamSetUpdateEpsetAction(SMnode *pMnode, SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans);
SStreamObj *mndGetStreamObj(SMnode *pMnode, int64_t streamId);
int32_t extractNodeEpset(SMnode *pMnode, SEpSet *pEpSet, bool *hasEpset, int32_t taskId, int32_t nodeId);

2
source/dnode/mnode/impl/inc/mndSubscribe.h
View File

@ -30,7 +30,7 @@ SMqSubscribeObj *mndAcquireSubscribe(SMnode *pMnode, const char *CGroup, const c
SMqSubscribeObj *mndAcquireSubscribeByKey(SMnode *pMnode, const char *key);
void mndReleaseSubscribe(SMnode *pMnode, SMqSubscribeObj *pSub);
int32_t mndMakeSubscribeKey(char *key, const char *cgroup, const char *topicName);
void mndMakeSubscribeKey(char *key, const char *cgroup, const char *topicName);
int32_t mndDropSubByTopic(SMnode *pMnode, STrans *pTrans, const char *topic);
int32_t mndSetDropSubCommitLogs(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub);

2
source/dnode/mnode/impl/inc/mndTrans.h
View File

@ -78,8 +78,10 @@ int32_t mndTransAppendUndoAction(STrans *pTrans, STransAction *pAction);
void mndTransSetRpcRsp(STrans *pTrans, void *pCont, int32_t contLen);
void mndTransSetCb(STrans *pTrans, ETrnFunc startFunc, ETrnFunc stopFunc, void *param, int32_t paramLen);
void mndTransSetDbName(STrans *pTrans, const char *dbname, const char *stbname);
void mndTransSetArbGroupId(STrans *pTrans, int32_t groupId);
void mndTransSetSerial(STrans *pTrans);
void mndTransSetParallel(STrans *pTrans);
void mndTransSetChangeless(STrans *pTrans);
void mndTransSetOper(STrans *pTrans, EOperType oper);
int32_t mndTransCheckConflict(SMnode *pMnode, STrans *pTrans);
#ifndef BUILD_NO_CALL

7
source/dnode/mnode/impl/inc/mndVgroup.h
View File

@ -29,6 +29,7 @@ void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup);
SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup);
SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw);
SEpSet mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup);
SEpSet mndGetVgroupEpsetById(SMnode *pMnode, int32_t vgId);
int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId);
void mndSortVnodeGid(SVgObj *pVgroup);
int64_t mndGetVnodesMemory(SMnode *pMnode, int32_t dnodeId);
@ -44,7 +45,7 @@ int32_t mndAddAlterVnodeAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pV
int32_t mndAddDropVnodeAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, bool isRedo);
int32_t mndSetMoveVgroupsInfoToTrans(SMnode *, STrans *pTrans, int32_t dropDnodeId, bool force, bool unsafe);
int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb, SVgObj *pVgroup,
SArray *pArray);
SArray *pArray, SVgObj* pNewVgroup);
int32_t mndBuildCompactVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int64_t compactTs,
STimeWindow tw);
int32_t mndBuildRaftAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb, SVgObj *pVgroup,
@ -54,8 +55,8 @@ void *mndBuildCreateVnodeReq(SMnode *, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *p
void *mndBuildDropVnodeReq(SMnode *, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen);
bool mndVgroupInDb(SVgObj *pVgroup, int64_t dbUid);
bool mndVgroupInDnode(SVgObj *pVgroup, int32_t dnodeId);
int32_t mndBuildRestoreAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *db, SVgObj *pVgroup,
SDnodeObj *pDnode);
int32_t mndBuildRestoreAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *db, SVgObj *pVgroup,
SDnodeObj *pDnode);
int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgroup);

1111
source/dnode/mnode/impl/src/mndArbGroup.c Normal file
View File

File diff suppressed because it is too large Load Diff

799
source/dnode/mnode/impl/src/mndConsumer.c
View File

File diff suppressed because it is too large Load Diff

170
source/dnode/mnode/impl/src/mndDb.c
View File

@ -16,6 +16,7 @@
#define _DEFAULT_SOURCE
#include "mndDb.h"
#include "audit.h"
#include "mndArbGroup.h"
#include "mndCluster.h"
#include "mndDnode.h"
#include "mndIndex.h"
@ -35,7 +36,7 @@
#include "tjson.h"
#define DB_VER_NUMBER 1
#define DB_RESERVE_SIZE 33
#define DB_RESERVE_SIZE 32
static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw);
static int32_t mndDbActionInsert(SSdb *pSdb, SDbObj *pDb);
@ -144,6 +145,7 @@ SSdbRaw *mndDbActionEncode(SDbObj *pDb) {
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.s3ChunkSize, _OVER)
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.s3KeepLocal, _OVER)
SDB_SET_INT8(pRaw, dataPos, pDb->cfg.s3Compact, _OVER)
SDB_SET_INT8(pRaw, dataPos, pDb->cfg.withArbitrator, _OVER)
SDB_SET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER)
SDB_SET_DATALEN(pRaw, dataPos, _OVER)
@ -238,6 +240,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) {
SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.s3ChunkSize, _OVER)
SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.s3KeepLocal, _OVER)
SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.s3Compact, _OVER)
SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.withArbitrator, _OVER)
SDB_GET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER)
taosInitRWLatch(&pDb->lock);
@ -330,6 +333,7 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) {
pOld->cfg.s3ChunkSize = pNew->cfg.s3ChunkSize;
pOld->cfg.s3KeepLocal = pNew->cfg.s3KeepLocal;
pOld->cfg.s3Compact = pNew->cfg.s3Compact;
pOld->cfg.withArbitrator = pNew->cfg.withArbitrator;
pOld->compactStartTime = pNew->compactStartTime;
taosWUnLockLatch(&pOld->lock);
return 0;
@ -404,7 +408,11 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
if (pCfg->precision < TSDB_MIN_PRECISION && pCfg->precision > TSDB_MAX_PRECISION) return -1;
if (pCfg->compression < TSDB_MIN_COMP_LEVEL || pCfg->compression > TSDB_MAX_COMP_LEVEL) return -1;
if (pCfg->replications < TSDB_MIN_DB_REPLICA || pCfg->replications > TSDB_MAX_DB_REPLICA) return -1;
#ifdef TD_ENTERPRISE
if ((pCfg->replications == 2) ^ (pCfg->withArbitrator == TSDB_MAX_DB_WITH_ARBITRATOR)) return -1;
#else
if (pCfg->replications != 1 && pCfg->replications != 3) return -1;
#endif
if (pCfg->strict < TSDB_DB_STRICT_OFF || pCfg->strict > TSDB_DB_STRICT_ON) return -1;
if (pCfg->schemaless < TSDB_DB_SCHEMALESS_OFF || pCfg->schemaless > TSDB_DB_SCHEMALESS_ON) return -1;
if (pCfg->cacheLast < TSDB_CACHE_MODEL_NONE || pCfg->cacheLast > TSDB_CACHE_MODEL_BOTH) return -1;
@ -433,33 +441,48 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
return terrno;
}
static int32_t mndCheckInChangeDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
static int32_t mndCheckInChangeDbCfg(SMnode *pMnode, SDbCfg *pOldCfg, SDbCfg *pNewCfg) {
terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
if (pCfg->buffer < TSDB_MIN_BUFFER_PER_VNODE || pCfg->buffer > TSDB_MAX_BUFFER_PER_VNODE) return -1;
if (pCfg->pages < TSDB_MIN_PAGES_PER_VNODE || pCfg->pages > TSDB_MAX_PAGES_PER_VNODE) return -1;
if (pCfg->pageSize < TSDB_MIN_PAGESIZE_PER_VNODE || pCfg->pageSize > TSDB_MAX_PAGESIZE_PER_VNODE) return -1;
if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) return -1;
if (pCfg->daysToKeep0 < TSDB_MIN_KEEP || pCfg->daysToKeep0 > TSDB_MAX_KEEP) return -1;
if (pCfg->daysToKeep1 < TSDB_MIN_KEEP || pCfg->daysToKeep1 > TSDB_MAX_KEEP) return -1;
if (pCfg->daysToKeep2 < TSDB_MIN_KEEP || pCfg->daysToKeep2 > TSDB_MAX_KEEP) return -1;
if (pCfg->daysToKeep0 < pCfg->daysPerFile) return -1;
if (pCfg->daysToKeep0 > pCfg->daysToKeep1) return -1;
if (pCfg->daysToKeep1 > pCfg->daysToKeep2) return -1;
if (pCfg->keepTimeOffset < TSDB_MIN_KEEP_TIME_OFFSET || pCfg->keepTimeOffset > TSDB_MAX_KEEP_TIME_OFFSET) return -1;
if (pCfg->walFsyncPeriod < TSDB_MIN_FSYNC_PERIOD || pCfg->walFsyncPeriod > TSDB_MAX_FSYNC_PERIOD) return -1;
if (pCfg->walLevel < TSDB_MIN_WAL_LEVEL || pCfg->walLevel > TSDB_MAX_WAL_LEVEL) return -1;
if (pCfg->cacheLast < TSDB_CACHE_MODEL_NONE || pCfg->cacheLast > TSDB_CACHE_MODEL_BOTH) return -1;
if (pCfg->cacheLastSize < TSDB_MIN_DB_CACHE_SIZE || pCfg->cacheLastSize > TSDB_MAX_DB_CACHE_SIZE) return -1;
if (pCfg->replications < TSDB_MIN_DB_REPLICA || pCfg->replications > TSDB_MAX_DB_REPLICA) return -1;
if (pCfg->replications != 1 && pCfg->replications != 3) return -1;
if (pCfg->sstTrigger < TSDB_MIN_STT_TRIGGER || pCfg->sstTrigger > TSDB_MAX_STT_TRIGGER) return -1;
if (pCfg->minRows < TSDB_MIN_MINROWS_FBLOCK || pCfg->minRows > TSDB_MAX_MINROWS_FBLOCK) return -1;
if (pCfg->maxRows < TSDB_MIN_MAXROWS_FBLOCK || pCfg->maxRows > TSDB_MAX_MAXROWS_FBLOCK) return -1;
if (pCfg->minRows > pCfg->maxRows) return -1;
if (pCfg->walRetentionPeriod < TSDB_DB_MIN_WAL_RETENTION_PERIOD) return -1;
if (pCfg->walRetentionSize < TSDB_DB_MIN_WAL_RETENTION_SIZE) return -1;
if (pCfg->strict < TSDB_DB_STRICT_OFF || pCfg->strict > TSDB_DB_STRICT_ON) return -1;
if (pCfg->replications > mndGetDnodeSize(pMnode)) {
if (pNewCfg->buffer < TSDB_MIN_BUFFER_PER_VNODE || pNewCfg->buffer > TSDB_MAX_BUFFER_PER_VNODE) return -1;
if (pNewCfg->pages < TSDB_MIN_PAGES_PER_VNODE || pNewCfg->pages > TSDB_MAX_PAGES_PER_VNODE) return -1;
if (pNewCfg->pageSize < TSDB_MIN_PAGESIZE_PER_VNODE || pNewCfg->pageSize > TSDB_MAX_PAGESIZE_PER_VNODE) return -1;
if (pNewCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pNewCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) return -1;
if (pNewCfg->daysToKeep0 < TSDB_MIN_KEEP || pNewCfg->daysToKeep0 > TSDB_MAX_KEEP) return -1;
if (pNewCfg->daysToKeep1 < TSDB_MIN_KEEP || pNewCfg->daysToKeep1 > TSDB_MAX_KEEP) return -1;
if (pNewCfg->daysToKeep2 < TSDB_MIN_KEEP || pNewCfg->daysToKeep2 > TSDB_MAX_KEEP) return -1;
if (pNewCfg->daysToKeep0 < pNewCfg->daysPerFile) return -1;
if (pNewCfg->daysToKeep0 > pNewCfg->daysToKeep1) return -1;
if (pNewCfg->daysToKeep1 > pNewCfg->daysToKeep2) return -1;
if (pNewCfg->keepTimeOffset < TSDB_MIN_KEEP_TIME_OFFSET || pNewCfg->keepTimeOffset > TSDB_MAX_KEEP_TIME_OFFSET)
return -1;
if (pNewCfg->walFsyncPeriod < TSDB_MIN_FSYNC_PERIOD || pNewCfg->walFsyncPeriod > TSDB_MAX_FSYNC_PERIOD) return -1;
if (pNewCfg->walLevel < TSDB_MIN_WAL_LEVEL || pNewCfg->walLevel > TSDB_MAX_WAL_LEVEL) return -1;
if (pNewCfg->cacheLast < TSDB_CACHE_MODEL_NONE || pNewCfg->cacheLast > TSDB_CACHE_MODEL_BOTH) return -1;
if (pNewCfg->cacheLastSize < TSDB_MIN_DB_CACHE_SIZE || pNewCfg->cacheLastSize > TSDB_MAX_DB_CACHE_SIZE) return -1;
if (pNewCfg->replications < TSDB_MIN_DB_REPLICA || pNewCfg->replications > TSDB_MAX_DB_REPLICA) return -1;
#ifdef TD_ENTERPRISE
if ((pNewCfg->replications == 2) ^ (pNewCfg->withArbitrator == TSDB_MAX_DB_WITH_ARBITRATOR)) return -1;
if (pNewCfg->replications == 2 && pNewCfg->withArbitrator == TSDB_MAX_DB_WITH_ARBITRATOR) {
if (pOldCfg->replications != 1 && pOldCfg->replications != 2) {
terrno = TSDB_CODE_OPS_NOT_SUPPORT;
return -1;
}
}
if (pNewCfg->replications != 2 && pOldCfg->replications == 2) {
terrno = TSDB_CODE_OPS_NOT_SUPPORT;
return -1;
}
#else
if (pNewCfg->replications != 1 && pNewCfg->replications != 3) return -1;
#endif
if (pNewCfg->sstTrigger < TSDB_MIN_STT_TRIGGER || pNewCfg->sstTrigger > TSDB_MAX_STT_TRIGGER) return -1;
if (pNewCfg->minRows < TSDB_MIN_MINROWS_FBLOCK || pNewCfg->minRows > TSDB_MAX_MINROWS_FBLOCK) return -1;
if (pNewCfg->maxRows < TSDB_MIN_MAXROWS_FBLOCK || pNewCfg->maxRows > TSDB_MAX_MAXROWS_FBLOCK) return -1;
if (pNewCfg->minRows > pNewCfg->maxRows) return -1;
if (pNewCfg->walRetentionPeriod < TSDB_DB_MIN_WAL_RETENTION_PERIOD) return -1;
if (pNewCfg->walRetentionSize < TSDB_DB_MIN_WAL_RETENTION_SIZE) return -1;
if (pNewCfg->strict < TSDB_DB_STRICT_OFF || pNewCfg->strict > TSDB_DB_STRICT_ON) return -1;
if (pNewCfg->replications > mndGetDnodeSize(pMnode)) {
terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
return -1;
}
@ -505,6 +528,7 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) {
if (pCfg->s3ChunkSize < 0) pCfg->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE;
if (pCfg->s3KeepLocal <= 0) pCfg->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
if (pCfg->s3Compact <= 0) pCfg->s3Compact = TSDB_DEFAULT_S3_COMPACT;
if (pCfg->withArbitrator < 0) pCfg->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR;
}
static int32_t mndSetCreateDbPrepareAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
@ -536,6 +560,15 @@ static int32_t mndSetCreateDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pD
if (sdbSetRawStatus(pVgRaw, SDB_STATUS_UPDATE) != 0) return -1;
}
if (pDb->cfg.withArbitrator) {
for (int32_t v = 0; v < pDb->cfg.numOfVgroups; ++v) {
SVgObj *pVgObj = pVgroups + v;
SArbGroup arbGroup = {0};
mndArbGroupInitFromVgObj(pVgObj, &arbGroup);
if (mndSetCreateArbGroupRedoLogs(pTrans, &arbGroup) != 0) return -1;
}
}
return 0;
}
@ -552,6 +585,15 @@ static int32_t mndSetCreateDbUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pD
if (sdbSetRawStatus(pVgRaw, SDB_STATUS_DROPPED) != 0) return -1;
}
if (pDb->cfg.withArbitrator) {
for (int32_t v = 0; v < pDb->cfg.numOfVgroups; ++v) {
SVgObj *pVgObj = pVgroups + v;
SArbGroup arbGroup = {0};
mndArbGroupInitFromVgObj(pVgObj, &arbGroup);
if (mndSetCreateArbGroupUndoLogs(pTrans, &arbGroup) != 0) return -1;
}
}
return 0;
}
@ -569,6 +611,15 @@ static int32_t mndSetCreateDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *
if (sdbSetRawStatus(pVgRaw, SDB_STATUS_READY) != 0) return -1;
}
if (pDb->cfg.withArbitrator) {
for (int32_t v = 0; v < pDb->cfg.numOfVgroups; ++v) {
SVgObj *pVgObj = pVgroups + v;
SArbGroup arbGroup = {0};
mndArbGroupInitFromVgObj(pVgObj, &arbGroup);
if (mndSetCreateArbGroupCommitLogs(pTrans, &arbGroup) != 0) return -1;
}
}
if (pUserDuped) {
SSdbRaw *pUserRaw = mndUserActionEncode(pUserDuped);
if (pUserRaw == NULL) return -1;
@ -653,6 +704,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate,
.s3KeepLocal = pCreate->s3KeepLocal,
.s3Compact = pCreate->s3Compact,
.tsdbPageSize = pCreate->tsdbPageSize,
.withArbitrator = pCreate->withArbitrator,
};
dbObj.cfg.numOfRetensions = pCreate->numOfRetensions;
@ -929,6 +981,12 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) {
terrno = 0;
}
if (pAlter->withArbitrator >= TSDB_MIN_DB_WITH_ARBITRATOR && pAlter->withArbitrator != pDb->cfg.withArbitrator) {
pDb->cfg.withArbitrator = pAlter->withArbitrator;
pDb->vgVersion++;
terrno = 0;
}
return terrno;
}
@ -967,12 +1025,24 @@ static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *
if (pIter == NULL) break;
if (mndVgroupInDb(pVgroup, pNewDb->uid)) {
if (mndBuildAlterVgroupAction(pMnode, pTrans, pOldDb, pNewDb, pVgroup, pArray) != 0) {
SVgObj newVgroup = {0};
if (mndBuildAlterVgroupAction(pMnode, pTrans, pOldDb, pNewDb, pVgroup, pArray, &newVgroup) != 0) {
sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pVgroup);
taosArrayDestroy(pArray);
return -1;
}
if (pNewDb->cfg.withArbitrator != pOldDb->cfg.withArbitrator) {
if (pNewDb->cfg.withArbitrator) {
SArbGroup arbGroup = {0};
mndArbGroupInitFromVgObj(&newVgroup, &arbGroup);
if (mndSetCreateArbGroupCommitLogs(pTrans, &arbGroup) != 0) return -1;
} else {
SArbGroup arbGroup = {0};
mndArbGroupInitFromVgObj(pVgroup, &arbGroup);
if (mndSetDropArbGroupCommitLogs(pTrans, &arbGroup) != 0) return -1;
}
}
}
sdbRelease(pSdb, pVgroup);
@ -1049,7 +1119,7 @@ static int32_t mndProcessAlterDbReq(SRpcMsg *pReq) {
goto _OVER;
}
code = mndCheckInChangeDbCfg(pMnode, &dbObj.cfg);
code = mndCheckInChangeDbCfg(pMnode, &pDb->cfg, &dbObj.cfg);
if (code != 0) goto _OVER;
dbObj.cfgVersion++;
@ -1120,6 +1190,7 @@ static void mndDumpDbCfgInfo(SDbCfgRsp *cfgRsp, SDbObj *pDb) {
cfgRsp->s3ChunkSize = pDb->cfg.s3ChunkSize;
cfgRsp->s3KeepLocal = pDb->cfg.s3KeepLocal;
cfgRsp->s3Compact = pDb->cfg.s3Compact;
cfgRsp->withArbitrator = pDb->cfg.withArbitrator;
}
static int32_t mndProcessGetDbCfgReq(SRpcMsg *pReq) {
@ -1177,6 +1248,25 @@ static int32_t mndSetDropDbPrepareLogs(SMnode *pMnode, STrans *pTrans, SDbObj *p
if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) return -1;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1;
SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL;
while (1) {
SArbGroup *pArbGroup = NULL;
pIter = sdbFetch(pSdb, SDB_ARBGROUP, pIter, (void **)&pArbGroup);
if (pIter == NULL) break;
if (pArbGroup->dbUid == pDb->uid) {
if (mndSetDropArbGroupPrepareLogs(pTrans,pArbGroup) != 0) {
sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pArbGroup);
return -1;
}
}
sdbRelease(pSdb, pArbGroup);
}
return 0;
}
@ -1189,6 +1279,22 @@ static int32_t mndSetDropDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pD
SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL;
while (1) {
SArbGroup *pArbGroup = NULL;
pIter = sdbFetch(pSdb, SDB_ARBGROUP, pIter, (void **)&pArbGroup);
if (pIter == NULL) break;
if (pArbGroup->dbUid == pDb->uid) {
if (mndSetDropArbGroupCommitLogs(pTrans,pArbGroup) != 0) {
sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pArbGroup);
return -1;
}
}
sdbRelease(pSdb, pArbGroup);
}
while (1) {
SVgObj *pVgroup = NULL;
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
@ -1429,7 +1535,7 @@ static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SArray *pVgList) {
pEp->port = pDnode->port;
}
mndReleaseDnode(pMnode, pDnode);
if (pVgid->syncState == TAOS_SYNC_STATE_LEADER) {
if (pVgid->syncState == TAOS_SYNC_STATE_LEADER || pVgid->syncState == TAOS_SYNC_STATE_ASSIGNED_LEADER) {
vgInfo.epSet.inUse = gid;
}
}
@ -1908,7 +2014,8 @@ bool mndIsDbReady(SMnode *pMnode, SDbObj *pDb) {
if (pVgroup->dbUid == pDb->uid && pVgroup->replica > 1) {
bool hasLeader = false;
for (int32_t i = 0; i < pVgroup->replica; ++i) {
if (pVgroup->vnodeGid[i].syncState == TAOS_SYNC_STATE_LEADER) {
if (pVgroup->vnodeGid[i].syncState == TAOS_SYNC_STATE_LEADER ||
pVgroup->vnodeGid[i].syncState == TAOS_SYNC_STATE_ASSIGNED_LEADER) {
hasLeader = true;
}
}
@ -2105,6 +2212,7 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb,
colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.s3KeepLocal, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.s3Compact, false);
colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.withArbitrator, false);
}
taosMemoryFree(buf);

73
source/dnode/mnode/impl/src/mndDef.c
View File

@ -72,7 +72,9 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) {
if (tEncodeI32(pEncoder, innerSz) < 0) return -1;
for (int32_t j = 0; j < innerSz; j++) {
SStreamTask *pTask = taosArrayGetP(pArray, j);
pTask->ver = SSTREAM_TASK_VER;
if (pTask->ver < SSTREAM_TASK_SUBTABLE_CHANGED_VER){
pTask->ver = SSTREAM_TASK_VER;
}
if (tEncodeStreamTask(pEncoder, pTask) < 0) return -1;
}
}
@ -249,7 +251,9 @@ void *tDecodeSMqVgEp(const void *buf, SMqVgEp *pVgEp, int8_t sver) {
return (void *)buf;
}
SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char *cgroup) {
static void *topicNameDup(void *p) { return taosStrdup((char *)p); }
SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char *cgroup, int8_t updateType, char *topic, SCMSubscribeReq *subscribe) {
SMqConsumerObj *pConsumer = taosMemoryCalloc(1, sizeof(SMqConsumerObj));
if (pConsumer == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -264,36 +268,64 @@ SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char *cgroup) {
pConsumer->hbStatus = 0;
taosInitRWLatch(&pConsumer->lock);
pConsumer->createTime = taosGetTimestampMs();
pConsumer->updateType = updateType;
pConsumer->currentTopics = taosArrayInit(0, sizeof(void *));
pConsumer->rebNewTopics = taosArrayInit(0, sizeof(void *));
pConsumer->rebRemovedTopics = taosArrayInit(0, sizeof(void *));
pConsumer->assignedTopics = taosArrayInit(0, sizeof(void *));
if (updateType == CONSUMER_ADD_REB){
pConsumer->rebNewTopics = taosArrayInit(0, sizeof(void *));
if(pConsumer->rebNewTopics == NULL){
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto END;
}
if (pConsumer->currentTopics == NULL || pConsumer->rebNewTopics == NULL || pConsumer->rebRemovedTopics == NULL ||
pConsumer->assignedTopics == NULL) {
taosArrayDestroy(pConsumer->currentTopics);
taosArrayDestroy(pConsumer->rebNewTopics);
taosArrayDestroy(pConsumer->rebRemovedTopics);
taosArrayDestroy(pConsumer->assignedTopics);
taosMemoryFree(pConsumer);
return NULL;
char* topicTmp = taosStrdup(topic);
taosArrayPush(pConsumer->rebNewTopics, &topicTmp);
}else if (updateType == CONSUMER_REMOVE_REB) {
pConsumer->rebRemovedTopics = taosArrayInit(0, sizeof(void *));
if(pConsumer->rebRemovedTopics == NULL){
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto END;
}
char* topicTmp = taosStrdup(topic);
taosArrayPush(pConsumer->rebRemovedTopics, &topicTmp);
}else if (updateType == CONSUMER_INSERT_SUB){
tstrncpy(pConsumer->clientId, subscribe->clientId, tListLen(pConsumer->clientId));
pConsumer->withTbName = subscribe->withTbName;
pConsumer->autoCommit = subscribe->autoCommit;
pConsumer->autoCommitInterval = subscribe->autoCommitInterval;
pConsumer->resetOffsetCfg = subscribe->resetOffsetCfg;
pConsumer->rebNewTopics = taosArrayDup(subscribe->topicNames, topicNameDup);
if (pConsumer->rebNewTopics == NULL){
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto END;
}
pConsumer->assignedTopics = subscribe->topicNames;
subscribe->topicNames = NULL;
}else if (updateType == CONSUMER_UPDATE_SUB){
pConsumer->assignedTopics = subscribe->topicNames;
subscribe->topicNames = NULL;
}
pConsumer->createTime = taosGetTimestampMs();
return pConsumer;
END:
tDeleteSMqConsumerObj(pConsumer);
return NULL;
}
void tDeleteSMqConsumerObj(SMqConsumerObj *pConsumer, bool delete) {
void tClearSMqConsumerObj(SMqConsumerObj *pConsumer) {
if (pConsumer == NULL) return;
taosArrayDestroyP(pConsumer->currentTopics, (FDelete)taosMemoryFree);
taosArrayDestroyP(pConsumer->rebNewTopics, (FDelete)taosMemoryFree);
taosArrayDestroyP(pConsumer->rebRemovedTopics, (FDelete)taosMemoryFree);
taosArrayDestroyP(pConsumer->assignedTopics, (FDelete)taosMemoryFree);
if (delete) {
taosMemoryFree(pConsumer);
}
}
void tDeleteSMqConsumerObj(SMqConsumerObj *pConsumer) {
tClearSMqConsumerObj(pConsumer);
taosMemoryFree(pConsumer);
}
int32_t tEncodeSMqConsumerObj(void **buf, const SMqConsumerObj *pConsumer) {
@ -548,6 +580,7 @@ SMqSubscribeObj *tCloneSubscribeObj(const SMqSubscribeObj *pSub) {
}
void tDeleteSubscribeObj(SMqSubscribeObj *pSub) {
if (pSub == NULL) return;
void *pIter = NULL;
while (1) {
pIter = taosHashIterate(pSub->consumerHash, pIter);

94
source/dnode/mnode/impl/src/mndDnode.c
View File

@ -33,7 +33,7 @@
#include "taos_monitor.h"
#define TSDB_DNODE_VER_NUMBER 2
#define TSDB_DNODE_RESERVE_SIZE 64
#define TSDB_DNODE_RESERVE_SIZE 40
static const char *offlineReason[] = {
"",
@ -183,11 +183,10 @@ static SSdbRaw *mndDnodeActionEncode(SDnodeObj *pDnode) {
SDB_SET_INT64(pRaw, dataPos, pDnode->updateTime, _OVER)
SDB_SET_INT16(pRaw, dataPos, pDnode->port, _OVER)
SDB_SET_BINARY(pRaw, dataPos, pDnode->fqdn, TSDB_FQDN_LEN, _OVER)
SDB_SET_BINARY(pRaw, dataPos, pDnode->machineId, TSDB_MACHINE_ID_LEN, _OVER)
SDB_SET_RESERVE(pRaw, dataPos, TSDB_DNODE_RESERVE_SIZE, _OVER)
SDB_SET_INT16(pRaw, dataPos, TSDB_ACTIVE_KEY_LEN, _OVER)
SDB_SET_BINARY(pRaw, dataPos, pDnode->active, TSDB_ACTIVE_KEY_LEN, _OVER)
SDB_SET_INT16(pRaw, dataPos, TSDB_CONN_ACTIVE_KEY_LEN, _OVER)
SDB_SET_BINARY(pRaw, dataPos, pDnode->connActive, TSDB_CONN_ACTIVE_KEY_LEN, _OVER)
SDB_SET_INT16(pRaw, dataPos, 0, _OVER) // forward/backward compatible
SDB_SET_INT16(pRaw, dataPos, 0, _OVER) // forward/backward compatible
SDB_SET_DATALEN(pRaw, dataPos, _OVER);
terrno = 0;
@ -227,13 +226,14 @@ static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw) {
SDB_GET_INT64(pRaw, dataPos, &pDnode->updateTime, _OVER)
SDB_GET_INT16(pRaw, dataPos, &pDnode->port, _OVER)
SDB_GET_BINARY(pRaw, dataPos, pDnode->fqdn, TSDB_FQDN_LEN, _OVER)
SDB_GET_BINARY(pRaw, dataPos, pDnode->machineId, TSDB_MACHINE_ID_LEN, _OVER)
SDB_GET_RESERVE(pRaw, dataPos, TSDB_DNODE_RESERVE_SIZE, _OVER)
if (sver > 1) {
int16_t keyLen = 0;
SDB_GET_INT16(pRaw, dataPos, &keyLen, _OVER)
SDB_GET_BINARY(pRaw, dataPos, pDnode->active, keyLen, _OVER)
SDB_GET_BINARY(pRaw, dataPos, NULL, keyLen, _OVER)
SDB_GET_INT16(pRaw, dataPos, &keyLen, _OVER)
SDB_GET_BINARY(pRaw, dataPos, pDnode->connActive, keyLen, _OVER)
SDB_GET_BINARY(pRaw, dataPos, NULL, keyLen, _OVER)
}
terrno = 0;
@ -271,12 +271,7 @@ static int32_t mndDnodeActionUpdate(SSdb *pSdb, SDnodeObj *pOld, SDnodeObj *pNew
mTrace("dnode:%d, perform update action, old row:%p new row:%p", pOld->id, pOld, pNew);
pOld->updateTime = pNew->updateTime;
#ifdef TD_ENTERPRISE
if (strncmp(pOld->active, pNew->active, TSDB_ACTIVE_KEY_LEN) != 0) {
strncpy(pOld->active, pNew->active, TSDB_ACTIVE_KEY_LEN);
}
if (strncmp(pOld->connActive, pNew->connActive, TSDB_CONN_ACTIVE_KEY_LEN) != 0) {
strncpy(pOld->connActive, pNew->connActive, TSDB_CONN_ACTIVE_KEY_LEN);
}
tstrncpy(pOld->machineId, pNew->machineId, TSDB_MACHINE_ID_LEN + 1);
#endif
return 0;
}
@ -311,6 +306,17 @@ SEpSet mndGetDnodeEpset(SDnodeObj *pDnode) {
return epSet;
}
SEpSet mndGetDnodeEpsetById(SMnode *pMnode, int32_t dnodeId) {
SEpSet epSet = {0};
SDnodeObj *pDnode = mndAcquireDnode(pMnode, dnodeId);
if (!pDnode) return epSet;
epSet = mndGetDnodeEpset(pDnode);
mndReleaseDnode(pMnode, pDnode);
return epSet;
}
static SDnodeObj *mndAcquireDnodeByEp(SMnode *pMnode, char *pEpStr) {
SSdb *pSdb = pMnode->pSdb;
@ -610,7 +616,7 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) {
for(int32_t j = 0; j < tagSize; j++){
SJson* item = tjsonGetArrayItem(arrayTag, j);
*(labels + j) = taosMemoryMalloc(MONITOR_TAG_NAME_LEN);
*(labels + j) = taosMemoryMalloc(MONITOR_TAG_NAME_LEN);
tjsonGetStringValue(item, "name", *(labels + j));
*(sample_labels + j) = taosMemoryMalloc(MONITOR_TAG_VALUE_LEN);
@ -626,7 +632,7 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) {
for(int32_t j = 0; j < metricLen; j++){
SJson *item = tjsonGetArrayItem(metrics, j);
char name[MONITOR_METRIC_NAME_LEN] = {0};
char name[MONITOR_METRIC_NAME_LEN] = {0};
tjsonGetStringValue(item, "name", name);
double value = 0;
@ -636,7 +642,7 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) {
tjsonGetDoubleValue(item, "type", &type);
int32_t metricNameLen = strlen(name) + strlen(tableName) + 2;
char* metricName = taosMemoryMalloc(metricNameLen);
char* metricName = taosMemoryMalloc(metricNameLen);
memset(metricName, 0, metricNameLen);
sprintf(metricName, "%s:%s", tableName, name);
@ -669,7 +675,7 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) {
else{
mTrace("get metric from registry:%p", metric);
}
if(type == 0){
taos_counter_add(metric, value, (const char**)sample_labels);
}
@ -689,7 +695,7 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) {
taosMemoryFreeClear(labels);
}
}
}
code = 0;
@ -705,6 +711,35 @@ _OVER:
*/
}
static int32_t mndUpdateDnodeObj(SMnode *pMnode, SDnodeObj *pDnode) {
int32_t code = 0;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, NULL, "update-dnode-obj");
if (pTrans == NULL) {
code = terrno;
goto _exit;
}
pDnode->updateTime = taosGetTimestampMs();
SSdbRaw *pCommitRaw = mndDnodeActionEncode(pDnode);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
code = terrno;
goto _exit;
}
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
code = terrno;
goto _exit;
}
_exit:
mndTransDrop(pTrans);
return code;
}
static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
SStatusReq statusReq = {0};
@ -776,7 +811,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
SVgObj *pVgroup = mndAcquireVgroup(pMnode, pVload->vgId);
if (pVgroup != NULL) {
if (pVload->syncState == TAOS_SYNC_STATE_LEADER) {
if (pVload->syncState == TAOS_SYNC_STATE_LEADER || pVload->syncState == TAOS_SYNC_STATE_ASSIGNED_LEADER) {
pVgroup->cacheUsage = pVload->cacheUsage;
pVgroup->numOfCachedTables = pVload->numOfCachedTables;
pVgroup->numOfTables = pVload->numOfTables;
@ -874,8 +909,9 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
pDnode->numOfDiskCfg = statusReq.numOfDiskCfg;
pDnode->memAvail = statusReq.memAvail;
pDnode->memTotal = statusReq.memTotal;
if (pDnode->machineId[0] == 0 && statusReq.machineId[0] != 0) {
if (memcmp(pDnode->machineId, statusReq.machineId, TSDB_MACHINE_ID_LEN) != 0) {
tstrncpy(pDnode->machineId, statusReq.machineId, TSDB_MACHINE_ID_LEN + 1);
mndUpdateDnodeObj(pMnode, pDnode);
}
SStatusRsp statusRsp = {0};
@ -1409,24 +1445,6 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) {
if (strcasecmp(cfgReq.config, "resetlog") == 0) {
strcpy(dcfgReq.config, "resetlog");
#ifdef TD_ENTERPRISE
} else if (strncasecmp(cfgReq.config, "supportvnodes", 13) == 0) {
int32_t optLen = strlen("supportvnodes");
int32_t flag = -1;
int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag);
if (code < 0) return code;
if (flag < 0 || flag > 4096) {
mError("dnode:%d, failed to config supportVnodes since value:%d. Valid range: [0, 4096]", cfgReq.dnodeId, flag);
terrno = TSDB_CODE_OUT_OF_RANGE;
goto _err_out;
}
if (flag == 0) {
flag = tsNumOfCores * 2;
}
flag = TMAX(flag, 2);
strcpy(dcfgReq.config, "supportvnodes");
snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag);
} else if (strncasecmp(cfgReq.config, "s3blocksize", 11) == 0) {
int32_t optLen = strlen("s3blocksize");
int32_t flag = -1;

1
source/dnode/mnode/impl/src/mndIndex.c
View File

@ -597,6 +597,7 @@ static int32_t mndSetUpdateIdxStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStb
taosRUnLockLatch(&pOld->lock);
pNew->pTags = NULL;
pNew->pColumns = NULL;
pNew->updateTime = taosGetTimestampMs();
pNew->lock = 0;

42
source/dnode/mnode/impl/src/mndMain.c
View File

@ -15,6 +15,7 @@
#define _DEFAULT_SOURCE
#include "mndAcct.h"
#include "mndArbGroup.h"
#include "mndCluster.h"
#include "mndCompact.h"
#include "mndCompactDetail.h"
@ -148,6 +149,22 @@ static void mndPullupS3MigrateDb(SMnode *pMnode) {
tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
}
static int32_t mndPullupArbHeartbeat(SMnode *pMnode) {
mTrace("pullup arb hb");
int32_t contLen = 0;
void *pReq = mndBuildTimerMsg(&contLen);
SRpcMsg rpcMsg = {.msgType = TDMT_MND_ARB_HEARTBEAT_TIMER, .pCont = pReq, .contLen = contLen, .info.noResp = 1};
return tmsgPutToQueue(&pMnode->msgCb, ARB_QUEUE, &rpcMsg);
}
static int32_t mndPullupArbCheckSync(SMnode *pMnode) {
mTrace("pullup arb sync");
int32_t contLen = 0;
void *pReq = mndBuildTimerMsg(&contLen);
SRpcMsg rpcMsg = {.msgType = TDMT_MND_ARB_CHECK_SYNC_TIMER, .pCont = pReq, .contLen = contLen, .info.noResp = 1};
return tmsgPutToQueue(&pMnode->msgCb, ARB_QUEUE, &rpcMsg);
}
static void mndCalMqRebalance(SMnode *pMnode) {
int32_t contLen = 0;
void *pReq = mndBuildTimerMsg(&contLen);
@ -404,6 +421,18 @@ static void *mndThreadFp(void *param) {
continue;
}
mndDoTimerPullupTask(pMnode, sec);
if (sec % (tsArbHeartBeatIntervalSec) == 0) {
if (mndPullupArbHeartbeat(pMnode) != 0) {
mError("failed to pullup arb heartbeat, since:%s", terrstr());
}
}
if (sec % (tsArbCheckSyncIntervalSec) == 0) {
if (mndPullupArbCheckSync(pMnode) != 0) {
mError("failed to pullup arb check sync, since:%s", terrstr());
}
}
}
return NULL;
@ -526,6 +555,7 @@ static int32_t mndInitSteps(SMnode *pMnode) {
if (mndAllocStep(pMnode, "mnode-mnode", mndInitMnode, mndCleanupMnode) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-qnode", mndInitQnode, mndCleanupQnode) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-snode", mndInitSnode, mndCleanupSnode) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-arbgroup", mndInitArbGroup, mndCleanupArbGroup) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-dnode", mndInitDnode, mndCleanupDnode) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-user", mndInitUser, mndCleanupUser) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-grant", mndInitGrant, mndCleanupGrant) != 0) return -1;
@ -707,6 +737,13 @@ ESyncRole mndGetRole(SMnode *pMnode) {
return syncGetRole(rid);
}
int64_t mndGetTerm(SMnode *pMnode) {
int64_t rid = pMnode->syncMgmt.sync;
return syncGetTerm(rid);
}
int32_t mndGetArbToken(SMnode *pMnode, char *outToken) { return syncGetArbToken(pMnode->syncMgmt.sync, outToken); }
void mndStop(SMnode *pMnode) {
mndSetStop(pMnode);
mndSyncStop(pMnode);
@ -782,7 +819,8 @@ _OVER:
pMsg->msgType == TDMT_MND_COMPACT_TIMER || pMsg->msgType == TDMT_MND_NODECHECK_TIMER ||
pMsg->msgType == TDMT_MND_GRANT_HB_TIMER || pMsg->msgType == TDMT_MND_STREAM_CHECKPOINT_CANDIDITATE ||
pMsg->msgType == TDMT_MND_STREAM_CHECKPOINT_TIMER || pMsg->msgType == TDMT_MND_STREAM_REQ_CHKPT ||
pMsg->msgType == TDMT_MND_S3MIGRATE_DB_TIMER) {
pMsg->msgType == TDMT_MND_S3MIGRATE_DB_TIMER ||
pMsg->msgType == TDMT_MND_ARB_HEARTBEAT_TIMER || pMsg->msgType == TDMT_MND_ARB_CHECK_SYNC_TIMER) {
mTrace("timer not process since mnode restored:%d stopped:%d, sync restored:%d role:%s ", pMnode->restored,
pMnode->stopped, state.restored, syncStr(state.state));
return -1;
@ -965,7 +1003,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
pVnDesc->dnode_id = pVgid->dnodeId;
tstrncpy(pVnDesc->vnode_role, syncStr(pVgid->syncState), sizeof(pVnDesc->vnode_role));
pVnDesc->syncState = pVgid->syncState;
if (pVgid->syncState == TAOS_SYNC_STATE_LEADER) {
if (pVgid->syncState == TAOS_SYNC_STATE_LEADER || pVgid->syncState == TAOS_SYNC_STATE_ASSIGNED_LEADER) {
tstrncpy(desc.status, "ready", sizeof(desc.status));
pClusterInfo->vgroups_alive++;
}

4
source/dnode/mnode/impl/src/mndQnode.c
View File

@ -189,7 +189,7 @@ int32_t mndSetCreateQnodeCommitLogs(STrans *pTrans, SQnodeObj *pObj) {
return 0;
}
bool mndQnodeInDnode(SQnodeObj *pQnode, int32_t dnodeId) {
bool mndQnodeInDnode(SQnodeObj *pQnode, int32_t dnodeId) {
return pQnode->pDnode->id == dnodeId;
}
@ -434,7 +434,7 @@ _OVER:
}
mndReleaseQnode(pMnode, pObj);
tFreeSMCreateQnodeReq(&dropReq);
tFreeSDDropQnodeReq(&dropReq);
return code;
}

2
source/dnode/mnode/impl/src/mndShow.c
View File

@ -67,6 +67,8 @@ static int32_t convertToRetrieveType(char *name, int32_t len) {
type = TSDB_MGMT_TABLE_QNODE;
} else if (strncasecmp(name, TSDB_INS_TABLE_SNODES, len) == 0) {
type = TSDB_MGMT_TABLE_SNODE;
} else if (strncasecmp(name, TSDB_INS_TABLE_ARBGROUPS, len) == 0) {
type = TSDB_MGMT_TABLE_ARBGROUP;
} else if (strncasecmp(name, TSDB_INS_TABLE_CLUSTER, len) == 0) {
type = TSDB_MGMT_TABLE_CLUSTER;
} else if (strncasecmp(name, TSDB_INS_TABLE_DATABASES, len) == 0) {

6
source/dnode/mnode/impl/src/mndStb.c
View File

@ -1145,14 +1145,16 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) {
} else if (createReq.tagVer > 0 || createReq.colVer > 0) {
int32_t tagDelta = createReq.tagVer - pStb->tagVer;
int32_t colDelta = createReq.colVer - pStb->colVer;
int32_t verDelta = tagDelta + colDelta;
mInfo("stb:%s, already exist while create, input tagVer:%d colVer:%d, exist tagVer:%d colVer:%d",
createReq.name, createReq.tagVer, createReq.colVer, pStb->tagVer, pStb->colVer);
if (tagDelta <= 0 && colDelta <= 0) {
mInfo("stb:%s, schema version is not incremented and nothing needs to be done", createReq.name);
code = 0;
goto _OVER;
} else if ((tagDelta == 1 || colDelta == 1) && (verDelta == 1)) {
} else if ((tagDelta == 1 && colDelta == 0) ||
(tagDelta == 0 && colDelta == 1) ||
(pStb->colVer == 1 && createReq.colVer > 1) ||
(pStb->tagVer == 1 && createReq.tagVer > 1)) {
isAlter = true;
mInfo("stb:%s, schema version is only increased by 1 number, do alter operation", createReq.name);
} else {

86
source/dnode/mnode/impl/src/mndStream.c
View File

@ -434,7 +434,9 @@ int32_t mndPersistTaskDeployReq(STrans *pTrans, SStreamTask *pTask) {
SEncoder encoder;
tEncoderInit(&encoder, NULL, 0);
pTask->ver = SSTREAM_TASK_VER;
if (pTask->ver < SSTREAM_TASK_SUBTABLE_CHANGED_VER){
pTask->ver = SSTREAM_TASK_VER;
}
tEncodeStreamTask(&encoder, pTask);
int32_t size = encoder.pos;
@ -513,6 +515,7 @@ static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStre
createReq.numOfColumns = pStream->outputSchema.nCols;
createReq.numOfTags = 1; // group id
createReq.pColumns = taosArrayInit_s(sizeof(SField), createReq.numOfColumns);
// build fields
for (int32_t i = 0; i < createReq.numOfColumns; i++) {
SField *pField = taosArrayGet(createReq.pColumns, i);
@ -586,12 +589,15 @@ static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStre
mndFreeStb(&stbObj);
mndReleaseStb(pMnode, pStb);
mndReleaseDb(pMnode, pDb);
mDebug("stream:%s create dst stable:%s, cols:%d", pStream->name, pStream->targetSTbName, pStream->outputSchema.nCols);
return 0;
_OVER:
tFreeSMCreateStbReq(&createReq);
mndReleaseStb(pMnode, pStb);
mndReleaseDb(pMnode, pDb);
mDebug("stream:%s failed to create dst stable:%s, code:%s", pStream->name, pStream->targetSTbName, tstrerror(terrno));
return -1;
}
@ -647,7 +653,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
terrno = TSDB_CODE_MND_INVALID_PLATFORM;
goto _OVER;
#endif
mInfo("stream:%s, start to create, sql:%s", createReq.name, createReq.sql);
mInfo("stream:%s, start to create stream, sql:%s", createReq.name, createReq.sql);
if (mndCheckCreateStreamReq(&createReq) != 0) {
mError("stream:%s, failed to create since %s", createReq.name, terrstr());
@ -684,17 +690,20 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
goto _OVER;
}
STrans *pTrans = doCreateTrans(pMnode, &streamObj, pReq, MND_STREAM_CREATE_NAME, "create stream tasks on dnodes");
STrans *pTrans = doCreateTrans(pMnode, &streamObj, pReq, TRN_CONFLICT_DB, MND_STREAM_CREATE_NAME, "create stream tasks on dnodes");
if (pTrans == NULL) {
goto _OVER;
}
// create stb for stream
if (createReq.createStb == STREAM_CREATE_STABLE_TRUE &&
mndCreateStbForStream(pMnode, pTrans, &streamObj, pReq->info.conn.user) < 0) {
mError("trans:%d, failed to create stb for stream %s since %s", pTrans->id, createReq.name, terrstr());
mndTransDrop(pTrans);
goto _OVER;
if (createReq.createStb == STREAM_CREATE_STABLE_TRUE) {
if (mndCreateStbForStream(pMnode, pTrans, &streamObj, pReq->info.conn.user) < 0) {
mError("trans:%d, failed to create stb for stream %s since %s", pTrans->id, createReq.name, terrstr());
mndTransDrop(pTrans);
goto _OVER;
}
} else {
mDebug("stream:%s no need create stable", createReq.name);
}
// schedule stream task for stream obj
@ -724,7 +733,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
// add into buffer firstly
// to make sure when the hb from vnode arrived, the newly created tasks have been in the task map already.
taosThreadMutexLock(&execInfo.lock);
mDebug("stream stream:%s tasks register into node list", createReq.name);
mDebug("stream stream:%s start to register tasks into task_node_list", createReq.name);
saveStreamTasksInfo(&streamObj, &execInfo);
taosThreadMutexUnlock(&execInfo.lock);
@ -890,7 +899,7 @@ static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStre
return -1;
}
STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, MND_STREAM_CHECKPOINT_NAME, "gen checkpoint for stream");
STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_CHECKPOINT_NAME, "gen checkpoint for stream");
if (pTrans == NULL) {
mError("failed to checkpoint of stream name%s, checkpointId: %" PRId64 ", reason:%s", pStream->name, checkpointId,
tstrerror(TSDB_CODE_MND_TRANS_CONFLICT));
@ -1143,7 +1152,7 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
return -1;
}
STrans *pTrans = doCreateTrans(pMnode, pStream, pReq, MND_STREAM_DROP_NAME, "drop stream");
STrans *pTrans = doCreateTrans(pMnode, pStream, pReq, TRN_CONFLICT_NOTHING, MND_STREAM_DROP_NAME, "drop stream");
if (pTrans == NULL) {
mError("stream:%s, failed to drop since %s", dropReq.name, terrstr());
sdbRelease(pMnode->pSdb, pStream);
@ -1508,6 +1517,8 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
}
}
pBlock->info.rows = numOfRows;
destroyStreamTaskIter(pIter);
taosRUnLockLatch(&pStream->lock);
@ -1571,7 +1582,7 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) {
return -1;
}
STrans *pTrans = doCreateTrans(pMnode, pStream, pReq, MND_STREAM_PAUSE_NAME, "pause the stream");
STrans *pTrans = doCreateTrans(pMnode, pStream, pReq, TRN_CONFLICT_NOTHING, MND_STREAM_PAUSE_NAME, "pause the stream");
if (pTrans == NULL) {
mError("stream:%s failed to pause stream since %s", pauseReq.name, terrstr());
sdbRelease(pMnode->pSdb, pStream);
@ -1660,7 +1671,7 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) {
return -1;
}
STrans *pTrans = doCreateTrans(pMnode, pStream, pReq, MND_STREAM_RESUME_NAME, "resume the stream");
STrans *pTrans = doCreateTrans(pMnode, pStream, pReq, TRN_CONFLICT_NOTHING, MND_STREAM_RESUME_NAME, "resume the stream");
if (pTrans == NULL) {
mError("stream:%s, failed to resume stream since %s", resumeReq.name, terrstr());
sdbRelease(pMnode->pSdb, pStream);
@ -1792,7 +1803,7 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange
// here create only one trans
if (pTrans == NULL) {
pTrans = doCreateTrans(pMnode, pStream, NULL, MND_STREAM_TASK_UPDATE_NAME, "update task epsets");
pTrans = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_TASK_UPDATE_NAME, "update task epsets");
if (pTrans == NULL) {
sdbRelease(pSdb, pStream);
sdbCancelFetch(pSdb, pIter);
@ -1813,7 +1824,7 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange
mDebug("stream:0x%" PRIx64 " %s involved node changed, create update trans, transId:%d", pStream->uid,
pStream->name, pTrans->id);
int32_t code = mndStreamSetUpdateEpsetAction(pStream, pChangeInfo, pTrans);
int32_t code = mndStreamSetUpdateEpsetAction(pMnode, pStream, pChangeInfo, pTrans);
// todo: not continue, drop all and retry again
if (code != TSDB_CODE_SUCCESS) {
@ -2144,41 +2155,60 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) {
SStreamObj *pStream = mndGetStreamObj(pMnode, req.streamId);
if (pStream == NULL) {
mError("failed to find the stream:0x%" PRIx64 " not handle the checkpoint req", req.streamId);
terrno = TSDB_CODE_MND_STREAM_NOT_EXIST;
taosThreadMutexUnlock(&execInfo.lock);
mWarn("failed to find the stream:0x%" PRIx64 ", not handle the checkpoint req, try to acquire in buf", req.streamId);
return -1;
// not in meta-store yet, try to acquire the task in exec buffer
// the checkpoint req arrives too soon before the completion of the create stream trans.
STaskId id = {.streamId = req.streamId, .taskId = req.taskId};
void* p = taosHashGet(execInfo.pTaskMap, &id, sizeof(id));
if (p == NULL) {
mError("failed to find the stream:0x%" PRIx64 " in buf, not handle the checkpoint req", req.streamId);
terrno = TSDB_CODE_MND_STREAM_NOT_EXIST;
taosThreadMutexUnlock(&execInfo.lock);
return -1;
} else {
mDebug("s-task:0x%" PRIx64 "-0x%x in buf not in mnode/meta, create stream trans may not complete yet",
req.streamId, req.taskId);
}
}
int32_t numOfTasks = mndGetNumOfStreamTasks(pStream);
int32_t numOfTasks = (pStream == NULL)? 0: mndGetNumOfStreamTasks(pStream);
SArray **pReqTaskList = (SArray **)taosHashGet(execInfo.pTransferStateStreams, &req.streamId, sizeof(req.streamId));
if (pReqTaskList == NULL) {
SArray *pList = taosArrayInit(4, sizeof(int32_t));
doAddTaskId(pList, req.taskId, pStream->uid, numOfTasks);
doAddTaskId(pList, req.taskId, req.streamId, numOfTasks);
taosHashPut(execInfo.pTransferStateStreams, &req.streamId, sizeof(int64_t), &pList, sizeof(void *));
pReqTaskList = (SArray **)taosHashGet(execInfo.pTransferStateStreams, &req.streamId, sizeof(req.streamId));
} else {
doAddTaskId(*pReqTaskList, req.taskId, pStream->uid, numOfTasks);
doAddTaskId(*pReqTaskList, req.taskId, req.streamId, numOfTasks);
}
int32_t total = taosArrayGetSize(*pReqTaskList);
if (total == numOfTasks) { // all tasks has send the reqs
int64_t checkpointId = mndStreamGenChkpId(pMnode);
mDebug("stream:0x%" PRIx64 " all tasks req, start checkpointId:%" PRId64, pStream->uid, checkpointId);
mInfo("stream:0x%" PRIx64 " all tasks req checkpoint, start checkpointId:%" PRId64, req.streamId, checkpointId);
// TODO:handle error
int32_t code = mndProcessStreamCheckpointTrans(pMnode, pStream, checkpointId, 0, false);
if (pStream != NULL) { // TODO:handle error
int32_t code = mndProcessStreamCheckpointTrans(pMnode, pStream, checkpointId, 0, false);
} else {
// todo: wait for the create stream trans completed, and launch the checkpoint trans
// SStreamObj *pStream = mndGetStreamObj(pMnode, req.streamId);
// sleep(500ms)
}
// remove this entry
taosHashRemove(execInfo.pTransferStateStreams, &req.streamId, sizeof(int64_t));
int32_t numOfStreams = taosHashGetSize(execInfo.pTransferStateStreams);
mDebug("stream:0x%" PRIx64 " removed, remain streams:%d fill-history not completed", pStream->uid, numOfStreams);
mDebug("stream:0x%" PRIx64 " removed, remain streams:%d fill-history not completed", req.streamId, numOfStreams);
}
if (pStream != NULL) {
mndReleaseStream(pMnode, pStream);
}
mndReleaseStream(pMnode, pStream);
taosThreadMutexUnlock(&execInfo.lock);
{

4
source/dnode/mnode/impl/src/mndStreamHb.c
View File

@ -66,7 +66,7 @@ static void addIntoCheckpointList(SArray* pList, const SFailedCheckpointInfo* pI
}
int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) {
STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, MND_STREAM_TASK_RESET_NAME, " reset from failed checkpoint");
STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_TASK_RESET_NAME, " reset from failed checkpoint");
if (pTrans == NULL) {
return terrno;
}
@ -156,7 +156,7 @@ static int32_t mndDropOrphanTasks(SMnode* pMnode, SArray* pList) {
}
SStreamObj dummyObj = {.uid = pTask->streamId, .sourceDb = "", .targetSTbName = ""};
STrans* pTrans = doCreateTrans(pMnode, &dummyObj, NULL, MND_STREAM_DROP_NAME, "drop stream");
STrans* pTrans = doCreateTrans(pMnode, &dummyObj, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_DROP_NAME, "drop stream");
if (pTrans == NULL) {
mError("failed to create trans to drop orphan tasks since %s", terrstr());
return -1;

4
source/dnode/mnode/impl/src/mndStreamTrans.c
View File

@ -161,8 +161,8 @@ int32_t mndAddtoCheckpointWaitingList(SStreamObj* pStream, int64_t checkpointId)
return TSDB_CODE_SUCCESS;
}
STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, const char *name, const char *pMsg) {
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, name);
STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, ETrnConflct conflict, const char *name, const char *pMsg) {
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, conflict, pReq, name);
if (pTrans == NULL) {
mError("failed to build trans:%s, reason: %s", name, tstrerror(TSDB_CODE_OUT_OF_MEMORY));
terrno = TSDB_CODE_OUT_OF_MEMORY;

22
source/dnode/mnode/impl/src/mndStreamUtil.c
View File

@ -115,6 +115,7 @@ SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady) {
char buf[256] = {0};
EPSET_TO_STR(&entry.epset, buf);
mDebug("take node snapshot, nodeId:%d %s", entry.nodeId, buf);
taosArrayPush(pVgroupListSnapshot, &entry);
sdbRelease(pSdb, pVgroup);
@ -300,7 +301,10 @@ static int32_t doSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTa
return code;
}
mDebug("pause node:%d, epset:%d", pTask->info.nodeId, epset.numOfEps);
char buf[256] = {0};
EPSET_TO_STR(&epset, buf);
mDebug("pause stream task in node:%d, epset:%s", pTask->info.nodeId, buf);
code = setTransAction(pTrans, pReq, sizeof(SVPauseStreamTaskReq), TDMT_STREAM_TASK_PAUSE, &epset, 0);
if (code != 0) {
taosMemoryFree(pReq);
@ -462,14 +466,22 @@ static int32_t doBuildStreamTaskUpdateMsg(void **pBuf, int32_t *pLen, SVgroupCha
return TSDB_CODE_SUCCESS;
}
static int32_t doSetUpdateTaskAction(STrans *pTrans, SStreamTask *pTask, SVgroupChangeInfo *pInfo) {
static int32_t doSetUpdateTaskAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask, SVgroupChangeInfo *pInfo) {
void *pBuf = NULL;
int32_t len = 0;
streamTaskUpdateEpsetInfo(pTask, pInfo->pUpdateNodeList);
doBuildStreamTaskUpdateMsg(&pBuf, &len, pInfo, pTask->info.nodeId, &pTask->id, pTrans->id);
int32_t code = setTransAction(pTrans, pBuf, len, TDMT_VND_STREAM_TASK_UPDATE, &pTask->info.epSet, 0);
SEpSet epset = {0};
bool hasEpset = false;
int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId);
if (code != TSDB_CODE_SUCCESS || !hasEpset) {
terrno = code;
return code;
}
code = setTransAction(pTrans, pBuf, len, TDMT_VND_STREAM_TASK_UPDATE, &epset, TSDB_CODE_VND_INVALID_VGROUP_ID);
if (code != TSDB_CODE_SUCCESS) {
taosMemoryFree(pBuf);
}
@ -478,14 +490,14 @@ static int32_t doSetUpdateTaskAction(STrans *pTrans, SStreamTask *pTask, SVgroup
}
// build trans to update the epset
int32_t mndStreamSetUpdateEpsetAction(SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans) {
int32_t mndStreamSetUpdateEpsetAction(SMnode *pMnode, SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans) {
mDebug("stream:0x%" PRIx64 " set tasks epset update action", pStream->uid);
taosWLockLatch(&pStream->lock);
SStreamTaskIter *pIter = createStreamTaskIter(pStream);
while (streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pIter);
int32_t code = doSetUpdateTaskAction(pTrans, pTask, pInfo);
int32_t code = doSetUpdateTaskAction(pMnode, pTrans, pTask, pInfo);
if (code != TSDB_CODE_SUCCESS) {
destroyStreamTaskIter(pIter);
taosWUnLockLatch(&pStream->lock);

975
source/dnode/mnode/impl/src/mndSubscribe.c
View File

File diff suppressed because it is too large Load Diff

1
source/dnode/mnode/impl/src/mndSync.c
View File

@ -409,6 +409,7 @@ SSyncFSM *mndSyncMakeFsm(SMnode *pMnode) {
pFsm->FpApplyQueueItems = mndApplyQueueItems;
pFsm->FpReConfigCb = NULL;
pFsm->FpBecomeLeaderCb = mndBecomeLeader;
pFsm->FpBecomeAssignedLeaderCb = NULL;
pFsm->FpBecomeFollowerCb = mndBecomeFollower;
pFsm->FpBecomeLearnerCb = mndBecomeLearner;
pFsm->FpGetSnapshot = mndSyncGetSnapshot;

321
source/dnode/mnode/impl/src/mndTopic.c
View File

@ -353,6 +353,66 @@ static int32_t extractTopicTbInfo(SNode *pAst, SMqTopicObj *pTopic) {
return 0;
}
static int32_t sendCheckInfoToVnode(STrans *pTrans, SMnode *pMnode, SMqTopicObj *topicObj){
STqCheckInfo info;
memcpy(info.topic, topicObj->name, TSDB_TOPIC_FNAME_LEN);
info.ntbUid = topicObj->ntbUid;
info.colIdList = topicObj->ntbColIds;
// broadcast forbid alter info
void *pIter = NULL;
SSdb *pSdb = pMnode->pSdb;
SVgObj *pVgroup = NULL;
int32_t code = 0;
void *buf = NULL;
while (1) {
// iterate vg
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
if (pIter == NULL) break;
if (!mndVgroupInDb(pVgroup, topicObj->dbUid)) {
sdbRelease(pSdb, pVgroup);
continue;
}
// encoder check alter info
int32_t len;
tEncodeSize(tEncodeSTqCheckInfo, &info, len, code);
if (code != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto END;
}
buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len);
void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
SEncoder encoder;
tEncoderInit(&encoder, abuf, len);
code = tEncodeSTqCheckInfo(&encoder, &info);
if (code < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto END;
}
tEncoderClear(&encoder);
((SMsgHead *)buf)->vgId = htonl(pVgroup->vgId);
// add redo action
STransAction action = {0};
action.epSet = mndGetVgroupEpset(pMnode, pVgroup);
action.pCont = buf;
action.contLen = sizeof(SMsgHead) + len;
action.msgType = TDMT_VND_TMQ_ADD_CHECKINFO;
code = mndTransAppendRedoAction(pTrans, &action);
if (code != 0) {
goto END;
}
sdbRelease(pSdb, pVgroup);
buf = NULL;
}
END:
taosMemoryFree(buf);
sdbRelease(pSdb, pVgroup);
sdbCancelFetch(pSdb, pIter);
return code;
}
static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *pCreate, SDbObj *pDb,
const char *userName) {
mInfo("start to create topic:%s", pCreate->name);
@ -396,13 +456,6 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *
topicObj.withMeta = pCreate->withMeta;
if (pCreate->subType == TOPIC_SUB_TYPE__COLUMN) {
if (pCreate->withMeta) {
terrno = TSDB_CODE_MND_INVALID_TOPIC_OPTION;
mError("topic:%s, failed to create since %s", pCreate->name, terrstr());
code = terrno;
goto _OUT;
}
topicObj.ast = taosStrdup(pCreate->ast);
topicObj.astLen = strlen(pCreate->ast) + 1;
@ -474,59 +527,9 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
if (topicObj.ntbUid != 0) {
STqCheckInfo info;
memcpy(info.topic, topicObj.name, TSDB_TOPIC_FNAME_LEN);
info.ntbUid = topicObj.ntbUid;
info.colIdList = topicObj.ntbColIds;
// broadcast forbid alter info
void *pIter = NULL;
SSdb *pSdb = pMnode->pSdb;
SVgObj *pVgroup = NULL;
while (1) {
// iterate vg
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
if (pIter == NULL) break;
if (!mndVgroupInDb(pVgroup, topicObj.dbUid)) {
sdbRelease(pSdb, pVgroup);
continue;
}
// encoder check alter info
int32_t len;
tEncodeSize(tEncodeSTqCheckInfo, &info, len, code);
if (code < 0) {
sdbRelease(pSdb, pVgroup);
sdbCancelFetch(pSdb, pIter);
goto _OUT;
}
void *buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len);
void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
SEncoder encoder;
tEncoderInit(&encoder, abuf, len);
if (tEncodeSTqCheckInfo(&encoder, &info) < 0) {
taosMemoryFree(buf);
sdbRelease(pSdb, pVgroup);
sdbCancelFetch(pSdb, pIter);
code = -1;
goto _OUT;
}
tEncoderClear(&encoder);
((SMsgHead *)buf)->vgId = htonl(pVgroup->vgId);
// add redo action
STransAction action = {0};
action.epSet = mndGetVgroupEpset(pMnode, pVgroup);
action.pCont = buf;
action.contLen = sizeof(SMsgHead) + len;
action.msgType = TDMT_VND_TMQ_ADD_CHECKINFO;
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
taosMemoryFree(buf);
sdbRelease(pSdb, pVgroup);
sdbCancelFetch(pSdb, pIter);
code = -1;
goto _OUT;
}
buf = NULL;
sdbRelease(pSdb, pVgroup);
code = sendCheckInfoToVnode(pTrans, pMnode, &topicObj);
if (code != 0){
goto _OUT;
}
}
@ -618,7 +621,7 @@ static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq) {
tNameFromString(&topicName, createTopicReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
//reuse this function for topic
auditRecord(pReq, pMnode->clusterId, "createTopic", dbname.dbname, topicName.dbname,
auditRecord(pReq, pMnode->clusterId, "createTopic", dbname.dbname, topicName.dbname,
createTopicReq.sql, strlen(createTopicReq.sql));
_OVER:
@ -653,9 +656,104 @@ _OVER:
return code;
}
static bool checkTopic(SArray *topics, char *topicName){
int32_t sz = taosArrayGetSize(topics);
for (int32_t i = 0; i < sz; i++) {
char *name = taosArrayGetP(topics, i);
if (strcmp(name, topicName) == 0) {
return true;
}
}
return false;
}
static int32_t mndDropConsumerByTopic(SMnode *pMnode, STrans *pTrans, char *topicName){
int32_t code = 0;
SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL;
SMqConsumerObj *pConsumer = NULL;
while (1) {
pIter = sdbFetch(pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer);
if (pIter == NULL) {
break;
}
bool found = checkTopic(pConsumer->assignedTopics, topicName);
if (found){
if (pConsumer->status == MQ_CONSUMER_STATUS_LOST) {
code = mndSetConsumerDropLogs(pTrans, pConsumer);
if (code != 0) {
goto end;
}
sdbRelease(pSdb, pConsumer);
continue;
}
mError("topic:%s, failed to drop since subscribed by consumer:0x%" PRIx64 ", in consumer group %s",
topicName, pConsumer->consumerId, pConsumer->cgroup);
code = TSDB_CODE_MND_TOPIC_SUBSCRIBED;
goto end;
}
if (checkTopic(pConsumer->rebNewTopics, topicName) || checkTopic(pConsumer->rebRemovedTopics, topicName)) {
code = TSDB_CODE_MND_TOPIC_SUBSCRIBED;
mError("topic:%s, failed to drop since subscribed by consumer:%" PRId64 ", in consumer group %s (reb new)",
topicName, pConsumer->consumerId, pConsumer->cgroup);
goto end;
}
sdbRelease(pSdb, pConsumer);
}
end:
sdbRelease(pSdb, pConsumer);
sdbCancelFetch(pSdb, pIter);
return code;
}
static int32_t mndDropCheckInfoByTopic(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic){
// broadcast to all vnode
void *pIter = NULL;
SVgObj *pVgroup = NULL;
int32_t code = 0;
SSdb *pSdb = pMnode->pSdb;
void *buf = NULL;
while (1) {
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
if (pIter == NULL) break;
if (!mndVgroupInDb(pVgroup, pTopic->dbUid)) {
sdbRelease(pSdb, pVgroup);
continue;
}
buf = taosMemoryCalloc(1, sizeof(SMsgHead) + TSDB_TOPIC_FNAME_LEN);
if (buf == NULL){
code = TSDB_CODE_OUT_OF_MEMORY;
goto end;
}
void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
((SMsgHead *)buf)->vgId = htonl(pVgroup->vgId);
memcpy(abuf, pTopic->name, TSDB_TOPIC_FNAME_LEN);
STransAction action = {0};
action.epSet = mndGetVgroupEpset(pMnode, pVgroup);
action.pCont = buf;
action.contLen = sizeof(SMsgHead) + TSDB_TOPIC_FNAME_LEN;
action.msgType = TDMT_VND_TMQ_DEL_CHECKINFO;
code = mndTransAppendRedoAction(pTrans, &action);
if (code != 0) {
taosMemoryFree(buf);
goto end;
}
sdbRelease(pSdb, pVgroup);
}
end:
sdbRelease(pSdb, pVgroup);
sdbCancelFetch(pSdb, pIter);
return code;
}
static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb;
SMDropTopicReq dropReq = {0};
int32_t code = 0;
SMqTopicObj *pTopic = NULL;
@ -705,68 +803,9 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) {
goto end;
}
void *pIter = NULL;
SMqConsumerObj *pConsumer;
while (1) {
pIter = sdbFetch(pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer);
if (pIter == NULL) {
break;
}
bool found = false;
int32_t sz = taosArrayGetSize(pConsumer->assignedTopics);
for (int32_t i = 0; i < sz; i++) {
char *name = taosArrayGetP(pConsumer->assignedTopics, i);
if (strcmp(name, pTopic->name) == 0) {
found = true;
break;
}
}
if (found){
if (pConsumer->status == MQ_CONSUMER_STATUS_LOST) {
mndDropConsumerFromSdb(pMnode, pConsumer->consumerId, &pReq->info);
mndReleaseConsumer(pMnode, pConsumer);
continue;
}
mndReleaseConsumer(pMnode, pConsumer);
sdbCancelFetch(pSdb, pIter);
terrno = TSDB_CODE_MND_TOPIC_SUBSCRIBED;
mError("topic:%s, failed to drop since subscribed by consumer:0x%" PRIx64 ", in consumer group %s",
dropReq.name, pConsumer->consumerId, pConsumer->cgroup);
code = -1;
goto end;
}
sz = taosArrayGetSize(pConsumer->rebNewTopics);
for (int32_t i = 0; i < sz; i++) {
char *name = taosArrayGetP(pConsumer->rebNewTopics, i);
if (strcmp(name, pTopic->name) == 0) {
mndReleaseConsumer(pMnode, pConsumer);
sdbCancelFetch(pSdb, pIter);
terrno = TSDB_CODE_MND_TOPIC_SUBSCRIBED;
mError("topic:%s, failed to drop since subscribed by consumer:%" PRId64 ", in consumer group %s (reb new)",
dropReq.name, pConsumer->consumerId, pConsumer->cgroup);
code = -1;
goto end;
}
}
sz = taosArrayGetSize(pConsumer->rebRemovedTopics);
for (int32_t i = 0; i < sz; i++) {
char *name = taosArrayGetP(pConsumer->rebRemovedTopics, i);
if (strcmp(name, pTopic->name) == 0) {
mndReleaseConsumer(pMnode, pConsumer);
sdbCancelFetch(pSdb, pIter);
terrno = TSDB_CODE_MND_TOPIC_SUBSCRIBED;
mError("topic:%s, failed to drop since subscribed by consumer:%" PRId64 ", in consumer group %s (reb remove)",
dropReq.name, pConsumer->consumerId, pConsumer->cgroup);
code = -1;
goto end;
}
}
sdbRelease(pSdb, pConsumer);
code = mndDropConsumerByTopic(pMnode, pTrans, dropReq.name);
if (code != 0) {
goto end;
}
code = mndDropSubByTopic(pMnode, pTrans, dropReq.name);
@ -776,36 +815,9 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) {
}
if (pTopic->ntbUid != 0) {
// broadcast to all vnode
pIter = NULL;
SVgObj *pVgroup = NULL;
while (1) {
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
if (pIter == NULL) break;
if (!mndVgroupInDb(pVgroup, pTopic->dbUid)) {
sdbRelease(pSdb, pVgroup);
continue;
}
void *buf = taosMemoryCalloc(1, sizeof(SMsgHead) + TSDB_TOPIC_FNAME_LEN);
void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
((SMsgHead *)buf)->vgId = htonl(pVgroup->vgId);
memcpy(abuf, pTopic->name, TSDB_TOPIC_FNAME_LEN);
STransAction action = {0};
action.epSet = mndGetVgroupEpset(pMnode, pVgroup);
action.pCont = buf;
action.contLen = sizeof(SMsgHead) + TSDB_TOPIC_FNAME_LEN;
action.msgType = TDMT_VND_TMQ_DEL_CHECKINFO;
code = mndTransAppendRedoAction(pTrans, &action);
if (code != 0) {
taosMemoryFree(buf);
sdbRelease(pSdb, pVgroup);
sdbCancelFetch(pSdb, pIter);
goto end;
}
sdbRelease(pSdb, pVgroup);
code = mndDropCheckInfoByTopic(pMnode, pTrans, pTopic);
if (code != 0) {
goto end;
}
}
@ -822,7 +834,6 @@ end:
SName name = {0};
tNameFromString(&name, dropReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
//reuse this function for topic
auditRecord(pReq, pMnode->clusterId, "dropTopic", name.dbname, name.tname, dropReq.sql, dropReq.sqlLen);

103
source/dnode/mnode/impl/src/mndTrans.c
View File

@ -15,11 +15,11 @@
#define _DEFAULT_SOURCE
#include "mndTrans.h"
#include "mndSubscribe.h"
#include "mndDb.h"
#include "mndPrivilege.h"
#include "mndShow.h"
#include "mndStb.h"
#include "mndSubscribe.h"
#include "mndSync.h"
#include "mndUser.h"
@ -733,10 +733,14 @@ void mndTransSetDbName(STrans *pTrans, const char *dbname, const char *stbname)
}
}
void mndTransSetArbGroupId(STrans *pTrans, int32_t groupId) { pTrans->arbGroupId = groupId; }
void mndTransSetSerial(STrans *pTrans) { pTrans->exec = TRN_EXEC_SERIAL; }
void mndTransSetParallel(STrans *pTrans) { pTrans->exec = TRN_EXEC_PARALLEL; }
void mndTransSetChangeless(STrans *pTrans) { pTrans->changeless = true; }
void mndTransSetOper(STrans *pTrans, EOperType oper) { pTrans->oper = oper; }
static int32_t mndTransSync(SMnode *pMnode, STrans *pTrans) {
@ -801,16 +805,23 @@ static bool mndCheckTransConflict(SMnode *pMnode, STrans *pNew) {
if (pNew->conflict == TRN_CONFLICT_TOPIC) {
if (pTrans->conflict == TRN_CONFLICT_GLOBAL) conflict = true;
if (pTrans->conflict == TRN_CONFLICT_TOPIC || pTrans->conflict == TRN_CONFLICT_TOPIC_INSIDE) {
if (strcasecmp(pNew->dbname, pTrans->dbname) == 0 ) conflict = true;
if (strcasecmp(pNew->dbname, pTrans->dbname) == 0) conflict = true;
}
}
if (pNew->conflict == TRN_CONFLICT_TOPIC_INSIDE) {
if (pTrans->conflict == TRN_CONFLICT_GLOBAL) conflict = true;
if (pTrans->conflict == TRN_CONFLICT_TOPIC ) {
if (strcasecmp(pNew->dbname, pTrans->dbname) == 0 ) conflict = true;
if (pTrans->conflict == TRN_CONFLICT_TOPIC) {
if (strcasecmp(pNew->dbname, pTrans->dbname) == 0) conflict = true;
}
if (pTrans->conflict == TRN_CONFLICT_TOPIC_INSIDE) {
if (strcasecmp(pNew->dbname, pTrans->dbname) == 0 && strcasecmp(pNew->stbname, pTrans->stbname) == 0) conflict = true;
if (strcasecmp(pNew->dbname, pTrans->dbname) == 0 && strcasecmp(pNew->stbname, pTrans->stbname) == 0)
conflict = true;
}
}
if (pNew->conflict == TRN_CONFLICT_ARBGROUP) {
if (pTrans->conflict == TRN_CONFLICT_GLOBAL) conflict = true;
if (pTrans->conflict == TRN_CONFLICT_ARBGROUP) {
if (pNew->arbGroupId == pTrans->arbGroupId) conflict = true;
}
}
@ -846,16 +857,70 @@ int32_t mndTransCheckConflict(SMnode *pMnode, STrans *pTrans) {
return 0;
}
static bool mndTransActionsOfSameType(SArray *pActions) {
int32_t size = taosArrayGetSize(pActions);
ETrnAct lastActType = TRANS_ACTION_NULL;
bool same = true;
for (int32_t i = 0; i < size; ++i) {
STransAction *pAction = taosArrayGet(pActions, i);
if (i > 0) {
if (lastActType != pAction->actionType) {
same = false;
break;
}
}
lastActType = pAction->actionType;
}
return same;
}
static int32_t mndTransCheckParallelActions(SMnode *pMnode, STrans *pTrans) {
if (pTrans->exec == TRN_EXEC_PARALLEL) {
if (mndTransActionsOfSameType(pTrans->redoActions) == false) {
terrno = TSDB_CODE_MND_TRANS_INVALID_STAGE;
mError("trans:%d, types of parallel redo actions are not the same", pTrans->id);
return -1;
}
if (pTrans->policy == TRN_POLICY_ROLLBACK) {
if (mndTransActionsOfSameType(pTrans->undoActions) == false) {
terrno = TSDB_CODE_MND_TRANS_INVALID_STAGE;
mError("trans:%d, types of parallel undo actions are not the same", pTrans->id);
return -1;
}
}
}
return 0;
}
static int32_t mndTransCheckCommitActions(SMnode *pMnode, STrans *pTrans) {
if (!pTrans->changeless && taosArrayGetSize(pTrans->commitActions) <= 0) {
terrno = TSDB_CODE_MND_TRANS_CLOG_IS_NULL;
mError("trans:%d, commit actions of non-changeless trans are empty", pTrans->id);
return -1;
}
if (mndTransActionsOfSameType(pTrans->commitActions) == false) {
terrno = TSDB_CODE_MND_TRANS_INVALID_STAGE;
mError("trans:%d, types of commit actions are not the same", pTrans->id);
return -1;
}
return 0;
}
int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) {
if(pTrans == NULL) return -1;
if (pTrans == NULL) return -1;
if (mndTransCheckConflict(pMnode, pTrans) != 0) {
return -1;
}
if (taosArrayGetSize(pTrans->commitActions) <= 0) {
terrno = TSDB_CODE_MND_TRANS_CLOG_IS_NULL;
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
if (mndTransCheckParallelActions(pMnode, pTrans) != 0) {
return -1;
}
if (mndTransCheckCommitActions(pMnode, pTrans) != 0) {
return -1;
}
@ -1142,6 +1207,7 @@ static int32_t mndTransSendSingleMsg(SMnode *pMnode, STrans *pTrans, STransActio
return -1;
}
rpcMsg.info.traceId.rootId = pTrans->mTraceId;
rpcMsg.info.notFreeAhandle = 1;
memcpy(rpcMsg.pCont, pAction->pCont, pAction->contLen);
@ -1156,7 +1222,7 @@ static int32_t mndTransSendSingleMsg(SMnode *pMnode, STrans *pTrans, STransActio
int32_t code = tmsgSendReq(&pAction->epSet, &rpcMsg);
if (code == 0) {
pAction->msgSent = 1;
//pAction->msgReceived = 0;
// pAction->msgReceived = 0;
pAction->errCode = TSDB_CODE_ACTION_IN_PROGRESS;
mInfo("trans:%d, %s:%d is sent, %s", pTrans->id, mndTransStr(pAction->stage), pAction->id, detail);
@ -1253,7 +1319,7 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA
for (int32_t action = 0; action < numOfActions; ++action) {
STransAction *pAction = taosArrayGet(pArray, action);
mDebug("trans:%d, %s:%d Sent:%d, Received:%d, errCode:0x%x, acceptableCode:0x%x, retryCode:0x%x",
mDebug("trans:%d, %s:%d Sent:%d, Received:%d, errCode:0x%x, acceptableCode:0x%x, retryCode:0x%x",
pTrans->id, mndTransStr(pAction->stage), pAction->id, pAction->msgSent, pAction->msgReceived,
pAction->errCode, pAction->acceptableCode, pAction->retryCode);
if (pAction->msgSent) {
@ -1262,7 +1328,7 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA
mndTransResetAction(pMnode, pTrans, pAction);
mInfo("trans:%d, %s:%d reset", pTrans->id, mndTransStr(pAction->stage), pAction->id);
}
}
}
}
}
return TSDB_CODE_ACTION_IN_PROGRESS;
@ -1271,24 +1337,25 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA
static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans, bool topHalf) {
int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->redoActions, topHalf);
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("failed to execute redoActions since:%s, code:0x%x", terrstr(), terrno);
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS && code != TSDB_CODE_MND_TRANS_CTX_SWITCH) {
mError("trans:%d, failed to execute redoActions since:%s, code:0x%x, topHalf:%d", pTrans->id, terrstr(), terrno,
topHalf);
}
return code;
}
static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans, bool topHalf) {
int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->undoActions, topHalf);
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("failed to execute undoActions since %s", terrstr());
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS && code != TSDB_CODE_MND_TRANS_CTX_SWITCH) {
mError("trans:%d, failed to execute undoActions since %s. topHalf:%d", pTrans->id, terrstr(), topHalf);
}
return code;
}
static int32_t mndTransExecuteCommitActions(SMnode *pMnode, STrans *pTrans, bool topHalf) {
int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->commitActions, topHalf);
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("failed to execute commitActions since %s", terrstr());
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS && code != TSDB_CODE_MND_TRANS_CTX_SWITCH) {
mError("trans:%d, failed to execute commitActions since %s. topHalf:%d", pTrans->id, terrstr(), topHalf);
}
return code;
}

169
source/dnode/mnode/impl/src/mndVgroup.c
View File

@ -16,6 +16,7 @@
#define _DEFAULT_SOURCE
#include "mndVgroup.h"
#include "audit.h"
#include "mndArbGroup.h"
#include "mndDb.h"
#include "mndDnode.h"
#include "mndMnode.h"
@ -354,7 +355,7 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg
mInfo(
"vgId:%d, build create vnode req, replica:%d selfIndex:%d learnerReplica:%d learnerSelfIndex:%d strict:%d "
"changeVersion:%d",
createReq.vgId, createReq.replica, createReq.selfIndex, createReq.learnerReplica, createReq.learnerReplica,
createReq.vgId, createReq.replica, createReq.selfIndex, createReq.learnerReplica, createReq.learnerSelfIndex,
createReq.strict, createReq.changeVersion);
for (int32_t i = 0; i < createReq.replica; ++i) {
mInfo("vgId:%d, replica:%d ep:%s:%u", createReq.vgId, i, createReq.replicas[i].fqdn, createReq.replicas[i].port);
@ -743,10 +744,6 @@ void mndSortVnodeGid(SVgObj *pVgroup) {
}
static int32_t mndGetAvailableDnode(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup, SArray *pArray) {
SSdb *pSdb = pMnode->pSdb;
int32_t allocedVnodes = 0;
void *pIter = NULL;
mDebug("start to sort %d dnodes", (int32_t)taosArrayGetSize(pArray));
taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes);
for (int32_t i = 0; i < (int32_t)taosArrayGetSize(pArray); ++i) {
@ -888,7 +885,7 @@ SEpSet mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup) {
SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId);
if (pDnode == NULL) continue;
if (pVgid->syncState == TAOS_SYNC_STATE_LEADER) {
if (pVgid->syncState == TAOS_SYNC_STATE_LEADER || pVgid->syncState == TAOS_SYNC_STATE_ASSIGNED_LEADER) {
epset.inUse = epset.numOfEps;
}
@ -900,6 +897,29 @@ SEpSet mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup) {
return epset;
}
SEpSet mndGetVgroupEpsetById(SMnode *pMnode, int32_t vgId) {
SEpSet epset = {0};
SVgObj * pVgroup = mndAcquireVgroup(pMnode, vgId);
if (!pVgroup) return epset;
for (int32_t v = 0; v < pVgroup->replica; ++v) {
const SVnodeGid *pVgid = &pVgroup->vnodeGid[v];
SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId);
if (pDnode == NULL) continue;
if (pVgid->syncState == TAOS_SYNC_STATE_LEADER || pVgid->syncState == TAOS_SYNC_STATE_ASSIGNED_LEADER) {
epset.inUse = epset.numOfEps;
}
addEpIntoEpSet(&epset, pDnode->fqdn, pDnode->port);
mndReleaseDnode(pMnode, pDnode);
}
mndReleaseVgroup(pMnode, pVgroup);
return epset;
}
static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb;
@ -963,7 +983,8 @@ static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p
strcpy(role, "dropping");
} else if (online) {
char *star = "";
if (pVgroup->vnodeGid[i].syncState == TAOS_SYNC_STATE_LEADER) {
if (pVgroup->vnodeGid[i].syncState == TAOS_SYNC_STATE_LEADER ||
pVgroup->vnodeGid[i].syncState == TAOS_SYNC_STATE_ASSIGNED_LEADER) {
if (!pVgroup->vnodeGid[i].syncRestore && !pVgroup->vnodeGid[i].syncCanRead) {
star = "**";
} else if (!pVgroup->vnodeGid[i].syncRestore && pVgroup->vnodeGid[i].syncCanRead) {
@ -2293,7 +2314,7 @@ int32_t mndAddVgroupBalanceToTrans(SMnode *pMnode, SVgObj *pVgroup, STrans *pTra
return -1;
}
int32_t dnodeId = pVgroup->vnodeGid[0].dnodeId;
int32_t dnodeId = 0;
for (int i = 0; i < replica; i++) {
if (pVgroup->vnodeGid[i].syncState == TAOS_SYNC_STATE_LEADER) {
@ -2326,24 +2347,7 @@ int32_t mndAddVgroupBalanceToTrans(SMnode *pMnode, SVgObj *pVgroup, STrans *pTra
return -1;
}
if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, pVgroup) != 0) {
mError("trans:%d, vgid:%d failed to be balanced to dnode:%d", pTrans->id, vgid, dnodeId);
return -1;
}
mndReleaseDb(pMnode, pDb);
SSdbRaw *pRaw = mndVgroupActionEncode(pVgroup);
if (pRaw == NULL) {
mError("trans:%d, vgid:%d failed to encode action to dnode:%d", pTrans->id, vgid, dnodeId);
return -1;
}
if (mndTransAppendCommitlog(pTrans, pRaw) != 0) {
sdbFreeRaw(pRaw);
mError("trans:%d, vgid:%d failed to append commit log dnode:%d", pTrans->id, vgid, dnodeId);
return -1;
}
(void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
} else {
mInfo("trans:%d, vgid:%d cant be balanced to dnode:%d, exist:%d, online:%d", pTrans->id, vgid, dnodeId, exist,
online);
@ -2394,81 +2398,102 @@ static int32_t mndCheckDnodeMemory(SMnode *pMnode, SDbObj *pOldDb, SDbObj *pNewD
}
int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb, SVgObj *pVgroup,
SArray *pArray) {
SVgObj newVgroup = {0};
memcpy(&newVgroup, pVgroup, sizeof(SVgObj));
SArray *pArray, SVgObj *pNewVgroup) {
memcpy(pNewVgroup, pVgroup, sizeof(SVgObj));
if (pVgroup->replica <= 0 || pVgroup->replica == pNewDb->cfg.replications) {
if (mndAddAlterVnodeConfigAction(pMnode, pTrans, pNewDb, pVgroup) != 0) return -1;
if (mndCheckDnodeMemory(pMnode, pOldDb, pNewDb, &newVgroup, pVgroup, pArray) != 0) return -1;
if (mndCheckDnodeMemory(pMnode, pOldDb, pNewDb, pNewVgroup, pVgroup, pArray) != 0) return -1;
return 0;
}
mndTransSetSerial(pTrans);
if (newVgroup.replica == 1 && pNewDb->cfg.replications == 3) {
if (pNewVgroup->replica == 1 && pNewDb->cfg.replications == 3) {
mInfo("db:%s, vgId:%d, will add 2 vnodes, vn:0 dnode:%d", pVgroup->dbName, pVgroup->vgId,
pVgroup->vnodeGid[0].dnodeId);
// add second
if (mndAddVnodeToVgroup(pMnode, pTrans, &newVgroup, pArray) != 0) return -1;
if (mndAddVnodeToVgroup(pMnode, pTrans, pNewVgroup, pArray) != 0) return -1;
// learner stage
newVgroup.vnodeGid[0].nodeRole = TAOS_SYNC_ROLE_VOTER;
newVgroup.vnodeGid[1].nodeRole = TAOS_SYNC_ROLE_LEARNER;
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[0].dnodeId) != 0)
pNewVgroup->vnodeGid[0].nodeRole = TAOS_SYNC_ROLE_VOTER;
pNewVgroup->vnodeGid[1].nodeRole = TAOS_SYNC_ROLE_LEARNER;
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, pNewVgroup, pNewVgroup->vnodeGid[0].dnodeId) != 0)
return -1;
if (mndAddCreateVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, &newVgroup.vnodeGid[1]) != 0) return -1;
if (mndAddCreateVnodeAction(pMnode, pTrans, pNewDb, pNewVgroup, &pNewVgroup->vnodeGid[1]) != 0) return -1;
// follower stage
newVgroup.vnodeGid[1].nodeRole = TAOS_SYNC_ROLE_VOTER;
if (mndAddAlterVnodeTypeAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[1].dnodeId) != 0) return -1;
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[0].dnodeId) != 0)
pNewVgroup->vnodeGid[1].nodeRole = TAOS_SYNC_ROLE_VOTER;
if (mndAddAlterVnodeTypeAction(pMnode, pTrans, pNewDb, pNewVgroup, pNewVgroup->vnodeGid[1].dnodeId) != 0) return -1;
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, pNewVgroup, pNewVgroup->vnodeGid[0].dnodeId) != 0)
return -1;
if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, &newVgroup) != 0) return -1;
if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, pNewVgroup) != 0) return -1;
// add third
if (mndAddVnodeToVgroup(pMnode, pTrans, &newVgroup, pArray) != 0) return -1;
if (mndAddVnodeToVgroup(pMnode, pTrans, pNewVgroup, pArray) != 0) return -1;
newVgroup.vnodeGid[0].nodeRole = TAOS_SYNC_ROLE_VOTER;
newVgroup.vnodeGid[1].nodeRole = TAOS_SYNC_ROLE_VOTER;
newVgroup.vnodeGid[2].nodeRole = TAOS_SYNC_ROLE_VOTER;
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[0].dnodeId) != 0)
pNewVgroup->vnodeGid[0].nodeRole = TAOS_SYNC_ROLE_VOTER;
pNewVgroup->vnodeGid[1].nodeRole = TAOS_SYNC_ROLE_VOTER;
pNewVgroup->vnodeGid[2].nodeRole = TAOS_SYNC_ROLE_VOTER;
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, pNewVgroup, pNewVgroup->vnodeGid[0].dnodeId) != 0)
return -1;
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[1].dnodeId) != 0)
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, pNewVgroup, pNewVgroup->vnodeGid[1].dnodeId) != 0)
return -1;
if (mndAddCreateVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, &newVgroup.vnodeGid[2]) != 0) return -1;
if (mndAddCreateVnodeAction(pMnode, pTrans, pNewDb, pNewVgroup, &pNewVgroup->vnodeGid[2]) != 0) return -1;
if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, &newVgroup) != 0) return -1;
} else if (newVgroup.replica == 3 && pNewDb->cfg.replications == 1) {
if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, pNewVgroup) != 0) return -1;
} else if (pNewVgroup->replica == 3 && pNewDb->cfg.replications == 1) {
mInfo("db:%s, vgId:%d, will remove 2 vnodes, vn:0 dnode:%d vn:1 dnode:%d vn:2 dnode:%d", pVgroup->dbName,
pVgroup->vgId, pVgroup->vnodeGid[0].dnodeId, pVgroup->vnodeGid[1].dnodeId, pVgroup->vnodeGid[2].dnodeId);
SVnodeGid del1 = {0};
SVnodeGid del2 = {0};
if (mndRemoveVnodeFromVgroup(pMnode, pTrans, &newVgroup, pArray, &del1) != 0) return -1;
if (mndAddDropVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, &del1, true) != 0) return -1;
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[0].dnodeId) != 0)
if (mndRemoveVnodeFromVgroup(pMnode, pTrans, pNewVgroup, pArray, &del1) != 0) return -1;
if (mndAddDropVnodeAction(pMnode, pTrans, pNewDb, pNewVgroup, &del1, true) != 0) return -1;
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, pNewVgroup, pNewVgroup->vnodeGid[0].dnodeId) != 0)
return -1;
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[1].dnodeId) != 0)
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, pNewVgroup, pNewVgroup->vnodeGid[1].dnodeId) != 0)
return -1;
if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, &newVgroup) != 0) return -1;
if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, pNewVgroup) != 0) return -1;
if (mndRemoveVnodeFromVgroup(pMnode, pTrans, &newVgroup, pArray, &del2) != 0) return -1;
if (mndAddDropVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, &del2, true) != 0) return -1;
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[0].dnodeId) != 0)
if (mndRemoveVnodeFromVgroup(pMnode, pTrans, pNewVgroup, pArray, &del2) != 0) return -1;
if (mndAddDropVnodeAction(pMnode, pTrans, pNewDb, pNewVgroup, &del2, true) != 0) return -1;
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, pNewVgroup, pNewVgroup->vnodeGid[0].dnodeId) != 0)
return -1;
if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, &newVgroup) != 0) return -1;
if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, pNewVgroup) != 0) return -1;
} else if (pNewVgroup->replica == 1 && pNewDb->cfg.replications == 2) {
mInfo("db:%s, vgId:%d, will add 1 vnode, vn:0 dnode:%d", pVgroup->dbName, pVgroup->vgId,
pVgroup->vnodeGid[0].dnodeId);
// add second
if (mndAddVnodeToVgroup(pMnode, pTrans, pNewVgroup, pArray) != 0) return -1;
// learner stage
pNewVgroup->vnodeGid[0].nodeRole = TAOS_SYNC_ROLE_VOTER;
pNewVgroup->vnodeGid[1].nodeRole = TAOS_SYNC_ROLE_LEARNER;
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, pNewVgroup, pNewVgroup->vnodeGid[0].dnodeId) != 0)
return -1;
if (mndAddCreateVnodeAction(pMnode, pTrans, pNewDb, pNewVgroup, &pNewVgroup->vnodeGid[1]) != 0) return -1;
// follower stage
pNewVgroup->vnodeGid[1].nodeRole = TAOS_SYNC_ROLE_VOTER;
if (mndAddAlterVnodeTypeAction(pMnode, pTrans, pNewDb, pNewVgroup, pNewVgroup->vnodeGid[1].dnodeId) != 0) return -1;
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, pNewVgroup, pNewVgroup->vnodeGid[0].dnodeId) != 0)
return -1;
if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, pNewVgroup) != 0) return -1;
} else {
return -1;
}
mndSortVnodeGid(&newVgroup);
mndSortVnodeGid(pNewVgroup);
{
SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup);
SSdbRaw *pVgRaw = mndVgroupActionEncode(pNewVgroup);
if (pVgRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) {
sdbFreeRaw(pVgRaw);
@ -2627,7 +2652,7 @@ int32_t mndBuildRestoreAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj
}
}
if (mndAddCreateVnodeAction(pMnode, pTrans, db, &newVgroup, &newVgroup.vnodeGid[selected]) != 0) return -1;
} else if (newVgroup.replica == 3) {
} else if (newVgroup.replica == 2 || newVgroup.replica == 3) {
for (int i = 0; i < newVgroup.replica; i++) {
if (newVgroup.vnodeGid[i].dnodeId == pDnode->id) {
newVgroup.vnodeGid[i].nodeRole = TAOS_SYNC_ROLE_LEARNER;
@ -2721,6 +2746,12 @@ int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgro
}
#endif
if (pDb->cfg.withArbitrator) {
code = TSDB_CODE_OPS_NOT_SUPPORT;
mError("vgId:%d, db:%s, with arbitrator, split vgroup not allowed", pVgroup->vgId, pVgroup->dbName);
goto _OVER;
}
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "split-vgroup");
if (pTrans == NULL) goto _OVER;
mndTransSetSerial(pTrans);
@ -2818,13 +2849,15 @@ int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgro
// adjust vgroup replica
if (pDb->cfg.replications != newVg1.replica) {
if (mndBuildAlterVgroupAction(pMnode, pTrans, pDb, pDb, &newVg1, pArray) != 0) goto _OVER;
SVgObj tmpGroup = {0};
if (mndBuildAlterVgroupAction(pMnode, pTrans, pDb, pDb, &newVg1, pArray, &tmpGroup) != 0) goto _OVER;
} else {
if (mndAddVgStatusAction(pTrans, &newVg1, SDB_STATUS_READY, TRN_STAGE_COMMIT_ACTION) < 0) goto _OVER;
}
if (pDb->cfg.replications != newVg2.replica) {
if (mndBuildAlterVgroupAction(pMnode, pTrans, pDb, pDb, &newVg2, pArray) != 0) goto _OVER;
SVgObj tmpGroup = {0};
if (mndBuildAlterVgroupAction(pMnode, pTrans, pDb, pDb, &newVg2, pArray, &tmpGroup) != 0) goto _OVER;
} else {
if (mndAddVgStatusAction(pTrans, &newVg2, SDB_STATUS_READY, TRN_STAGE_COMMIT_ACTION) < 0) goto _OVER;
}
@ -2912,10 +2945,22 @@ static int32_t mndBalanceVgroupBetweenDnode(SMnode *pMnode, STrans *pTrans, SDno
}
SDbObj *pDb = mndAcquireDb(pMnode, pVgroup->dbName);
if (pDb == NULL) {
mError("vgId:%d, balance vgroup can't find db obj dbName:%s", pVgroup->vgId, pVgroup->dbName);
goto _OUT;
}
if (pDb->cfg.withArbitrator) {
mInfo("vgId:%d, db:%s, with arbitrator, balance vgroup not allowed", pVgroup->vgId, pVgroup->dbName);
goto _OUT;
}
code = mndSetBalanceVgroupInfoToTrans(pMnode, pTrans, pDb, pVgroup, pSrc, pDst);
if (code == 0) {
code = taosHashPut(pBalancedVgroups, &pVgroup->vgId, sizeof(int32_t), &pVgroup->vgId, sizeof(int32_t));
}
_OUT:
mndReleaseDb(pMnode, pDb);
sdbRelease(pSdb, pVgroup);
sdbCancelFetch(pSdb, pIter);

1
source/dnode/mnode/impl/test/CMakeLists.txt
View File

@ -1,6 +1,7 @@
enable_testing()
add_subdirectory(acct)
add_subdirectory(arbgroup)
#add_subdirectory(db)
#add_subdirectory(dnode)
add_subdirectory(func)

13
source/dnode/mnode/impl/test/arbgroup/CMakeLists.txt Normal file
View File

@ -0,0 +1,13 @@
SET(CMAKE_CXX_STANDARD 11)
aux_source_directory(. MNODE_ARBGROUP_TEST_SRC)
add_executable(arbgroupTest ${MNODE_ARBGROUP_TEST_SRC})
target_link_libraries(
arbgroupTest
PRIVATE dnode nodes planner gtest qcom
)
add_test(
NAME arbgroupTest
COMMAND arbgroupTest
)

286
source/dnode/mnode/impl/test/arbgroup/arbgroup.cpp Normal file
View File

@ -0,0 +1,286 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
// #include <iostream>
#include <string>
#include <gtest/gtest.h>
// #include "nodes.h"
// #include "planner.h"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wwrite-strings"
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
#pragma GCC diagnostic ignored "-Wsign-compare"
// #include <libs/transport/trpc.h>
#include "../../inc/mndArbGroup.h"
namespace {
void generateArbToken(int32_t nodeId, int32_t vgId, char* buf) {
memset(buf, 0, TSDB_ARB_TOKEN_SIZE);
int32_t randVal = taosSafeRand() % 1000;
int64_t currentMs = taosGetTimestampMs();
snprintf(buf, TSDB_ARB_TOKEN_SIZE, "d%d#g%d#%" PRId64 "#%d", nodeId, vgId, currentMs, randVal);
}
} // namespace
class ArbgroupTest : public testing::Test {
protected:
static void SetUpTestSuite() {
std::cout << "setup env for arbgroupTest suite" << std::endl;
}
static void TearDownTestSuite() { std::cout << "tearDown env for arbgroupTest suite" << std::endl; }
virtual void SetUp() override {}
virtual void TearDown() override {}
};
int main(int argc, char** argv) {
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
TEST_F(ArbgroupTest, 01_encode_decode_sdb) {
SArbGroup group = {0};
group.vgId = 5;
group.dbUid = 1234;
group.members[0].info.dnodeId = 1;
generateArbToken(1, 5, group.members[0].state.token);
group.members[1].info.dnodeId = 2;
generateArbToken(2, 5, group.members[1].state.token);
group.isSync = 1;
group.assignedLeader.dnodeId = 1;
generateArbToken(1, 5, group.assignedLeader.token);
group.version = 2234;
// --------------------------------------------------------------------------------
SSdbRaw* pRaw = mndArbGroupActionEncode(&group);
ASSERT_NE(pRaw, nullptr);
SSdbRow* pRow = mndArbGroupActionDecode(pRaw);
ASSERT_NE(pRow, nullptr);
SArbGroup* pNewGroup = (SArbGroup*)sdbGetRowObj(pRow);
EXPECT_EQ(group.vgId, pNewGroup->vgId);
EXPECT_EQ(group.dbUid, pNewGroup->dbUid);
EXPECT_EQ(group.members[0].info.dnodeId, pNewGroup->members[0].info.dnodeId);
EXPECT_EQ(group.members[1].info.dnodeId, pNewGroup->members[1].info.dnodeId);
EXPECT_EQ(group.isSync, pNewGroup->isSync);
EXPECT_EQ(group.assignedLeader.dnodeId, pNewGroup->assignedLeader.dnodeId);
EXPECT_EQ(std::string(group.members[0].state.token), std::string(pNewGroup->members[0].state.token));
EXPECT_EQ(std::string(group.members[1].state.token), std::string(pNewGroup->members[1].state.token));
EXPECT_EQ(std::string(group.assignedLeader.token), std::string(pNewGroup->assignedLeader.token));
EXPECT_EQ(group.version, pNewGroup->version);
taosMemoryFree(pRow);
taosMemoryFree(pRaw);
}
TEST_F(ArbgroupTest, 02_process_heart_beat_rsp) {
const int32_t dnodeId = 1;
const int32_t vgId = 5;
SArbGroup group = {0};
group.vgId = vgId;
group.dbUid = 1234;
group.members[0].info.dnodeId = dnodeId;
generateArbToken(dnodeId, vgId, group.members[0].state.token);
group.members[0].state.lastHbMs = 1000;
group.members[0].state.responsedHbSeq = 100;
group.members[0].state.nextHbSeq = 102;
group.members[1].info.dnodeId = 2;
generateArbToken(2, vgId, group.members[1].state.token);
group.isSync = 1;
group.assignedLeader.dnodeId = dnodeId;
strncpy(group.assignedLeader.token, group.members[0].state.token, TSDB_ARB_TOKEN_SIZE);
taosThreadMutexInit(&group.mutex, NULL);
// --------------------------------------------------------------------------------
{ // expired hb => skip
SVArbHbRspMember rspMember = {0};
rspMember.vgId = vgId;
rspMember.hbSeq = group.members[0].state.responsedHbSeq - 1;
strncpy(rspMember.memberToken, group.members[0].state.token, TSDB_ARB_TOKEN_SIZE);
int32_t nowMs = group.members[0].state.lastHbMs + 10;
SArbGroup newGroup = {0};
bool updateToken = mndUpdateArbGroupByHeartBeat(&group, &rspMember, nowMs, dnodeId, &newGroup);
EXPECT_FALSE(updateToken);
EXPECT_NE(group.members[0].state.responsedHbSeq, rspMember.hbSeq);
EXPECT_NE(group.members[0].state.lastHbMs, nowMs);
}
{ // old token
SVArbHbRspMember rspMember = {0};
rspMember.vgId = vgId;
rspMember.hbSeq = group.members[0].state.responsedHbSeq + 1;
strncpy(rspMember.memberToken, group.members[0].state.token, TSDB_ARB_TOKEN_SIZE);
int32_t nowMs = group.members[0].state.lastHbMs + 10;
SArbGroup newGroup = {0};
bool updateToken = mndUpdateArbGroupByHeartBeat(&group, &rspMember, nowMs, dnodeId, &newGroup);
EXPECT_FALSE(updateToken);
EXPECT_EQ(group.members[0].state.responsedHbSeq, rspMember.hbSeq);
EXPECT_EQ(group.members[0].state.lastHbMs, nowMs);
}
{ // new token
SVArbHbRspMember rspMember = {0};
rspMember.vgId = vgId;
rspMember.hbSeq = group.members[0].state.responsedHbSeq + 1;
generateArbToken(dnodeId, vgId, rspMember.memberToken);
int32_t nowMs = group.members[0].state.lastHbMs + 10;
SArbGroup newGroup = {0};
bool updateToken = mndUpdateArbGroupByHeartBeat(&group, &rspMember, nowMs, dnodeId, &newGroup);
EXPECT_TRUE(updateToken);
EXPECT_EQ(group.members[0].state.responsedHbSeq, rspMember.hbSeq);
EXPECT_EQ(group.members[0].state.lastHbMs, nowMs);
EXPECT_EQ(std::string(newGroup.members[0].state.token), std::string(rspMember.memberToken));
EXPECT_FALSE(newGroup.isSync);
EXPECT_EQ(newGroup.assignedLeader.dnodeId, 0);
EXPECT_EQ(std::string(newGroup.assignedLeader.token).size(), 0);
}
taosThreadMutexDestroy(&group.mutex);
}
TEST_F(ArbgroupTest, 03_process_check_sync_rsp) {
const int32_t dnodeId = 1;
const int32_t vgId = 5;
SArbGroup group = {0};
group.vgId = vgId;
group.dbUid = 1234;
group.members[0].info.dnodeId = dnodeId;
generateArbToken(dnodeId, vgId, group.members[0].state.token);
group.members[0].state.lastHbMs = 1000;
group.members[0].state.responsedHbSeq = 100;
group.members[0].state.nextHbSeq = 102;
group.members[1].info.dnodeId = 2;
generateArbToken(2, vgId, group.members[1].state.token);
group.isSync = 0;
taosThreadMutexInit(&group.mutex, NULL);
// --------------------------------------------------------------------------------
{ // token mismatch => skip
char member0Token[TSDB_ARB_TOKEN_SIZE] = {0};
strncpy(member0Token, group.members[0].state.token, TSDB_ARB_TOKEN_SIZE);
char member1Token[TSDB_ARB_TOKEN_SIZE] = {0};
generateArbToken(2, 5, member1Token);
bool newIsSync = false;
SArbGroup newGroup = {0};
bool updateIsSync = mndUpdateArbGroupByCheckSync(&group, vgId, member0Token, member1Token, newIsSync, &newGroup);
EXPECT_FALSE(updateIsSync);
}
{ // newIsSync
char member0Token[TSDB_ARB_TOKEN_SIZE] = {0};
strncpy(member0Token, group.members[0].state.token, TSDB_ARB_TOKEN_SIZE);
char member1Token[TSDB_ARB_TOKEN_SIZE] = {0};
strncpy(member1Token, group.members[1].state.token, TSDB_ARB_TOKEN_SIZE);
bool newIsSync = true;
SArbGroup newGroup = {0};
bool updateIsSync = mndUpdateArbGroupByCheckSync(&group, vgId, member0Token, member1Token, newIsSync, &newGroup);
EXPECT_TRUE(updateIsSync);
EXPECT_TRUE(newGroup.isSync);
}
taosThreadMutexDestroy(&group.mutex);
}
TEST_F(ArbgroupTest, 04_process_set_assigned_leader){
const int32_t dnodeId = 1;
const int32_t vgId = 5;
SArbGroup group = {0};
group.vgId = vgId;
group.dbUid = 1234;
group.members[0].info.dnodeId = dnodeId;
generateArbToken(dnodeId, vgId, group.members[0].state.token);
group.members[0].state.lastHbMs = 1000;
group.members[0].state.responsedHbSeq = 100;
group.members[0].state.nextHbSeq = 102;
group.members[1].info.dnodeId = 2;
generateArbToken(2, vgId, group.members[1].state.token);
group.isSync = 1;
group.assignedLeader.dnodeId = dnodeId;
strncpy(group.assignedLeader.token, group.members[0].state.token, TSDB_ARB_TOKEN_SIZE);
taosThreadMutexInit(&group.mutex, NULL);
// --------------------------------------------------------------------------------
{ // token mismatch => skip
char memberToken[TSDB_ARB_TOKEN_SIZE] = {0};
generateArbToken(dnodeId, vgId, memberToken);
int32_t errcode = TSDB_CODE_SUCCESS;
SArbGroup newGroup = {0};
bool updateAssigned = mndUpdateArbGroupBySetAssignedLeader(&group, vgId, memberToken, errcode, &newGroup);
EXPECT_FALSE(updateAssigned);
}
{ // errcode != TSDB_CODE_SUCCESS
char memberToken[TSDB_ARB_TOKEN_SIZE] = {0};
strncpy(memberToken, group.assignedLeader.token, TSDB_ARB_TOKEN_SIZE);
int32_t errcode = TSDB_CODE_MND_ARB_TOKEN_MISMATCH;
SArbGroup newGroup = {0};
bool updateAssigned = mndUpdateArbGroupBySetAssignedLeader(&group, vgId, memberToken, errcode, &newGroup);
EXPECT_FALSE(updateAssigned);
}
{ // errcode == TSDB_CODE_SUCCESS
char memberToken[TSDB_ARB_TOKEN_SIZE] = {0};
strncpy(memberToken, group.assignedLeader.token, TSDB_ARB_TOKEN_SIZE);
int32_t errcode = TSDB_CODE_SUCCESS;
SArbGroup newGroup = {0};
bool updateAssigned = mndUpdateArbGroupBySetAssignedLeader(&group, vgId, memberToken, errcode, &newGroup);
EXPECT_TRUE(updateAssigned);
EXPECT_FALSE(newGroup.isSync);
}
taosThreadMutexDestroy(&group.mutex);
}
#pragma GCC diagnostic pop

3
source/dnode/mnode/sdb/inc/sdb.h
View File

@ -153,7 +153,8 @@ typedef enum {
SDB_COMPACT = 24,
SDB_COMPACT_DETAIL = 25,
SDB_GRANT = 26, // grant log
SDB_MAX = 27
SDB_ARBGROUP = 27,
SDB_MAX = 28
} ESdbType;
typedef struct SSdbRaw {

4
source/dnode/mnode/sdb/src/sdbHash.c
View File

@ -70,7 +70,9 @@ const char *sdbTableName(ESdbType type) {
return "compact";
case SDB_COMPACT_DETAIL:
return "compact_detail";
default:
case SDB_ARBGROUP:
return "arb_group";
default:
return "undefine";
}
}

3
source/dnode/vnode/inc/vnode.h
View File

@ -76,6 +76,9 @@ int32_t vnodeGetTableList(void *pVnode, int8_t type, SArray *pList);
int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list);
int32_t vnodeIsCatchUp(SVnode *pVnode);
ESyncRole vnodeGetRole(SVnode *pVnode);
int32_t vnodeGetArbToken(SVnode *pVnode, char *outToken);
int32_t vnodeUpdateArbTerm(SVnode *pVnode, int64_t arbTerm);
int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list);
int32_t vnodeGetCtbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg);

4
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -106,7 +106,7 @@ typedef struct SQueryNode SQueryNode;
#define VND_INFO_FNAME "vnode.json"
#define VND_INFO_FNAME_TMP "vnode_tmp.json"
#define VNODE_METRIC_SQL_COUNT "taos_sql_req:count"
#define VNODE_METRIC_SQL_COUNT "taosd_sql_req:count"
#define VNODE_METRIC_TAG_NAME_SQL_TYPE "sql_type"
#define VNODE_METRIC_TAG_NAME_CLUSTER_ID "cluster_id"
@ -136,7 +136,7 @@ typedef SVCreateTSmaReq SSmaCfg;
SMTbCursor* metaOpenTbCursor(void* pVnode);
void metaCloseTbCursor(SMTbCursor* pTbCur);
void metaPauseTbCursor(SMTbCursor* pTbCur);
void metaResumeTbCursor(SMTbCursor* pTbCur, int8_t first);
void metaResumeTbCursor(SMTbCursor* pTbCur, int8_t first, int8_t move);
int32_t metaTbCursorNext(SMTbCursor* pTbCur, ETableType jumpTableType);
int32_t metaTbCursorPrev(SMTbCursor* pTbCur, ETableType jumpTableType);

Some files were not shown because too many files have changed in this diff Show More