Merge branch '3.0' into enh/TS-3812-3.0
This commit is contained in:
kailixu
commit
7445b9ff50
|
|
@ -314,9 +314,9 @@ def pre_test_build_win() {
|
|||
cd %WIN_CONNECTOR_ROOT%
|
||||
python.exe -m pip install --upgrade pip
|
||||
python -m pip uninstall taospy -y
|
||||
python -m pip install taospy==2.7.10
|
||||
python -m pip install taospy==2.7.12
|
||||
python -m pip uninstall taos-ws-py -y
|
||||
python -m pip install taos-ws-py==0.2.8
|
||||
python -m pip install taos-ws-py==0.2.9
|
||||
xcopy /e/y/i/f %WIN_INTERNAL_ROOT%\\debug\\build\\lib\\taos.dll C:\\Windows\\System32
|
||||
'''
|
||||
return 1
|
||||
|
|
|
|||
|
|
@ -142,8 +142,15 @@ TDengine currently supports timestamp, number, character, Boolean type, and the
|
|||
| BINARY | byte array |
|
||||
| NCHAR | java.lang.String |
|
||||
| JSON | java.lang.String |
|
||||
| VARBINARY | byte[] |
|
||||
| GEOMETRY | byte[] |
|
||||
|
||||
**Note**: Only TAG supports JSON types
|
||||
Due to historical reasons, the BINARY type data in TDengine is not truly binary data and is no longer recommended for use. Please use VARBINARY type instead.
|
||||
GEOMETRY type is binary data in little endian byte order, which complies with the WKB specification. For detailed information, please refer to [Data Type] (/tao-sql/data-type/#Data Types)
|
||||
For WKB specifications, please refer to [Well Known Binary (WKB)]（ https://libgeos.org/specifications/wkb/ ）
|
||||
For Java connector, the jts library can be used to easily create GEOMETRY type objects, serialize them, and write them to TDengine. Here is an example [Geometry example]（https://github.com/taosdata/TDengine/blob/3.0/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/GeometryDemo.java）
|
||||
|
||||
|
||||
## Installation Steps
|
||||
|
||||
|
|
@ -354,7 +361,7 @@ The configuration parameters in properties are as follows.
|
|||
- TSDBDriver.PROPERTY_KEY_CONFIG_DIR: only works when using JDBC native connection. Client configuration file directory path, default value `/etc/taos` on Linux OS, default value `C:/TDengine/cfg` on Windows OS, default value `/etc/taos` on macOS.
|
||||
- TSDBDriver.PROPERTY_KEY_CHARSET: In the character set used by the client, the default value is the system character set.
|
||||
- TSDBDriver.PROPERTY_KEY_LOCALE: this only takes effect when using JDBC native connection. Client language environment, the default value is system current locale.
|
||||
- TSDBDriver.PROPERTY_KEY_TIME_ZONE: only takes effect when using JDBC native connection. In the time zone used by the client, the default value is the system's current time zone.
|
||||
- TSDBDriver.PROPERTY_KEY_TIME_ZONE: only takes effect when using JDBC native connection. In the time zone used by the client, the default value is the system's current time zone. Due to historical reasons, we only support some specifications of the POSIX standard, such as UTC-8 (representing timezone Shanghai in China), GMT-7, Europe/Paris.
|
||||
- TSDBDriver.HTTP_CONNECT_TIMEOUT: REST connection timeout in milliseconds, the default value is 60000 ms. It only takes effect when using JDBC REST connection.
|
||||
- TSDBDriver.HTTP_SOCKET_TIMEOUT: socket timeout in milliseconds, the default value is 60000 ms. It only takes effect when using JDBC REST connection and batchfetch is false.
|
||||
- TSDBDriver.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT: message transmission timeout in milliseconds, the default value is 60000 ms. It only takes effect when using JDBC REST connection and batchfetch is true.
|
||||
|
|
@ -456,13 +463,15 @@ public class ParameterBindingDemo {
|
|||
|
||||
private static final String host = "127.0.0.1";
|
||||
private static final Random random = new Random(System.currentTimeMillis());
|
||||
private static final int BINARY_COLUMN_SIZE = 20;
|
||||
private static final int BINARY_COLUMN_SIZE = 50;
|
||||
private static final String[] schemaList = {
|
||||
"create table stable1(ts timestamp, f1 tinyint, f2 smallint, f3 int, f4 bigint) tags(t1 tinyint, t2 smallint, t3 int, t4 bigint)",
|
||||
"create table stable2(ts timestamp, f1 float, f2 double) tags(t1 float, t2 double)",
|
||||
"create table stable3(ts timestamp, f1 bool) tags(t1 bool)",
|
||||
"create table stable4(ts timestamp, f1 binary(" + BINARY_COLUMN_SIZE + ")) tags(t1 binary(" + BINARY_COLUMN_SIZE + "))",
|
||||
"create table stable5(ts timestamp, f1 nchar(" + BINARY_COLUMN_SIZE + ")) tags(t1 nchar(" + BINARY_COLUMN_SIZE + "))"
|
||||
"create table stable5(ts timestamp, f1 nchar(" + BINARY_COLUMN_SIZE + ")) tags(t1 nchar(" + BINARY_COLUMN_SIZE + "))",
|
||||
"create table stable6(ts timestamp, f1 varbinary(" + BINARY_COLUMN_SIZE + ")) tags(t1 varbinary(" + BINARY_COLUMN_SIZE + "))",
|
||||
"create table stable7(ts timestamp, f1 geometry(" + BINARY_COLUMN_SIZE + ")) tags(t1 geometry(" + BINARY_COLUMN_SIZE + "))",
|
||||
};
|
||||
private static final int numOfSubTable = 10, numOfRow = 10;
|
||||
|
||||
|
|
@ -474,21 +483,20 @@ public class ParameterBindingDemo {
|
|||
init(conn);
|
||||
|
||||
bindInteger(conn);
|
||||
|
||||
bindFloat(conn);
|
||||
|
||||
bindBoolean(conn);
|
||||
|
||||
bindBytes(conn);
|
||||
|
||||
bindString(conn);
|
||||
bindVarbinary(conn);
|
||||
bindGeometry(conn);
|
||||
|
||||
clean(conn);
|
||||
conn.close();
|
||||
}
|
||||
|
||||
private static void init(Connection conn) throws SQLException {
|
||||
clean(conn);
|
||||
try (Statement stmt = conn.createStatement()) {
|
||||
stmt.execute("drop database if exists test_parabind");
|
||||
stmt.execute("create database if not exists test_parabind");
|
||||
stmt.execute("use test_parabind");
|
||||
for (int i = 0; i < schemaList.length; i++) {
|
||||
|
|
@ -496,6 +504,11 @@ public class ParameterBindingDemo {
|
|||
}
|
||||
}
|
||||
}
|
||||
private static void clean(Connection conn) throws SQLException {
|
||||
try (Statement stmt = conn.createStatement()) {
|
||||
stmt.execute("drop database if exists test_parabind");
|
||||
}
|
||||
}
|
||||
|
||||
private static void bindInteger(Connection conn) throws SQLException {
|
||||
String sql = "insert into ? using stable1 tags(?,?,?,?) values(?,?,?,?,?)";
|
||||
|
|
@ -674,10 +687,84 @@ public class ParameterBindingDemo {
|
|||
pstmt.columnDataExecuteBatch();
|
||||
}
|
||||
}
|
||||
|
||||
private static void bindVarbinary(Connection conn) throws SQLException {
|
||||
String sql = "insert into ? using stable6 tags(?) values(?,?)";
|
||||
|
||||
try (TSDBPreparedStatement pstmt = conn.prepareStatement(sql).unwrap(TSDBPreparedStatement.class)) {
|
||||
|
||||
for (int i = 1; i <= numOfSubTable; i++) {
|
||||
// set table name
|
||||
pstmt.setTableName("t6_" + i);
|
||||
// set tags
|
||||
byte[] bTag = new byte[]{0,2,3,4,5};
|
||||
bTag[0] = (byte) i;
|
||||
pstmt.setTagVarbinary(0, bTag);
|
||||
|
||||
// set columns
|
||||
ArrayList<Long> tsList = new ArrayList<>();
|
||||
long current = System.currentTimeMillis();
|
||||
for (int j = 0; j < numOfRow; j++)
|
||||
tsList.add(current + j);
|
||||
pstmt.setTimestamp(0, tsList);
|
||||
|
||||
ArrayList<byte[]> f1List = new ArrayList<>();
|
||||
for (int j = 0; j < numOfRow; j++) {
|
||||
byte[] v = new byte[]{0,2,3,4,5,6};
|
||||
v[0] = (byte)j;
|
||||
f1List.add(v);
|
||||
}
|
||||
pstmt.setVarbinary(1, f1List, BINARY_COLUMN_SIZE);
|
||||
|
||||
// add column
|
||||
pstmt.columnDataAddBatch();
|
||||
}
|
||||
// execute
|
||||
pstmt.columnDataExecuteBatch();
|
||||
}
|
||||
}
|
||||
|
||||
private static void bindGeometry(Connection conn) throws SQLException {
|
||||
String sql = "insert into ? using stable7 tags(?) values(?,?)";
|
||||
|
||||
try (TSDBPreparedStatement pstmt = conn.prepareStatement(sql).unwrap(TSDBPreparedStatement.class)) {
|
||||
|
||||
byte[] g1 = StringUtils.hexToBytes("0101000000000000000000F03F0000000000000040");
|
||||
byte[] g2 = StringUtils.hexToBytes("0102000020E610000002000000000000000000F03F000000000000004000000000000008400000000000001040");
|
||||
List<byte[]> listGeo = new ArrayList<>();
|
||||
listGeo.add(g1);
|
||||
listGeo.add(g2);
|
||||
|
||||
for (int i = 1; i <= 2; i++) {
|
||||
// set table name
|
||||
pstmt.setTableName("t7_" + i);
|
||||
// set tags
|
||||
pstmt.setTagGeometry(0, listGeo.get(i - 1));
|
||||
|
||||
// set columns
|
||||
ArrayList<Long> tsList = new ArrayList<>();
|
||||
long current = System.currentTimeMillis();
|
||||
for (int j = 0; j < numOfRow; j++)
|
||||
tsList.add(current + j);
|
||||
pstmt.setTimestamp(0, tsList);
|
||||
|
||||
ArrayList<byte[]> f1List = new ArrayList<>();
|
||||
for (int j = 0; j < numOfRow; j++) {
|
||||
f1List.add(listGeo.get(i - 1));
|
||||
}
|
||||
pstmt.setGeometry(1, f1List, BINARY_COLUMN_SIZE);
|
||||
|
||||
// add column
|
||||
pstmt.columnDataAddBatch();
|
||||
}
|
||||
// execute
|
||||
pstmt.columnDataExecuteBatch();
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Note**: both setString and setNString require the user to declare the width of the corresponding column in the size parameter of the table definition
|
||||
**Note**: both String and byte[] require the user to declare the width of the corresponding column in the size parameter of the table definition
|
||||
|
||||
The methods to set VALUES columns:
|
||||
|
||||
|
|
@ -692,6 +779,8 @@ public void setByte(int columnIndex, ArrayList<Byte> list) throws SQLException
|
|||
public void setShort(int columnIndex, ArrayList<Short> list) throws SQLException
|
||||
public void setString(int columnIndex, ArrayList<String> list, int size) throws SQLException
|
||||
public void setNString(int columnIndex, ArrayList<String> list, int size) throws SQLException
|
||||
public void setVarbinary(int columnIndex, ArrayList<byte[]> list, int size) throws SQLException
|
||||
public void setGeometry(int columnIndex, ArrayList<byte[]> list, int size) throws SQLException
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
|
|
@ -880,6 +969,9 @@ public void setTagFloat(int index, float value)
|
|||
public void setTagDouble(int index, double value)
|
||||
public void setTagString(int index, String value)
|
||||
public void setTagNString(int index, String value)
|
||||
public void setTagJson(int index, String value)
|
||||
public void setTagVarbinary(int index, byte[] value)
|
||||
public void setTagGeometry(int index, byte[] value)
|
||||
```
|
||||
|
||||
### Schemaless Writing
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@
|
|||
<dependency>
|
||||
<groupId>com.taosdata.jdbc</groupId>
|
||||
<artifactId>taos-jdbcdriver</artifactId>
|
||||
<version>3.2.4</version>
|
||||
<version>3.2.7-SNAPSHOT</version>
|
||||
</dependency>
|
||||
<!-- ANCHOR_END: dep-->
|
||||
<dependency>
|
||||
|
|
|
|||
|
|
@ -142,8 +142,14 @@ TDengine 目前支持时间戳、数字、字符、布尔类型，与 Java 对
|
|||
| BINARY | byte array |
|
||||
| NCHAR | java.lang.String |
|
||||
| JSON | java.lang.String |
|
||||
| VARBINARY | byte[] |
|
||||
| GEOMETRY | byte[] |
|
||||
|
||||
**注意**：JSON 类型仅在 tag 中支持。
|
||||
由于历史原因，TDengine中的BINARY底层不是真正的二进制数据，已不建议使用。请用VARBINARY类型代替。
|
||||
GEOMETRY类型是little endian字节序的二进制数据，符合WKB规范。详细信息请参考 [数据类型](/taos-sql/data-type/#数据类型)
|
||||
WKB规范请参考[Well-Known Binary (WKB)](https://libgeos.org/specifications/wkb/)
|
||||
对于java连接器，可以使用jts库来方便的创建GEOMETRY类型对象，序列化后写入TDengine，这里有一个样例[Geometry示例](https://github.com/taosdata/TDengine/blob/3.0/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/GeometryDemo.java)
|
||||
|
||||
## 安装步骤
|
||||
|
||||
|
|
@ -357,7 +363,7 @@ properties 中的配置参数如下：
|
|||
- TSDBDriver.PROPERTY_KEY_CONFIG_DIR：仅在使用 JDBC 原生连接时生效。客户端配置文件目录路径，Linux OS 上默认值 `/etc/taos`，Windows OS 上默认值 `C:/TDengine/cfg`。
|
||||
- TSDBDriver.PROPERTY_KEY_CHARSET：客户端使用的字符集，默认值为系统字符集。
|
||||
- TSDBDriver.PROPERTY_KEY_LOCALE：仅在使用 JDBC 原生连接时生效。 客户端语言环境，默认值系统当前 locale。
|
||||
- TSDBDriver.PROPERTY_KEY_TIME_ZONE：仅在使用 JDBC 原生连接时生效。 客户端使用的时区，默认值为系统当前时区。
|
||||
- TSDBDriver.PROPERTY_KEY_TIME_ZONE：仅在使用 JDBC 原生连接时生效。 客户端使用的时区，默认值为系统当前时区。因为历史的原因，我们只支持POSIX标准的部分规范，如UTC-8(代表中国上上海), GMT-8，Asia/Shanghai 这几种形式。
|
||||
- TSDBDriver.HTTP_CONNECT_TIMEOUT: 连接超时时间，单位 ms， 默认值为 60000。仅在 REST 连接时生效。
|
||||
- TSDBDriver.HTTP_SOCKET_TIMEOUT: socket 超时时间，单位 ms，默认值为 60000。仅在 REST 连接且 batchfetch 设置为 false 时生效。
|
||||
- TSDBDriver.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT: 消息超时时间, 单位 ms， 默认值为 60000。 仅在 REST 连接且 batchfetch 设置为 true 时生效。
|
||||
|
|
@ -459,13 +465,15 @@ public class ParameterBindingDemo {
|
|||
|
||||
private static final String host = "127.0.0.1";
|
||||
private static final Random random = new Random(System.currentTimeMillis());
|
||||
private static final int BINARY_COLUMN_SIZE = 30;
|
||||
private static final int BINARY_COLUMN_SIZE = 50;
|
||||
private static final String[] schemaList = {
|
||||
"create table stable1(ts timestamp, f1 tinyint, f2 smallint, f3 int, f4 bigint) tags(t1 tinyint, t2 smallint, t3 int, t4 bigint)",
|
||||
"create table stable2(ts timestamp, f1 float, f2 double) tags(t1 float, t2 double)",
|
||||
"create table stable3(ts timestamp, f1 bool) tags(t1 bool)",
|
||||
"create table stable4(ts timestamp, f1 binary(" + BINARY_COLUMN_SIZE + ")) tags(t1 binary(" + BINARY_COLUMN_SIZE + "))",
|
||||
"create table stable5(ts timestamp, f1 nchar(" + BINARY_COLUMN_SIZE + ")) tags(t1 nchar(" + BINARY_COLUMN_SIZE + "))"
|
||||
"create table stable5(ts timestamp, f1 nchar(" + BINARY_COLUMN_SIZE + ")) tags(t1 nchar(" + BINARY_COLUMN_SIZE + "))",
|
||||
"create table stable6(ts timestamp, f1 varbinary(" + BINARY_COLUMN_SIZE + ")) tags(t1 varbinary(" + BINARY_COLUMN_SIZE + "))",
|
||||
"create table stable7(ts timestamp, f1 geometry(" + BINARY_COLUMN_SIZE + ")) tags(t1 geometry(" + BINARY_COLUMN_SIZE + "))",
|
||||
};
|
||||
private static final int numOfSubTable = 10, numOfRow = 10;
|
||||
|
||||
|
|
@ -477,21 +485,20 @@ public class ParameterBindingDemo {
|
|||
init(conn);
|
||||
|
||||
bindInteger(conn);
|
||||
|
||||
bindFloat(conn);
|
||||
|
||||
bindBoolean(conn);
|
||||
|
||||
bindBytes(conn);
|
||||
|
||||
bindString(conn);
|
||||
bindVarbinary(conn);
|
||||
bindGeometry(conn);
|
||||
|
||||
clean(conn);
|
||||
conn.close();
|
||||
}
|
||||
|
||||
private static void init(Connection conn) throws SQLException {
|
||||
clean(conn);
|
||||
try (Statement stmt = conn.createStatement()) {
|
||||
stmt.execute("drop database if exists test_parabind");
|
||||
stmt.execute("create database if not exists test_parabind");
|
||||
stmt.execute("use test_parabind");
|
||||
for (int i = 0; i < schemaList.length; i++) {
|
||||
|
|
@ -499,6 +506,11 @@ public class ParameterBindingDemo {
|
|||
}
|
||||
}
|
||||
}
|
||||
private static void clean(Connection conn) throws SQLException {
|
||||
try (Statement stmt = conn.createStatement()) {
|
||||
stmt.execute("drop database if exists test_parabind");
|
||||
}
|
||||
}
|
||||
|
||||
private static void bindInteger(Connection conn) throws SQLException {
|
||||
String sql = "insert into ? using stable1 tags(?,?,?,?) values(?,?,?,?,?)";
|
||||
|
|
@ -677,10 +689,84 @@ public class ParameterBindingDemo {
|
|||
pstmt.columnDataExecuteBatch();
|
||||
}
|
||||
}
|
||||
|
||||
private static void bindVarbinary(Connection conn) throws SQLException {
|
||||
String sql = "insert into ? using stable6 tags(?) values(?,?)";
|
||||
|
||||
try (TSDBPreparedStatement pstmt = conn.prepareStatement(sql).unwrap(TSDBPreparedStatement.class)) {
|
||||
|
||||
for (int i = 1; i <= numOfSubTable; i++) {
|
||||
// set table name
|
||||
pstmt.setTableName("t6_" + i);
|
||||
// set tags
|
||||
byte[] bTag = new byte[]{0,2,3,4,5};
|
||||
bTag[0] = (byte) i;
|
||||
pstmt.setTagVarbinary(0, bTag);
|
||||
|
||||
// set columns
|
||||
ArrayList<Long> tsList = new ArrayList<>();
|
||||
long current = System.currentTimeMillis();
|
||||
for (int j = 0; j < numOfRow; j++)
|
||||
tsList.add(current + j);
|
||||
pstmt.setTimestamp(0, tsList);
|
||||
|
||||
ArrayList<byte[]> f1List = new ArrayList<>();
|
||||
for (int j = 0; j < numOfRow; j++) {
|
||||
byte[] v = new byte[]{0,2,3,4,5,6};
|
||||
v[0] = (byte)j;
|
||||
f1List.add(v);
|
||||
}
|
||||
pstmt.setVarbinary(1, f1List, BINARY_COLUMN_SIZE);
|
||||
|
||||
// add column
|
||||
pstmt.columnDataAddBatch();
|
||||
}
|
||||
// execute
|
||||
pstmt.columnDataExecuteBatch();
|
||||
}
|
||||
}
|
||||
|
||||
private static void bindGeometry(Connection conn) throws SQLException {
|
||||
String sql = "insert into ? using stable7 tags(?) values(?,?)";
|
||||
|
||||
try (TSDBPreparedStatement pstmt = conn.prepareStatement(sql).unwrap(TSDBPreparedStatement.class)) {
|
||||
|
||||
byte[] g1 = StringUtils.hexToBytes("0101000000000000000000F03F0000000000000040");
|
||||
byte[] g2 = StringUtils.hexToBytes("0102000020E610000002000000000000000000F03F000000000000004000000000000008400000000000001040");
|
||||
List<byte[]> listGeo = new ArrayList<>();
|
||||
listGeo.add(g1);
|
||||
listGeo.add(g2);
|
||||
|
||||
for (int i = 1; i <= 2; i++) {
|
||||
// set table name
|
||||
pstmt.setTableName("t7_" + i);
|
||||
// set tags
|
||||
pstmt.setTagGeometry(0, listGeo.get(i - 1));
|
||||
|
||||
// set columns
|
||||
ArrayList<Long> tsList = new ArrayList<>();
|
||||
long current = System.currentTimeMillis();
|
||||
for (int j = 0; j < numOfRow; j++)
|
||||
tsList.add(current + j);
|
||||
pstmt.setTimestamp(0, tsList);
|
||||
|
||||
ArrayList<byte[]> f1List = new ArrayList<>();
|
||||
for (int j = 0; j < numOfRow; j++) {
|
||||
f1List.add(listGeo.get(i - 1));
|
||||
}
|
||||
pstmt.setGeometry(1, f1List, BINARY_COLUMN_SIZE);
|
||||
|
||||
// add column
|
||||
pstmt.columnDataAddBatch();
|
||||
}
|
||||
// execute
|
||||
pstmt.columnDataExecuteBatch();
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**注**：setString 和 setNString 都要求用户在 size 参数里声明表定义中对应列的列宽
|
||||
**注**：字符串和数组类型都要求用户在 size 参数里声明表定义中对应列的列宽。
|
||||
|
||||
用于设定 VALUES 数据列的取值的方法总共有：
|
||||
|
||||
|
|
@ -695,6 +781,8 @@ public void setByte(int columnIndex, ArrayList<Byte> list) throws SQLException
|
|||
public void setShort(int columnIndex, ArrayList<Short> list) throws SQLException
|
||||
public void setString(int columnIndex, ArrayList<String> list, int size) throws SQLException
|
||||
public void setNString(int columnIndex, ArrayList<String> list, int size) throws SQLException
|
||||
public void setVarbinary(int columnIndex, ArrayList<byte[]> list, int size) throws SQLException
|
||||
public void setGeometry(int columnIndex, ArrayList<byte[]> list, int size) throws SQLException
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
|
|
@ -883,6 +971,9 @@ public void setTagFloat(int index, float value)
|
|||
public void setTagDouble(int index, double value)
|
||||
public void setTagString(int index, String value)
|
||||
public void setTagNString(int index, String value)
|
||||
public void setTagJson(int index, String value)
|
||||
public void setTagVarbinary(int index, byte[] value)
|
||||
public void setTagGeometry(int index, byte[] value)
|
||||
```
|
||||
|
||||
### 无模式写入
|
||||
|
|
|
|||
|
|
@ -11,13 +11,20 @@
|
|||
|
||||
<properties>
|
||||
<project.assembly.dir>src/main/resources/assembly</project.assembly.dir>
|
||||
<java.version>1.8</java.version>
|
||||
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.taosdata.jdbc</groupId>
|
||||
<artifactId>taos-jdbcdriver</artifactId>
|
||||
<version>3.0.0</version>
|
||||
<version>3.2.7</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.locationtech.jts</groupId>
|
||||
<artifactId>jts-core</artifactId>
|
||||
<version>1.19.0</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
|
|
@ -68,12 +75,12 @@
|
|||
</execution>
|
||||
|
||||
<execution>
|
||||
<id>SubscribeDemo</id>
|
||||
<id>GeometryDemo</id>
|
||||
<configuration>
|
||||
<finalName>SubscribeDemo</finalName>
|
||||
<finalName>GeometryDemo</finalName>
|
||||
<archive>
|
||||
<manifest>
|
||||
<mainClass>com.taosdata.example.SubscribeDemo</mainClass>
|
||||
<mainClass>com.taosdata.example.GeometryDemo</mainClass>
|
||||
</manifest>
|
||||
</archive>
|
||||
<descriptorRefs>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,94 @@
|
|||
package com.taosdata.example;
|
||||
|
||||
import com.taosdata.jdbc.tmq.ConsumerRecord;
|
||||
import com.taosdata.jdbc.tmq.ConsumerRecords;
|
||||
import com.taosdata.jdbc.tmq.ReferenceDeserializer;
|
||||
import com.taosdata.jdbc.tmq.TaosConsumer;
|
||||
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Timestamp;
|
||||
import java.time.Duration;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
|
||||
public abstract class ConsumerLoop {
|
||||
private final TaosConsumer<ResultBean> consumer;
|
||||
private final List<String> topics;
|
||||
private final AtomicBoolean shutdown;
|
||||
private final CountDownLatch shutdownLatch;
|
||||
|
||||
public ConsumerLoop() throws SQLException {
|
||||
Properties config = new Properties();
|
||||
config.setProperty("td.connect.type", "jni");
|
||||
config.setProperty("bootstrap.servers", "localhost:6030");
|
||||
config.setProperty("td.connect.user", "root");
|
||||
config.setProperty("td.connect.pass", "taosdata");
|
||||
config.setProperty("auto.offset.reset", "earliest");
|
||||
config.setProperty("msg.with.table.name", "true");
|
||||
config.setProperty("enable.auto.commit", "true");
|
||||
config.setProperty("auto.commit.interval.ms", "1000");
|
||||
config.setProperty("group.id", "group1");
|
||||
config.setProperty("client.id", "1");
|
||||
config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.ConsumerLoop$ResultDeserializer");
|
||||
config.setProperty("value.deserializer.encoding", "UTF-8");
|
||||
config.setProperty("experimental.snapshot.enable", "true");
|
||||
|
||||
this.consumer = new TaosConsumer<>(config);
|
||||
this.topics = Collections.singletonList("topic_speed");
|
||||
this.shutdown = new AtomicBoolean(false);
|
||||
this.shutdownLatch = new CountDownLatch(1);
|
||||
}
|
||||
|
||||
public abstract void process(ResultBean result);
|
||||
|
||||
public void pollData() throws SQLException {
|
||||
try {
|
||||
consumer.subscribe(topics);
|
||||
|
||||
while (!shutdown.get()) {
|
||||
ConsumerRecords<ResultBean> records = consumer.poll(Duration.ofMillis(100));
|
||||
for (ConsumerRecord<ResultBean> record : records) {
|
||||
ResultBean bean = record.value();
|
||||
process(bean);
|
||||
}
|
||||
}
|
||||
consumer.unsubscribe();
|
||||
} finally {
|
||||
consumer.close();
|
||||
shutdownLatch.countDown();
|
||||
}
|
||||
}
|
||||
|
||||
public void shutdown() throws InterruptedException {
|
||||
shutdown.set(true);
|
||||
shutdownLatch.await();
|
||||
}
|
||||
|
||||
public static class ResultDeserializer extends ReferenceDeserializer<ResultBean> {
|
||||
|
||||
}
|
||||
|
||||
public static class ResultBean {
|
||||
private Timestamp ts;
|
||||
private int speed;
|
||||
|
||||
public Timestamp getTs() {
|
||||
return ts;
|
||||
}
|
||||
|
||||
public void setTs(Timestamp ts) {
|
||||
this.ts = ts;
|
||||
}
|
||||
|
||||
public int getSpeed() {
|
||||
return speed;
|
||||
}
|
||||
|
||||
public void setSpeed(int speed) {
|
||||
this.speed = speed;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,190 @@
|
|||
package com.taosdata.example;
|
||||
|
||||
import com.taosdata.jdbc.TSDBPreparedStatement;
|
||||
import org.locationtech.jts.geom.*;
|
||||
import org.locationtech.jts.io.ByteOrderValues;
|
||||
import org.locationtech.jts.io.ParseException;
|
||||
import org.locationtech.jts.io.WKBReader;
|
||||
import org.locationtech.jts.io.WKBWriter;
|
||||
|
||||
import java.sql.*;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Properties;
|
||||
|
||||
public class GeometryDemo {
|
||||
private static String host = "localhost";
|
||||
private static final String dbName = "test";
|
||||
private static final String tbName = "weather";
|
||||
private static final String user = "root";
|
||||
private static final String password = "taosdata";
|
||||
|
||||
private Connection connection;
|
||||
|
||||
public static void main(String[] args) throws SQLException {
|
||||
for (int i = 0; i < args.length; i++) {
|
||||
if ("-host".equalsIgnoreCase(args[i]) && i < args.length - 1)
|
||||
host = args[++i];
|
||||
}
|
||||
if (host == null) {
|
||||
printHelp();
|
||||
}
|
||||
GeometryDemo demo = new GeometryDemo();
|
||||
demo.init();
|
||||
demo.createDatabase();
|
||||
demo.useDatabase();
|
||||
demo.dropTable();
|
||||
demo.createTable();
|
||||
|
||||
demo.insert();
|
||||
demo.stmtInsert();
|
||||
demo.select();
|
||||
|
||||
demo.dropTable();
|
||||
demo.close();
|
||||
}
|
||||
|
||||
private void init() {
|
||||
final String url = "jdbc:TAOS://" + host + ":6030/?user=" + user + "&password=" + password;
|
||||
// get connection
|
||||
try {
|
||||
Properties properties = new Properties();
|
||||
properties.setProperty("charset", "UTF-8");
|
||||
properties.setProperty("locale", "en_US.UTF-8");
|
||||
properties.setProperty("timezone", "UTC-8");
|
||||
System.out.println("get connection starting...");
|
||||
connection = DriverManager.getConnection(url, properties);
|
||||
if (connection != null)
|
||||
System.out.println("[ OK ] Connection established.");
|
||||
} catch (SQLException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
private void createDatabase() {
|
||||
String sql = "create database if not exists " + dbName;
|
||||
execute(sql);
|
||||
}
|
||||
|
||||
private void useDatabase() {
|
||||
String sql = "use " + dbName;
|
||||
execute(sql);
|
||||
}
|
||||
|
||||
private void dropTable() {
|
||||
final String sql = "drop table if exists " + dbName + "." + tbName + "";
|
||||
execute(sql);
|
||||
}
|
||||
|
||||
private void createTable() {
|
||||
final String sql = "create table if not exists " + dbName + "." + tbName + " (ts timestamp, temperature float, humidity int, location geometry(50))";
|
||||
execute(sql);
|
||||
}
|
||||
|
||||
private void insert() {
|
||||
final String sql = "insert into " + dbName + "." + tbName + " (ts, temperature, humidity, location) values(now, 20.5, 34, 'POINT(1 2)')";
|
||||
execute(sql);
|
||||
}
|
||||
|
||||
private void stmtInsert() throws SQLException {
|
||||
TSDBPreparedStatement preparedStatement = (TSDBPreparedStatement) connection.prepareStatement("insert into " + dbName + "." + tbName + " values (?, ?, ?, ?)");
|
||||
|
||||
long current = System.currentTimeMillis();
|
||||
ArrayList<Long> tsList = new ArrayList<>();
|
||||
tsList.add(current);
|
||||
tsList.add(current + 1);
|
||||
preparedStatement.setTimestamp(0, tsList);
|
||||
ArrayList<Float> tempList = new ArrayList<>();
|
||||
tempList.add(20.1F);
|
||||
tempList.add(21.2F);
|
||||
preparedStatement.setFloat(1, tempList);
|
||||
ArrayList<Integer> humList = new ArrayList<>();
|
||||
humList.add(30);
|
||||
humList.add(31);
|
||||
preparedStatement.setInt(2, humList);
|
||||
|
||||
|
||||
ArrayList<byte[]> list = new ArrayList<>();
|
||||
GeometryFactory gf = new GeometryFactory();
|
||||
Point p1 = gf.createPoint(new Coordinate(1,2));
|
||||
p1.setSRID(1234);
|
||||
|
||||
// NOTE: TDengine current version only support 2D dimension and little endian byte order
|
||||
WKBWriter w = new WKBWriter(2, ByteOrderValues.LITTLE_ENDIAN, true);
|
||||
byte[] wkb = w.write(p1);
|
||||
list.add(wkb);
|
||||
|
||||
Coordinate[] coordinates = { new Coordinate(10, 20),
|
||||
new Coordinate(30, 40)};
|
||||
LineString lineString = gf.createLineString(coordinates);
|
||||
lineString.setSRID(2345);
|
||||
byte[] wkb2 = w.write(lineString);
|
||||
list.add(wkb2);
|
||||
|
||||
preparedStatement.setGeometry(3, list, 50);
|
||||
|
||||
preparedStatement.columnDataAddBatch();
|
||||
preparedStatement.columnDataExecuteBatch();
|
||||
}
|
||||
|
||||
private void select() {
|
||||
final String sql = "select * from " + dbName + "." + tbName;
|
||||
executeQuery(sql);
|
||||
}
|
||||
|
||||
private void close() {
|
||||
try {
|
||||
if (connection != null) {
|
||||
this.connection.close();
|
||||
System.out.println("connection closed.");
|
||||
}
|
||||
} catch (SQLException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
private void executeQuery(String sql) {
|
||||
long start = System.currentTimeMillis();
|
||||
try (Statement statement = connection.createStatement()) {
|
||||
ResultSet resultSet = statement.executeQuery(sql);
|
||||
long end = System.currentTimeMillis();
|
||||
printSql(sql, true, (end - start));
|
||||
|
||||
while (resultSet.next()){
|
||||
byte[] result1 = resultSet.getBytes(4);
|
||||
WKBReader reader = new WKBReader();
|
||||
Geometry g1 = reader.read(result1);
|
||||
System.out.println("GEO OBJ: " + g1 + ", SRID: " + g1.getSRID());
|
||||
}
|
||||
|
||||
} catch (SQLException e) {
|
||||
long end = System.currentTimeMillis();
|
||||
printSql(sql, false, (end - start));
|
||||
e.printStackTrace();
|
||||
} catch (ParseException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
private void printSql(String sql, boolean succeed, long cost) {
|
||||
System.out.println("[ " + (succeed ? "OK" : "ERROR!") + " ] time cost: " + cost + " ms, execute statement ====> " + sql);
|
||||
}
|
||||
|
||||
private void execute(String sql) {
|
||||
long start = System.currentTimeMillis();
|
||||
try (Statement statement = connection.createStatement()) {
|
||||
boolean execute = statement.execute(sql);
|
||||
long end = System.currentTimeMillis();
|
||||
printSql(sql, true, (end - start));
|
||||
} catch (SQLException e) {
|
||||
long end = System.currentTimeMillis();
|
||||
printSql(sql, false, (end - start));
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
private static void printHelp() {
|
||||
System.out.println("Usage: java -jar JDBCDemo.jar -host <hostname>");
|
||||
System.exit(0);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,316 @@
|
|||
package com.taosdata.example;
|
||||
|
||||
import com.taosdata.jdbc.TSDBPreparedStatement;
|
||||
import com.taosdata.jdbc.utils.StringUtils;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Random;
|
||||
|
||||
public class ParameterBindingDemo {
|
||||
|
||||
private static final String host = "127.0.0.1";
|
||||
private static final Random random = new Random(System.currentTimeMillis());
|
||||
private static final int BINARY_COLUMN_SIZE = 50;
|
||||
private static final String[] schemaList = {
|
||||
"create table stable1(ts timestamp, f1 tinyint, f2 smallint, f3 int, f4 bigint) tags(t1 tinyint, t2 smallint, t3 int, t4 bigint)",
|
||||
"create table stable2(ts timestamp, f1 float, f2 double) tags(t1 float, t2 double)",
|
||||
"create table stable3(ts timestamp, f1 bool) tags(t1 bool)",
|
||||
"create table stable4(ts timestamp, f1 binary(" + BINARY_COLUMN_SIZE + ")) tags(t1 binary(" + BINARY_COLUMN_SIZE + "))",
|
||||
"create table stable5(ts timestamp, f1 nchar(" + BINARY_COLUMN_SIZE + ")) tags(t1 nchar(" + BINARY_COLUMN_SIZE + "))",
|
||||
"create table stable6(ts timestamp, f1 varbinary(" + BINARY_COLUMN_SIZE + ")) tags(t1 varbinary(" + BINARY_COLUMN_SIZE + "))",
|
||||
"create table stable7(ts timestamp, f1 geometry(" + BINARY_COLUMN_SIZE + ")) tags(t1 geometry(" + BINARY_COLUMN_SIZE + "))",
|
||||
};
|
||||
private static final int numOfSubTable = 10, numOfRow = 10;
|
||||
|
||||
public static void main(String[] args) throws SQLException {
|
||||
|
||||
String jdbcUrl = "jdbc:TAOS://" + host + ":6030/";
|
||||
Connection conn = DriverManager.getConnection(jdbcUrl, "root", "taosdata");
|
||||
|
||||
init(conn);
|
||||
|
||||
bindInteger(conn);
|
||||
bindFloat(conn);
|
||||
bindBoolean(conn);
|
||||
bindBytes(conn);
|
||||
bindString(conn);
|
||||
bindVarbinary(conn);
|
||||
bindGeometry(conn);
|
||||
|
||||
clean(conn);
|
||||
conn.close();
|
||||
}
|
||||
|
||||
private static void init(Connection conn) throws SQLException {
|
||||
clean(conn);
|
||||
try (Statement stmt = conn.createStatement()) {
|
||||
stmt.execute("create database if not exists test_parabind");
|
||||
stmt.execute("use test_parabind");
|
||||
for (int i = 0; i < schemaList.length; i++) {
|
||||
stmt.execute(schemaList[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
private static void clean(Connection conn) throws SQLException {
|
||||
try (Statement stmt = conn.createStatement()) {
|
||||
stmt.execute("drop database if exists test_parabind");
|
||||
}
|
||||
}
|
||||
|
||||
private static void bindInteger(Connection conn) throws SQLException {
|
||||
String sql = "insert into ? using stable1 tags(?,?,?,?) values(?,?,?,?,?)";
|
||||
|
||||
try (TSDBPreparedStatement pstmt = conn.prepareStatement(sql).unwrap(TSDBPreparedStatement.class)) {
|
||||
|
||||
for (int i = 1; i <= numOfSubTable; i++) {
|
||||
// set table name
|
||||
pstmt.setTableName("t1_" + i);
|
||||
// set tags
|
||||
pstmt.setTagByte(0, Byte.parseByte(Integer.toString(random.nextInt(Byte.MAX_VALUE))));
|
||||
pstmt.setTagShort(1, Short.parseShort(Integer.toString(random.nextInt(Short.MAX_VALUE))));
|
||||
pstmt.setTagInt(2, random.nextInt(Integer.MAX_VALUE));
|
||||
pstmt.setTagLong(3, random.nextLong());
|
||||
// set columns
|
||||
ArrayList<Long> tsList = new ArrayList<>();
|
||||
long current = System.currentTimeMillis();
|
||||
for (int j = 0; j < numOfRow; j++)
|
||||
tsList.add(current + j);
|
||||
pstmt.setTimestamp(0, tsList);
|
||||
|
||||
ArrayList<Byte> f1List = new ArrayList<>();
|
||||
for (int j = 0; j < numOfRow; j++)
|
||||
f1List.add(Byte.parseByte(Integer.toString(random.nextInt(Byte.MAX_VALUE))));
|
||||
pstmt.setByte(1, f1List);
|
||||
|
||||
ArrayList<Short> f2List = new ArrayList<>();
|
||||
for (int j = 0; j < numOfRow; j++)
|
||||
f2List.add(Short.parseShort(Integer.toString(random.nextInt(Short.MAX_VALUE))));
|
||||
pstmt.setShort(2, f2List);
|
||||
|
||||
ArrayList<Integer> f3List = new ArrayList<>();
|
||||
for (int j = 0; j < numOfRow; j++)
|
||||
f3List.add(random.nextInt(Integer.MAX_VALUE));
|
||||
pstmt.setInt(3, f3List);
|
||||
|
||||
ArrayList<Long> f4List = new ArrayList<>();
|
||||
for (int j = 0; j < numOfRow; j++)
|
||||
f4List.add(random.nextLong());
|
||||
pstmt.setLong(4, f4List);
|
||||
|
||||
// add column
|
||||
pstmt.columnDataAddBatch();
|
||||
}
|
||||
// execute column
|
||||
pstmt.columnDataExecuteBatch();
|
||||
}
|
||||
}
|
||||
|
||||
private static void bindFloat(Connection conn) throws SQLException {
|
||||
String sql = "insert into ? using stable2 tags(?,?) values(?,?,?)";
|
||||
|
||||
TSDBPreparedStatement pstmt = conn.prepareStatement(sql).unwrap(TSDBPreparedStatement.class);
|
||||
|
||||
for (int i = 1; i <= numOfSubTable; i++) {
|
||||
// set table name
|
||||
pstmt.setTableName("t2_" + i);
|
||||
// set tags
|
||||
pstmt.setTagFloat(0, random.nextFloat());
|
||||
pstmt.setTagDouble(1, random.nextDouble());
|
||||
// set columns
|
||||
ArrayList<Long> tsList = new ArrayList<>();
|
||||
long current = System.currentTimeMillis();
|
||||
for (int j = 0; j < numOfRow; j++)
|
||||
tsList.add(current + j);
|
||||
pstmt.setTimestamp(0, tsList);
|
||||
|
||||
ArrayList<Float> f1List = new ArrayList<>();
|
||||
for (int j = 0; j < numOfRow; j++)
|
||||
f1List.add(random.nextFloat());
|
||||
pstmt.setFloat(1, f1List);
|
||||
|
||||
ArrayList<Double> f2List = new ArrayList<>();
|
||||
for (int j = 0; j < numOfRow; j++)
|
||||
f2List.add(random.nextDouble());
|
||||
pstmt.setDouble(2, f2List);
|
||||
|
||||
// add column
|
||||
pstmt.columnDataAddBatch();
|
||||
}
|
||||
// execute
|
||||
pstmt.columnDataExecuteBatch();
|
||||
// close if no try-with-catch statement is used
|
||||
pstmt.close();
|
||||
}
|
||||
|
||||
private static void bindBoolean(Connection conn) throws SQLException {
|
||||
String sql = "insert into ? using stable3 tags(?) values(?,?)";
|
||||
|
||||
try (TSDBPreparedStatement pstmt = conn.prepareStatement(sql).unwrap(TSDBPreparedStatement.class)) {
|
||||
for (int i = 1; i <= numOfSubTable; i++) {
|
||||
// set table name
|
||||
pstmt.setTableName("t3_" + i);
|
||||
// set tags
|
||||
pstmt.setTagBoolean(0, random.nextBoolean());
|
||||
// set columns
|
||||
ArrayList<Long> tsList = new ArrayList<>();
|
||||
long current = System.currentTimeMillis();
|
||||
for (int j = 0; j < numOfRow; j++)
|
||||
tsList.add(current + j);
|
||||
pstmt.setTimestamp(0, tsList);
|
||||
|
||||
ArrayList<Boolean> f1List = new ArrayList<>();
|
||||
for (int j = 0; j < numOfRow; j++)
|
||||
f1List.add(random.nextBoolean());
|
||||
pstmt.setBoolean(1, f1List);
|
||||
|
||||
// add column
|
||||
pstmt.columnDataAddBatch();
|
||||
}
|
||||
// execute
|
||||
pstmt.columnDataExecuteBatch();
|
||||
}
|
||||
}
|
||||
|
||||
private static void bindBytes(Connection conn) throws SQLException {
|
||||
String sql = "insert into ? using stable4 tags(?) values(?,?)";
|
||||
|
||||
try (TSDBPreparedStatement pstmt = conn.prepareStatement(sql).unwrap(TSDBPreparedStatement.class)) {
|
||||
|
||||
for (int i = 1; i <= numOfSubTable; i++) {
|
||||
// set table name
|
||||
pstmt.setTableName("t4_" + i);
|
||||
// set tags
|
||||
pstmt.setTagString(0, new String("abc"));
|
||||
|
||||
// set columns
|
||||
ArrayList<Long> tsList = new ArrayList<>();
|
||||
long current = System.currentTimeMillis();
|
||||
for (int j = 0; j < numOfRow; j++)
|
||||
tsList.add(current + j);
|
||||
pstmt.setTimestamp(0, tsList);
|
||||
|
||||
ArrayList<String> f1List = new ArrayList<>();
|
||||
for (int j = 0; j < numOfRow; j++) {
|
||||
f1List.add(new String("abc"));
|
||||
}
|
||||
pstmt.setString(1, f1List, BINARY_COLUMN_SIZE);
|
||||
|
||||
// add column
|
||||
pstmt.columnDataAddBatch();
|
||||
}
|
||||
// execute
|
||||
pstmt.columnDataExecuteBatch();
|
||||
}
|
||||
}
|
||||
|
||||
private static void bindString(Connection conn) throws SQLException {
|
||||
String sql = "insert into ? using stable5 tags(?) values(?,?)";
|
||||
|
||||
try (TSDBPreparedStatement pstmt = conn.prepareStatement(sql).unwrap(TSDBPreparedStatement.class)) {
|
||||
|
||||
for (int i = 1; i <= numOfSubTable; i++) {
|
||||
// set table name
|
||||
pstmt.setTableName("t5_" + i);
|
||||
// set tags
|
||||
pstmt.setTagNString(0, "California.SanFrancisco");
|
||||
|
||||
// set columns
|
||||
ArrayList<Long> tsList = new ArrayList<>();
|
||||
long current = System.currentTimeMillis();
|
||||
for (int j = 0; j < numOfRow; j++)
|
||||
tsList.add(current + j);
|
||||
pstmt.setTimestamp(0, tsList);
|
||||
|
||||
ArrayList<String> f1List = new ArrayList<>();
|
||||
for (int j = 0; j < numOfRow; j++) {
|
||||
f1List.add("California.LosAngeles");
|
||||
}
|
||||
pstmt.setNString(1, f1List, BINARY_COLUMN_SIZE);
|
||||
|
||||
// add column
|
||||
pstmt.columnDataAddBatch();
|
||||
}
|
||||
// execute
|
||||
pstmt.columnDataExecuteBatch();
|
||||
}
|
||||
}
|
||||
|
||||
private static void bindVarbinary(Connection conn) throws SQLException {
|
||||
String sql = "insert into ? using stable6 tags(?) values(?,?)";
|
||||
|
||||
try (TSDBPreparedStatement pstmt = conn.prepareStatement(sql).unwrap(TSDBPreparedStatement.class)) {
|
||||
|
||||
for (int i = 1; i <= numOfSubTable; i++) {
|
||||
// set table name
|
||||
pstmt.setTableName("t6_" + i);
|
||||
// set tags
|
||||
byte[] bTag = new byte[]{0,2,3,4,5};
|
||||
bTag[0] = (byte) i;
|
||||
pstmt.setTagVarbinary(0, bTag);
|
||||
|
||||
// set columns
|
||||
ArrayList<Long> tsList = new ArrayList<>();
|
||||
long current = System.currentTimeMillis();
|
||||
for (int j = 0; j < numOfRow; j++)
|
||||
tsList.add(current + j);
|
||||
pstmt.setTimestamp(0, tsList);
|
||||
|
||||
ArrayList<byte[]> f1List = new ArrayList<>();
|
||||
for (int j = 0; j < numOfRow; j++) {
|
||||
byte[] v = new byte[]{0,2,3,4,5,6};
|
||||
v[0] = (byte)j;
|
||||
f1List.add(v);
|
||||
}
|
||||
pstmt.setVarbinary(1, f1List, BINARY_COLUMN_SIZE);
|
||||
|
||||
// add column
|
||||
pstmt.columnDataAddBatch();
|
||||
}
|
||||
// execute
|
||||
pstmt.columnDataExecuteBatch();
|
||||
}
|
||||
}
|
||||
|
||||
private static void bindGeometry(Connection conn) throws SQLException {
|
||||
String sql = "insert into ? using stable7 tags(?) values(?,?)";
|
||||
|
||||
try (TSDBPreparedStatement pstmt = conn.prepareStatement(sql).unwrap(TSDBPreparedStatement.class)) {
|
||||
|
||||
byte[] g1 = StringUtils.hexToBytes("0101000000000000000000F03F0000000000000040");
|
||||
byte[] g2 = StringUtils.hexToBytes("0102000020E610000002000000000000000000F03F000000000000004000000000000008400000000000001040");
|
||||
List<byte[]> listGeo = new ArrayList<>();
|
||||
listGeo.add(g1);
|
||||
listGeo.add(g2);
|
||||
|
||||
for (int i = 1; i <= 2; i++) {
|
||||
// set table name
|
||||
pstmt.setTableName("t7_" + i);
|
||||
// set tags
|
||||
pstmt.setTagGeometry(0, listGeo.get(i - 1));
|
||||
|
||||
// set columns
|
||||
ArrayList<Long> tsList = new ArrayList<>();
|
||||
long current = System.currentTimeMillis();
|
||||
for (int j = 0; j < numOfRow; j++)
|
||||
tsList.add(current + j);
|
||||
pstmt.setTimestamp(0, tsList);
|
||||
|
||||
ArrayList<byte[]> f1List = new ArrayList<>();
|
||||
for (int j = 0; j < numOfRow; j++) {
|
||||
f1List.add(listGeo.get(i - 1));
|
||||
}
|
||||
pstmt.setGeometry(1, f1List, BINARY_COLUMN_SIZE);
|
||||
|
||||
// add column
|
||||
pstmt.columnDataAddBatch();
|
||||
}
|
||||
// execute
|
||||
pstmt.columnDataExecuteBatch();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,74 +0,0 @@
|
|||
package com.taosdata.example;
|
||||
|
||||
import com.taosdata.jdbc.TSDBConnection;
|
||||
import com.taosdata.jdbc.TSDBDriver;
|
||||
import com.taosdata.jdbc.TSDBResultSet;
|
||||
import com.taosdata.jdbc.TSDBSubscribe;
|
||||
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.ResultSetMetaData;
|
||||
import java.util.Properties;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
public class SubscribeDemo {
|
||||
private static final String usage = "java -jar SubscribeDemo.jar -host <hostname> -database <database name> -topic <topic> -sql <sql>";
|
||||
|
||||
public static void main(String[] args) {
|
||||
// parse args from command line
|
||||
String host = "", database = "", topic = "", sql = "";
|
||||
for (int i = 0; i < args.length; i++) {
|
||||
if ("-host".equalsIgnoreCase(args[i]) && i < args.length - 1) {
|
||||
host = args[++i];
|
||||
}
|
||||
if ("-database".equalsIgnoreCase(args[i]) && i < args.length - 1) {
|
||||
database = args[++i];
|
||||
}
|
||||
if ("-topic".equalsIgnoreCase(args[i]) && i < args.length - 1) {
|
||||
topic = args[++i];
|
||||
}
|
||||
if ("-sql".equalsIgnoreCase(args[i]) && i < args.length - 1) {
|
||||
sql = args[++i];
|
||||
}
|
||||
}
|
||||
if (host.isEmpty() || database.isEmpty() || topic.isEmpty() || sql.isEmpty()) {
|
||||
System.out.println(usage);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
Properties properties = new Properties();
|
||||
properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
|
||||
properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
|
||||
properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
|
||||
final String url = "jdbc:TAOS://" + host + ":6030/" + database + "?user=root&password=taosdata";
|
||||
// get TSDBConnection
|
||||
TSDBConnection connection = (TSDBConnection) DriverManager.getConnection(url, properties);
|
||||
// create TSDBSubscribe
|
||||
TSDBSubscribe sub = connection.subscribe(topic, sql, false);
|
||||
|
||||
int total = 0;
|
||||
while (true) {
|
||||
TSDBResultSet rs = sub.consume();
|
||||
int count = 0;
|
||||
ResultSetMetaData meta = rs.getMetaData();
|
||||
while (rs.next()) {
|
||||
for (int i = 1; i <= meta.getColumnCount(); i++) {
|
||||
System.out.print(meta.getColumnLabel(i) + ": " + rs.getString(i) + "\t");
|
||||
}
|
||||
System.out.println();
|
||||
count++;
|
||||
}
|
||||
total += count;
|
||||
// System.out.printf("%d rows consumed, total %d\n", count, total);
|
||||
if (total >= 10)
|
||||
break;
|
||||
TimeUnit.SECONDS.sleep(1);
|
||||
}
|
||||
sub.close(false);
|
||||
connection.close();
|
||||
} catch (Exception e) {
|
||||
System.out.println("host: " + host + ", database: " + database + ", topic: " + topic + ", sql: " + sql);
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,170 @@
|
|||
package com.taosdata.example;
|
||||
|
||||
import com.taosdata.jdbc.ws.TSWSPreparedStatement;
|
||||
|
||||
import java.sql.*;
|
||||
import java.util.Random;
|
||||
|
||||
public class WSParameterBindingDemo {
|
||||
private static final String host = "127.0.0.1";
|
||||
private static final Random random = new Random(System.currentTimeMillis());
|
||||
private static final int BINARY_COLUMN_SIZE = 30;
|
||||
private static final String[] schemaList = {
|
||||
"create table stable1(ts timestamp, f1 tinyint, f2 smallint, f3 int, f4 bigint) tags(t1 tinyint, t2 smallint, t3 int, t4 bigint)",
|
||||
"create table stable2(ts timestamp, f1 float, f2 double) tags(t1 float, t2 double)",
|
||||
"create table stable3(ts timestamp, f1 bool) tags(t1 bool)",
|
||||
"create table stable4(ts timestamp, f1 binary(" + BINARY_COLUMN_SIZE + ")) tags(t1 binary(" + BINARY_COLUMN_SIZE + "))",
|
||||
"create table stable5(ts timestamp, f1 nchar(" + BINARY_COLUMN_SIZE + ")) tags(t1 nchar(" + BINARY_COLUMN_SIZE + "))"
|
||||
};
|
||||
private static final int numOfSubTable = 10, numOfRow = 10;
|
||||
|
||||
public static void main(String[] args) throws SQLException {
|
||||
|
||||
String jdbcUrl = "jdbc:TAOS-RS://" + host + ":6041/?batchfetch=true";
|
||||
Connection conn = DriverManager.getConnection(jdbcUrl, "root", "taosdata");
|
||||
|
||||
init(conn);
|
||||
|
||||
bindInteger(conn);
|
||||
|
||||
bindFloat(conn);
|
||||
|
||||
bindBoolean(conn);
|
||||
|
||||
bindBytes(conn);
|
||||
|
||||
bindString(conn);
|
||||
|
||||
conn.close();
|
||||
}
|
||||
|
||||
private static void init(Connection conn) throws SQLException {
|
||||
try (Statement stmt = conn.createStatement()) {
|
||||
stmt.execute("drop database if exists test_ws_parabind");
|
||||
stmt.execute("create database if not exists test_ws_parabind");
|
||||
stmt.execute("use test_ws_parabind");
|
||||
for (int i = 0; i < schemaList.length; i++) {
|
||||
stmt.execute(schemaList[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void bindInteger(Connection conn) throws SQLException {
|
||||
String sql = "insert into ? using stable1 tags(?,?,?,?) values(?,?,?,?,?)";
|
||||
|
||||
try (TSWSPreparedStatement pstmt = conn.prepareStatement(sql).unwrap(TSWSPreparedStatement.class)) {
|
||||
|
||||
for (int i = 1; i <= numOfSubTable; i++) {
|
||||
// set table name
|
||||
pstmt.setTableName("t1_" + i);
|
||||
// set tags
|
||||
pstmt.setTagByte(1, Byte.parseByte(Integer.toString(random.nextInt(Byte.MAX_VALUE))));
|
||||
pstmt.setTagShort(2, Short.parseShort(Integer.toString(random.nextInt(Short.MAX_VALUE))));
|
||||
pstmt.setTagInt(3, random.nextInt(Integer.MAX_VALUE));
|
||||
pstmt.setTagLong(4, random.nextLong());
|
||||
// set columns
|
||||
long current = System.currentTimeMillis();
|
||||
for (int j = 0; j < numOfRow; j++) {
|
||||
pstmt.setTimestamp(1, new Timestamp(current + j));
|
||||
pstmt.setByte(2, Byte.parseByte(Integer.toString(random.nextInt(Byte.MAX_VALUE))));
|
||||
pstmt.setShort(3, Short.parseShort(Integer.toString(random.nextInt(Short.MAX_VALUE))));
|
||||
pstmt.setInt(4, random.nextInt(Integer.MAX_VALUE));
|
||||
pstmt.setLong(5, random.nextLong());
|
||||
pstmt.addBatch();
|
||||
}
|
||||
pstmt.executeBatch();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void bindFloat(Connection conn) throws SQLException {
|
||||
String sql = "insert into ? using stable2 tags(?,?) values(?,?,?)";
|
||||
|
||||
try(TSWSPreparedStatement pstmt = conn.prepareStatement(sql).unwrap(TSWSPreparedStatement.class)) {
|
||||
|
||||
for (int i = 1; i <= numOfSubTable; i++) {
|
||||
// set table name
|
||||
pstmt.setTableName("t2_" + i);
|
||||
// set tags
|
||||
pstmt.setTagFloat(1, random.nextFloat());
|
||||
pstmt.setTagDouble(2, random.nextDouble());
|
||||
// set columns
|
||||
long current = System.currentTimeMillis();
|
||||
for (int j = 0; j < numOfRow; j++) {
|
||||
pstmt.setTimestamp(1, new Timestamp(current + j));
|
||||
pstmt.setFloat(2, random.nextFloat());
|
||||
pstmt.setDouble(3, random.nextDouble());
|
||||
pstmt.addBatch();
|
||||
}
|
||||
pstmt.executeBatch();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void bindBoolean(Connection conn) throws SQLException {
|
||||
String sql = "insert into ? using stable3 tags(?) values(?,?)";
|
||||
|
||||
try (TSWSPreparedStatement pstmt = conn.prepareStatement(sql).unwrap(TSWSPreparedStatement.class)) {
|
||||
for (int i = 1; i <= numOfSubTable; i++) {
|
||||
// set table name
|
||||
pstmt.setTableName("t3_" + i);
|
||||
// set tags
|
||||
pstmt.setTagBoolean(1, random.nextBoolean());
|
||||
// set columns
|
||||
long current = System.currentTimeMillis();
|
||||
for (int j = 0; j < numOfRow; j++) {
|
||||
pstmt.setTimestamp(1, new Timestamp(current + j));
|
||||
pstmt.setBoolean(2, random.nextBoolean());
|
||||
pstmt.addBatch();
|
||||
}
|
||||
pstmt.executeBatch();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void bindBytes(Connection conn) throws SQLException {
|
||||
String sql = "insert into ? using stable4 tags(?) values(?,?)";
|
||||
|
||||
try (TSWSPreparedStatement pstmt = conn.prepareStatement(sql).unwrap(TSWSPreparedStatement.class)) {
|
||||
|
||||
for (int i = 1; i <= numOfSubTable; i++) {
|
||||
// set table name
|
||||
pstmt.setTableName("t4_" + i);
|
||||
// set tags
|
||||
pstmt.setTagString(1, new String("abc"));
|
||||
|
||||
// set columns
|
||||
long current = System.currentTimeMillis();
|
||||
for (int j = 0; j < numOfRow; j++) {
|
||||
pstmt.setTimestamp(1, new Timestamp(current + j));
|
||||
pstmt.setString(2, "abc");
|
||||
pstmt.addBatch();
|
||||
}
|
||||
pstmt.executeBatch();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void bindString(Connection conn) throws SQLException {
|
||||
String sql = "insert into ? using stable5 tags(?) values(?,?)";
|
||||
|
||||
try (TSWSPreparedStatement pstmt = conn.prepareStatement(sql).unwrap(TSWSPreparedStatement.class)) {
|
||||
|
||||
for (int i = 1; i <= numOfSubTable; i++) {
|
||||
// set table name
|
||||
pstmt.setTableName("t5_" + i);
|
||||
// set tags
|
||||
pstmt.setTagNString(1, "California.SanFrancisco");
|
||||
|
||||
// set columns
|
||||
long current = System.currentTimeMillis();
|
||||
for (int j = 0; j < numOfRow; j++) {
|
||||
pstmt.setTimestamp(0, new Timestamp(current + j));
|
||||
pstmt.setNString(1, "California.SanFrancisco");
|
||||
pstmt.addBatch();
|
||||
}
|
||||
pstmt.executeBatch();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -135,13 +135,11 @@ static inline int STupleKeyCmpr(const void* pKey1, int kLen1, const void* pKey2,
|
|||
}
|
||||
|
||||
enum {
|
||||
TMQ_MSG_TYPE__DUMMY = 0,
|
||||
TMQ_MSG_TYPE__POLL_DATA_RSP,
|
||||
TMQ_MSG_TYPE__POLL_DATA_RSP = 0,
|
||||
TMQ_MSG_TYPE__POLL_META_RSP,
|
||||
TMQ_MSG_TYPE__EP_RSP,
|
||||
TMQ_MSG_TYPE__POLL_DATA_META_RSP,
|
||||
TMQ_MSG_TYPE__WALINFO_RSP,
|
||||
TMQ_MSG_TYPE__END_RSP,
|
||||
};
|
||||
|
||||
enum {
|
||||
|
|
|
|||
|
|
@ -3199,32 +3199,6 @@ int32_t tEncodeSTqCheckInfo(SEncoder* pEncoder, const STqCheckInfo* pInfo);
|
|||
int32_t tDecodeSTqCheckInfo(SDecoder* pDecoder, STqCheckInfo* pInfo);
|
||||
void tDeleteSTqCheckInfo(STqCheckInfo* pInfo);
|
||||
|
||||
typedef struct {
|
||||
char topic[TSDB_TOPIC_FNAME_LEN];
|
||||
} STqDelCheckInfoReq;
|
||||
|
||||
typedef struct {
|
||||
int32_t vgId;
|
||||
int64_t offset;
|
||||
char topicName[TSDB_TOPIC_FNAME_LEN];
|
||||
char cgroup[TSDB_CGROUP_LEN];
|
||||
} SMqOffset;
|
||||
|
||||
typedef struct {
|
||||
int64_t consumerId;
|
||||
int32_t num;
|
||||
SMqOffset* offsets;
|
||||
} SMqCMCommitOffsetReq;
|
||||
|
||||
typedef struct {
|
||||
int32_t reserved;
|
||||
} SMqCMCommitOffsetRsp;
|
||||
|
||||
int32_t tEncodeSMqOffset(SEncoder* encoder, const SMqOffset* pOffset);
|
||||
int32_t tDecodeSMqOffset(SDecoder* decoder, SMqOffset* pOffset);
|
||||
int32_t tEncodeSMqCMCommitOffsetReq(SEncoder* encoder, const SMqCMCommitOffsetReq* pReq);
|
||||
int32_t tDecodeSMqCMCommitOffsetReq(SDecoder* decoder, SMqCMCommitOffsetReq* pReq);
|
||||
|
||||
// tqOffset
|
||||
enum {
|
||||
TMQ_OFFSET__RESET_NONE = -3,
|
||||
|
|
|
|||
|
|
@ -236,6 +236,7 @@ bool fmIsInterpPseudoColumnFunc(int32_t funcId);
|
|||
bool fmIsGroupKeyFunc(int32_t funcId);
|
||||
bool fmIsBlockDistFunc(int32_t funcId);
|
||||
bool fmIsConstantResFunc(SFunctionNode* pFunc);
|
||||
bool fmIsSkipScanCheckFunc(int32_t funcId);
|
||||
|
||||
void getLastCacheDataType(SDataType* pType);
|
||||
SFunctionNode* createFunction(const char* pName, SNodeList* pParameterList);
|
||||
|
|
|
|||
|
|
@ -293,6 +293,7 @@ typedef struct SPartitionLogicNode {
|
|||
SNodeList* pPartitionKeys;
|
||||
SNodeList* pTags;
|
||||
SNode* pSubtable;
|
||||
SNodeList* pAggFuncs;
|
||||
|
||||
bool needBlockOutputTsOrder; // if true, partition output block will have ts order maintained
|
||||
int32_t pkTsColId;
|
||||
|
|
|
|||
|
|
@ -22,6 +22,15 @@
|
|||
extern "C" {
|
||||
#endif
|
||||
|
||||
#if defined(WINDOWS) && !defined(__USE_PTHREAD)
|
||||
#include <windows.h>
|
||||
#define __USE_WIN_THREAD
|
||||
// https://learn.microsoft.com/en-us/windows/win32/winprog/using-the-windows-headers
|
||||
// #ifndef _WIN32_WINNT
|
||||
// #define _WIN32_WINNT 0x0600
|
||||
// #endif
|
||||
#endif
|
||||
|
||||
#if !defined(WINDOWS) && !defined(_ALPINE)
|
||||
#ifndef __USE_XOPEN2K
|
||||
#define TD_USE_SPINLOCK_AS_MUTEX
|
||||
|
|
@ -29,6 +38,22 @@ typedef pthread_mutex_t pthread_spinlock_t;
|
|||
#endif
|
||||
#endif
|
||||
|
||||
#ifdef __USE_WIN_THREAD
|
||||
typedef pthread_t TdThread; // pthread api
|
||||
typedef pthread_spinlock_t TdThreadSpinlock; // pthread api
|
||||
typedef CRITICAL_SECTION TdThreadMutex; // windows api
|
||||
typedef HANDLE TdThreadMutexAttr; // windows api
|
||||
typedef struct {
|
||||
SRWLOCK lock;
|
||||
int8_t excl;
|
||||
} TdThreadRwlock; // pthread api
|
||||
typedef pthread_attr_t TdThreadAttr; // pthread api
|
||||
typedef pthread_once_t TdThreadOnce; // pthread api
|
||||
typedef HANDLE TdThreadRwlockAttr; // windows api
|
||||
typedef CONDITION_VARIABLE TdThreadCond; // windows api
|
||||
typedef HANDLE TdThreadCondAttr; // windows api
|
||||
typedef pthread_key_t TdThreadKey; // pthread api
|
||||
#else
|
||||
typedef pthread_t TdThread;
|
||||
typedef pthread_spinlock_t TdThreadSpinlock;
|
||||
typedef pthread_mutex_t TdThreadMutex;
|
||||
|
|
@ -40,11 +65,14 @@ typedef pthread_rwlockattr_t TdThreadRwlockAttr;
|
|||
typedef pthread_cond_t TdThreadCond;
|
||||
typedef pthread_condattr_t TdThreadCondAttr;
|
||||
typedef pthread_key_t TdThreadKey;
|
||||
#endif
|
||||
|
||||
#define taosThreadCleanupPush pthread_cleanup_push
|
||||
#define taosThreadCleanupPop pthread_cleanup_pop
|
||||
|
||||
#ifdef WINDOWS
|
||||
#if defined(WINDOWS) && !defined(__USE_PTHREAD)
|
||||
#define TD_PTHREAD_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER_FORBID
|
||||
#elif defined(WINDOWS)
|
||||
#define TD_PTHREAD_MUTEX_INITIALIZER (TdThreadMutex)(-1)
|
||||
#else
|
||||
#define TD_PTHREAD_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER
|
||||
|
|
|
|||
|
|
@ -33,17 +33,14 @@ adapterName="taosadapter"
|
|||
benchmarkName="taosBenchmark"
|
||||
dumpName="taosdump"
|
||||
demoName="taosdemo"
|
||||
xname="taosx"
|
||||
|
||||
clientName2="taos"
|
||||
serverName2="${clientName2}d"
|
||||
configFile2="${clientName2}.cfg"
|
||||
productName2="TDengine"
|
||||
emailName2="taosdata.com"
|
||||
xname2="${clientName2}x"
|
||||
adapterName2="${clientName2}adapter"
|
||||
|
||||
explorerName="${clientName2}-explorer"
|
||||
benchmarkName2="${clientName2}Benchmark"
|
||||
demoName2="${clientName2}demo"
|
||||
dumpName2="${clientName2}dump"
|
||||
|
|
@ -218,8 +215,6 @@ function install_bin() {
|
|||
${csudo}rm -f ${bin_link_dir}/${demoName2} || :
|
||||
${csudo}rm -f ${bin_link_dir}/${benchmarkName2} || :
|
||||
${csudo}rm -f ${bin_link_dir}/${dumpName2} || :
|
||||
${csudo}rm -f ${bin_link_dir}/${xname2} || :
|
||||
${csudo}rm -f ${bin_link_dir}/${explorerName} || :
|
||||
${csudo}rm -f ${bin_link_dir}/set_core || :
|
||||
${csudo}rm -f ${bin_link_dir}/TDinsight.sh || :
|
||||
|
||||
|
|
@ -233,8 +228,6 @@ function install_bin() {
|
|||
[ -x ${install_main_dir}/bin/${benchmarkName2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${benchmarkName2} ${bin_link_dir}/${demoName2} || :
|
||||
[ -x ${install_main_dir}/bin/${benchmarkName2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${benchmarkName2} ${bin_link_dir}/${benchmarkName2} || :
|
||||
[ -x ${install_main_dir}/bin/${dumpName2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${dumpName2} ${bin_link_dir}/${dumpName2} || :
|
||||
[ -x ${install_main_dir}/bin/${xname2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${xname2} ${bin_link_dir}/${xname2} || :
|
||||
[ -x ${install_main_dir}/bin/${explorerName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${explorerName} ${bin_link_dir}/${explorerName} || :
|
||||
[ -x ${install_main_dir}/bin/TDinsight.sh ] && ${csudo}ln -sf ${install_main_dir}/bin/TDinsight.sh ${bin_link_dir}/TDinsight.sh || :
|
||||
if [ "$clientName2" == "${clientName}" ]; then
|
||||
[ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || :
|
||||
|
|
@ -703,26 +696,6 @@ function clean_service_on_systemd() {
|
|||
# if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then
|
||||
# ${csudo}rm -f ${service_config_dir}/${serverName2}.service
|
||||
# fi
|
||||
x_service_config="${service_config_dir}/${xName2}.service"
|
||||
if [ -e "$x_service_config" ]; then
|
||||
if systemctl is-active --quiet ${xName2}; then
|
||||
echo "${productName2} ${xName2} is running, stopping it..."
|
||||
${csudo}systemctl stop ${xName2} &>/dev/null || echo &>/dev/null
|
||||
fi
|
||||
${csudo}systemctl disable ${xName2} &>/dev/null || echo &>/dev/null
|
||||
${csudo}rm -f ${x_service_config}
|
||||
fi
|
||||
|
||||
explorer_service_config="${service_config_dir}/${explorerName2}.service"
|
||||
if [ -e "$explorer_service_config" ]; then
|
||||
if systemctl is-active --quiet ${explorerName2}; then
|
||||
echo "${productName2} ${explorerName2} is running, stopping it..."
|
||||
${csudo}systemctl stop ${explorerName2} &>/dev/null || echo &>/dev/null
|
||||
fi
|
||||
${csudo}systemctl disable ${explorerName2} &>/dev/null || echo &>/dev/null
|
||||
${csudo}rm -f ${explorer_service_config}
|
||||
${csudo}rm -f /etc/${clientName2}/explorer.toml
|
||||
fi
|
||||
}
|
||||
|
||||
function install_service_on_systemd() {
|
||||
|
|
|
|||
|
|
@ -2553,6 +2553,14 @@ static void fetchCallback(void* pResult, void* param, int32_t code) {
|
|||
}
|
||||
|
||||
void taosAsyncFetchImpl(SRequestObj* pRequest, __taos_async_fn_t fp, void* param) {
|
||||
if (pRequest->syncQuery && pRequest->body.param != param) {
|
||||
if (pRequest->body.param) {
|
||||
tsem_destroy(&((SSyncQueryParam *)pRequest->body.param)->sem);
|
||||
}
|
||||
taosMemoryFree(pRequest->body.param);
|
||||
pRequest->syncQuery = false;
|
||||
}
|
||||
|
||||
pRequest->body.fetchFp = fp;
|
||||
pRequest->body.param = param;
|
||||
|
||||
|
|
|
|||
|
|
@ -29,8 +29,6 @@
|
|||
|
||||
#define OFFSET_IS_RESET_OFFSET(_of) ((_of) < 0)
|
||||
|
||||
typedef void (*__tmq_askep_fn_t)(tmq_t* pTmq, int32_t code, SDataBuf* pBuf, void* pParam);
|
||||
|
||||
struct SMqMgmt {
|
||||
int8_t inited;
|
||||
tmr_h timer;
|
||||
|
|
@ -42,11 +40,13 @@ volatile int32_t tmqInitRes = 0; // initialize rsp code
|
|||
static struct SMqMgmt tmqMgmt = {0};
|
||||
|
||||
typedef struct {
|
||||
int32_t code;
|
||||
int8_t tmqRspType;
|
||||
int32_t epoch;
|
||||
} SMqRspWrapper;
|
||||
|
||||
typedef struct {
|
||||
int32_t code;
|
||||
int8_t tmqRspType;
|
||||
int32_t epoch;
|
||||
SMqAskEpRsp msg;
|
||||
|
|
@ -133,7 +133,6 @@ enum {
|
|||
|
||||
enum {
|
||||
TMQ_DELAYED_TASK__ASK_EP = 1,
|
||||
TMQ_DELAYED_TASK__REPORT,
|
||||
TMQ_DELAYED_TASK__COMMIT,
|
||||
};
|
||||
|
||||
|
|
@ -165,6 +164,7 @@ typedef struct {
|
|||
} SMqClientTopic;
|
||||
|
||||
typedef struct {
|
||||
int32_t code;
|
||||
int8_t tmqRspType;
|
||||
int32_t epoch; // epoch can be used to guard the vgHandle
|
||||
int32_t vgId;
|
||||
|
|
@ -189,8 +189,8 @@ typedef struct {
|
|||
|
||||
typedef struct {
|
||||
int64_t refId;
|
||||
bool sync;
|
||||
void* pParam;
|
||||
__tmq_askep_fn_t pUserFn;
|
||||
} SMqAskEpCbParam;
|
||||
|
||||
typedef struct {
|
||||
|
|
@ -252,13 +252,12 @@ typedef struct SSyncCommitInfo {
|
|||
int32_t code;
|
||||
} SSyncCommitInfo;
|
||||
|
||||
static int32_t doAskEp(tmq_t* tmq);
|
||||
static int32_t syncAskEp(tmq_t* tmq);
|
||||
static int32_t makeTopicVgroupKey(char* dst, const char* topicName, int32_t vg);
|
||||
static int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet);
|
||||
static int32_t doSendCommitMsg(tmq_t* tmq, int32_t vgId, SEpSet* epSet, STqOffsetVal* offset, const char* pTopicName, SMqCommitCbParamSet* pParamSet);
|
||||
static void commitRspCountDown(SMqCommitCbParamSet* pParamSet, int64_t consumerId, const char* pTopic, int32_t vgId);
|
||||
static void asyncAskEp(tmq_t* pTmq, __tmq_askep_fn_t askEpFn, void* param);
|
||||
static void addToQueueCallbackFn(tmq_t* pTmq, int32_t code, SDataBuf* pDataBuf, void* param);
|
||||
static void askEp(tmq_t* pTmq, void* param, bool sync, bool updateEpset);
|
||||
|
||||
tmq_conf_t* tmq_conf_new() {
|
||||
tmq_conf_t* conf = taosMemoryCalloc(1, sizeof(tmq_conf_t));
|
||||
|
|
@ -848,7 +847,7 @@ int32_t tmqHandleAllDelayedTask(tmq_t* pTmq) {
|
|||
|
||||
while (pTaskType != NULL) {
|
||||
if (*pTaskType == TMQ_DELAYED_TASK__ASK_EP) {
|
||||
asyncAskEp(pTmq, addToQueueCallbackFn, NULL);
|
||||
askEp(pTmq, NULL, false, false);
|
||||
|
||||
int64_t* pRefId = taosMemoryMalloc(sizeof(int64_t));
|
||||
*pRefId = pTmq->refId;
|
||||
|
|
@ -865,7 +864,8 @@ int32_t tmqHandleAllDelayedTask(tmq_t* pTmq) {
|
|||
tscDebug("consumer:0x%" PRIx64 " next commit to vnode(s) in %.2fs", pTmq->consumerId,
|
||||
pTmq->autoCommitInterval / 1000.0);
|
||||
taosTmrReset(tmqAssignDelayedCommitTask, pTmq->autoCommitInterval, pRefId, tmqMgmt.timer, &pTmq->commitTimer);
|
||||
} else if (*pTaskType == TMQ_DELAYED_TASK__REPORT) {
|
||||
} else {
|
||||
tscError("consumer:0x%" PRIx64 " invalid task type:%d", pTmq->consumerId, *pTaskType);
|
||||
}
|
||||
|
||||
taosFreeQitem(pTaskType);
|
||||
|
|
@ -876,10 +876,8 @@ int32_t tmqHandleAllDelayedTask(tmq_t* pTmq) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
static void* tmqFreeRspWrapper(SMqRspWrapper* rspWrapper) {
|
||||
if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__END_RSP) {
|
||||
// do nothing
|
||||
} else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__EP_RSP) {
|
||||
static void tmqFreeRspWrapper(SMqRspWrapper* rspWrapper) {
|
||||
if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__EP_RSP) {
|
||||
SMqAskEpRspWrapper* pEpRspWrapper = (SMqAskEpRspWrapper*)rspWrapper;
|
||||
tDeleteSMqAskEpRsp(&pEpRspWrapper->msg);
|
||||
} else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_RSP) {
|
||||
|
|
@ -907,8 +905,6 @@ static void* tmqFreeRspWrapper(SMqRspWrapper* rspWrapper) {
|
|||
taosArrayDestroy(pRsp->taosxRsp.createTableLen);
|
||||
taosArrayDestroyP(pRsp->taosxRsp.createTableReq, taosMemoryFree);
|
||||
}
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
void tmqClearUnhandleMsg(tmq_t* tmq) {
|
||||
|
|
@ -1222,7 +1218,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
|
|||
}
|
||||
|
||||
int32_t retryCnt = 0;
|
||||
while (TSDB_CODE_MND_CONSUMER_NOT_READY == doAskEp(tmq)) {
|
||||
while (TSDB_CODE_MND_CONSUMER_NOT_READY == syncAskEp(tmq)) {
|
||||
if (retryCnt++ > MAX_RETRY_COUNT) {
|
||||
tscError("consumer:0x%" PRIx64 ", mnd not ready for subscribe, retry more than 2 minutes", tmq->consumerId);
|
||||
code = TSDB_CODE_MND_CONSUMER_NOT_READY;
|
||||
|
|
@ -1305,33 +1301,19 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
|
|||
uint64_t requestId = pParam->requestId;
|
||||
tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId);
|
||||
if (tmq == NULL) {
|
||||
taosMemoryFree(pParam);
|
||||
taosMemoryFreeClear(pMsg->pData);
|
||||
taosMemoryFreeClear(pMsg->pEpSet);
|
||||
terrno = TSDB_CODE_TMQ_CONSUMER_CLOSED;
|
||||
return -1;
|
||||
code = TSDB_CODE_TMQ_CONSUMER_CLOSED;
|
||||
goto FAIL;
|
||||
}
|
||||
|
||||
if (code != 0) {
|
||||
if (code == TSDB_CODE_TMQ_CONSUMER_MISMATCH) {
|
||||
atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__RECOVER);
|
||||
tscDebug("consumer:0x%" PRIx64 " wait for the re-balance, set status to be RECOVER",
|
||||
tmq->consumerId);
|
||||
} else if (code == TSDB_CODE_TQ_NO_COMMITTED_OFFSET) {
|
||||
SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper), DEF_QITEM, 0);
|
||||
if (pRspWrapper == NULL) {
|
||||
tscWarn("consumer:0x%" PRIx64 " msg from vgId:%d discarded, since out of memory, reqId:0x%" PRIx64,
|
||||
tmq->consumerId, vgId, requestId);
|
||||
goto END;
|
||||
}
|
||||
|
||||
pRspWrapper->tmqRspType = TMQ_MSG_TYPE__END_RSP;
|
||||
taosWriteQitem(tmq->mqueue, pRspWrapper);
|
||||
} else{
|
||||
tscError("consumer:0x%" PRIx64 " msg from vgId:%d discarded, since %s, reqId:0x%" PRIx64, tmq->consumerId,
|
||||
vgId, tstrerror(code), requestId);
|
||||
}
|
||||
SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper), DEF_QITEM, 0);
|
||||
if (pRspWrapper == NULL) {
|
||||
tscWarn("consumer:0x%" PRIx64 " msg discard from vgId:%d, since out of memory", tmq->consumerId, vgId);
|
||||
taosReleaseRef(tmqMgmt.rsetId, refId);
|
||||
code = TSDB_CODE_OUT_OF_MEMORY;
|
||||
goto FAIL;
|
||||
}
|
||||
|
||||
if(code != 0){
|
||||
goto END;
|
||||
}
|
||||
|
||||
|
|
@ -1340,30 +1322,19 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
|
|||
if (msgEpoch < clientEpoch) {
|
||||
// do not write into queue since updating epoch reset
|
||||
tscWarn("consumer:0x%" PRIx64
|
||||
" msg discard from vgId:%d since from earlier epoch, rsp epoch %d, current epoch %d, reqId:0x%" PRIx64,
|
||||
" msg discard from vgId:%d since from earlier epoch, rsp epoch %d, current epoch %d, reqId:0x%" PRIx64,
|
||||
tmq->consumerId, vgId, msgEpoch, clientEpoch, requestId);
|
||||
code = -1;
|
||||
goto END;
|
||||
}
|
||||
|
||||
ASSERT(msgEpoch == clientEpoch);
|
||||
|
||||
// handle meta rsp
|
||||
int8_t rspType = ((SMqRspHead*)pMsg->pData)->mqMsgType;
|
||||
|
||||
SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper), DEF_QITEM, 0);
|
||||
if (pRspWrapper == NULL) {
|
||||
tscWarn("consumer:0x%" PRIx64 " msg discard from vgId:%d, since out of memory", tmq->consumerId, vgId);
|
||||
code = TSDB_CODE_OUT_OF_MEMORY;
|
||||
goto END;
|
||||
}
|
||||
|
||||
pRspWrapper->tmqRspType = rspType;
|
||||
pRspWrapper->reqId = requestId;
|
||||
pRspWrapper->pEpset = pMsg->pEpSet;
|
||||
pMsg->pEpSet = NULL;
|
||||
pRspWrapper->vgId = vgId;
|
||||
strcpy(pRspWrapper->topicName, pParam->topicName);
|
||||
|
||||
if (rspType == TMQ_MSG_TYPE__POLL_DATA_RSP) {
|
||||
SDecoder decoder;
|
||||
|
|
@ -1392,22 +1363,22 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
|
|||
tscError("consumer:0x%" PRIx64 " invalid rsp msg received, type:%d ignored", tmq->consumerId, rspType);
|
||||
}
|
||||
|
||||
END:
|
||||
pRspWrapper->code = code;
|
||||
pRspWrapper->vgId = vgId;
|
||||
strcpy(pRspWrapper->topicName, pParam->topicName);
|
||||
taosWriteQitem(tmq->mqueue, pRspWrapper);
|
||||
|
||||
int32_t total = taosQueueItemSize(tmq->mqueue);
|
||||
tscDebug("consumer:0x%" PRIx64 " put poll res into mqueue, type:%d, vgId:%d, total in queue:%d, reqId:0x%" PRIx64,
|
||||
tmq->consumerId, rspType, vgId, total, requestId);
|
||||
|
||||
END:
|
||||
if(code != 0){
|
||||
setVgIdle(tmq, pParam->topicName, vgId);
|
||||
}
|
||||
|
||||
tsem_post(&tmq->rspSem);
|
||||
taosReleaseRef(tmqMgmt.rsetId, refId);
|
||||
|
||||
FAIL:
|
||||
tsem_post(&tmq->rspSem);
|
||||
taosMemoryFree(pParam);
|
||||
taosMemoryFreeClear(pMsg->pData);
|
||||
taosMemoryFreeClear(pMsg->pEpSet);
|
||||
if(pMsg) taosMemoryFreeClear(pMsg->pData);
|
||||
if(pMsg) taosMemoryFreeClear(pMsg->pEpSet);
|
||||
|
||||
return code;
|
||||
}
|
||||
|
|
@ -1444,6 +1415,8 @@ static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopic
|
|||
STqOffsetVal offsetNew = {0};
|
||||
offsetNew.type = tmq->resetOffsetCfg;
|
||||
|
||||
tscInfo("consumer:0x%" PRIx64 ", update topic:%s, new numOfVgs:%d, num:%d, port:%d", tmq->consumerId, pTopic->topicName, vgNumGet, pVgEp->epSet.numOfEps,pVgEp->epSet.eps[pVgEp->epSet.inUse].port);
|
||||
|
||||
SMqClientVg clientVg = {
|
||||
.pollCnt = 0,
|
||||
.vgId = pVgEp->vgId,
|
||||
|
|
@ -1478,7 +1451,9 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp)
|
|||
bool set = false;
|
||||
|
||||
int32_t topicNumGet = taosArrayGetSize(pRsp->topics);
|
||||
if (epoch <= tmq->epoch) {
|
||||
if (topicNumGet <= 0 && epoch <= tmq->epoch) {
|
||||
tscInfo("consumer:0x%" PRIx64 " no update ep epoch from %d to epoch %d, incoming topics:%d",
|
||||
tmq->consumerId, tmq->epoch, epoch, topicNumGet);
|
||||
return false;
|
||||
}
|
||||
|
||||
|
|
@ -1546,48 +1521,6 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp)
|
|||
return set;
|
||||
}
|
||||
|
||||
int32_t askEpCallbackFn(void* param, SDataBuf* pMsg, int32_t code) {
|
||||
SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param;
|
||||
tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, pParam->refId);
|
||||
if (tmq == NULL) {
|
||||
code = TSDB_CODE_TMQ_CONSUMER_CLOSED;
|
||||
goto END;
|
||||
}
|
||||
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
tscError("consumer:0x%" PRIx64 ", get topic endpoint error, code:%s", tmq->consumerId, tstrerror(code));
|
||||
taosReleaseRef(tmqMgmt.rsetId, pParam->refId);
|
||||
goto END;
|
||||
}
|
||||
|
||||
SMqRspHead* head = pMsg->pData;
|
||||
int32_t epoch = atomic_load_32(&tmq->epoch);
|
||||
if (head->epoch <= epoch) {
|
||||
tscInfo("consumer:0x%" PRIx64 ", recv ep, msg epoch %d, current epoch %d, no need to update local ep",
|
||||
tmq->consumerId, head->epoch, epoch);
|
||||
|
||||
if (tmq->status == TMQ_CONSUMER_STATUS__RECOVER) {
|
||||
SMqAskEpRsp rsp;
|
||||
tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp);
|
||||
int8_t flag = (taosArrayGetSize(rsp.topics) == 0) ? TMQ_CONSUMER_STATUS__NO_TOPIC : TMQ_CONSUMER_STATUS__READY;
|
||||
atomic_store_8(&tmq->status, flag);
|
||||
tDeleteSMqAskEpRsp(&rsp);
|
||||
}
|
||||
|
||||
} else {
|
||||
tscInfo("consumer:0x%" PRIx64 ", recv ep, msg epoch %d, current epoch %d, update local ep", tmq->consumerId,
|
||||
head->epoch, epoch);
|
||||
}
|
||||
taosReleaseRef(tmqMgmt.rsetId, pParam->refId);
|
||||
|
||||
END:
|
||||
pParam->pUserFn(tmq, code, pMsg, pParam->pParam);
|
||||
taosMemoryFree(pMsg->pEpSet);
|
||||
taosMemoryFree(pMsg->pData);
|
||||
taosMemoryFree(pParam);
|
||||
return code;
|
||||
}
|
||||
|
||||
void tmqBuildConsumeReqImpl(SMqPollReq* pReq, tmq_t* tmq, int64_t timeout, SMqClientTopic* pTopic, SMqClientVg* pVg) {
|
||||
int32_t groupLen = strlen(tmq->groupId);
|
||||
memcpy(pReq->subKey, tmq->groupId, groupLen);
|
||||
|
|
@ -1722,10 +1655,9 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p
|
|||
int64_t transporterId = 0;
|
||||
char offsetFormatBuf[TSDB_OFFSET_LEN] = {0};
|
||||
tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pVg->offsetInfo.endOffset);
|
||||
|
||||
tscDebug("consumer:0x%" PRIx64 " send poll to %s vgId:%d, epoch %d, req:%s, reqId:0x%" PRIx64, pTmq->consumerId,
|
||||
pTopic->topicName, pVg->vgId, pTmq->epoch, offsetFormatBuf, req.reqId);
|
||||
code = asyncSendMsgToServer(pTmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo);
|
||||
tscDebug("consumer:0x%" PRIx64 " send poll to %s vgId:%d, code:%d, epoch %d, req:%s, reqId:0x%" PRIx64, pTmq->consumerId,
|
||||
pTopic->topicName, pVg->vgId, code, pTmq->epoch, offsetFormatBuf, req.reqId);
|
||||
if(code != 0){
|
||||
goto FAIL;
|
||||
}
|
||||
|
|
@ -1735,11 +1667,9 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p
|
|||
pTmq->pollCnt++;
|
||||
|
||||
return 0;
|
||||
|
||||
FAIL:
|
||||
taosMemoryFreeClear(pParam);
|
||||
taosMemoryFreeClear(msg);
|
||||
return code;
|
||||
return tmqPollCb(pParam, NULL, code);
|
||||
}
|
||||
|
||||
// broadcast the poll request to all related vnodes
|
||||
|
|
@ -1776,8 +1706,6 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {
|
|||
atomic_store_32(&pVg->vgSkipCnt, 0);
|
||||
code = doTmqPollImpl(tmq, pTopic, pVg, timeout);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
|
||||
tsem_post(&tmq->rspSem);
|
||||
goto end;
|
||||
}
|
||||
}
|
||||
|
|
@ -1789,22 +1717,6 @@ end:
|
|||
return code;
|
||||
}
|
||||
|
||||
static int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspWrapper* rspWrapper) {
|
||||
if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__EP_RSP) {
|
||||
if (rspWrapper->epoch > atomic_load_32(&tmq->epoch)) {
|
||||
SMqAskEpRspWrapper* pEpRspWrapper = (SMqAskEpRspWrapper*)rspWrapper;
|
||||
SMqAskEpRsp* rspMsg = &pEpRspWrapper->msg;
|
||||
doUpdateLocalEp(tmq, rspWrapper->epoch, rspMsg);
|
||||
tDeleteSMqAskEpRsp(rspMsg);
|
||||
} else {
|
||||
tmqFreeRspWrapper(rspWrapper);
|
||||
}
|
||||
} else {
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void updateVgInfo(SMqClientVg* pVg, STqOffsetVal* reqOffset, STqOffsetVal* rspOffset, int64_t sver, int64_t ever, int64_t consumerId, bool hasData){
|
||||
if (!pVg->seekUpdated) {
|
||||
tscDebug("consumer:0x%" PRIx64" local offset is update, since seekupdate not set", consumerId);
|
||||
|
|
@ -1839,11 +1751,28 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
|
|||
|
||||
tscDebug("consumer:0x%" PRIx64 " handle rsp, type:%d", tmq->consumerId, pRspWrapper->tmqRspType);
|
||||
|
||||
if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__END_RSP) {
|
||||
if (pRspWrapper->code != 0) {
|
||||
SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper;
|
||||
if (pRspWrapper->code == TSDB_CODE_TMQ_CONSUMER_MISMATCH) {
|
||||
atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__RECOVER);
|
||||
tscDebug("consumer:0x%" PRIx64 " wait for the re-balance, set status to be RECOVER", tmq->consumerId);
|
||||
} else if (pRspWrapper->code == TSDB_CODE_TQ_NO_COMMITTED_OFFSET) {
|
||||
terrno = pRspWrapper->code;
|
||||
tscError("consumer:0x%" PRIx64 " unexpected rsp from poll, code:%s", tmq->consumerId, tstrerror(pRspWrapper->code));
|
||||
taosFreeQitem(pRspWrapper);
|
||||
return NULL;
|
||||
} else{
|
||||
if(pRspWrapper->code == TSDB_CODE_VND_INVALID_VGROUP_ID){ // for vnode transform
|
||||
askEp(tmq, NULL, false, true);
|
||||
}
|
||||
tscError("consumer:0x%" PRIx64 " msg from vgId:%d discarded, since %s", tmq->consumerId, pollRspWrapper->vgId, tstrerror(pRspWrapper->code));
|
||||
taosWLockLatch(&tmq->lock);
|
||||
SMqClientVg* pVg = getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId);
|
||||
pVg->emptyBlockReceiveTs = taosGetTimestampMs();
|
||||
taosWUnLockLatch(&tmq->lock);
|
||||
}
|
||||
setVgIdle(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId);
|
||||
taosFreeQitem(pRspWrapper);
|
||||
terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET;
|
||||
tscError("consumer:0x%" PRIx64 " unexpected rsp from poll, code:%s", tmq->consumerId, tstrerror(terrno));
|
||||
return NULL;
|
||||
} else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_RSP) {
|
||||
SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper;
|
||||
|
||||
|
|
@ -1878,9 +1807,10 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
|
|||
tscDebug("consumer:0x%" PRIx64 " empty block received, vgId:%d, offset:%s, vg total:%" PRId64
|
||||
", total:%" PRId64 ", reqId:0x%" PRIx64,
|
||||
tmq->consumerId, pVg->vgId, buf, pVg->numOfRows, tmq->totalRows, pollRspWrapper->reqId);
|
||||
pRspWrapper = tmqFreeRspWrapper(pRspWrapper);
|
||||
pVg->emptyBlockReceiveTs = taosGetTimestampMs();
|
||||
taosFreeQitem(pollRspWrapper);
|
||||
tmqFreeRspWrapper(pRspWrapper);
|
||||
taosFreeQitem(pRspWrapper);
|
||||
taosWUnLockLatch(&tmq->lock);
|
||||
} else { // build rsp
|
||||
int64_t numOfRows = 0;
|
||||
SMqRspObj* pRsp = tmqBuildRspFromWrapper(pollRspWrapper, pVg, &numOfRows);
|
||||
|
|
@ -1890,20 +1820,18 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
|
|||
", vg total:%" PRId64 ", total:%" PRId64 ", reqId:0x%" PRIx64,
|
||||
tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, numOfRows, pVg->numOfRows, tmq->totalRows,
|
||||
pollRspWrapper->reqId);
|
||||
taosFreeQitem(pollRspWrapper);
|
||||
taosFreeQitem(pRspWrapper);
|
||||
taosWUnLockLatch(&tmq->lock);
|
||||
return pRsp;
|
||||
}
|
||||
taosWUnLockLatch(&tmq->lock);
|
||||
} else {
|
||||
tscInfo("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d",
|
||||
tmq->consumerId, pollRspWrapper->vgId, pDataRsp->head.epoch, consumerEpoch);
|
||||
setVgIdle(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId);
|
||||
pRspWrapper = tmqFreeRspWrapper(pRspWrapper);
|
||||
taosFreeQitem(pollRspWrapper);
|
||||
tmqFreeRspWrapper(pRspWrapper);
|
||||
taosFreeQitem(pRspWrapper);
|
||||
}
|
||||
} else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_META_RSP) {
|
||||
// todo handle the wal range and epset for each vgroup
|
||||
SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper;
|
||||
int32_t consumerEpoch = atomic_load_32(&tmq->epoch);
|
||||
|
||||
|
|
@ -1924,15 +1852,15 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
|
|||
updateVgInfo(pVg, &pollRspWrapper->metaRsp.rspOffset, &pollRspWrapper->metaRsp.rspOffset, pollRspWrapper->metaRsp.head.walsver, pollRspWrapper->metaRsp.head.walever, tmq->consumerId, true);
|
||||
// build rsp
|
||||
SMqMetaRspObj* pRsp = tmqBuildMetaRspFromWrapper(pollRspWrapper);
|
||||
taosFreeQitem(pollRspWrapper);
|
||||
taosFreeQitem(pRspWrapper);
|
||||
taosWUnLockLatch(&tmq->lock);
|
||||
return pRsp;
|
||||
} else {
|
||||
tscInfo("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d",
|
||||
tmq->consumerId, pollRspWrapper->vgId, pollRspWrapper->metaRsp.head.epoch, consumerEpoch);
|
||||
setVgIdle(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId);
|
||||
pRspWrapper = tmqFreeRspWrapper(pRspWrapper);
|
||||
taosFreeQitem(pollRspWrapper);
|
||||
tmqFreeRspWrapper(pRspWrapper);
|
||||
taosFreeQitem(pRspWrapper);
|
||||
}
|
||||
} else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP) {
|
||||
SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper;
|
||||
|
|
@ -1956,8 +1884,8 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
|
|||
tscDebug("consumer:0x%" PRIx64 " taosx empty block received, vgId:%d, vg total:%" PRId64 ", reqId:0x%" PRIx64,
|
||||
tmq->consumerId, pVg->vgId, pVg->numOfRows, pollRspWrapper->reqId);
|
||||
pVg->emptyBlockReceiveTs = taosGetTimestampMs();
|
||||
pRspWrapper = tmqFreeRspWrapper(pRspWrapper);
|
||||
taosFreeQitem(pollRspWrapper);
|
||||
tmqFreeRspWrapper(pRspWrapper);
|
||||
taosFreeQitem(pRspWrapper);
|
||||
taosWUnLockLatch(&tmq->lock);
|
||||
continue;
|
||||
} else {
|
||||
|
|
@ -1982,20 +1910,25 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
|
|||
tmq->consumerId, pVg->vgId, buf, pollRspWrapper->dataRsp.blockNum, numOfRows, pVg->numOfRows,
|
||||
tmq->totalRows, pollRspWrapper->reqId);
|
||||
|
||||
taosFreeQitem(pollRspWrapper);
|
||||
taosFreeQitem(pRspWrapper);
|
||||
taosWUnLockLatch(&tmq->lock);
|
||||
return pRsp;
|
||||
} else {
|
||||
tscInfo("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d",
|
||||
tmq->consumerId, pollRspWrapper->vgId, pollRspWrapper->taosxRsp.head.epoch, consumerEpoch);
|
||||
setVgIdle(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId);
|
||||
pRspWrapper = tmqFreeRspWrapper(pRspWrapper);
|
||||
taosFreeQitem(pollRspWrapper);
|
||||
tmqFreeRspWrapper(pRspWrapper);
|
||||
taosFreeQitem(pRspWrapper);
|
||||
}
|
||||
} else {
|
||||
tscDebug("consumer:0x%" PRIx64 " not data msg received", tmq->consumerId);
|
||||
tmqHandleNoPollRsp(tmq, pRspWrapper);
|
||||
} else if(pRspWrapper->tmqRspType == TMQ_MSG_TYPE__EP_RSP){
|
||||
tscDebug("consumer:0x%" PRIx64 " ep msg received", tmq->consumerId);
|
||||
SMqAskEpRspWrapper* pEpRspWrapper = (SMqAskEpRspWrapper*)pRspWrapper;
|
||||
SMqAskEpRsp* rspMsg = &pEpRspWrapper->msg;
|
||||
doUpdateLocalEp(tmq, pEpRspWrapper->epoch, rspMsg);
|
||||
tmqFreeRspWrapper(pRspWrapper);
|
||||
taosFreeQitem(pRspWrapper);
|
||||
} else {
|
||||
tscError("consumer:0x%" PRIx64 " invalid msg received:%d", tmq->consumerId, pRspWrapper->tmqRspType);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -2018,7 +1951,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) {
|
|||
|
||||
while (atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__RECOVER) {
|
||||
int32_t retryCnt = 0;
|
||||
while (TSDB_CODE_MND_CONSUMER_NOT_READY == doAskEp(tmq)) {
|
||||
while (TSDB_CODE_MND_CONSUMER_NOT_READY == syncAskEp(tmq)) {
|
||||
if (retryCnt++ > 40) {
|
||||
return NULL;
|
||||
}
|
||||
|
|
@ -2407,55 +2340,64 @@ end:
|
|||
}
|
||||
}
|
||||
|
||||
void defaultAskEpCb(tmq_t* pTmq, int32_t code, SDataBuf* pDataBuf, void* param) {
|
||||
SAskEpInfo* pInfo = param;
|
||||
pInfo->code = code;
|
||||
int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) {
|
||||
SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param;
|
||||
tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, pParam->refId);
|
||||
if (tmq == NULL) {
|
||||
code = TSDB_CODE_TMQ_CONSUMER_CLOSED;
|
||||
goto FAIL;
|
||||
}
|
||||
|
||||
if (pTmq == NULL || code != TSDB_CODE_SUCCESS){
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
tscError("consumer:0x%" PRIx64 ", get topic endpoint error, code:%s", tmq->consumerId, tstrerror(code));
|
||||
goto END;
|
||||
}
|
||||
|
||||
SMqRspHead* head = pDataBuf->pData;
|
||||
SMqAskEpRsp rsp;
|
||||
tDecodeSMqAskEpRsp(POINTER_SHIFT(pDataBuf->pData, sizeof(SMqRspHead)), &rsp);
|
||||
doUpdateLocalEp(pTmq, head->epoch, &rsp);
|
||||
tDeleteSMqAskEpRsp(&rsp);
|
||||
SMqRspHead* head = pMsg->pData;
|
||||
int32_t epoch = atomic_load_32(&tmq->epoch);
|
||||
tscInfo("consumer:0x%" PRIx64 ", recv ep, msg epoch %d, current epoch %d", tmq->consumerId,
|
||||
head->epoch, epoch);
|
||||
if(pParam->sync){
|
||||
SMqAskEpRsp rsp = {0};
|
||||
tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp);
|
||||
doUpdateLocalEp(tmq, head->epoch, &rsp);
|
||||
tDeleteSMqAskEpRsp(&rsp);
|
||||
}else{
|
||||
SMqAskEpRspWrapper* pWrapper = taosAllocateQitem(sizeof(SMqAskEpRspWrapper), DEF_QITEM, 0);
|
||||
if (pWrapper == NULL) {
|
||||
code = TSDB_CODE_OUT_OF_MEMORY;
|
||||
goto END;
|
||||
}
|
||||
|
||||
pWrapper->tmqRspType = TMQ_MSG_TYPE__EP_RSP;
|
||||
pWrapper->epoch = head->epoch;
|
||||
memcpy(&pWrapper->msg, pMsg->pData, sizeof(SMqRspHead));
|
||||
tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pWrapper->msg);
|
||||
|
||||
taosWriteQitem(tmq->mqueue, pWrapper);
|
||||
}
|
||||
|
||||
END:
|
||||
tsem_post(&pInfo->sem);
|
||||
taosReleaseRef(tmqMgmt.rsetId, pParam->refId);
|
||||
|
||||
FAIL:
|
||||
if(pParam->sync){
|
||||
SAskEpInfo* pInfo = pParam->pParam;
|
||||
pInfo->code = code;
|
||||
tsem_post(&pInfo->sem);
|
||||
}
|
||||
|
||||
taosMemoryFree(pMsg->pEpSet);
|
||||
taosMemoryFree(pMsg->pData);
|
||||
taosMemoryFree(pParam);
|
||||
return code;
|
||||
}
|
||||
|
||||
void addToQueueCallbackFn(tmq_t* pTmq, int32_t code, SDataBuf* pDataBuf, void* param) {
|
||||
if (pTmq == NULL){
|
||||
terrno = TSDB_CODE_TMQ_CONSUMER_CLOSED;
|
||||
return;
|
||||
}
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
terrno = code;
|
||||
return;
|
||||
}
|
||||
|
||||
SMqAskEpRspWrapper* pWrapper = taosAllocateQitem(sizeof(SMqAskEpRspWrapper), DEF_QITEM, 0);
|
||||
if (pWrapper == NULL) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return;
|
||||
}
|
||||
|
||||
SMqRspHead* head = pDataBuf->pData;
|
||||
|
||||
pWrapper->tmqRspType = TMQ_MSG_TYPE__EP_RSP;
|
||||
pWrapper->epoch = head->epoch;
|
||||
memcpy(&pWrapper->msg, pDataBuf->pData, sizeof(SMqRspHead));
|
||||
tDecodeSMqAskEpRsp(POINTER_SHIFT(pDataBuf->pData, sizeof(SMqRspHead)), &pWrapper->msg);
|
||||
|
||||
taosWriteQitem(pTmq->mqueue, pWrapper);
|
||||
}
|
||||
|
||||
int32_t doAskEp(tmq_t* pTmq) {
|
||||
int32_t syncAskEp(tmq_t* pTmq) {
|
||||
SAskEpInfo* pInfo = taosMemoryMalloc(sizeof(SAskEpInfo));
|
||||
tsem_init(&pInfo->sem, 0, 0);
|
||||
|
||||
asyncAskEp(pTmq, defaultAskEpCb, pInfo);
|
||||
askEp(pTmq, pInfo, true, false);
|
||||
tsem_wait(&pInfo->sem);
|
||||
|
||||
int32_t code = pInfo->code;
|
||||
|
|
@ -2464,10 +2406,10 @@ int32_t doAskEp(tmq_t* pTmq) {
|
|||
return code;
|
||||
}
|
||||
|
||||
void asyncAskEp(tmq_t* pTmq, __tmq_askep_fn_t askEpFn, void* param) {
|
||||
void askEp(tmq_t* pTmq, void* param, bool sync, bool updateEpSet) {
|
||||
SMqAskEpReq req = {0};
|
||||
req.consumerId = pTmq->consumerId;
|
||||
req.epoch = pTmq->epoch;
|
||||
req.epoch = updateEpSet ? -1 :pTmq->epoch;
|
||||
strcpy(req.cgroup, pTmq->groupId);
|
||||
int code = 0;
|
||||
SMqAskEpCbParam* pParam = NULL;
|
||||
|
|
@ -2501,7 +2443,7 @@ void asyncAskEp(tmq_t* pTmq, __tmq_askep_fn_t askEpFn, void* param) {
|
|||
}
|
||||
|
||||
pParam->refId = pTmq->refId;
|
||||
pParam->pUserFn = askEpFn;
|
||||
pParam->sync = sync;
|
||||
pParam->pParam = param;
|
||||
|
||||
SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
|
||||
|
|
@ -2515,7 +2457,7 @@ void asyncAskEp(tmq_t* pTmq, __tmq_askep_fn_t askEpFn, void* param) {
|
|||
sendInfo->requestId = generateRequestId();
|
||||
sendInfo->requestObjRefId = 0;
|
||||
sendInfo->param = pParam;
|
||||
sendInfo->fp = askEpCallbackFn;
|
||||
sendInfo->fp = askEpCb;
|
||||
sendInfo->msgType = TDMT_MND_TMQ_ASK_EP;
|
||||
|
||||
SEpSet epSet = getEpSet_s(&pTmq->pTscObj->pAppInfo->mgmtEp);
|
||||
|
|
@ -2530,7 +2472,7 @@ void asyncAskEp(tmq_t* pTmq, __tmq_askep_fn_t askEpFn, void* param) {
|
|||
FAIL:
|
||||
taosMemoryFreeClear(pParam);
|
||||
taosMemoryFreeClear(pReq);
|
||||
askEpFn(pTmq, code, NULL, param);
|
||||
askEpCb(pParam, NULL, code);
|
||||
}
|
||||
|
||||
int32_t makeTopicVgroupKey(char* dst, const char* topicName, int32_t vg) {
|
||||
|
|
|
|||
|
|
@ -478,7 +478,7 @@ static int32_t taosAddSystemCfg(SConfig *pCfg) {
|
|||
if (cfgAddTimezone(pCfg, "timezone", tsTimezoneStr, CFG_SCOPE_BOTH) != 0) return -1;
|
||||
if (cfgAddLocale(pCfg, "locale", tsLocale, CFG_SCOPE_BOTH) != 0) return -1;
|
||||
if (cfgAddCharset(pCfg, "charset", tsCharset, CFG_SCOPE_BOTH) != 0) return -1;
|
||||
if (cfgAddBool(pCfg, "assert", 1, CFG_SCOPE_BOTH) != 0) return -1;
|
||||
if (cfgAddBool(pCfg, "assert", tsAssert, CFG_SCOPE_BOTH) != 0) return -1;
|
||||
if (cfgAddBool(pCfg, "enableCoreFile", 1, CFG_SCOPE_BOTH) != 0) return -1;
|
||||
if (cfgAddFloat(pCfg, "numOfCores", tsNumOfCores, 1, 100000, CFG_SCOPE_BOTH) != 0) return -1;
|
||||
|
||||
|
|
|
|||
|
|
@ -5430,43 +5430,7 @@ int32_t tDeserializeSServerStatusRsp(void *buf, int32_t bufLen, SServerStatusRsp
|
|||
tDecoderClear(&decoder);
|
||||
return 0;
|
||||
}
|
||||
int32_t tEncodeSMqOffset(SEncoder *encoder, const SMqOffset *pOffset) {
|
||||
if (tEncodeI32(encoder, pOffset->vgId) < 0) return -1;
|
||||
if (tEncodeI64(encoder, pOffset->offset) < 0) return -1;
|
||||
if (tEncodeCStr(encoder, pOffset->topicName) < 0) return -1;
|
||||
if (tEncodeCStr(encoder, pOffset->cgroup) < 0) return -1;
|
||||
return encoder->pos;
|
||||
}
|
||||
|
||||
int32_t tDecodeSMqOffset(SDecoder *decoder, SMqOffset *pOffset) {
|
||||
if (tDecodeI32(decoder, &pOffset->vgId) < 0) return -1;
|
||||
if (tDecodeI64(decoder, &pOffset->offset) < 0) return -1;
|
||||
if (tDecodeCStrTo(decoder, pOffset->topicName) < 0) return -1;
|
||||
if (tDecodeCStrTo(decoder, pOffset->cgroup) < 0) return -1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tEncodeSMqCMCommitOffsetReq(SEncoder *encoder, const SMqCMCommitOffsetReq *pReq) {
|
||||
if (tStartEncode(encoder) < 0) return -1;
|
||||
if (tEncodeI32(encoder, pReq->num) < 0) return -1;
|
||||
for (int32_t i = 0; i < pReq->num; i++) {
|
||||
tEncodeSMqOffset(encoder, &pReq->offsets[i]);
|
||||
}
|
||||
tEndEncode(encoder);
|
||||
return encoder->pos;
|
||||
}
|
||||
|
||||
int32_t tDecodeSMqCMCommitOffsetReq(SDecoder *decoder, SMqCMCommitOffsetReq *pReq) {
|
||||
if (tStartDecode(decoder) < 0) return -1;
|
||||
if (tDecodeI32(decoder, &pReq->num) < 0) return -1;
|
||||
pReq->offsets = (SMqOffset *)tDecoderMalloc(decoder, sizeof(SMqOffset) * pReq->num);
|
||||
if (pReq->offsets == NULL) return -1;
|
||||
for (int32_t i = 0; i < pReq->num; i++) {
|
||||
tDecodeSMqOffset(decoder, &pReq->offsets[i]);
|
||||
}
|
||||
tEndDecode(decoder);
|
||||
return 0;
|
||||
}
|
||||
int32_t tSerializeSExplainRsp(void *buf, int32_t bufLen, SExplainRsp *pRsp) {
|
||||
SEncoder encoder = {0};
|
||||
tEncoderInit(&encoder, buf, bufLen);
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@
|
|||
#define _DEFAULT_SOURCE
|
||||
#include "mndConsumer.h"
|
||||
#include "mndPrivilege.h"
|
||||
#include "mndVgroup.h"
|
||||
#include "mndShow.h"
|
||||
#include "mndSubscribe.h"
|
||||
#include "mndTopic.h"
|
||||
|
|
@ -401,6 +402,9 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) {
|
|||
|
||||
SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, data->topicName);
|
||||
if(pSub == NULL){
|
||||
#ifdef TMQ_DEBUG
|
||||
ASSERT(0);
|
||||
#endif
|
||||
continue;
|
||||
}
|
||||
taosWLockLatch(&pSub->lock);
|
||||
|
|
@ -498,7 +502,9 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) {
|
|||
SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, topic);
|
||||
// txn guarantees pSub is created
|
||||
if(pSub == NULL) {
|
||||
#ifdef TMQ_DEBUG
|
||||
ASSERT(0);
|
||||
#endif
|
||||
continue;
|
||||
}
|
||||
taosRLockLatch(&pSub->lock);
|
||||
|
|
@ -509,7 +515,9 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) {
|
|||
// 2.1 fetch topic schema
|
||||
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic);
|
||||
if(pTopic == NULL) {
|
||||
#ifdef TMQ_DEBUG
|
||||
ASSERT(0);
|
||||
#endif
|
||||
taosRUnLockLatch(&pSub->lock);
|
||||
mndReleaseSubscribe(pMnode, pSub);
|
||||
continue;
|
||||
|
|
@ -542,6 +550,14 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) {
|
|||
SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, j);
|
||||
char offsetKey[TSDB_PARTITION_KEY_LEN];
|
||||
mndMakePartitionKey(offsetKey, pConsumer->cgroup, topic, pVgEp->vgId);
|
||||
|
||||
if(epoch == -1){
|
||||
SVgObj *pVgroup = mndAcquireVgroup(pMnode, pVgEp->vgId);
|
||||
if(pVgroup){
|
||||
pVgEp->epSet = mndGetVgroupEpset(pMnode, pVgroup);
|
||||
mndReleaseVgroup(pMnode, pVgroup);
|
||||
}
|
||||
}
|
||||
// 2.2.1 build vg ep
|
||||
SMqSubVgEp vgEp = {
|
||||
.epSet = pVgEp->epSet,
|
||||
|
|
@ -648,7 +664,7 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
|
|||
}
|
||||
|
||||
// check topic existence
|
||||
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_TOPIC, pMsg, "subscribe");
|
||||
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pMsg, "subscribe");
|
||||
if (pTrans == NULL) {
|
||||
goto _over;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@
|
|||
|
||||
#define _DEFAULT_SOURCE
|
||||
#include "mndStb.h"
|
||||
#include "audit.h"
|
||||
#include "mndDb.h"
|
||||
#include "mndDnode.h"
|
||||
#include "mndIndex.h"
|
||||
|
|
@ -31,7 +32,6 @@
|
|||
#include "mndUser.h"
|
||||
#include "mndVgroup.h"
|
||||
#include "tname.h"
|
||||
#include "audit.h"
|
||||
|
||||
#define STB_VER_NUMBER 1
|
||||
#define STB_RESERVE_SIZE 64
|
||||
|
|
@ -858,7 +858,19 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat
|
|||
}
|
||||
return 0;
|
||||
}
|
||||
static int32_t mndGenIdxNameForFirstTag(char *fullname, char *dbname, char *tagname) {
|
||||
char randStr[24] = {0};
|
||||
|
||||
int8_t start = 8;
|
||||
int8_t end = sizeof(randStr) - 1;
|
||||
// gen rand str len [base:end]
|
||||
// note: ignore rand performance issues
|
||||
int64_t len = taosRand() % (end - start + 1) + start;
|
||||
|
||||
taosRandStr2(randStr, len);
|
||||
sprintf(fullname, "%s.%s_%s", dbname, tagname, randStr);
|
||||
return 0;
|
||||
}
|
||||
static int32_t mndCreateStb(SMnode *pMnode, SRpcMsg *pReq, SMCreateStbReq *pCreate, SDbObj *pDb) {
|
||||
SStbObj stbObj = {0};
|
||||
int32_t code = -1;
|
||||
|
|
@ -871,11 +883,8 @@ static int32_t mndCreateStb(SMnode *pMnode, SRpcMsg *pReq, SMCreateStbReq *pCrea
|
|||
mInfo("trans:%d, used to create stb:%s", pTrans->id, pCreate->name);
|
||||
if (mndBuildStbFromReq(pMnode, &stbObj, pCreate, pDb) != 0) goto _OVER;
|
||||
|
||||
char randStr[24] = {0};
|
||||
taosRandStr2(randStr, tListLen(randStr) - 1);
|
||||
SSchema *pSchema = &(stbObj.pTags[0]);
|
||||
sprintf(fullIdxName, "%s.%s_%s", pDb->name, pSchema->name, randStr);
|
||||
|
||||
mndGenIdxNameForFirstTag(fullIdxName, pDb->name, pSchema->name);
|
||||
SSIdx idx = {0};
|
||||
if (mndAcquireGlobalIdx(pMnode, fullIdxName, SDB_IDX, &idx) == 0 && idx.pIdx != NULL) {
|
||||
terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;
|
||||
|
|
@ -1066,78 +1075,75 @@ static int32_t mndBuildStbFromAlter(SStbObj *pStb, SStbObj *pDst, SMCreateStbReq
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static char* mndAuditFieldTypeStr(int32_t type){
|
||||
switch (type)
|
||||
{
|
||||
case TSDB_DATA_TYPE_NULL:
|
||||
return "null";
|
||||
case TSDB_DATA_TYPE_BOOL:
|
||||
return "bool";
|
||||
case TSDB_DATA_TYPE_TINYINT:
|
||||
return "tinyint";
|
||||
case TSDB_DATA_TYPE_SMALLINT:
|
||||
return "smallint";
|
||||
case TSDB_DATA_TYPE_INT:
|
||||
return "int";
|
||||
case TSDB_DATA_TYPE_BIGINT:
|
||||
return "bigint";
|
||||
case TSDB_DATA_TYPE_FLOAT:
|
||||
return "float";
|
||||
case TSDB_DATA_TYPE_DOUBLE:
|
||||
return "double";
|
||||
case TSDB_DATA_TYPE_VARCHAR:
|
||||
return "varchar";
|
||||
case TSDB_DATA_TYPE_TIMESTAMP:
|
||||
return "timestamp";
|
||||
case TSDB_DATA_TYPE_NCHAR:
|
||||
return "nchar";
|
||||
case TSDB_DATA_TYPE_UTINYINT:
|
||||
return "utinyint";
|
||||
case TSDB_DATA_TYPE_USMALLINT:
|
||||
return "usmallint";
|
||||
case TSDB_DATA_TYPE_UINT:
|
||||
return "uint";
|
||||
case TSDB_DATA_TYPE_UBIGINT:
|
||||
return "ubigint";
|
||||
case TSDB_DATA_TYPE_JSON:
|
||||
return "json";
|
||||
case TSDB_DATA_TYPE_VARBINARY:
|
||||
return "varbinary";
|
||||
case TSDB_DATA_TYPE_DECIMAL:
|
||||
return "decimal";
|
||||
case TSDB_DATA_TYPE_BLOB:
|
||||
return "blob";
|
||||
case TSDB_DATA_TYPE_MEDIUMBLOB:
|
||||
return "mediumblob";
|
||||
case TSDB_DATA_TYPE_GEOMETRY:
|
||||
return "geometry";
|
||||
static char *mndAuditFieldTypeStr(int32_t type) {
|
||||
switch (type) {
|
||||
case TSDB_DATA_TYPE_NULL:
|
||||
return "null";
|
||||
case TSDB_DATA_TYPE_BOOL:
|
||||
return "bool";
|
||||
case TSDB_DATA_TYPE_TINYINT:
|
||||
return "tinyint";
|
||||
case TSDB_DATA_TYPE_SMALLINT:
|
||||
return "smallint";
|
||||
case TSDB_DATA_TYPE_INT:
|
||||
return "int";
|
||||
case TSDB_DATA_TYPE_BIGINT:
|
||||
return "bigint";
|
||||
case TSDB_DATA_TYPE_FLOAT:
|
||||
return "float";
|
||||
case TSDB_DATA_TYPE_DOUBLE:
|
||||
return "double";
|
||||
case TSDB_DATA_TYPE_VARCHAR:
|
||||
return "varchar";
|
||||
case TSDB_DATA_TYPE_TIMESTAMP:
|
||||
return "timestamp";
|
||||
case TSDB_DATA_TYPE_NCHAR:
|
||||
return "nchar";
|
||||
case TSDB_DATA_TYPE_UTINYINT:
|
||||
return "utinyint";
|
||||
case TSDB_DATA_TYPE_USMALLINT:
|
||||
return "usmallint";
|
||||
case TSDB_DATA_TYPE_UINT:
|
||||
return "uint";
|
||||
case TSDB_DATA_TYPE_UBIGINT:
|
||||
return "ubigint";
|
||||
case TSDB_DATA_TYPE_JSON:
|
||||
return "json";
|
||||
case TSDB_DATA_TYPE_VARBINARY:
|
||||
return "varbinary";
|
||||
case TSDB_DATA_TYPE_DECIMAL:
|
||||
return "decimal";
|
||||
case TSDB_DATA_TYPE_BLOB:
|
||||
return "blob";
|
||||
case TSDB_DATA_TYPE_MEDIUMBLOB:
|
||||
return "mediumblob";
|
||||
case TSDB_DATA_TYPE_GEOMETRY:
|
||||
return "geometry";
|
||||
|
||||
default:
|
||||
return "error";
|
||||
default:
|
||||
return "error";
|
||||
}
|
||||
}
|
||||
|
||||
static void mndAuditFieldStr(char* detail, SArray *arr, int32_t len, int32_t max){
|
||||
static void mndAuditFieldStr(char *detail, SArray *arr, int32_t len, int32_t max) {
|
||||
int32_t detialLen = strlen(detail);
|
||||
int32_t fieldLen = 0;
|
||||
for (int32_t i = 0; i < len; ++i) {
|
||||
SField *pField = taosArrayGet(arr, i);
|
||||
char field[TSDB_COL_NAME_LEN + 20] = {0};
|
||||
char field[TSDB_COL_NAME_LEN + 20] = {0};
|
||||
fieldLen = strlen(", ");
|
||||
if(detialLen > 0 && detialLen < max-fieldLen-1) {
|
||||
if (detialLen > 0 && detialLen < max - fieldLen - 1) {
|
||||
strcat(detail, ", ");
|
||||
detialLen += fieldLen;
|
||||
}
|
||||
else{
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
sprintf(field, "%s:%s", pField->name, mndAuditFieldTypeStr(pField->type));
|
||||
fieldLen = strlen(field);
|
||||
if(detialLen < max-fieldLen-1) {
|
||||
if (detialLen < max - fieldLen - 1) {
|
||||
strcat(detail, field);
|
||||
detialLen += fieldLen;
|
||||
}
|
||||
else{
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
|
@ -1252,14 +1258,17 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) {
|
|||
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
|
||||
|
||||
char detail[AUDIT_DETAIL_MAX] = {0};
|
||||
sprintf(detail, "colVer:%d, delay1:%" PRId64 ", delay2:%" PRId64 ", deleteMark1:%" PRId64 ", "
|
||||
"deleteMark2:%" PRId64 ", igExists:%d, numOfColumns:%d, numOfFuncs:%d, numOfTags:%d, "
|
||||
"source:%d, suid:%" PRId64 ", tagVer:%d, ttl:%d, "
|
||||
sprintf(detail,
|
||||
"colVer:%d, delay1:%" PRId64 ", delay2:%" PRId64 ", deleteMark1:%" PRId64
|
||||
", "
|
||||
"deleteMark2:%" PRId64
|
||||
", igExists:%d, numOfColumns:%d, numOfFuncs:%d, numOfTags:%d, "
|
||||
"source:%d, suid:%" PRId64
|
||||
", tagVer:%d, ttl:%d, "
|
||||
"watermark1:%" PRId64 ", watermark2:%" PRId64,
|
||||
createReq.colVer, createReq.delay1, createReq.delay2, createReq.deleteMark1,
|
||||
createReq.deleteMark2, createReq.igExists, createReq.numOfColumns, createReq.numOfFuncs, createReq.numOfTags,
|
||||
createReq.source, createReq.suid, createReq.tagVer, createReq.ttl,
|
||||
createReq.watermark1, createReq.watermark2);
|
||||
createReq.colVer, createReq.delay1, createReq.delay2, createReq.deleteMark1, createReq.deleteMark2,
|
||||
createReq.igExists, createReq.numOfColumns, createReq.numOfFuncs, createReq.numOfTags, createReq.source,
|
||||
createReq.suid, createReq.tagVer, createReq.ttl, createReq.watermark1, createReq.watermark2);
|
||||
|
||||
mndAuditFieldStr(detail, createReq.pColumns, createReq.numOfColumns, AUDIT_DETAIL_MAX);
|
||||
mndAuditFieldStr(detail, createReq.pTags, createReq.numOfTags, AUDIT_DETAIL_MAX);
|
||||
|
|
@ -2338,8 +2347,7 @@ static int32_t mndProcessAlterStbReq(SRpcMsg *pReq) {
|
|||
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
|
||||
|
||||
char detail[2000] = {0};
|
||||
sprintf(detail, "alterType:%d, numOfFields:%d, ttl:%d" ,
|
||||
alterReq.alterType, alterReq.numOfFields, alterReq.ttl);
|
||||
sprintf(detail, "alterType:%d, numOfFields:%d, ttl:%d", alterReq.alterType, alterReq.numOfFields, alterReq.ttl);
|
||||
|
||||
SName name = {0};
|
||||
tNameFromString(&name, alterReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
|
||||
|
|
@ -2608,8 +2616,7 @@ static int32_t mndProcessDropStbReq(SRpcMsg *pReq) {
|
|||
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
|
||||
|
||||
char detail[2000] = {0};
|
||||
sprintf(detail, "igNotExists:%d, source:%d" ,
|
||||
dropReq.igNotExists, dropReq.source);
|
||||
sprintf(detail, "igNotExists:%d, source:%d", dropReq.igNotExists, dropReq.source);
|
||||
|
||||
SName name = {0};
|
||||
tNameFromString(&name, dropReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
|
||||
|
|
@ -3369,7 +3376,7 @@ static int32_t buildSysDbColsInfo(SSDataBlock *p, int8_t buildWhichDBs, char *tb
|
|||
return p->info.rows;
|
||||
}
|
||||
|
||||
static int8_t determineBuildColForWhichDBs(const char* db) {
|
||||
static int8_t determineBuildColForWhichDBs(const char *db) {
|
||||
int8_t buildWhichDBs;
|
||||
if (!db[0])
|
||||
buildWhichDBs = BUILD_COL_FOR_ALL_DB;
|
||||
|
|
@ -3387,11 +3394,11 @@ static int8_t determineBuildColForWhichDBs(const char* db) {
|
|||
}
|
||||
|
||||
static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
|
||||
uint8_t buildWhichDBs;
|
||||
uint8_t buildWhichDBs;
|
||||
SMnode *pMnode = pReq->info.node;
|
||||
SSdb *pSdb = pMnode->pSdb;
|
||||
SStbObj *pStb = NULL;
|
||||
int32_t numOfRows = 0;
|
||||
int32_t numOfRows = 0;
|
||||
|
||||
buildWhichDBs = determineBuildColForWhichDBs(pShow->db);
|
||||
|
||||
|
|
|
|||
|
|
@ -771,6 +771,29 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
static int32_t sendDeleteSubToVnode(SMqSubscribeObj *pSub, STrans *pTrans){
|
||||
// iter all vnode to delete handle
|
||||
int32_t sz = taosArrayGetSize(pSub->unassignedVgs);
|
||||
for (int32_t i = 0; i < sz; i++) {
|
||||
SMqVgEp *pVgEp = taosArrayGetP(pSub->unassignedVgs, i);
|
||||
SMqVDeleteReq *pReq = taosMemoryCalloc(1, sizeof(SMqVDeleteReq));
|
||||
pReq->head.vgId = htonl(pVgEp->vgId);
|
||||
pReq->vgId = pVgEp->vgId;
|
||||
pReq->consumerId = -1;
|
||||
memcpy(pReq->subKey, pSub->key, TSDB_SUBSCRIBE_KEY_LEN);
|
||||
STransAction action = {0};
|
||||
action.epSet = pVgEp->epSet;
|
||||
action.pCont = pReq;
|
||||
action.contLen = sizeof(SMqVDeleteReq);
|
||||
action.msgType = TDMT_VND_TMQ_DELETE_SUB;
|
||||
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
|
||||
taosMemoryFree(pReq);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg) {
|
||||
SMnode *pMnode = pMsg->info.node;
|
||||
SMDropCgroupReq dropReq = {0};
|
||||
|
|
@ -831,6 +854,11 @@ static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg) {
|
|||
|
||||
mInfo("trans:%d, used to drop cgroup:%s on topic %s", pTrans->id, dropReq.cgroup, dropReq.topic);
|
||||
|
||||
code = sendDeleteSubToVnode(pSub, pTrans);
|
||||
if (code != 0) {
|
||||
goto end;
|
||||
}
|
||||
|
||||
if (mndSetDropSubCommitLogs(pMnode, pTrans, pSub) < 0) {
|
||||
mError("cgroup %s on topic:%s, failed to drop since %s", dropReq.cgroup, dropReq.topic, terrstr());
|
||||
code = -1;
|
||||
|
|
@ -847,7 +875,11 @@ end:
|
|||
mndReleaseSubscribe(pMnode, pSub);
|
||||
mndTransDrop(pTrans);
|
||||
|
||||
return code;
|
||||
if (code != 0) {
|
||||
mError("cgroup %s on topic:%s, failed to drop", dropReq.cgroup, dropReq.topic);
|
||||
return code;
|
||||
}
|
||||
return TSDB_CODE_ACTION_IN_PROGRESS;
|
||||
}
|
||||
|
||||
void mndCleanupSubscribe(SMnode *pMnode) {}
|
||||
|
|
@ -1113,25 +1145,10 @@ int32_t mndDropSubByTopic(SMnode *pMnode, STrans *pTrans, const char *topicName)
|
|||
sdbCancelFetch(pSdb, pIter);
|
||||
return -1;
|
||||
}
|
||||
int32_t sz = taosArrayGetSize(pSub->unassignedVgs);
|
||||
for (int32_t i = 0; i < sz; i++) {
|
||||
SMqVgEp *pVgEp = taosArrayGetP(pSub->unassignedVgs, i);
|
||||
SMqVDeleteReq *pReq = taosMemoryCalloc(1, sizeof(SMqVDeleteReq));
|
||||
pReq->head.vgId = htonl(pVgEp->vgId);
|
||||
pReq->vgId = pVgEp->vgId;
|
||||
pReq->consumerId = -1;
|
||||
memcpy(pReq->subKey, pSub->key, TSDB_SUBSCRIBE_KEY_LEN);
|
||||
STransAction action = {0};
|
||||
action.epSet = pVgEp->epSet;
|
||||
action.pCont = pReq;
|
||||
action.contLen = sizeof(SMqVDeleteReq);
|
||||
action.msgType = TDMT_VND_TMQ_DELETE_SUB;
|
||||
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
|
||||
taosMemoryFree(pReq);
|
||||
sdbRelease(pSdb, pSub);
|
||||
sdbCancelFetch(pSdb, pIter);
|
||||
return -1;
|
||||
}
|
||||
if (sendDeleteSubToVnode(pSub, pTrans) != 0) {
|
||||
sdbRelease(pSdb, pSub);
|
||||
sdbCancelFetch(pSdb, pIter);
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (mndSetDropSubRedoLogs(pMnode, pTrans, pSub) < 0) {
|
||||
|
|
|
|||
|
|
@ -485,12 +485,6 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *
|
|||
topicObj.astLen = strlen(pCreate->ast) + 1;
|
||||
}
|
||||
}
|
||||
/*} else if (pCreate->subType == TOPIC_SUB_TYPE__DB) {*/
|
||||
/*topicObj.ast = NULL;*/
|
||||
/*topicObj.astLen = 0;*/
|
||||
/*topicObj.physicalPlan = NULL;*/
|
||||
/*topicObj.withTbName = 1;*/
|
||||
/*topicObj.withSchema = 1;*/
|
||||
|
||||
SSdbRaw *pCommitRaw = mndTopicActionEncode(&topicObj);
|
||||
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
|
||||
|
|
|
|||
|
|
@ -201,9 +201,9 @@ int64_t mndGetIpWhiteVer(SMnode *pMnode) {
|
|||
taosThreadRwlockUnlock(&ipWhiteMgt.rw);
|
||||
mDebug("ip-white-list on mnode ver: %" PRId64 "", ver);
|
||||
|
||||
// if (mndEnableIpWhiteList(pMnode) == 0 || tsEnableWhiteList == false) {
|
||||
// return 0;
|
||||
// }
|
||||
if (mndEnableIpWhiteList(pMnode) == 0 || tsEnableWhiteList == false) {
|
||||
return 0;
|
||||
}
|
||||
return ver;
|
||||
}
|
||||
|
||||
|
|
@ -1277,7 +1277,7 @@ int32_t mndProcessGetUserWhiteListReq(SRpcMsg *pReq) {
|
|||
goto _OVER;
|
||||
}
|
||||
|
||||
// code = mndSetUserWhiteListRsp(pMnode, pUser, &wlRsp);
|
||||
code = mndSetUserWhiteListRsp(pMnode, pUser, &wlRsp);
|
||||
if (code) {
|
||||
goto _OVER;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -65,7 +65,8 @@ set(
|
|||
"src/tq/tqSink.c"
|
||||
"src/tq/tqCommit.c"
|
||||
"src/tq/tqStreamTask.c"
|
||||
"src/tq/tqSnapshot.c"
|
||||
"src/tq/tqHandleSnapshot.c"
|
||||
"src/tq/tqCheckInfoSnapshot.c"
|
||||
"src/tq/tqOffsetSnapshot.c"
|
||||
"src/tq/tqStreamStateSnap.c"
|
||||
"src/tq/tqStreamTaskSnap.c"
|
||||
|
|
|
|||
|
|
@ -88,14 +88,10 @@ typedef struct {
|
|||
int64_t snapshotVer;
|
||||
SWalReader* pWalReader;
|
||||
SWalRef* pRef;
|
||||
// STqPushHandle pushHandle; // push
|
||||
STqExecHandle execHandle; // exec
|
||||
SRpcMsg* msg;
|
||||
tq_handle_status status;
|
||||
} STqHandle;
|
||||
typedef struct {
|
||||
int64_t snapshotVer;
|
||||
} SStreamHandle;
|
||||
|
||||
struct STQ {
|
||||
SVnode* pVnode;
|
||||
|
|
@ -138,7 +134,7 @@ int32_t tqMetaOpen(STQ* pTq);
|
|||
int32_t tqMetaClose(STQ* pTq);
|
||||
int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle);
|
||||
int32_t tqMetaDeleteHandle(STQ* pTq, const char* key);
|
||||
int32_t tqMetaRestoreHandle(STQ* pTq);
|
||||
//int32_t tqMetaRestoreHandle(STQ* pTq);
|
||||
int32_t tqMetaSaveCheckInfo(STQ* pTq, const char* key, const void* value, int32_t vLen);
|
||||
int32_t tqMetaDeleteCheckInfo(STQ* pTq, const char* key);
|
||||
int32_t tqMetaRestoreCheckInfo(STQ* pTq);
|
||||
|
|
|
|||
|
|
@ -69,6 +69,8 @@ typedef struct STqSnapReader STqSnapReader;
|
|||
typedef struct STqSnapWriter STqSnapWriter;
|
||||
typedef struct STqOffsetReader STqOffsetReader;
|
||||
typedef struct STqOffsetWriter STqOffsetWriter;
|
||||
typedef struct STqCheckInfoReader STqCheckInfoReader;
|
||||
typedef struct STqCheckInfoWriter STqCheckInfoWriter;
|
||||
typedef struct SStreamTaskReader SStreamTaskReader;
|
||||
typedef struct SStreamTaskWriter SStreamTaskWriter;
|
||||
typedef struct SStreamStateReader SStreamStateReader;
|
||||
|
|
@ -308,6 +310,14 @@ int32_t tqSnapRead(STqSnapReader* pReader, uint8_t** ppData);
|
|||
int32_t tqSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapWriter** ppWriter);
|
||||
int32_t tqSnapWriterClose(STqSnapWriter** ppWriter, int8_t rollback);
|
||||
int32_t tqSnapWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData);
|
||||
// STqCheckInfoshotReader ==
|
||||
int32_t tqCheckInfoReaderOpen(STQ* pTq, int64_t sver, int64_t ever, STqCheckInfoReader** ppReader);
|
||||
int32_t tqCheckInfoReaderClose(STqCheckInfoReader** ppReader);
|
||||
int32_t tqCheckInfoRead(STqCheckInfoReader* pReader, uint8_t** ppData);
|
||||
// STqCheckInfoshotWriter ======================================
|
||||
int32_t tqCheckInfoWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqCheckInfoWriter** ppWriter);
|
||||
int32_t tqCheckInfoWriterClose(STqCheckInfoWriter** ppWriter, int8_t rollback);
|
||||
int32_t tqCheckInfoWrite(STqCheckInfoWriter* pWriter, uint8_t* pData, uint32_t nData);
|
||||
// STqOffsetReader ========================================
|
||||
int32_t tqOffsetReaderOpen(STQ* pTq, int64_t sver, int64_t ever, STqOffsetReader** ppReader);
|
||||
int32_t tqOffsetReaderClose(STqOffsetReader** ppReader);
|
||||
|
|
@ -503,6 +513,7 @@ enum {
|
|||
SNAP_DATA_STREAM_TASK_CHECKPOINT = 10,
|
||||
SNAP_DATA_STREAM_STATE = 11,
|
||||
SNAP_DATA_STREAM_STATE_BACKEND = 12,
|
||||
SNAP_DATA_TQ_CHECKINFO = 13,
|
||||
};
|
||||
|
||||
struct SSnapDataHdr {
|
||||
|
|
|
|||
|
|
@ -670,7 +670,14 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
|
|||
STqHandle* pHandle = NULL;
|
||||
while (1) {
|
||||
pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey));
|
||||
if (pHandle || tqMetaGetHandle(pTq, req.subKey) < 0) {
|
||||
if (pHandle) {
|
||||
break;
|
||||
}
|
||||
taosRLockLatch(&pTq->lock);
|
||||
ret = tqMetaGetHandle(pTq, req.subKey);
|
||||
taosRUnLockLatch(&pTq->lock);
|
||||
|
||||
if (ret < 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
|
@ -690,7 +697,9 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
|
|||
tqDestroyTqHandle(&handle);
|
||||
goto end;
|
||||
}
|
||||
taosWLockLatch(&pTq->lock);
|
||||
ret = tqMetaSaveHandle(pTq, req.subKey, &handle);
|
||||
taosWUnLockLatch(&pTq->lock);
|
||||
} else {
|
||||
while(1){
|
||||
taosWLockLatch(&pTq->lock);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,196 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "meta.h"
|
||||
#include "tdbInt.h"
|
||||
#include "tq.h"
|
||||
|
||||
// STqCheckInfoReader ========================================
|
||||
struct STqCheckInfoReader {
|
||||
STQ* pTq;
|
||||
int64_t sver;
|
||||
int64_t ever;
|
||||
TBC* pCur;
|
||||
};
|
||||
|
||||
int32_t tqCheckInfoReaderOpen(STQ* pTq, int64_t sver, int64_t ever, STqCheckInfoReader** ppReader) {
|
||||
int32_t code = 0;
|
||||
STqCheckInfoReader* pReader = NULL;
|
||||
|
||||
// alloc
|
||||
pReader = (STqCheckInfoReader*)taosMemoryCalloc(1, sizeof(STqCheckInfoReader));
|
||||
if (pReader == NULL) {
|
||||
code = TSDB_CODE_OUT_OF_MEMORY;
|
||||
goto _err;
|
||||
}
|
||||
pReader->pTq = pTq;
|
||||
pReader->sver = sver;
|
||||
pReader->ever = ever;
|
||||
|
||||
// impl
|
||||
code = tdbTbcOpen(pTq->pCheckStore, &pReader->pCur, NULL);
|
||||
if (code) {
|
||||
taosMemoryFree(pReader);
|
||||
goto _err;
|
||||
}
|
||||
|
||||
code = tdbTbcMoveToFirst(pReader->pCur);
|
||||
if (code) {
|
||||
taosMemoryFree(pReader);
|
||||
goto _err;
|
||||
}
|
||||
|
||||
tqInfo("vgId:%d, vnode checkinfo tq reader opened", TD_VID(pTq->pVnode));
|
||||
|
||||
*ppReader = pReader;
|
||||
return code;
|
||||
|
||||
_err:
|
||||
tqError("vgId:%d, vnode checkinfo tq reader open failed since %s", TD_VID(pTq->pVnode), tstrerror(code));
|
||||
*ppReader = NULL;
|
||||
return code;
|
||||
}
|
||||
|
||||
int32_t tqCheckInfoReaderClose(STqCheckInfoReader** ppReader) {
|
||||
int32_t code = 0;
|
||||
|
||||
tdbTbcClose((*ppReader)->pCur);
|
||||
taosMemoryFree(*ppReader);
|
||||
*ppReader = NULL;
|
||||
|
||||
return code;
|
||||
}
|
||||
|
||||
int32_t tqCheckInfoRead(STqCheckInfoReader* pReader, uint8_t** ppData) {
|
||||
int32_t code = 0;
|
||||
void* pKey = NULL;
|
||||
void* pVal = NULL;
|
||||
int32_t kLen = 0;
|
||||
int32_t vLen = 0;
|
||||
|
||||
if (tdbTbcNext(pReader->pCur, &pKey, &kLen, &pVal, &vLen)) {
|
||||
goto _exit;
|
||||
}
|
||||
|
||||
*ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + vLen);
|
||||
if (*ppData == NULL) {
|
||||
code = TSDB_CODE_OUT_OF_MEMORY;
|
||||
goto _err;
|
||||
}
|
||||
|
||||
SSnapDataHdr* pHdr = (SSnapDataHdr*)(*ppData);
|
||||
pHdr->type = SNAP_DATA_TQ_CHECKINFO;
|
||||
pHdr->size = vLen;
|
||||
memcpy(pHdr->data, pVal, vLen);
|
||||
|
||||
_exit:
|
||||
tdbFree(pKey);
|
||||
tdbFree(pVal);
|
||||
|
||||
tqInfo("vgId:%d, vnode check info tq read data, vLen:%d", TD_VID(pReader->pTq->pVnode), vLen);
|
||||
return code;
|
||||
|
||||
_err:
|
||||
tdbFree(pKey);
|
||||
tdbFree(pVal);
|
||||
|
||||
tqError("vgId:%d, vnode check info tq read data failed since %s", TD_VID(pReader->pTq->pVnode), tstrerror(code));
|
||||
return code;
|
||||
}
|
||||
|
||||
// STqCheckInfoWriter ========================================
|
||||
struct STqCheckInfoWriter {
|
||||
STQ* pTq;
|
||||
int64_t sver;
|
||||
int64_t ever;
|
||||
TXN* txn;
|
||||
};
|
||||
|
||||
int32_t tqCheckInfoWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqCheckInfoWriter** ppWriter) {
|
||||
int32_t code = 0;
|
||||
STqCheckInfoWriter* pWriter;
|
||||
|
||||
// alloc
|
||||
pWriter = (STqCheckInfoWriter*)taosMemoryCalloc(1, sizeof(*pWriter));
|
||||
if (pWriter == NULL) {
|
||||
code = TSDB_CODE_OUT_OF_MEMORY;
|
||||
goto _err;
|
||||
}
|
||||
pWriter->pTq = pTq;
|
||||
pWriter->sver = sver;
|
||||
pWriter->ever = ever;
|
||||
|
||||
if (tdbBegin(pTq->pMetaDB, &pWriter->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) {
|
||||
code = -1;
|
||||
taosMemoryFree(pWriter);
|
||||
goto _err;
|
||||
}
|
||||
|
||||
*ppWriter = pWriter;
|
||||
return code;
|
||||
|
||||
_err:
|
||||
tqError("vgId:%d, tq check info writer open failed since %s", TD_VID(pTq->pVnode), tstrerror(code));
|
||||
*ppWriter = NULL;
|
||||
return code;
|
||||
}
|
||||
|
||||
int32_t tqCheckInfoWriterClose(STqCheckInfoWriter** ppWriter, int8_t rollback) {
|
||||
int32_t code = 0;
|
||||
STqCheckInfoWriter* pWriter = *ppWriter;
|
||||
STQ* pTq = pWriter->pTq;
|
||||
|
||||
if (rollback) {
|
||||
tdbAbort(pWriter->pTq->pMetaDB, pWriter->txn);
|
||||
} else {
|
||||
code = tdbCommit(pWriter->pTq->pMetaDB, pWriter->txn);
|
||||
if (code) goto _err;
|
||||
code = tdbPostCommit(pWriter->pTq->pMetaDB, pWriter->txn);
|
||||
if (code) goto _err;
|
||||
}
|
||||
|
||||
taosMemoryFree(pWriter);
|
||||
*ppWriter = NULL;
|
||||
|
||||
return code;
|
||||
|
||||
_err:
|
||||
tqError("vgId:%d, tq check info writer close failed since %s", TD_VID(pTq->pVnode), tstrerror(code));
|
||||
return code;
|
||||
}
|
||||
|
||||
int32_t tqCheckInfoWrite(STqCheckInfoWriter* pWriter, uint8_t* pData, uint32_t nData) {
|
||||
int32_t code = 0;
|
||||
STQ* pTq = pWriter->pTq;
|
||||
STqCheckInfo info = {0};
|
||||
SDecoder decoder;
|
||||
SDecoder* pDecoder = &decoder;
|
||||
|
||||
tDecoderInit(pDecoder, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
|
||||
code = tDecodeSTqCheckInfo(pDecoder, &info);
|
||||
if (code) goto _err;
|
||||
code = taosHashPut(pTq->pCheckInfo, info.topic, strlen(info.topic), &info, sizeof(STqCheckInfo));
|
||||
if (code) goto _err;
|
||||
code = tqMetaSaveCheckInfo(pTq, info.topic, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
|
||||
if (code) goto _err;
|
||||
tDecoderClear(pDecoder);
|
||||
|
||||
return code;
|
||||
|
||||
_err:
|
||||
tDecoderClear(pDecoder);
|
||||
tqError("vgId:%d, vnode check info tq write failed since %s", TD_VID(pTq->pVnode), tstrerror(code));
|
||||
return code;
|
||||
}
|
||||
|
|
@ -75,29 +75,13 @@ int32_t tqSnapReaderClose(STqSnapReader** ppReader) {
|
|||
|
||||
int32_t tqSnapRead(STqSnapReader* pReader, uint8_t** ppData) {
|
||||
int32_t code = 0;
|
||||
const void* pKey = NULL;
|
||||
const void* pVal = NULL;
|
||||
void* pKey = NULL;
|
||||
void* pVal = NULL;
|
||||
int32_t kLen = 0;
|
||||
int32_t vLen = 0;
|
||||
SDecoder decoder;
|
||||
STqHandle handle;
|
||||
|
||||
*ppData = NULL;
|
||||
for (;;) {
|
||||
if (tdbTbcGet(pReader->pCur, &pKey, &kLen, &pVal, &vLen)) {
|
||||
goto _exit;
|
||||
}
|
||||
|
||||
tDecoderInit(&decoder, (uint8_t*)pVal, vLen);
|
||||
tDecodeSTqHandle(&decoder, &handle);
|
||||
tDecoderClear(&decoder);
|
||||
|
||||
if (handle.snapshotVer <= pReader->sver && handle.snapshotVer >= pReader->ever) {
|
||||
tdbTbcMoveToNext(pReader->pCur);
|
||||
break;
|
||||
} else {
|
||||
tdbTbcMoveToNext(pReader->pCur);
|
||||
}
|
||||
if (tdbTbcNext(pReader->pCur, &pKey, &kLen, &pVal, &vLen)) {
|
||||
goto _exit;
|
||||
}
|
||||
|
||||
*ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + vLen);
|
||||
|
|
@ -111,13 +95,15 @@ int32_t tqSnapRead(STqSnapReader* pReader, uint8_t** ppData) {
|
|||
pHdr->size = vLen;
|
||||
memcpy(pHdr->data, pVal, vLen);
|
||||
|
||||
tqInfo("vgId:%d, vnode snapshot tq read data, version:%" PRId64 " subKey: %s vLen:%d", TD_VID(pReader->pTq->pVnode),
|
||||
handle.snapshotVer, handle.subKey, vLen);
|
||||
|
||||
_exit:
|
||||
tdbFree(pKey);
|
||||
tdbFree(pVal);
|
||||
tqInfo("vgId:%d, vnode snapshot tq read data, vLen:%d", TD_VID(pReader->pTq->pVnode), vLen);
|
||||
return code;
|
||||
|
||||
_err:
|
||||
tdbFree(pKey);
|
||||
tdbFree(pVal);
|
||||
tqError("vgId:%d, vnode snapshot tq read data failed since %s", TD_VID(pReader->pTq->pVnode), tstrerror(code));
|
||||
return code;
|
||||
}
|
||||
|
|
@ -173,20 +159,13 @@ int32_t tqSnapWriterClose(STqSnapWriter** ppWriter, int8_t rollback) {
|
|||
if (code) goto _err;
|
||||
}
|
||||
|
||||
int vgId = TD_VID(pWriter->pTq->pVnode);
|
||||
|
||||
taosMemoryFree(pWriter);
|
||||
*ppWriter = NULL;
|
||||
|
||||
// restore from metastore
|
||||
if (tqMetaRestoreHandle(pTq) < 0) {
|
||||
goto _err;
|
||||
}
|
||||
|
||||
return code;
|
||||
|
||||
_err:
|
||||
tqError("vgId:%d, tq snapshot writer close failed since %s", vgId, tstrerror(code));
|
||||
tqError("vgId:%d, tq snapshot writer close failed since %s", TD_VID(pTq->pVnode), tstrerror(code));
|
||||
return code;
|
||||
}
|
||||
|
||||
|
|
@ -200,7 +179,9 @@ int32_t tqSnapWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
|
|||
tDecoderInit(pDecoder, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
|
||||
code = tDecodeSTqHandle(pDecoder, &handle);
|
||||
if (code) goto _err;
|
||||
taosWLockLatch(&pTq->lock);
|
||||
code = tqMetaSaveHandle(pTq, handle.subKey, &handle);
|
||||
taosWUnLockLatch(&pTq->lock);
|
||||
if (code < 0) goto _err;
|
||||
tDecoderClear(pDecoder);
|
||||
|
||||
|
|
@ -388,34 +388,34 @@ int32_t tqCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle){
|
|||
return taosHashPut(pTq->pHandle, handle->subKey, strlen(handle->subKey), handle, sizeof(STqHandle));
|
||||
}
|
||||
|
||||
int32_t tqMetaRestoreHandle(STQ* pTq) {
|
||||
int code = 0;
|
||||
TBC* pCur = NULL;
|
||||
if (tdbTbcOpen(pTq->pExecStore, &pCur, NULL) < 0) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
void* pKey = NULL;
|
||||
int kLen = 0;
|
||||
void* pVal = NULL;
|
||||
int vLen = 0;
|
||||
|
||||
tdbTbcMoveToFirst(pCur);
|
||||
|
||||
while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) {
|
||||
STqHandle handle = {0};
|
||||
code = restoreHandle(pTq, pVal, vLen, &handle);
|
||||
if (code < 0) {
|
||||
tqDestroyTqHandle(&handle);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
tdbFree(pKey);
|
||||
tdbFree(pVal);
|
||||
tdbTbcClose(pCur);
|
||||
return code;
|
||||
}
|
||||
//int32_t tqMetaRestoreHandle(STQ* pTq) {
|
||||
// int code = 0;
|
||||
// TBC* pCur = NULL;
|
||||
// if (tdbTbcOpen(pTq->pExecStore, &pCur, NULL) < 0) {
|
||||
// return -1;
|
||||
// }
|
||||
//
|
||||
// void* pKey = NULL;
|
||||
// int kLen = 0;
|
||||
// void* pVal = NULL;
|
||||
// int vLen = 0;
|
||||
//
|
||||
// tdbTbcMoveToFirst(pCur);
|
||||
//
|
||||
// while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) {
|
||||
// STqHandle handle = {0};
|
||||
// code = restoreHandle(pTq, pVal, vLen, &handle);
|
||||
// if (code < 0) {
|
||||
// tqDestroyTqHandle(&handle);
|
||||
// break;
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// tdbFree(pKey);
|
||||
// tdbFree(pVal);
|
||||
// tdbTbcClose(pCur);
|
||||
// return code;
|
||||
//}
|
||||
|
||||
int32_t tqMetaGetHandle(STQ* pTq, const char* key) {
|
||||
void* pVal = NULL;
|
||||
|
|
|
|||
|
|
@ -159,6 +159,7 @@ int32_t tqOffsetSnapWrite(STqOffsetWriter* pWriter, uint8_t* pData, uint32_t nDa
|
|||
taosCloseFile(&pFile);
|
||||
return -1;
|
||||
}
|
||||
taosCloseFile(&pFile);
|
||||
} else {
|
||||
return -1;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1087,6 +1087,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
|
|||
int32_t vgId = TD_VID(pTq->pVnode);
|
||||
|
||||
// update the table list for each consumer handle
|
||||
taosWLockLatch(&pTq->lock);
|
||||
while (1) {
|
||||
pIter = taosHashIterate(pTq->pHandle, pIter);
|
||||
if (pIter == NULL) {
|
||||
|
|
@ -1116,6 +1117,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
|
|||
tqError("qGetTableList in tqUpdateTbUidList error:%d handle %s consumer:0x%" PRIx64, ret, pTqHandle->subKey, pTqHandle->consumerId);
|
||||
taosArrayDestroy(list);
|
||||
taosHashCancelIterate(pTq->pHandle, pIter);
|
||||
taosWUnLockLatch(&pTq->lock);
|
||||
return ret;
|
||||
}
|
||||
tqReaderSetTbUidList(pTqHandle->execHandle.pTqReader, list, NULL);
|
||||
|
|
@ -1125,7 +1127,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
taosWUnLockLatch(&pTq->lock);
|
||||
// update the table list handle for each stream scanner/wal reader
|
||||
taosWLockLatch(&pTq->pStreamMeta->lock);
|
||||
while (1) {
|
||||
|
|
|
|||
|
|
@ -34,6 +34,8 @@ struct SVSnapReader {
|
|||
STqSnapReader *pTqSnapReader;
|
||||
int8_t tqOffsetDone;
|
||||
STqOffsetReader *pTqOffsetReader;
|
||||
int8_t tqCheckInfoDone;
|
||||
STqCheckInfoReader *pTqCheckInfoReader;
|
||||
// stream
|
||||
int8_t streamTaskDone;
|
||||
SStreamTaskReader *pStreamTaskReader;
|
||||
|
|
@ -81,6 +83,18 @@ void vnodeSnapReaderClose(SVSnapReader *pReader) {
|
|||
metaSnapReaderClose(&pReader->pMetaReader);
|
||||
}
|
||||
|
||||
if (pReader->pTqSnapReader) {
|
||||
tqSnapReaderClose(&pReader->pTqSnapReader);
|
||||
}
|
||||
|
||||
if (pReader->pTqOffsetReader) {
|
||||
tqOffsetReaderClose(&pReader->pTqOffsetReader);
|
||||
}
|
||||
|
||||
if (pReader->pTqCheckInfoReader) {
|
||||
tqCheckInfoReaderClose(&pReader->pTqCheckInfoReader);
|
||||
}
|
||||
|
||||
taosMemoryFree(pReader);
|
||||
}
|
||||
|
||||
|
|
@ -181,6 +195,7 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
|
|||
}
|
||||
|
||||
// TQ ================
|
||||
vInfo("vgId:%d tq transform start", vgId);
|
||||
if (!pReader->tqHandleDone) {
|
||||
if (pReader->pTqSnapReader == NULL) {
|
||||
code = tqSnapReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->ever, &pReader->pTqSnapReader);
|
||||
|
|
@ -200,6 +215,25 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
|
|||
}
|
||||
}
|
||||
}
|
||||
if (!pReader->tqCheckInfoDone) {
|
||||
if (pReader->pTqCheckInfoReader == NULL) {
|
||||
code = tqCheckInfoReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->ever, &pReader->pTqCheckInfoReader);
|
||||
if (code < 0) goto _err;
|
||||
}
|
||||
|
||||
code = tqCheckInfoRead(pReader->pTqCheckInfoReader, ppData);
|
||||
if (code) {
|
||||
goto _err;
|
||||
} else {
|
||||
if (*ppData) {
|
||||
goto _exit;
|
||||
} else {
|
||||
pReader->tqCheckInfoDone = 1;
|
||||
code = tqCheckInfoReaderClose(&pReader->pTqCheckInfoReader);
|
||||
if (code) goto _err;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!pReader->tqOffsetDone) {
|
||||
if (pReader->pTqOffsetReader == NULL) {
|
||||
code = tqOffsetReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->ever, &pReader->pTqOffsetReader);
|
||||
|
|
@ -334,6 +368,7 @@ struct SVSnapWriter {
|
|||
// tq
|
||||
STqSnapWriter *pTqSnapWriter;
|
||||
STqOffsetWriter *pTqOffsetWriter;
|
||||
STqCheckInfoWriter *pTqCheckInfoWriter;
|
||||
// stream
|
||||
SStreamTaskWriter *pStreamTaskWriter;
|
||||
SStreamStateWriter *pStreamStateWriter;
|
||||
|
|
@ -411,6 +446,21 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot *
|
|||
if (code) goto _exit;
|
||||
}
|
||||
|
||||
if (pWriter->pTqSnapWriter) {
|
||||
code = tqSnapWriterClose(&pWriter->pTqSnapWriter, rollback);
|
||||
if (code) goto _exit;
|
||||
}
|
||||
|
||||
if (pWriter->pTqCheckInfoWriter) {
|
||||
code = tqCheckInfoWriterClose(&pWriter->pTqCheckInfoWriter, rollback);
|
||||
if (code) goto _exit;
|
||||
}
|
||||
|
||||
if (pWriter->pTqOffsetWriter) {
|
||||
code = tqOffsetWriterClose(&pWriter->pTqOffsetWriter, rollback);
|
||||
if (code) goto _exit;
|
||||
}
|
||||
|
||||
if (pWriter->pStreamTaskWriter) {
|
||||
code = streamTaskSnapWriterClose(pWriter->pStreamTaskWriter, rollback);
|
||||
if (code) goto _exit;
|
||||
|
|
@ -519,8 +569,34 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
|
|||
if (code) goto _err;
|
||||
} break;
|
||||
case SNAP_DATA_TQ_HANDLE: {
|
||||
// tq handle
|
||||
if (pWriter->pTqSnapWriter == NULL) {
|
||||
code = tqSnapWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pTqSnapWriter);
|
||||
if (code) goto _err;
|
||||
}
|
||||
|
||||
code = tqSnapWrite(pWriter->pTqSnapWriter, pData, nData);
|
||||
if (code) goto _err;
|
||||
} break;
|
||||
case SNAP_DATA_TQ_CHECKINFO: {
|
||||
// tq checkinfo
|
||||
if (pWriter->pTqCheckInfoWriter == NULL) {
|
||||
code = tqCheckInfoWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pTqCheckInfoWriter);
|
||||
if (code) goto _err;
|
||||
}
|
||||
|
||||
code = tqCheckInfoWrite(pWriter->pTqCheckInfoWriter, pData, nData);
|
||||
if (code) goto _err;
|
||||
} break;
|
||||
case SNAP_DATA_TQ_OFFSET: {
|
||||
// tq offset
|
||||
if (pWriter->pTqOffsetWriter == NULL) {
|
||||
code = tqOffsetWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pTqOffsetWriter);
|
||||
if (code) goto _err;
|
||||
}
|
||||
|
||||
code = tqOffsetSnapWrite(pWriter->pTqOffsetWriter, pData, nData);
|
||||
if (code) goto _err;
|
||||
} break;
|
||||
case SNAP_DATA_STREAM_TASK:
|
||||
case SNAP_DATA_STREAM_TASK_CHECKPOINT: {
|
||||
|
|
|
|||
|
|
@ -52,6 +52,7 @@ extern "C" {
|
|||
#define FUNC_MGT_INTERP_PC_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(23)
|
||||
#define FUNC_MGT_GEOMETRY_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(24)
|
||||
#define FUNC_MGT_FORBID_SYSTABLE_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(25)
|
||||
#define FUNC_MGT_SKIP_SCAN_CHECK_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(26)
|
||||
|
||||
#define FUNC_MGT_TEST_MASK(val, mask) (((val) & (mask)) != 0)
|
||||
|
||||
|
|
|
|||
|
|
@ -3446,7 +3446,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
|
|||
{
|
||||
.name = "_group_key",
|
||||
.type = FUNCTION_TYPE_GROUP_KEY,
|
||||
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_KEEP_ORDER_FUNC,
|
||||
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_SKIP_SCAN_CHECK_FUNC,
|
||||
.translateFunc = translateGroupKey,
|
||||
.getEnvFunc = getGroupKeyFuncEnv,
|
||||
.initFunc = functionSetup,
|
||||
|
|
|
|||
|
|
@ -346,6 +346,10 @@ bool fmIsConstantResFunc(SFunctionNode* pFunc) {
|
|||
return true;
|
||||
}
|
||||
|
||||
bool fmIsSkipScanCheckFunc(int32_t funcId) {
|
||||
return isSpecificClassifyFunc(funcId, FUNC_MGT_SKIP_SCAN_CHECK_FUNC);
|
||||
}
|
||||
|
||||
void getLastCacheDataType(SDataType* pType) {
|
||||
pType->bytes = getFirstLastInfoSize(pType->bytes) + VARSTR_HEADER_SIZE;
|
||||
pType->type = TSDB_DATA_TYPE_BINARY;
|
||||
|
|
|
|||
|
|
@ -543,6 +543,7 @@ static int32_t logicPartitionCopy(const SPartitionLogicNode* pSrc, SPartitionLog
|
|||
CLONE_NODE_LIST_FIELD(pPartitionKeys);
|
||||
CLONE_NODE_LIST_FIELD(pTags);
|
||||
CLONE_NODE_FIELD(pSubtable);
|
||||
CLONE_NODE_LIST_FIELD(pAggFuncs);
|
||||
COPY_SCALAR_FIELD(needBlockOutputTsOrder);
|
||||
COPY_SCALAR_FIELD(pkTsColId);
|
||||
COPY_SCALAR_FIELD(pkTsColTbId);
|
||||
|
|
|
|||
|
|
@ -1205,6 +1205,7 @@ void nodesDestroyNode(SNode* pNode) {
|
|||
nodesDestroyList(pLogicNode->pPartitionKeys);
|
||||
nodesDestroyList(pLogicNode->pTags);
|
||||
nodesDestroyNode(pLogicNode->pSubtable);
|
||||
nodesDestroyList(pLogicNode->pAggFuncs);
|
||||
break;
|
||||
}
|
||||
case QUERY_NODE_LOGIC_PLAN_INDEF_ROWS_FUNC: {
|
||||
|
|
|
|||
|
|
@ -1256,6 +1256,10 @@ static int32_t createPartitionLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pS
|
|||
nodesCloneNode(nodesListGetNode(pCxt->pCurrRoot->pTargets, 0)));
|
||||
}
|
||||
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = nodesCollectFuncs(pSelect, SQL_CLAUSE_GROUP_BY, NULL, fmIsAggFunc, &pPartition->pAggFuncs);
|
||||
}
|
||||
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
pPartition->pPartitionKeys = nodesCloneList(pSelect->pPartitionByList);
|
||||
if (NULL == pPartition->pPartitionKeys) {
|
||||
|
|
|
|||
|
|
@ -171,11 +171,16 @@ static bool scanPathOptMayBeOptimized(SLogicNode* pNode) {
|
|||
}
|
||||
|
||||
static bool scanPathOptShouldGetFuncs(SLogicNode* pNode) {
|
||||
if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode)) {
|
||||
if (pNode->pParent && QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode->pParent)) {
|
||||
if (WINDOW_TYPE_INTERVAL == ((SWindowLogicNode*)pNode->pParent)->winType) return true;
|
||||
} else {
|
||||
return !scanPathOptHaveNormalCol(((SPartitionLogicNode*)pNode)->pPartitionKeys);
|
||||
}
|
||||
}
|
||||
|
||||
if ((QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode) &&
|
||||
WINDOW_TYPE_INTERVAL == ((SWindowLogicNode*)pNode)->winType) ||
|
||||
(QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode) && pNode->pParent &&
|
||||
QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode->pParent) &&
|
||||
WINDOW_TYPE_INTERVAL == ((SWindowLogicNode*)pNode->pParent)->winType)) {
|
||||
WINDOW_TYPE_INTERVAL == ((SWindowLogicNode*)pNode)->winType)) {
|
||||
return true;
|
||||
}
|
||||
if (QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pNode)) {
|
||||
|
|
@ -191,30 +196,17 @@ static SNodeList* scanPathOptGetAllFuncs(SLogicNode* pNode) {
|
|||
return ((SWindowLogicNode*)pNode)->pFuncs;
|
||||
case QUERY_NODE_LOGIC_PLAN_AGG:
|
||||
return ((SAggLogicNode*)pNode)->pAggFuncs;
|
||||
case QUERY_NODE_LOGIC_PLAN_PARTITION:
|
||||
return ((SPartitionLogicNode*)pNode)->pAggFuncs;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static bool scanPathOptNeedOptimizeDataRequire(const SFunctionNode* pFunc) {
|
||||
if (!fmIsSpecialDataRequiredFunc(pFunc->funcId)) {
|
||||
return false;
|
||||
}
|
||||
SNode* pPara = NULL;
|
||||
FOREACH(pPara, pFunc->pParameterList) {
|
||||
if (QUERY_NODE_COLUMN != nodeType(pPara) && QUERY_NODE_VALUE != nodeType(pPara)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool scanPathOptNeedDynOptimize(const SFunctionNode* pFunc) {
|
||||
if (!fmIsDynamicScanOptimizedFunc(pFunc->funcId)) {
|
||||
return false;
|
||||
}
|
||||
SNode* pPara = NULL;
|
||||
static bool scanPathOptIsSpecifiedFuncType(const SFunctionNode* pFunc, bool (*typeCheckFn)(int32_t)) {
|
||||
if (!typeCheckFn(pFunc->funcId)) return false;
|
||||
SNode* pPara;
|
||||
FOREACH(pPara, pFunc->pParameterList) {
|
||||
if (QUERY_NODE_COLUMN != nodeType(pPara) && QUERY_NODE_VALUE != nodeType(pPara)) {
|
||||
return false;
|
||||
|
|
@ -232,10 +224,12 @@ static int32_t scanPathOptGetRelatedFuncs(SScanLogicNode* pScan, SNodeList** pSd
|
|||
FOREACH(pNode, pAllFuncs) {
|
||||
SFunctionNode* pFunc = (SFunctionNode*)pNode;
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
if (scanPathOptNeedOptimizeDataRequire(pFunc)) {
|
||||
if (scanPathOptIsSpecifiedFuncType(pFunc, fmIsSpecialDataRequiredFunc)) {
|
||||
code = nodesListMakeStrictAppend(&pTmpSdrFuncs, nodesCloneNode(pNode));
|
||||
} else if (scanPathOptNeedDynOptimize(pFunc)) {
|
||||
} else if (scanPathOptIsSpecifiedFuncType(pFunc, fmIsDynamicScanOptimizedFunc)) {
|
||||
code = nodesListMakeStrictAppend(&pTmpDsoFuncs, nodesCloneNode(pNode));
|
||||
} else if (scanPathOptIsSpecifiedFuncType(pFunc, fmIsSkipScanCheckFunc)) {
|
||||
continue;
|
||||
} else {
|
||||
otherFunc = true;
|
||||
break;
|
||||
|
|
|
|||
|
|
@ -420,6 +420,7 @@ void transThreadOnce();
|
|||
|
||||
void transInit();
|
||||
void transCleanup();
|
||||
void transPrintEpSet(SEpSet* pEpSet);
|
||||
|
||||
void transFreeMsg(void* msg);
|
||||
int32_t transCompressMsg(char* msg, int32_t len);
|
||||
|
|
|
|||
|
|
@ -2221,7 +2221,9 @@ bool cliResetEpset(STransConnCtx* pCtx, STransMsg* pResp, bool hasEpSet) {
|
|||
}
|
||||
} else {
|
||||
if (!transEpSetIsEqual(&pCtx->epSet, &epSet)) {
|
||||
tDebug("epset not equal, retry new epset");
|
||||
tDebug("epset not equal, retry new epset1");
|
||||
transPrintEpSet(&pCtx->epSet);
|
||||
transPrintEpSet(&epSet);
|
||||
epsetAssign(&pCtx->epSet, &epSet);
|
||||
noDelay = false;
|
||||
} else {
|
||||
|
|
@ -2246,7 +2248,9 @@ bool cliResetEpset(STransConnCtx* pCtx, STransMsg* pResp, bool hasEpSet) {
|
|||
}
|
||||
} else {
|
||||
if (!transEpSetIsEqual(&pCtx->epSet, &epSet)) {
|
||||
tDebug("epset not equal, retry new epset");
|
||||
tDebug("epset not equal, retry new epset2");
|
||||
transPrintEpSet(&pCtx->epSet);
|
||||
transPrintEpSet(&epSet);
|
||||
epsetAssign(&pCtx->epSet, &epSet);
|
||||
noDelay = false;
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -456,10 +456,18 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset)
|
|||
return -1;
|
||||
}
|
||||
#ifdef WINDOWS
|
||||
size_t pos = _lseeki64(pFile->fd, 0, SEEK_CUR);
|
||||
_lseeki64(pFile->fd, offset, SEEK_SET);
|
||||
int64_t ret = _read(pFile->fd, buf, count);
|
||||
_lseeki64(pFile->fd, pos, SEEK_SET);
|
||||
DWORD ret = 0;
|
||||
OVERLAPPED ol = {0};
|
||||
ol.OffsetHigh = (uint32_t)((offset & 0xFFFFFFFF00000000LL) >> 0x20);
|
||||
ol.Offset = (uint32_t)(offset & 0xFFFFFFFFLL);
|
||||
|
||||
HANDLE handle = (HANDLE)_get_osfhandle(pFile->fd);
|
||||
SetLastError(0);
|
||||
BOOL result = ReadFile(handle, buf, count, &ret, &ol);
|
||||
if (!result && GetLastError() != ERROR_HANDLE_EOF) {
|
||||
errno = GetLastError();
|
||||
ret = -1;
|
||||
}
|
||||
#else
|
||||
int64_t ret = pread(pFile->fd, buf, count, offset);
|
||||
#endif
|
||||
|
|
@ -523,10 +531,18 @@ int64_t taosPWriteFile(TdFilePtr pFile, const void *buf, int64_t count, int64_t
|
|||
return 0;
|
||||
}
|
||||
#ifdef WINDOWS
|
||||
size_t pos = _lseeki64(pFile->fd, 0, SEEK_CUR);
|
||||
_lseeki64(pFile->fd, offset, SEEK_SET);
|
||||
int64_t ret = _write(pFile->fd, buf, count);
|
||||
_lseeki64(pFile->fd, pos, SEEK_SET);
|
||||
DWORD ret = 0;
|
||||
OVERLAPPED ol = {0};
|
||||
ol.OffsetHigh = (uint32_t)((offset & 0xFFFFFFFF00000000LL) >> 0x20);
|
||||
ol.Offset = (uint32_t)(offset & 0xFFFFFFFFLL);
|
||||
|
||||
HANDLE handle = (HANDLE)_get_osfhandle(pFile->fd);
|
||||
SetLastError(0);
|
||||
BOOL result = WriteFile(handle, buf, count, &ret, &ol);
|
||||
if (!result) {
|
||||
errno = GetLastError();
|
||||
ret = -1;
|
||||
}
|
||||
#else
|
||||
int64_t ret = pwrite(pFile->fd, buf, count, offset);
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -86,8 +86,9 @@ void taosRandStr(char* str, int32_t size) {
|
|||
}
|
||||
|
||||
void taosRandStr2(char* str, int32_t size) {
|
||||
const char* set = "abcdefghijklmnopqrstuvwxyz0123456789";
|
||||
int32_t len = 36;
|
||||
|
||||
const char* set = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ@";
|
||||
int32_t len = strlen(set);
|
||||
|
||||
for (int32_t i = 0; i < size; ++i) {
|
||||
str[i] = set[taosRand() % len];
|
||||
|
|
|
|||
|
|
@ -84,34 +84,98 @@ int32_t taosThreadAttrSetStackSize(TdThreadAttr *attr, size_t stacksize) {
|
|||
|
||||
int32_t taosThreadCancel(TdThread thread) { return pthread_cancel(thread); }
|
||||
|
||||
int32_t taosThreadCondDestroy(TdThreadCond *cond) { return pthread_cond_destroy(cond); }
|
||||
int32_t taosThreadCondDestroy(TdThreadCond *cond) {
|
||||
#ifdef __USE_WIN_THREAD
|
||||
return 0;
|
||||
#else
|
||||
return pthread_cond_destroy(cond);
|
||||
#endif
|
||||
}
|
||||
|
||||
int32_t taosThreadCondInit(TdThreadCond *cond, const TdThreadCondAttr *attr) { return pthread_cond_init(cond, attr); }
|
||||
int32_t taosThreadCondInit(TdThreadCond *cond, const TdThreadCondAttr *attr) {
|
||||
#ifdef __USE_WIN_THREAD
|
||||
InitializeConditionVariable(cond);
|
||||
return 0;
|
||||
#else
|
||||
return pthread_cond_init(cond, attr);
|
||||
#endif
|
||||
}
|
||||
|
||||
int32_t taosThreadCondSignal(TdThreadCond *cond) { return pthread_cond_signal(cond); }
|
||||
int32_t taosThreadCondSignal(TdThreadCond *cond) {
|
||||
#ifdef __USE_WIN_THREAD
|
||||
WakeConditionVariable(cond);
|
||||
return 0;
|
||||
#else
|
||||
return pthread_cond_signal(cond);
|
||||
#endif
|
||||
}
|
||||
|
||||
int32_t taosThreadCondBroadcast(TdThreadCond *cond) { return pthread_cond_broadcast(cond); }
|
||||
int32_t taosThreadCondBroadcast(TdThreadCond *cond) {
|
||||
#ifdef __USE_WIN_THREAD
|
||||
WakeAllConditionVariable(cond);
|
||||
return 0;
|
||||
#else
|
||||
return pthread_cond_broadcast(cond);
|
||||
#endif
|
||||
}
|
||||
|
||||
int32_t taosThreadCondWait(TdThreadCond *cond, TdThreadMutex *mutex) {
|
||||
#ifdef __USE_WIN_THREAD
|
||||
if (!SleepConditionVariableCS(cond, mutex, INFINITE)) {
|
||||
return EINVAL;
|
||||
}
|
||||
return 0;
|
||||
#else
|
||||
THREAD_PTR_CHECK(mutex)
|
||||
return pthread_cond_wait(cond, mutex);
|
||||
#endif
|
||||
}
|
||||
|
||||
int32_t taosThreadCondTimedWait(TdThreadCond *cond, TdThreadMutex *mutex, const struct timespec *abstime) {
|
||||
#ifdef __USE_WIN_THREAD
|
||||
if (!abstime) return EINVAL;
|
||||
if (SleepConditionVariableCS(cond, mutex, (DWORD)(abstime->tv_sec * 1e3 + abstime->tv_nsec / 1e6))) return 0;
|
||||
if (GetLastError() == ERROR_TIMEOUT) {
|
||||
return ETIMEDOUT;
|
||||
}
|
||||
return EINVAL;
|
||||
#else
|
||||
THREAD_PTR_CHECK(mutex)
|
||||
return pthread_cond_timedwait(cond, mutex, abstime);
|
||||
#endif
|
||||
}
|
||||
|
||||
int32_t taosThreadCondAttrDestroy(TdThreadCondAttr *attr) { return pthread_condattr_destroy(attr); }
|
||||
int32_t taosThreadCondAttrDestroy(TdThreadCondAttr *attr) {
|
||||
#ifdef __USE_WIN_THREAD
|
||||
return 0;
|
||||
#else
|
||||
return pthread_condattr_destroy(attr);
|
||||
#endif
|
||||
}
|
||||
|
||||
int32_t taosThreadCondAttrGetPshared(const TdThreadCondAttr *attr, int32_t *pshared) {
|
||||
#ifdef __USE_WIN_THREAD
|
||||
if (pshared) *pshared = PTHREAD_PROCESS_PRIVATE;
|
||||
return 0;
|
||||
#else
|
||||
return pthread_condattr_getpshared(attr, pshared);
|
||||
#endif
|
||||
}
|
||||
|
||||
int32_t taosThreadCondAttrInit(TdThreadCondAttr *attr) { return pthread_condattr_init(attr); }
|
||||
int32_t taosThreadCondAttrInit(TdThreadCondAttr *attr) {
|
||||
#ifdef __USE_WIN_THREAD
|
||||
return 0;
|
||||
#else
|
||||
return pthread_condattr_init(attr);
|
||||
#endif
|
||||
}
|
||||
|
||||
int32_t taosThreadCondAttrSetPshared(TdThreadCondAttr *attr, int32_t pshared) {
|
||||
#ifdef __USE_WIN_THREAD
|
||||
return 0;
|
||||
#else
|
||||
return pthread_condattr_setpshared(attr, pshared);
|
||||
#endif
|
||||
}
|
||||
|
||||
int32_t taosThreadDetach(TdThread thread) { return pthread_detach(thread); }
|
||||
|
|
@ -142,17 +206,37 @@ int32_t taosThreadKill(TdThread thread, int32_t sig) { return pthread_kill(threa
|
|||
// }
|
||||
|
||||
int32_t taosThreadMutexDestroy(TdThreadMutex *mutex) {
|
||||
#ifdef __USE_WIN_THREAD
|
||||
DeleteCriticalSection(mutex);
|
||||
return 0;
|
||||
#else
|
||||
THREAD_PTR_CHECK(mutex)
|
||||
return pthread_mutex_destroy(mutex);
|
||||
#endif
|
||||
}
|
||||
|
||||
int32_t taosThreadMutexInit(TdThreadMutex *mutex, const TdThreadMutexAttr *attr) {
|
||||
#ifdef __USE_WIN_THREAD
|
||||
/**
|
||||
* Windows Server 2003 and Windows XP: In low memory situations, InitializeCriticalSection can raise a
|
||||
* STATUS_NO_MEMORY exception. Starting with Windows Vista, this exception was eliminated and
|
||||
* InitializeCriticalSection always succeeds, even in low memory situations.
|
||||
*/
|
||||
InitializeCriticalSection(mutex);
|
||||
return 0;
|
||||
#else
|
||||
return pthread_mutex_init(mutex, attr);
|
||||
#endif
|
||||
}
|
||||
|
||||
int32_t taosThreadMutexLock(TdThreadMutex *mutex) {
|
||||
#ifdef __USE_WIN_THREAD
|
||||
EnterCriticalSection(mutex);
|
||||
return 0;
|
||||
#else
|
||||
THREAD_PTR_CHECK(mutex)
|
||||
return pthread_mutex_lock(mutex);
|
||||
#endif
|
||||
}
|
||||
|
||||
// int32_t taosThreadMutexTimedLock(TdThreadMutex * mutex, const struct timespec *abstime) {
|
||||
|
|
@ -160,19 +244,40 @@ int32_t taosThreadMutexLock(TdThreadMutex *mutex) {
|
|||
// }
|
||||
|
||||
int32_t taosThreadMutexTryLock(TdThreadMutex *mutex) {
|
||||
THREAD_PTR_CHECK(mutex)
|
||||
return pthread_mutex_trylock(mutex);
|
||||
#ifdef __USE_WIN_THREAD
|
||||
if (TryEnterCriticalSection(mutex)) return 0;
|
||||
return EBUSY;
|
||||
#else
|
||||
THREAD_PTR_CHECK(mutex)
|
||||
return pthread_mutex_trylock(mutex);
|
||||
#endif
|
||||
}
|
||||
|
||||
int32_t taosThreadMutexUnlock(TdThreadMutex *mutex) {
|
||||
#ifdef __USE_WIN_THREAD
|
||||
LeaveCriticalSection(mutex);
|
||||
return 0;
|
||||
#else
|
||||
THREAD_PTR_CHECK(mutex)
|
||||
return pthread_mutex_unlock(mutex);
|
||||
#endif
|
||||
}
|
||||
|
||||
int32_t taosThreadMutexAttrDestroy(TdThreadMutexAttr *attr) { return pthread_mutexattr_destroy(attr); }
|
||||
int32_t taosThreadMutexAttrDestroy(TdThreadMutexAttr *attr) {
|
||||
#ifdef __USE_WIN_THREAD
|
||||
return 0;
|
||||
#else
|
||||
return pthread_mutexattr_destroy(attr);
|
||||
#endif
|
||||
}
|
||||
|
||||
int32_t taosThreadMutexAttrGetPshared(const TdThreadMutexAttr *attr, int32_t *pshared) {
|
||||
#ifdef __USE_WIN_THREAD
|
||||
if (pshared) *pshared = PTHREAD_PROCESS_PRIVATE;
|
||||
return 0;
|
||||
#else
|
||||
return pthread_mutexattr_getpshared(attr, pshared);
|
||||
#endif
|
||||
}
|
||||
|
||||
// int32_t taosThreadMutexAttrGetRobust(const TdThreadMutexAttr * attr, int32_t * robust) {
|
||||
|
|
@ -180,13 +285,28 @@ int32_t taosThreadMutexAttrGetPshared(const TdThreadMutexAttr *attr, int32_t *ps
|
|||
// }
|
||||
|
||||
int32_t taosThreadMutexAttrGetType(const TdThreadMutexAttr *attr, int32_t *kind) {
|
||||
#ifdef __USE_WIN_THREAD
|
||||
if (kind) *kind = PTHREAD_MUTEX_NORMAL;
|
||||
return 0;
|
||||
#else
|
||||
return pthread_mutexattr_gettype(attr, kind);
|
||||
#endif
|
||||
}
|
||||
|
||||
int32_t taosThreadMutexAttrInit(TdThreadMutexAttr *attr) { return pthread_mutexattr_init(attr); }
|
||||
int32_t taosThreadMutexAttrInit(TdThreadMutexAttr *attr) {
|
||||
#ifdef __USE_WIN_THREAD
|
||||
return 0;
|
||||
#else
|
||||
return pthread_mutexattr_init(attr);
|
||||
#endif
|
||||
}
|
||||
|
||||
int32_t taosThreadMutexAttrSetPshared(TdThreadMutexAttr *attr, int32_t pshared) {
|
||||
#ifdef __USE_WIN_THREAD
|
||||
return 0;
|
||||
#else
|
||||
return pthread_mutexattr_setpshared(attr, pshared);
|
||||
#endif
|
||||
}
|
||||
|
||||
// int32_t taosThreadMutexAttrSetRobust(TdThreadMutexAttr * attr, int32_t robust) {
|
||||
|
|
@ -194,20 +314,46 @@ int32_t taosThreadMutexAttrSetPshared(TdThreadMutexAttr *attr, int32_t pshared)
|
|||
// }
|
||||
|
||||
int32_t taosThreadMutexAttrSetType(TdThreadMutexAttr *attr, int32_t kind) {
|
||||
#ifdef __USE_WIN_THREAD
|
||||
return 0;
|
||||
#else
|
||||
return pthread_mutexattr_settype(attr, kind);
|
||||
#endif
|
||||
}
|
||||
|
||||
int32_t taosThreadOnce(TdThreadOnce *onceControl, void (*initRoutine)(void)) {
|
||||
return pthread_once(onceControl, initRoutine);
|
||||
}
|
||||
|
||||
int32_t taosThreadRwlockDestroy(TdThreadRwlock *rwlock) { return pthread_rwlock_destroy(rwlock); }
|
||||
|
||||
int32_t taosThreadRwlockInit(TdThreadRwlock *rwlock, const TdThreadRwlockAttr *attr) {
|
||||
return pthread_rwlock_init(rwlock, attr);
|
||||
int32_t taosThreadRwlockDestroy(TdThreadRwlock *rwlock) {
|
||||
#ifdef __USE_WIN_THREAD
|
||||
/* SRWLock does not need explicit destruction so long as there are no waiting threads
|
||||
* See: https://docs.microsoft.com/windows/win32/api/synchapi/nf-synchapi-initializesrwlock#remarks
|
||||
*/
|
||||
return 0;
|
||||
#else
|
||||
return pthread_rwlock_destroy(rwlock);
|
||||
#endif
|
||||
}
|
||||
|
||||
int32_t taosThreadRwlockRdlock(TdThreadRwlock *rwlock) { return pthread_rwlock_rdlock(rwlock); }
|
||||
int32_t taosThreadRwlockInit(TdThreadRwlock *rwlock, const TdThreadRwlockAttr *attr) {
|
||||
#ifdef __USE_WIN_THREAD
|
||||
memset(rwlock, 0, sizeof(*rwlock));
|
||||
InitializeSRWLock(&rwlock->lock);
|
||||
return 0;
|
||||
#else
|
||||
return pthread_rwlock_init(rwlock, attr);
|
||||
#endif
|
||||
}
|
||||
|
||||
int32_t taosThreadRwlockRdlock(TdThreadRwlock *rwlock) {
|
||||
#ifdef __USE_WIN_THREAD
|
||||
AcquireSRWLockShared(&rwlock->lock);
|
||||
return 0;
|
||||
#else
|
||||
return pthread_rwlock_rdlock(rwlock);
|
||||
#endif
|
||||
}
|
||||
|
||||
// int32_t taosThreadRwlockTimedRdlock(TdThreadRwlock * rwlock, const struct timespec *abstime) {
|
||||
// return pthread_rwlock_timedrdlock(rwlock, abstime);
|
||||
|
|
@ -217,24 +363,79 @@ int32_t taosThreadRwlockRdlock(TdThreadRwlock *rwlock) { return pthread_rwlock_r
|
|||
// return pthread_rwlock_timedwrlock(rwlock, abstime);
|
||||
// }
|
||||
|
||||
int32_t taosThreadRwlockTryRdlock(TdThreadRwlock *rwlock) { return pthread_rwlock_tryrdlock(rwlock); }
|
||||
|
||||
int32_t taosThreadRwlockTryWrlock(TdThreadRwlock *rwlock) { return pthread_rwlock_trywrlock(rwlock); }
|
||||
|
||||
int32_t taosThreadRwlockUnlock(TdThreadRwlock *rwlock) { return pthread_rwlock_unlock(rwlock); }
|
||||
|
||||
int32_t taosThreadRwlockWrlock(TdThreadRwlock *rwlock) { return pthread_rwlock_wrlock(rwlock); }
|
||||
|
||||
int32_t taosThreadRwlockAttrDestroy(TdThreadRwlockAttr *attr) { return pthread_rwlockattr_destroy(attr); }
|
||||
|
||||
int32_t taosThreadRwlockAttrGetPshared(const TdThreadRwlockAttr *attr, int32_t *pshared) {
|
||||
return pthread_rwlockattr_getpshared(attr, pshared);
|
||||
int32_t taosThreadRwlockTryRdlock(TdThreadRwlock *rwlock) {
|
||||
#ifdef __USE_WIN_THREAD
|
||||
if (!TryAcquireSRWLockShared(&rwlock->lock)) return EBUSY;
|
||||
return 0;
|
||||
#else
|
||||
return pthread_rwlock_tryrdlock(rwlock);
|
||||
#endif
|
||||
}
|
||||
|
||||
int32_t taosThreadRwlockAttrInit(TdThreadRwlockAttr *attr) { return pthread_rwlockattr_init(attr); }
|
||||
int32_t taosThreadRwlockTryWrlock(TdThreadRwlock *rwlock) {
|
||||
#ifdef __USE_WIN_THREAD
|
||||
if (!TryAcquireSRWLockExclusive(&rwlock->lock)) return EBUSY;
|
||||
atomic_store_8(&rwlock->excl, 1);
|
||||
return 0;
|
||||
#else
|
||||
return pthread_rwlock_trywrlock(rwlock);
|
||||
#endif
|
||||
}
|
||||
|
||||
int32_t taosThreadRwlockUnlock(TdThreadRwlock *rwlock) {
|
||||
#ifdef __USE_WIN_THREAD
|
||||
if (1 == atomic_val_compare_exchange_8(&rwlock->excl, 1, 0)) {
|
||||
ReleaseSRWLockExclusive(&rwlock->lock);
|
||||
} else {
|
||||
ReleaseSRWLockShared(&rwlock->lock);
|
||||
}
|
||||
return 0;
|
||||
#else
|
||||
return pthread_rwlock_unlock(rwlock);
|
||||
#endif
|
||||
}
|
||||
|
||||
int32_t taosThreadRwlockWrlock(TdThreadRwlock *rwlock) {
|
||||
#ifdef __USE_WIN_THREAD
|
||||
AcquireSRWLockExclusive(&rwlock->lock);
|
||||
atomic_store_8(&rwlock->excl, 1);
|
||||
return 0;
|
||||
#else
|
||||
return pthread_rwlock_wrlock(rwlock);
|
||||
#endif
|
||||
}
|
||||
|
||||
int32_t taosThreadRwlockAttrDestroy(TdThreadRwlockAttr *attr) {
|
||||
#ifdef __USE_WIN_THREAD
|
||||
return 0;
|
||||
#else
|
||||
return pthread_rwlockattr_destroy(attr);
|
||||
#endif
|
||||
}
|
||||
|
||||
int32_t taosThreadRwlockAttrGetPshared(const TdThreadRwlockAttr *attr, int32_t *pshared) {
|
||||
#ifdef __USE_WIN_THREAD
|
||||
if (pshared) *pshared = PTHREAD_PROCESS_PRIVATE;
|
||||
return 0;
|
||||
#else
|
||||
return pthread_rwlockattr_getpshared(attr, pshared);
|
||||
#endif
|
||||
}
|
||||
|
||||
int32_t taosThreadRwlockAttrInit(TdThreadRwlockAttr *attr) {
|
||||
#ifdef __USE_WIN_THREAD
|
||||
return 0;
|
||||
#else
|
||||
return pthread_rwlockattr_init(attr);
|
||||
#endif
|
||||
}
|
||||
|
||||
int32_t taosThreadRwlockAttrSetPshared(TdThreadRwlockAttr *attr, int32_t pshared) {
|
||||
#ifdef __USE_WIN_THREAD
|
||||
return 0;
|
||||
#else
|
||||
return pthread_rwlockattr_setpshared(attr, pshared);
|
||||
#endif
|
||||
}
|
||||
|
||||
TdThread taosThreadSelf(void) { return pthread_self(); }
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@
|
|||
|
||||
static TdThread cacheRefreshWorker = {0};
|
||||
static TdThreadOnce cacheThreadInit = PTHREAD_ONCE_INIT;
|
||||
static TdThreadMutex guard = TD_PTHREAD_MUTEX_INITIALIZER;
|
||||
static TdThreadMutex guard;
|
||||
static SArray *pCacheArrayList = NULL;
|
||||
static bool stopRefreshWorker = false;
|
||||
static bool refreshWorkerNormalStopped = false;
|
||||
|
|
@ -155,6 +155,8 @@ static void *taosCacheTimedRefresh(void *handle);
|
|||
static void doInitRefreshThread(void) {
|
||||
pCacheArrayList = taosArrayInit(4, POINTER_BYTES);
|
||||
|
||||
taosThreadMutexInit(&guard, NULL);
|
||||
|
||||
TdThreadAttr thattr;
|
||||
taosThreadAttrInit(&thattr);
|
||||
taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE);
|
||||
|
|
|
|||
|
|
@ -221,7 +221,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_BUFSIZE, "Invalid func bufSize"
|
|||
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_COMMENT, "Invalid func comment")
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_RETRIEVE, "Invalid func retrieve msg")
|
||||
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST, "index already exists")
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST, "index already exists in db")
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_INDEX_NOT_EXIST, "index not exist")
|
||||
|
||||
|
||||
|
|
@ -312,7 +312,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_STREAMS, "Too many streams")
|
|||
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TARGET_TABLE, "Cannot write the same stable as other stream")
|
||||
|
||||
// mnode-sma
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_MND_SMA_ALREADY_EXIST, "index already exists")
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_MND_SMA_ALREADY_EXIST, "index already exists in db")
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_MND_SMA_NOT_EXIST, "index not exist")
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SMA_OPTION, "Invalid sma index option")
|
||||
|
||||
|
|
|
|||
|
|
@ -163,6 +163,8 @@
|
|||
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqSubscribeStb-r3.py -N 5
|
||||
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -i True
|
||||
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3 -i True
|
||||
,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform.py -N 2 -n 1
|
||||
,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeReplicate.py -M 3 -N 3 -n 3
|
||||
,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-19201.py
|
||||
,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-21561.py
|
||||
,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TS-3404.py
|
||||
|
|
|
|||
|
|
@ -292,11 +292,12 @@ if $rows != 1 then
|
|||
return -1
|
||||
endi
|
||||
|
||||
sql drop index $data[0][0]
|
||||
#$drop_name=`$data[0][0]`
|
||||
#sql drop index `$data[0][0]\`
|
||||
|
||||
if $rows != 0 then
|
||||
return -1
|
||||
endi
|
||||
#if $rows != 0 then
|
||||
# return -1
|
||||
#endi
|
||||
|
||||
|
||||
sql_error drop index t2
|
||||
|
|
@ -304,7 +305,7 @@ sql_error drop index t3
|
|||
|
||||
|
||||
|
||||
sql create index ti0 on $mtPrefix (t1)
|
||||
#sql create index ti0 on $mtPrefix (t1)
|
||||
|
||||
$i = $interval
|
||||
while $i < $limit
|
||||
|
|
|
|||
|
|
@ -45,9 +45,9 @@ class TMQCom:
|
|||
tdSql.init(conn.cursor())
|
||||
# tdSql.init(conn.cursor(), logSql) # output sql.txt file
|
||||
|
||||
def initConsumerTable(self,cdbName='cdb'):
|
||||
def initConsumerTable(self,cdbName='cdb', replicaVar=1):
|
||||
tdLog.info("create consume database, and consume info table, and consume result table")
|
||||
tdSql.query("create database if not exists %s vgroups 1"%(cdbName))
|
||||
tdSql.query("create database if not exists %s vgroups 1 replica %d"%(cdbName,replicaVar))
|
||||
tdSql.query("drop table if exists %s.consumeinfo "%(cdbName))
|
||||
tdSql.query("drop table if exists %s.consumeresult "%(cdbName))
|
||||
tdSql.query("drop table if exists %s.notifyinfo "%(cdbName))
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ sys.path.append("./7-tmq")
|
|||
from tmqCommon import *
|
||||
|
||||
class TDTestCase:
|
||||
updatecfgDict = {'debugFlag': 135}
|
||||
# updatecfgDict = {'debugFlag': 135}
|
||||
|
||||
def __init__(self):
|
||||
self.vgroups = 2
|
||||
|
|
@ -252,7 +252,6 @@ class TDTestCase:
|
|||
break
|
||||
|
||||
tdLog.info("all consumers status into 'lost'")
|
||||
|
||||
# drop consumer groups
|
||||
tdLog.info("drop all consumers")
|
||||
for i in range(len(groupIdList)):
|
||||
|
|
|
|||
|
|
@ -0,0 +1,173 @@
|
|||
|
||||
import taos
|
||||
import sys
|
||||
import time
|
||||
import socket
|
||||
import os
|
||||
import threading
|
||||
import math
|
||||
|
||||
from util.log import *
|
||||
from util.sql import *
|
||||
from util.cases import *
|
||||
from util.dnodes import *
|
||||
from util.common import *
|
||||
from util.cluster import *
|
||||
sys.path.append("./7-tmq")
|
||||
from tmqCommon import *
|
||||
|
||||
class TDTestCase:
|
||||
def __init__(self):
|
||||
self.vgroups = 1
|
||||
self.ctbNum = 10
|
||||
self.rowsPerTbl = 10000
|
||||
|
||||
def init(self, conn, logSql, replicaVar=1):
|
||||
self.replicaVar = int(replicaVar)
|
||||
tdLog.debug(f"start to excute {__file__}")
|
||||
tdSql.init(conn.cursor(), False)
|
||||
|
||||
def prepareTestEnv(self):
|
||||
tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ")
|
||||
paraDict = {'dbName': 'dbt',
|
||||
'dropFlag': 1,
|
||||
'event': '',
|
||||
'vgroups': 1,
|
||||
'stbName': 'stb',
|
||||
'colPrefix': 'c',
|
||||
'tagPrefix': 't',
|
||||
'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
|
||||
'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
|
||||
'ctbPrefix': 'ctb',
|
||||
'ctbStartIdx': 0,
|
||||
'ctbNum': 10,
|
||||
'rowsPerTbl': 10000,
|
||||
'batchNum': 10,
|
||||
'startTs': 1640966400000, # 2022-01-01 00:00:00.000
|
||||
'pollDelay': 20,
|
||||
'showMsg': 1,
|
||||
'showRow': 1,
|
||||
'snapshot': 0}
|
||||
|
||||
paraDict['vgroups'] = self.vgroups
|
||||
paraDict['ctbNum'] = self.ctbNum
|
||||
paraDict['rowsPerTbl'] = self.rowsPerTbl
|
||||
|
||||
tmqCom.initConsumerTable()
|
||||
tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], wal_retention_period=36000,vgroups=paraDict["vgroups"],replica=self.replicaVar)
|
||||
tdLog.info("create stb")
|
||||
tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"])
|
||||
tdLog.info("create ctb")
|
||||
tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
|
||||
ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
|
||||
tdLog.info("insert data")
|
||||
tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"],
|
||||
ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
|
||||
startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
|
||||
|
||||
tdLog.info("restart taosd to ensure that the data falls into the disk")
|
||||
# tdDnodes.stop(1)
|
||||
# tdDnodes.start(1)
|
||||
tdSql.query("flush database %s"%(paraDict['dbName']))
|
||||
return
|
||||
|
||||
def tmqCase1(self):
|
||||
tdLog.printNoPrefix("======== test case 1: ")
|
||||
paraDict = {'dbName': 'dbt',
|
||||
'dropFlag': 1,
|
||||
'event': '',
|
||||
'vgroups': 1,
|
||||
'stbName': 'stb',
|
||||
'colPrefix': 'c',
|
||||
'tagPrefix': 't',
|
||||
'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
|
||||
'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
|
||||
'ctbPrefix': 'ctbn',
|
||||
'ctbStartIdx': 0,
|
||||
'ctbNum': 10,
|
||||
'rowsPerTbl': 10000,
|
||||
'batchNum': 10,
|
||||
'startTs': 1640966400000, # 2022-01-01 00:00:00.000
|
||||
'pollDelay': 20,
|
||||
'showMsg': 1,
|
||||
'showRow': 1,
|
||||
'snapshot': 0}
|
||||
|
||||
paraDict['vgroups'] = self.vgroups
|
||||
paraDict['ctbNum'] = self.ctbNum
|
||||
paraDict['rowsPerTbl'] = self.rowsPerTbl
|
||||
|
||||
tdLog.info("create ctb")
|
||||
tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
|
||||
ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
|
||||
tdLog.info("insert data")
|
||||
pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict)
|
||||
|
||||
topicNameList = ['topic1']
|
||||
# expectRowsList = []
|
||||
tmqCom.initConsumerTable("cdb", self.replicaVar)
|
||||
|
||||
tdLog.info("create topics from stb with filter")
|
||||
queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
|
||||
# sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName'])
|
||||
sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
|
||||
tdLog.info("create topic sql: %s"%sqlString)
|
||||
tdSql.execute(sqlString)
|
||||
# tdSql.query(queryString)
|
||||
# expectRowsList.append(tdSql.getRows())
|
||||
|
||||
# init consume info, and start tmq_sim, then check consume result
|
||||
tdLog.info("insert consume info to consume processor")
|
||||
consumerId = 0
|
||||
expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2
|
||||
topicList = topicNameList[0]
|
||||
ifcheckdata = 1
|
||||
ifManualCommit = 1
|
||||
keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest'
|
||||
tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
|
||||
|
||||
tdLog.info("start consume processor")
|
||||
tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
|
||||
tdLog.info("wait the consume result")
|
||||
|
||||
tmqCom.getStartConsumeNotifyFromTmqsim()
|
||||
tmqCom.getStartCommitNotifyFromTmqsim()
|
||||
|
||||
tdSql.query("select * from information_schema.ins_vnodes")
|
||||
# tdLog.debug(tdSql.queryResult)
|
||||
tdDnodes = cluster.dnodes
|
||||
for result in tdSql.queryResult:
|
||||
if result[2] == 'dbt' and result[3] == 'leader':
|
||||
tdLog.debug("leader is %d"%(result[0] - 1))
|
||||
tdDnodes[result[0] - 1].stoptaosd()
|
||||
break
|
||||
|
||||
pInsertThread.join()
|
||||
expectRows = 1
|
||||
resultList = tmqCom.selectConsumeResult(expectRows)
|
||||
|
||||
if expectrowcnt > resultList[0]:
|
||||
tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt, resultList[0]))
|
||||
tdLog.exit("%d tmq consume rows error!"%consumerId)
|
||||
|
||||
# tmqCom.checkFileContent(consumerId, queryString)
|
||||
|
||||
time.sleep(10)
|
||||
for i in range(len(topicNameList)):
|
||||
tdSql.query("drop topic %s"%topicNameList[i])
|
||||
|
||||
tdLog.printNoPrefix("======== test case 1 end ...... ")
|
||||
|
||||
def run(self):
|
||||
tdSql.prepare()
|
||||
self.prepareTestEnv()
|
||||
self.tmqCase1()
|
||||
|
||||
def stop(self):
|
||||
tdSql.close()
|
||||
tdLog.success(f"{__file__} successfully executed")
|
||||
|
||||
event = threading.Event()
|
||||
|
||||
tdCases.addLinux(__file__, TDTestCase())
|
||||
tdCases.addWindows(__file__, TDTestCase())
|
||||
|
|
@ -0,0 +1,337 @@
|
|||
|
||||
import taos
|
||||
import sys
|
||||
import time
|
||||
import socket
|
||||
import os
|
||||
import threading
|
||||
import math
|
||||
|
||||
from util.log import *
|
||||
from util.sql import *
|
||||
from util.cases import *
|
||||
from util.dnodes import *
|
||||
from util.common import *
|
||||
from util.cluster import *
|
||||
sys.path.append("./7-tmq")
|
||||
from tmqCommon import *
|
||||
|
||||
class TDTestCase:
|
||||
def __init__(self):
|
||||
self.vgroups = 1
|
||||
self.ctbNum = 10
|
||||
self.rowsPerTbl = 10000
|
||||
|
||||
def init(self, conn, logSql, replicaVar=1):
|
||||
self.replicaVar = int(replicaVar)
|
||||
tdLog.debug(f"start to excute {__file__}")
|
||||
tdSql.init(conn.cursor(), False)
|
||||
|
||||
def getDataPath(self):
|
||||
selfPath = os.path.dirname(os.path.realpath(__file__))
|
||||
|
||||
return selfPath + '/../../../sim/dnode%d/data/vnode/vnode%d/wal/*';
|
||||
|
||||
def prepareTestEnv(self):
|
||||
tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ")
|
||||
paraDict = {'dbName': 'dbt',
|
||||
'dropFlag': 1,
|
||||
'event': '',
|
||||
'vgroups': 1,
|
||||
'stbName': 'stb',
|
||||
'colPrefix': 'c',
|
||||
'tagPrefix': 't',
|
||||
'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
|
||||
'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
|
||||
'ctbPrefix': 'ctb',
|
||||
'ctbStartIdx': 0,
|
||||
'ctbNum': 10,
|
||||
'rowsPerTbl': 10000,
|
||||
'batchNum': 10,
|
||||
'startTs': 1640966400000, # 2022-01-01 00:00:00.000
|
||||
'pollDelay': 60,
|
||||
'showMsg': 1,
|
||||
'showRow': 1,
|
||||
'snapshot': 0}
|
||||
|
||||
paraDict['vgroups'] = self.vgroups
|
||||
paraDict['ctbNum'] = self.ctbNum
|
||||
paraDict['rowsPerTbl'] = self.rowsPerTbl
|
||||
|
||||
tmqCom.initConsumerTable()
|
||||
tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], wal_retention_period=36000,vgroups=paraDict["vgroups"],replica=self.replicaVar)
|
||||
tdLog.info("create stb")
|
||||
tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"])
|
||||
# tdLog.info("create ctb")
|
||||
# tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
|
||||
# ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
|
||||
# tdLog.info("insert data")
|
||||
# tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"],
|
||||
# ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
|
||||
# startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
|
||||
|
||||
# tdLog.info("restart taosd to ensure that the data falls into the disk")
|
||||
# tdDnodes.stop(1)
|
||||
# tdDnodes.start(1)
|
||||
# tdSql.query("flush database %s"%(paraDict['dbName']))
|
||||
return
|
||||
|
||||
def restartAndRemoveWal(self):
|
||||
tdDnodes = cluster.dnodes
|
||||
tdSql.query("select * from information_schema.ins_vnodes")
|
||||
for result in tdSql.queryResult:
|
||||
if result[2] == 'dbt':
|
||||
tdLog.debug("dnode is %d"%(result[0]))
|
||||
dnodeId = result[0]
|
||||
vnodeId = result[1]
|
||||
|
||||
tdDnodes[dnodeId - 1].stoptaosd()
|
||||
time.sleep(1)
|
||||
dataPath = self.getDataPath()
|
||||
dataPath = dataPath%(dnodeId,vnodeId)
|
||||
os.system('rm -rf ' + dataPath)
|
||||
tdLog.debug("dataPath:%s"%dataPath)
|
||||
tdDnodes[dnodeId - 1].starttaosd()
|
||||
time.sleep(1)
|
||||
break
|
||||
tdLog.debug("restart dnode ok")
|
||||
|
||||
def redistributeVgroups(self):
|
||||
dnodesList = []
|
||||
tdSql.query("show dnodes")
|
||||
for result in tdSql.queryResult:
|
||||
dnodesList.append(result[0])
|
||||
|
||||
tdSql.query("select * from information_schema.ins_vnodes")
|
||||
vnodeId = 0
|
||||
for result in tdSql.queryResult:
|
||||
if result[2] == 'dbt':
|
||||
tdLog.debug("dnode is %d"%(result[0]))
|
||||
dnodesList.remove(result[0])
|
||||
vnodeId = result[1]
|
||||
break
|
||||
redistributeSql = "redistribute vgroup %d dnode %d" %(vnodeId, dnodesList[0])
|
||||
tdLog.debug("redistributeSql:%s"%(redistributeSql))
|
||||
tdSql.query(redistributeSql)
|
||||
tdLog.debug("redistributeSql ok")
|
||||
|
||||
def tmqCase1(self):
|
||||
tdLog.printNoPrefix("======== test case 1: ")
|
||||
paraDict = {'dbName': 'dbt',
|
||||
'dropFlag': 1,
|
||||
'event': '',
|
||||
'vgroups': 1,
|
||||
'stbName': 'stb',
|
||||
'colPrefix': 'c',
|
||||
'tagPrefix': 't',
|
||||
'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
|
||||
'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
|
||||
'ctbPrefix': 'ctb',
|
||||
'ctbStartIdx': 0,
|
||||
'ctbNum': 10,
|
||||
'rowsPerTbl': 10000,
|
||||
'batchNum': 10,
|
||||
'startTs': 1640966400000, # 2022-01-01 00:00:00.000
|
||||
'pollDelay': 60,
|
||||
'showMsg': 1,
|
||||
'showRow': 1,
|
||||
'snapshot': 0}
|
||||
|
||||
paraDict['vgroups'] = self.vgroups
|
||||
paraDict['ctbNum'] = self.ctbNum
|
||||
paraDict['rowsPerTbl'] = self.rowsPerTbl
|
||||
|
||||
topicNameList = ['topic1']
|
||||
# expectRowsList = []
|
||||
tmqCom.initConsumerTable()
|
||||
|
||||
tdLog.info("create topics from stb with filter")
|
||||
queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
|
||||
# sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName'])
|
||||
sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
|
||||
tdLog.info("create topic sql: %s"%sqlString)
|
||||
tdSql.execute(sqlString)
|
||||
# tdSql.query(queryString)
|
||||
# expectRowsList.append(tdSql.getRows())
|
||||
|
||||
# init consume info, and start tmq_sim, then check consume result
|
||||
tdLog.info("insert consume info to consume processor")
|
||||
consumerId = 0
|
||||
expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2
|
||||
topicList = topicNameList[0]
|
||||
ifcheckdata = 1
|
||||
ifManualCommit = 1
|
||||
keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest'
|
||||
tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
|
||||
|
||||
tdLog.info("start consume processor")
|
||||
tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
|
||||
tdLog.info("wait the consume result")
|
||||
|
||||
tdLog.info("create ctb1")
|
||||
tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
|
||||
ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
|
||||
tdLog.info("insert data")
|
||||
pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict)
|
||||
|
||||
tmqCom.getStartConsumeNotifyFromTmqsim()
|
||||
tmqCom.getStartCommitNotifyFromTmqsim()
|
||||
|
||||
#restart dnode & remove wal
|
||||
self.restartAndRemoveWal()
|
||||
|
||||
# redistribute vgroup
|
||||
self.redistributeVgroups();
|
||||
|
||||
tdLog.info("create ctb2")
|
||||
paraDict['ctbPrefix'] = "ctbn"
|
||||
tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
|
||||
ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
|
||||
tdLog.info("insert data")
|
||||
pInsertThread1 = tmqCom.asyncInsertDataByInterlace(paraDict)
|
||||
pInsertThread.join()
|
||||
pInsertThread1.join()
|
||||
|
||||
expectRows = 1
|
||||
resultList = tmqCom.selectConsumeResult(expectRows)
|
||||
|
||||
if expectrowcnt / 2 >= resultList[0]:
|
||||
tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0]))
|
||||
tdLog.exit("%d tmq consume rows error!"%consumerId)
|
||||
|
||||
# tmqCom.checkFileContent(consumerId, queryString)
|
||||
|
||||
time.sleep(10)
|
||||
for i in range(len(topicNameList)):
|
||||
tdSql.query("drop topic %s"%topicNameList[i])
|
||||
|
||||
tdLog.printNoPrefix("======== test case 1 end ...... ")
|
||||
|
||||
def tmqCase2(self):
|
||||
tdLog.printNoPrefix("======== test case 2: ")
|
||||
paraDict = {'dbName':'dbt'}
|
||||
|
||||
ntbName = "ntb"
|
||||
|
||||
topicNameList = ['topic2']
|
||||
tmqCom.initConsumerTable()
|
||||
|
||||
sqlString = "create table %s.%s(ts timestamp, i nchar(8))" %(paraDict['dbName'], ntbName)
|
||||
tdLog.info("create nomal table sql: %s"%sqlString)
|
||||
tdSql.execute(sqlString)
|
||||
|
||||
tdLog.info("create topics from nomal table")
|
||||
queryString = "select * from %s.%s"%(paraDict['dbName'], ntbName)
|
||||
sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
|
||||
tdLog.info("create topic sql: %s"%sqlString)
|
||||
tdSql.execute(sqlString)
|
||||
tdSql.query("flush database %s"%(paraDict['dbName']))
|
||||
#restart dnode & remove wal
|
||||
self.restartAndRemoveWal()
|
||||
|
||||
# redistribute vgroup
|
||||
self.redistributeVgroups();
|
||||
|
||||
sqlString = "alter table %s.%s modify column i nchar(16)" %(paraDict['dbName'], ntbName)
|
||||
tdLog.info("alter table sql: %s"%sqlString)
|
||||
tdSql.error(sqlString)
|
||||
|
||||
time.sleep(1)
|
||||
for i in range(len(topicNameList)):
|
||||
tdSql.query("drop topic %s"%topicNameList[i])
|
||||
|
||||
tdLog.printNoPrefix("======== test case 2 end ...... ")
|
||||
|
||||
def tmqCase3(self):
|
||||
tdLog.printNoPrefix("======== test case 3: ")
|
||||
paraDict = {'dbName': 'dbt',
|
||||
'dropFlag': 1,
|
||||
'event': '',
|
||||
'vgroups': 1,
|
||||
'stbName': 'stbn',
|
||||
'colPrefix': 'c',
|
||||
'tagPrefix': 't',
|
||||
'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
|
||||
'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
|
||||
'ctbPrefix': 'ctb',
|
||||
'ctbStartIdx': 0,
|
||||
'ctbNum': 10,
|
||||
'rowsPerTbl': 10000,
|
||||
'batchNum': 10,
|
||||
'startTs': 1640966400000, # 2022-01-01 00:00:00.000
|
||||
'pollDelay': 2,
|
||||
'showMsg': 1,
|
||||
'showRow': 1,
|
||||
'snapshot': 0}
|
||||
|
||||
paraDict['vgroups'] = self.vgroups
|
||||
paraDict['ctbNum'] = self.ctbNum
|
||||
paraDict['rowsPerTbl'] = self.rowsPerTbl
|
||||
|
||||
topicNameList = ['topic3']
|
||||
tmqCom.initConsumerTable()
|
||||
|
||||
tdLog.info("create stb")
|
||||
tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"])
|
||||
|
||||
tdLog.info("create ctb")
|
||||
tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
|
||||
ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
|
||||
tdLog.info("insert data")
|
||||
tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"],
|
||||
ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
|
||||
startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
|
||||
|
||||
tdLog.info("create topics from stb with filter")
|
||||
queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
|
||||
sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
|
||||
tdLog.info("create topic sql: %s"%sqlString)
|
||||
tdSql.execute(sqlString)
|
||||
|
||||
# init consume info, and start tmq_sim, then check consume result
|
||||
tdLog.info("insert consume info to consume processor")
|
||||
consumerId = 0
|
||||
expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"]
|
||||
topicList = topicNameList[0]
|
||||
ifcheckdata = 1
|
||||
ifManualCommit = 1
|
||||
keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest'
|
||||
tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
|
||||
|
||||
tdLog.info("start consume processor")
|
||||
tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
|
||||
tdLog.info("wait the consume result")
|
||||
|
||||
time.sleep(5)
|
||||
#restart dnode & remove wal
|
||||
self.restartAndRemoveWal()
|
||||
|
||||
# redistribute vgroup
|
||||
self.redistributeVgroups();
|
||||
|
||||
tdLog.info("start consume processor")
|
||||
tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
|
||||
tdLog.info("wait the consume result")
|
||||
|
||||
time.sleep(10)
|
||||
for i in range(len(topicNameList)):
|
||||
tdSql.query("drop topic %s"%topicNameList[i])
|
||||
|
||||
tdLog.printNoPrefix("======== test case 3 end ...... ")
|
||||
|
||||
def run(self):
|
||||
|
||||
tdSql.prepare()
|
||||
self.prepareTestEnv()
|
||||
self.tmqCase1()
|
||||
# self.tmqCase2()
|
||||
# self.tmqCase3()
|
||||
|
||||
def stop(self):
|
||||
tdSql.close()
|
||||
tdLog.success(f"{__file__} successfully executed")
|
||||
|
||||
event = threading.Event()
|
||||
|
||||
tdCases.addLinux(__file__, TDTestCase())
|
||||
tdCases.addWindows(__file__, TDTestCase())
|
||||
Loading…
Reference in New Issue