xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into feat/TS-5215-2

This commit is contained in:
Minglei Jin 2024-10-22 10:12:34 +08:00 committed by GitHub
parent aa14186f18 4527e96d2b
commit fe8b6c823e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
148 changed files with 15177 additions and 8851 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
cmake/cmake.options
View File

@ -144,6 +144,12 @@ option(
OFF
)
option(
BUILD_WITH_ANALYSIS
"If build with analysis"
ON
)
ENDIF ()
IF(NOT TD_ENTERPRISE)
@ -151,8 +157,15 @@ MESSAGE("switch s3 off with community version")
set(BUILD_S3 OFF)
set(BUILD_WITH_S3 OFF)
set(BUILD_WITH_COS OFF)
set(BUILD_WITH_ANALYSIS OFF)
ENDIF ()
IF(${BUILD_WITH_ANALYSIS})
message("build with analysis")
set(BUILD_S3 ON)
set(BUILD_WITH_S3 ON)
ENDIF()
IF(${BUILD_S3})
IF(${BUILD_WITH_S3})

3
cmake/stub_CMakeLists.txt.in
View File

@ -9,6 +9,5 @@ ExternalProject_Add(stub
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND ""
TEST_COMMAND ""
GIT_SHALLOW true
TEST_COMMAND ""
)

16
docs/examples/JDBC/SpringJdbcTemplate/readme.md
View File

@ -1,11 +1,11 @@
## TDengine Spring JDBC Template Demo
`Spring JDBC Template` 简化了原生 JDBC Connection 获取释放等操作，使得操作数据库更加方便。
`Spring JDBC Template` simplifies the operations of acquiring and releasing native JDBC Connections, making database operations more convenient.
### 配置
### Configuration
修改 `src/main/resources/applicationContext.xml` 文件中 TDengine 的配置信息：
Modify the TDengine configuration in the `src/main/resources/applicationContext.xml` file:
```xml
<bean id="dataSource" class="org.springframework.jdbc.datasource.DriverManagerDataSource">
@ -20,13 +20,15 @@
</bean>
```
### 打包运行
### Package and run
Navigate to the `TDengine/tests/examples/JDBC/SpringJdbcTemplate` directory and execute the following commands to generate an executable jar file.
进入 `TDengine/tests/examples/JDBC/SpringJdbcTemplate` 目录下，执行以下命令可以生成可执行 jar 包。
```shell
mvn clean package
```
打包成功之后，进入 `target/` 目录下，执行以下命令就可运行测试：
After successfully packaging, navigate to the `target/` directory and execute the following commands to run the tests:
```shell
java -jar target/SpringJdbcTemplate-1.0-SNAPSHOT-jar-with-dependencies.jar
```
```

10
docs/examples/JDBC/connectionPools/src/main/resources/log4j.properties
View File

@ -1,21 +1,21 @@
### 设置###
### Settings###
log4j.rootLogger=debug,stdout,DebugLog,ErrorLog
### 输出信息到控制抬 ###
### Output information to the console ###
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.Target=System.out
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=[%-5p] %d{yyyy-MM-dd HH:mm:ss,SSS} method:%l%n%m%n
### 输出DEBUG 级别以上的日志到=logs/debug.log
### Output logs of DEBUG level and above to logs/debug.log
log4j.appender.DebugLog=org.apache.log4j.DailyRollingFileAppender
log4j.appender.DebugLog.File=logs/debug.log
log4j.appender.DebugLog.Append=true
log4j.appender.DebugLog.Threshold=DEBUG
log4j.appender.DebugLog.layout=org.apache.log4j.PatternLayout
log4j.appender.DebugLog.layout.ConversionPattern=%-d{yyyy-MM-dd HH:mm:ss} [ %t:%r ] - [ %p ] %m%n
### 输出ERROR 级别以上的日志到=logs/error.log
### Output logs of ERROR level and above to logs/error.log
log4j.appender.ErrorLog=org.apache.log4j.DailyRollingFileAppender
log4j.appender.ErrorLog.File=logs/error.log
log4j.appender.ErrorLog.Append=true
log4j.appender.ErrorLog.Threshold=ERROR
log4j.appender.ErrorLog.layout=org.apache.log4j.PatternLayout
log4j.appender.ErrorLog.layout.ConversionPattern=%-d{yyyy-MM-dd HH:mm:ss} [ %t:%r ] - [ %p ] %m%n
log4j.appender.ErrorLog.layout.ConversionPattern=%-d{yyyy-MM-dd HH:mm:ss} [ %t:%r ] - [ %p ] %m%n

21
docs/examples/JDBC/connectionPools/src/main/resources/proxool.xml
View File

@ -1,27 +1,28 @@
<?xml version="1.0" encoding="UTF-8"?>
<something-else-entirely>
<proxool>
<!-- Alias for the data source -->
<alias>ds</alias>
<!--数据源的别名-->
<!-- URL connection string -->
<driver-url>jdbc:TAOS-RS://127.0.0.1:6041/log</driver-url>
<!--url连接串-->
<!-- Driver class -->
<driver-class>com.taosdata.jdbc.rs.RestfulDriver</driver-class>
<!--驱动类-->
<driver-properties>
<property name="user" value="root"/>
<property name="password" value="taosdata"/>
</driver-properties>
<!--最大连接数(默认5个),超过了这个连接数,再有请求时,就排在队列中等候,最大的等待请求数由maximum-new-connections决定 -->
<!-- Maximum connection count (default is 5). If this number is exceeded, new requests will be queued. The maximum number of queued requests is determined by maximum-new-connections -->
<maximum-connection-count>100</maximum-connection-count>
<!-- 定义连接池中的最大连接数 -->
<!-- Defines the maximum number of connections in the connection pool -->
<maximum-active-time>100</maximum-active-time>
<!--最少保持的空闲连接数(默认2个)-->
<!-- Minimum number of idle connections to maintain (default is 2) -->
<prototype-count>1</prototype-count>
<!--最小连接数(默认2个)-->
<!-- Minimum connection count (default is 2) -->
<minimum-connection-count>5</minimum-connection-count>
<!--proxool自动侦察各个连接状态的时间间隔(毫秒),侦察到空闲的连接就马上回收,超时的销毁 默认30秒-->
<!-- Interval (in milliseconds) for Proxool to automatically check the status of each connection. Idle connections are immediately reclaimed, and timed-out connections are destroyed. Default is 30 seconds -->
<house-keeping-sleep-time>30000</house-keeping-sleep-time>
<!--用于保持连接的测试语句 -->
<!-- Test statement used to maintain the connection -->
<house-keeping-test-sql>select server_version()</house-keeping-test-sql>
</proxool>
</something-else-entirely>
</something-else-entirely>

9
docs/examples/JDBC/consumer-demo/src/main/java/com/taosdata/Worker.java
View File

@ -35,17 +35,18 @@ public class Worker implements Runnable {
public void run() {
while (!Thread.interrupted()) {
try {
// 控制请求频率
// Control request rate
if (semaphore.tryAcquire()) {
ConsumerRecords<Bean> records = consumer.poll(Duration.ofMillis(sleepTime));
pool.submit(() -> {
RateLimiter limiter = RateLimiter.create(rate);
try {
for (ConsumerRecord<Bean> record : records) {
// 流量控制
// Traffic control
limiter.acquire();
// 业务处理数据
System.out.println("[" + LocalDateTime.now() + "] Thread id:" + Thread.currentThread().getId() + " -> " + record.value());
// Business data processing
System.out.println("[" + LocalDateTime.now() + "] Thread id:"
+ Thread.currentThread().getId() + " -> " + record.value());
}
} finally {
semaphore.release();

8
docs/examples/JDBC/mybatisplus-demo/readme
View File

@ -1,14 +1,14 @@
# 使用说明
# Instructions
## 创建使用db
## Create and use the database
```shell
$ taos
> create database mp_test
```
## 执行测试用例
## Execute test cases
```shell
$ mvn clean test
```
```

14
docs/examples/JDBC/springbootdemo/readme.md
View File

@ -1,6 +1,6 @@
## TDengine SpringBoot + Mybatis Demo
## 需要提前创建 test 数据库
## Need to create a test database in advance
```
$ taos -s 'create database if not exists test'
@ -8,7 +8,7 @@ $ taos -s 'create database if not exists test'
$ curl http://localhost:8080/weather/init
```
### 配置 application.properties
### Configure application.properties
```properties
# datasource config
spring.datasource.driver-class-name=com.taosdata.jdbc.TSDBDriver
@ -38,9 +38,9 @@ mybatis.mapper-locations=classpath:mapper/*.xml
logging.level.com.taosdata.jdbc.springbootdemo.dao=debug
```
### 主要功能
### Main functions
* 创建数据库和表
* Create databases and tables
```xml
<!-- weatherMapper.xml -->
<update id="createDB" >
@ -52,14 +52,14 @@ logging.level.com.taosdata.jdbc.springbootdemo.dao=debug
</update>
```
* 插入单条记录
* Insert a single record
```xml
<!-- weatherMapper.xml -->
<insert id="insert" parameterType="Weather" >
insert into test.weather (ts, temperature, humidity) values (now, #{temperature,jdbcType=INTEGER}, #{humidity,jdbcType=FLOAT})
</insert>
```
* 插入多条记录
* Insert multiple records
```xml
<!-- weatherMapper.xml -->
<insert id="batchInsert" parameterType="java.util.List" >
@ -69,7 +69,7 @@ logging.level.com.taosdata.jdbc.springbootdemo.dao=debug
</foreach>
</insert>
```
* 分页查询
* Pagination query
```xml
<!-- weatherMapper.xml -->
<?xml version="1.0" encoding="UTF-8"?>

7
docs/examples/JDBC/taosdemo/readme.md
View File

@ -1,11 +1,14 @@
```
cd tests/examples/JDBC/taosdemo
mvn clean package -Dmaven.test.skip=true
# 先建表，再插入的
# Create tables first, then insert data
java -jar target/taosdemo-2.0.1-jar-with-dependencies.jar -host <hostname> -database <db name> -doCreateTable true -superTableSQL "create table weather(ts timestamp, f1 int) tags(t1 nchar(4))" -numOfTables 1000 -numOfRowsPerTable 100000000 -numOfThreadsForInsert 10 -numOfTablesPerSQL 10 -numOfValuesPerSQL 100
# 不建表，直接插入的
# Insert data directly without creating tables
java -jar target/taosdemo-2.0.1-jar-with-dependencies.jar -host <hostname> -database <db name> -doCreateTable false -superTableSQL "create table weather(ts timestamp, f1 int) tags(t1 nchar(4))" -numOfTables 1000 -numOfRowsPerTable 100000000 -numOfThreadsForInsert 10 -numOfTablesPerSQL 10 -numOfValuesPerSQL 100
```
如果发生错误 Exception in thread "main" java.lang.UnsatisfiedLinkError: no taos in java.library.path
请检查是否安装 TDengine 客户端安装包或编译 TDengine 安装。如果确定已经安装过还出现这个错误，可以在命令行 java 后加 -Djava.library.path=/usr/lib 来指定寻找共享库的路径。
If you encounter the error Exception in thread "main" `java.lang.UnsatisfiedLinkError: no taos in java.library.path`, please check whether the TDengine client package is installed or TDengine is compiled and installed. If you are sure it is installed and still encounter this error, you can add `-Djava.library.path=/usr/lib` after the `java` command to specify the path to the shared library.

15
docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/TaosDemoApplication.java
View File

@ -24,14 +24,14 @@ public class TaosDemoApplication {
private static final Logger logger = LogManager.getLogger(TaosDemoApplication.class);
public static void main(String[] args) throws IOException {
// 读配置参数
// Read configuration parameters
JdbcTaosdemoConfig config = new JdbcTaosdemoConfig(args);
boolean isHelp = Arrays.asList(args).contains("--help");
if (isHelp || config.host == null || config.host.isEmpty()) {
JdbcTaosdemoConfig.printHelp();
System.exit(0);
}
// 初始化
//
final DataSource dataSource = DataSourceFactory.getInstance(config.host, config.port, config.user,
config.password);
if (config.executeSql != null && !config.executeSql.isEmpty()
@ -50,7 +50,7 @@ public class TaosDemoApplication {
final SuperTableService superTableService = new SuperTableService(dataSource);
final SubTableService subTableService = new SubTableService(dataSource);
// 创建数据库
// create database
long start = System.currentTimeMillis();
Map<String, String> databaseParam = new HashMap<>();
databaseParam.put("database", config.database);
@ -81,13 +81,13 @@ public class TaosDemoApplication {
config.prefixOfFields, config.numOfTags, config.prefixOfTags);
}
/**********************************************************************************/
// 建表
// create table
start = System.currentTimeMillis();
if (config.doCreateTable) {
superTableService.drop(superTableMeta.getDatabase(), superTableMeta.getName());
superTableService.create(superTableMeta);
if (!config.autoCreateTable) {
// 批量建子表
// create sub tables in batch
subTableService.createSubTable(superTableMeta, config.numOfTables, config.prefixOfTable,
config.numOfThreadsForCreate);
}
@ -95,7 +95,7 @@ public class TaosDemoApplication {
end = System.currentTimeMillis();
logger.info(">>> create table time cost : " + (end - start) + " ms.");
/**********************************************************************************/
// 插入
// insert data
long tableSize = config.numOfTables;
int threadSize = config.numOfThreadsForInsert;
long startTime = getProperStartTime(config.startTime, config.days);
@ -111,10 +111,9 @@ public class TaosDemoApplication {
end = System.currentTimeMillis();
logger.info("insert " + affectedRows + " rows, time cost: " + (end - start) + " ms");
/**********************************************************************************/
// 查询
/**********************************************************************************/
// 删除表
// drop table
if (config.dropTable) {
superTableService.drop(config.database, config.superTable);
}

15
docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SubTableMapper.java
View File

@ -9,21 +9,22 @@ import java.util.List;
@Repository
public interface SubTableMapper {
// 创建：子表
// Create: SubTable
void createUsingSuperTable(SubTableMeta subTableMeta);
// 插入：一张子表多个values
// Insert: Multiple records into one SubTable
int insertOneTableMultiValues(SubTableValue subTableValue);
// 插入：一张子表多个values, 自动建表
// Insert: Multiple records into one SubTable, auto create SubTables
int insertOneTableMultiValuesUsingSuperTable(SubTableValue subTableValue);
// 插入：多张表多个values
// Insert: Multiple records into multiple SubTable
int insertMultiTableMultiValues(List<SubTableValue> tables);
// 插入：多张表多个values，自动建表
// Insert: Multiple records into multiple SubTable, auto create SubTables
int insertMultiTableMultiValuesUsingSuperTable(List<SubTableValue> tables);
//<!-- TODO:修改子表标签值 alter table ${tablename} set tag tagName=newTagValue-->
// <!-- TODO: Modify SubTable tag value: alter table ${tablename} set tag
// tagName=newTagValue-->
}
}

22
docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SuperTableMapper.java
View File

@ -6,24 +6,26 @@ import org.springframework.stereotype.Repository;
@Repository
public interface SuperTableMapper {
// 创建超级表 create table if not exists xxx.xxx (f1 type1, f2 type2, ... ) tags( t1 type1, t2 type2 ...)
// Create super table: create table if not exists xxx.xxx (f1 type1, f2 type2,
// ... ) tags( t1 type1, t2 type2 ...)
void createSuperTable(SuperTableMeta tableMetadata);
// 删除超级表 drop table if exists xxx;
// Drop super table: drop table if exists xxx;
void dropSuperTable(String database, String name);
//<!-- TODO:查询所有超级表信息 show stables -->
// <!-- TODO: Query all super table information show stables -->
//<!-- TODO:查询表结构 describe stable -->
// <!-- TODO: Query table structure describe stable -->
//<!-- TODO:增加列 alter table ${tablename} add column fieldName dataType -->
// <!-- TODO: Add column alter table ${tablename} add column fieldName dataType
// -->
//<!-- TODO:删除列 alter table ${tablename} drop column fieldName -->
// <!-- TODO: Drop column alter table ${tablename} drop column fieldName -->
//<!-- TODO:添加标签 alter table ${tablename} add tag new_tagName tag_type -->
// <!-- TODO: Add tag alter table ${tablename} add tag new_tagName tag_type -->
//<!-- TODO:删除标签 alter table ${tablename} drop tag_name -->
//<!-- TODO:修改标签名 alter table ${tablename} change tag old_tagName new_tagName -->
// <!-- TODO: Drop tag alter table ${tablename} drop tag_name -->
// <!-- TODO: Change tag name alter table ${tablename} change tag old_tagName
// new_tagName -->
}

13
docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/TableMapper.java
View File

@ -9,19 +9,18 @@ import java.util.List;
@Repository
public interface TableMapper {
// 创建：普通表
// Create: Normal table
void create(TableMeta tableMeta);
// 插入：一张表多个value
// Insert: Multiple records into one table
int insertOneTableMultiValues(TableValue values);
// 插入: 一张表多个value，指定的列
// Insert: Multiple records into one table, specified columns
int insertOneTableMultiValuesWithColumns(TableValue values);
// 插入：多个表多个value
// Insert: Multiple records into multiple tables
int insertMultiTableMultiValues(List<TableValue> tables);
// 插入：多个表多个value, 指定的列
// Insert: Multiple records into multiple tables, specified columns
int insertMultiTableMultiValuesWithColumns(List<TableValue> tables);
}
}

4
docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/DatabaseService.java
View File

@ -14,12 +14,12 @@ public class DatabaseService {
this.databaseMapper = new DatabaseMapperImpl(dataSource);
}
// 建库，指定 name
// Create database with specified name
public void createDatabase(String database) {
databaseMapper.createDatabase(database);
}
// 建库，指定参数 keep,days,replica等
// Create database with specified parameters such as keep, days, replica, etc.
public void createDatabase(Map<String, String> map) {
if (map.isEmpty())
return;

60
docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SubTableService.java
View File

@ -27,7 +27,8 @@ public class SubTableService extends AbstractService {
this.mapper = new SubTableMapperImpl(datasource);
}
public void createSubTable(SuperTableMeta superTableMeta, long numOfTables, String prefixOfTable, int numOfThreadsForCreate) {
public void createSubTable(SuperTableMeta superTableMeta, long numOfTables, String prefixOfTable,
int numOfThreadsForCreate) {
ExecutorService executor = Executors.newFixedThreadPool(numOfThreadsForCreate);
for (long i = 0; i < numOfTables; i++) {
long tableIndex = i;
@ -35,54 +36,58 @@ public class SubTableService extends AbstractService {
}
executor.shutdown();
try {
executor.awaitTermination(Long.MAX_VALUE,TimeUnit.NANOSECONDS);
executor.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
public void createSubTable(SuperTableMeta superTableMeta, String tableName) {
// 构造数据
// Construct data
SubTableMeta meta = SubTableMetaGenerator.generate(superTableMeta, tableName);
createSubTable(meta);
}
// 创建一张子表，可以指定database，supertable，tablename，tag值
// Create a sub-table, specifying database, super table, table name, and tag
// values
public void createSubTable(SubTableMeta subTableMeta) {
mapper.createUsingSuperTable(subTableMeta);
}
/*************************************************************************************************************************/
// 插入：多线程，多表
// Insert: Multi-threaded, multiple tables
public int insert(List<SubTableValue> subTableValues, int threadSize, int frequency) {
ExecutorService executor = Executors.newFixedThreadPool(threadSize);
Future<Integer> future = executor.submit(() -> insert(subTableValues));
executor.shutdown();
//TODO：frequency
// TODO：frequency
return getAffectRows(future);
}
// 插入：单表，insert into xxx values(),()...
// Insert: Single table, insert into xxx values(),()...
public int insert(SubTableValue subTableValue) {
return mapper.insertOneTableMultiValues(subTableValue);
}
// 插入: 多表，insert into xxx values(),()... xxx values(),()...
// Insert: Multiple tables, insert into xxx values(),()... xxx values(),()...
public int insert(List<SubTableValue> subTableValues) {
return mapper.insertMultiTableMultiValues(subTableValues);
}
// 插入：单表，自动建表, insert into xxx using xxx tags(...) values(),()...
// Insert: Single table, auto-create table, insert into xxx using xxx tags(...)
// values(),()...
public int insertAutoCreateTable(SubTableValue subTableValue) {
return mapper.insertOneTableMultiValuesUsingSuperTable(subTableValue);
}
// 插入：多表，自动建表, insert into xxx using XXX tags(...) values(),()... xxx using XXX tags(...) values(),()...
// Insert: Multiple tables, auto-create tables, insert into xxx using XXX
// tags(...) values(),()... xxx using XXX tags(...) values(),()...
public int insertAutoCreateTable(List<SubTableValue> subTableValues) {
return mapper.insertMultiTableMultiValuesUsingSuperTable(subTableValues);
}
public int insertMultiThreads(SuperTableMeta superTableMeta, int threadSize, long tableSize, long startTime, long gap, JdbcTaosdemoConfig config) {
public int insertMultiThreads(SuperTableMeta superTableMeta, int threadSize, long tableSize, long startTime,
long gap, JdbcTaosdemoConfig config) {
List<FutureTask> taskList = new ArrayList<>();
List<Thread> threads = IntStream.range(0, threadSize)
.mapToObj(i -> {
@ -94,8 +99,7 @@ public class SubTableService extends AbstractService {
startTime, config.timeGap,
config.numOfRowsPerTable, config.numOfTablesPerSQL, config.numOfValuesPerSQL,
config.order, config.rate, config.range,
config.prefixOfTable, config.autoCreateTable)
);
config.prefixOfTable, config.autoCreateTable));
taskList.add(task);
return new Thread(task, "InsertThread-" + i);
}).collect(Collectors.toList());
@ -126,7 +130,7 @@ public class SubTableService extends AbstractService {
private class InsertTask implements Callable<Integer> {
private final long startTableInd; // included
private final long endTableInd; // excluded
private final long endTableInd; // excluded
private final long startTime;
private final long timeGap;
private final long numOfRowsPerTable;
@ -140,10 +144,10 @@ public class SubTableService extends AbstractService {
private final boolean autoCreateTable;
public InsertTask(SuperTableMeta superTableMeta, long startTableInd, long endTableInd,
long startTime, long timeGap,
long numOfRowsPerTable, long numOfTablesPerSQL, long numOfValuesPerSQL,
int order, int rate, long range,
String prefixOfTable, boolean autoCreateTable) {
long startTime, long timeGap,
long numOfRowsPerTable, long numOfTablesPerSQL, long numOfValuesPerSQL,
int order, int rate, long range,
String prefixOfTable, boolean autoCreateTable) {
this.superTableMeta = superTableMeta;
this.startTableInd = startTableInd;
this.endTableInd = endTableInd;
@ -159,7 +163,6 @@ public class SubTableService extends AbstractService {
this.autoCreateTable = autoCreateTable;
}
@Override
public Integer call() {
@ -171,23 +174,27 @@ public class SubTableService extends AbstractService {
int affectRows = 0;
// row
for (long rowCnt = 0; rowCnt < numOfRowsPerTable; ) {
for (long rowCnt = 0; rowCnt < numOfRowsPerTable;) {
long rowSize = numOfValuesPerSQL;
if (rowCnt + rowSize > numOfRowsPerTable) {
rowSize = numOfRowsPerTable - rowCnt;
}
//table
for (long tableCnt = startTableInd; tableCnt < endTableInd; ) {
// table
for (long tableCnt = startTableInd; tableCnt < endTableInd;) {
long tableSize = numOfTablesPerSQL;
if (tableCnt + tableSize > endTableInd) {
tableSize = endTableInd - tableCnt;
}
long startTime = this.startTime + rowCnt * timeGap;
// System.out.println(Thread.currentThread().getName() + " >>> " + "rowCnt: " + rowCnt + ", rowSize: " + rowSize + ", " + "tableCnt: " + tableCnt + ",tableSize: " + tableSize + ", " + "startTime: " + startTime + ",timeGap: " + timeGap + "");
// System.out.println(Thread.currentThread().getName() + " >>> " + "rowCnt: " +
// rowCnt + ", rowSize: " + rowSize + ", " + "tableCnt: " + tableCnt +
// ",tableSize: " + tableSize + ", " + "startTime: " + startTime + ",timeGap: "
// + timeGap + "");
/***********************************************/
// 生成数据
List<SubTableValue> data = SubTableValueGenerator.generate(superTableMeta, prefixOfTable, tableCnt, tableSize, rowSize, startTime, timeGap);
// 乱序
// Construct data
List<SubTableValue> data = SubTableValueGenerator.generate(superTableMeta, prefixOfTable, tableCnt,
tableSize, rowSize, startTime, timeGap);
// disorder
if (order != 0)
SubTableValueGenerator.disrupt(data, rate, range);
// insert
@ -205,5 +212,4 @@ public class SubTableService extends AbstractService {
}
}
}

2
docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SuperTableService.java
View File

@ -14,7 +14,7 @@ public class SuperTableService {
this.superTableMapper = new SuperTableMapperImpl(dataSource);
}
// 创建超级表，指定每个field的名称和类型，每个tag的名称和类型
// Create super table, specifying the name and type of each field and each tag
public void create(SuperTableMeta superTableMeta) {
superTableMapper.createSuperTable(superTableMeta);
}

5
docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/TableService.java
View File

@ -11,15 +11,14 @@ public class TableService extends AbstractService {
private TableMapper tableMapper;
//创建一张表
// Create a table
public void create(TableMeta tableMeta) {
tableMapper.create(tableMeta);
}
//创建多张表
// Create multiple tables
public void create(List<TableMeta> tables) {
tables.stream().forEach(this::create);
}
}

10
docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/FieldValueGenerator.java
View File

@ -11,7 +11,8 @@ public class FieldValueGenerator {
public static Random random = new Random(System.currentTimeMillis());
// 生成start到end的时间序列，时间戳为顺序，不含有乱序，field的value为随机生成
// Generate a time series from start to end, timestamps are in order without
// disorder, field values are randomly generated
public static List<RowValue> generate(long start, long end, long timeGap, List<FieldMeta> fieldMetaList) {
List<RowValue> values = new ArrayList<>();
@ -29,9 +30,12 @@ public class FieldValueGenerator {
return values;
}
// 生成start到end的时间序列，时间戳为顺序，含有乱序，rate为乱序的比例，range为乱序前跳范围，field的value为随机生成
// Generate a time series from start to end, timestamps are in order but include
// disorder, rate is the proportion of disorder, range is the jump range before
// disorder, field values are randomly generated
public static List<RowValue> disrupt(List<RowValue> values, int rate, long range) {
long timeGap = (long) (values.get(1).getFields().get(0).getValue()) - (long) (values.get(0).getFields().get(0).getValue());
long timeGap = (long) (values.get(1).getFields().get(0).getValue())
- (long) (values.get(0).getFields().get(0).getValue());
int bugSize = values.size() * rate / 100;
Set<Integer> bugIndSet = new HashSet<>();
while (bugIndSet.size() < bugSize) {

2
docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/SubTableMetaGenerator.java
View File

@ -9,7 +9,7 @@ import java.util.List;
public class SubTableMetaGenerator {
// 创建tableSize张子表，使用tablePrefix作为子表名的前缀，使用superTableMeta的元数据
// Create tableSize sub-tables, using tablePrefix as the prefix for sub-table names, and using the metadata from superTableMeta
// create table xxx using XXX tags(XXX)
public static List<SubTableMeta> generate(SuperTableMeta superTableMeta, int tableSize, String tablePrefix) {
List<SubTableMeta> subTableMetaList = new ArrayList<>();

13
docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/SuperTableMetaGenerator.java
View File

@ -10,10 +10,11 @@ import java.util.List;
public class SuperTableMetaGenerator {
// 创建超级表，使用指定SQL语句
// Create super table using the specified SQL statement
public static SuperTableMeta generate(String superTableSQL) {
SuperTableMeta tableMeta = new SuperTableMeta();
// for example : create table superTable (ts timestamp, temperature float, humidity int) tags(location nchar(64), groupId int)
// for example : create table superTable (ts timestamp, temperature float,
// humidity int) tags(location nchar(64), groupId int)
superTableSQL = superTableSQL.trim().toLowerCase();
if (!superTableSQL.startsWith("create"))
throw new RuntimeException("invalid create super table SQL");
@ -54,8 +55,9 @@ public class SuperTableMetaGenerator {
return tableMeta;
}
// 创建超级表,指定field和tag的个数
public static SuperTableMeta generate(String database, String name, int fieldSize, String fieldPrefix, int tagSize, String tagPrefix) {
// Create super table with specified number of fields and tags
public static SuperTableMeta generate(String database, String name, int fieldSize, String fieldPrefix, int tagSize,
String tagPrefix) {
if (fieldSize < 2 || tagSize < 1) {
throw new RuntimeException("create super table but fieldSize less than 2 or tagSize less than 1");
}
@ -66,7 +68,8 @@ public class SuperTableMetaGenerator {
List<FieldMeta> fields = new ArrayList<>();
fields.add(new FieldMeta("ts", "timestamp"));
for (int i = 1; i <= fieldSize; i++) {
fields.add(new FieldMeta(fieldPrefix + "" + i, TaosConstants.DATA_TYPES[i % TaosConstants.DATA_TYPES.length]));
fields.add(
new FieldMeta(fieldPrefix + "" + i, TaosConstants.DATA_TYPES[i % TaosConstants.DATA_TYPES.length]));
}
tableMetadata.setFields(fields);
// tags

2
docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/TagValueGenerator.java
View File

@ -9,7 +9,7 @@ import java.util.List;
public class TagValueGenerator {
// 创建标签值：使用tagMetas
// Create tag values using tagMetas
public static List<TagValue> generate(List<TagMeta> tagMetas) {
List<TagValue> tagValues = new ArrayList<>();
for (int i = 0; i < tagMetas.size(); i++) {

6
docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/utils/TimeStampUtil.java
View File

@ -41,17 +41,17 @@ public class TimeStampUtil {
if (start == 0)
start = now - size * timeGap;
// 如果size小于1异常
// If size is less than 1, throw an exception
if (size < 1)
throw new IllegalArgumentException("size less than 1.");
// 如果timeGap为1，已经超长，需要前移start
// If timeGap is 1 and it exceeds the limit, move start forward
if (start + size > now) {
start = now - size;
return new TimeTuple(start, now, 1);
}
long end = start + (long) (timeGap * size);
if (end > now) {
//压缩timeGap
// Compress timeGap
end = now;
double gap = (end - start) / (size * 1.0f);
if (gap < 1.0f) {

10
docs/examples/JDBC/taosdemo/src/main/resources/log4j.properties
View File

@ -1,21 +1,21 @@
### 设置###
### Settings ###
log4j.rootLogger=info,stdout
### 输出信息到控制抬 ###
### Output information to the console ###
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.Target=System.out
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=[%-5p] %d{yyyy-MM-dd HH:mm:ss,SSS} method:%l%n%m%n
### 输出DEBUG 级别以上的日志到=logs/debug.log
### Output logs of DEBUG level and above to logs/debug.log ###
log4j.appender.DebugLog=org.apache.log4j.DailyRollingFileAppender
log4j.appender.DebugLog.File=logs/debug.log
log4j.appender.DebugLog.Append=true
log4j.appender.DebugLog.Threshold=DEBUG
log4j.appender.DebugLog.layout=org.apache.log4j.PatternLayout
log4j.appender.DebugLog.layout.ConversionPattern=%-d{yyyy-MM-dd HH:mm:ss} [ %t:%r ] - [ %p ] %m%n
### 输出ERROR 级别以上的日志到=logs/error.log
### Output logs of ERROR level and above to logs/error.log ###
log4j.appender.ErrorLog=org.apache.log4j.DailyRollingFileAppender
log4j.appender.ErrorLog.File=logs/error.log
log4j.appender.ErrorLog.Append=true
log4j.appender.ErrorLog.Threshold=ERROR
log4j.appender.ErrorLog.layout=org.apache.log4j.PatternLayout
log4j.appender.ErrorLog.layout.ConversionPattern=%-d{yyyy-MM-dd HH:mm:ss} [ %t:%r ] - [ %p ] %m%n
log4j.appender.ErrorLog.layout.ConversionPattern=%-d{yyyy-MM-dd HH:mm:ss} [ %t:%r ] - [ %p ] %m%n

2
docs/zh/01-index.md
View File

@ -6,7 +6,7 @@ slug: /
TDengine 是一款[开源](https://www.taosdata.com/tdengine/open_source_time-series_database)、[高性能](https://www.taosdata.com/fast)、[云原生](https://www.taosdata.com/tdengine/cloud_native_time-series_database)的<a href="https://www.taosdata.com/" data-internallinksmanager029f6b8e52c="2" title="时序数据库" target="_blank" rel="noopener">时序数据库</a>（<a href="https://www.taosdata.com/time-series-database" data-internallinksmanager029f6b8e52c="9" title="Time Series DataBase" target="_blank" rel="noopener">Time Series Database</a>, <a href="https://www.taosdata.com/tsdb" data-internallinksmanager029f6b8e52c="8" title="TSDB" target="_blank" rel="noopener">TSDB</a>）, 它专为物联网、车联网、工业互联网、金融、IT 运维等场景优化设计。同时它还带有内建的缓存、流式计算、数据订阅等系统功能，能大幅减少系统设计的复杂度，降低研发和运营成本，是一款极简的时序数据处理平台。本文档是 TDengine 的用户手册，主要是介绍 TDengine 的基本概念、安装、使用、功能、开发接口、运营维护、TDengine 内核设计等等，它主要是面向架构师、开发工程师与系统管理员的。如果你对时序数据的基本概念、价值以及其所能带来的业务价值尚不了解，请参考[时序数据基础](./concept)
TDengine 充分利用了时序数据的特点，提出了“一个数据采集点一张表”与“超级表”的概念，设计了创新的存储引擎，让数据的写入、查询和存储效率都得到极大的提升。为正确理解并使用 TDengine，无论如何，请您仔细阅读[快速入门](./basic)一章。
TDengine 充分利用了时序数据的特点，提出了“一个数据采集点一张表”与“超级表”的概念，设计了创新的存储引擎，让数据的写入、查询和存储效率都得到极大的提升。为正确理解并使用 TDengine，无论如何，请您仔细阅读[数据模型](./basic/model)一章。
如果你是开发工程师，请一定仔细阅读[开发指南](./develop)一章，该部分对数据库连接、建模、插入数据、查询、流式计算、缓存、数据订阅、用户自定义函数等功能都做了详细介绍，并配有各种编程语言的示例代码。大部分情况下，你只要复制粘贴示例代码，针对自己的应用稍作改动，就能跑起来。对 REST API、各种编程语言的连接器（Connector）想做更多详细了解的话，请看[连接器](./reference/connector)一章。

4
docs/zh/14-reference/01-components/07-explorer.md
View File

@ -4,11 +4,11 @@ sidebar_label: taosExplorer
toc_max_heading_level: 4
---
taosExplorer 是一个为用户提供 TDengine 实例的可视化管理交互工具的 web 服务。本节主要讲述其安装和部署。它的各项功能都是基于简单易上手的图形界面，可以直接尝试，如果有需要也可以考高级功能和运维指南中的相关内容。为了确保访问 taosExplorer 的最佳体验，请使用 Chrome 79 及以上版本，或 Edge 79 及以上版本。
taosExplorer 是一个为用户提供 TDengine 实例的可视化管理交互工具的 web 服务，虽然它没有开源，但随开源版安装包免费提供。本节主要讲述其安装和部署。它的各项功能都是基于简单易上手的图形界面，可以直接尝试，如果有需要也可以参考高级功能和运维指南中的相关内容。为了确保访问 taosExplorer 的最佳体验，请使用 Chrome 79 及以上版本，或 Edge 79 及以上版本。
## 安装
taosEexplorer 无需单独安装，从 TDengine 3.3.0.0 版本开始，它随着 TDengine Enterprise Server 安装包一起发布，安装完成后，就可以看到 `taos-explorer` 服务。
taosEexplorer 无需单独安装，从 TDengine 3.3.0.0 版本开始，它随着 TDengine 安装包一起发布，安装完成后，就可以看到 `taos-explorer` 服务。如果按照 GitHub 里步骤自己编译 TDengine 源代码生成的安装包不包含 taosExplorer。
## 配置

4
docs/zh/14-reference/02-tools/10-taosbenchmark.md
View File

@ -477,7 +477,9 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\)
| 13 | BOOL | bool
| 14 | NCHAR | nchar
| 15 | VARCHAR | varchar
| 15 | JSON | json
| 16 | VARBINARY | varbinary
| 17 | GEOMETRY | geometry
| 18 | JSON | json
注意：taosBenchmark 配置文件中数据类型必须小写方可识别

2
include/common/systable.h
View File

@ -29,6 +29,8 @@ extern "C" {
#define TSDB_INS_TABLE_QNODES "ins_qnodes"
#define TSDB_INS_TABLE_BNODES "ins_bnodes" // no longer used
#define TSDB_INS_TABLE_SNODES "ins_snodes"
#define TSDB_INS_TABLE_ANODES "ins_anodes"
#define TSDB_INS_TABLE_ANODES_FULL "ins_anodes_full"
#define TSDB_INS_TABLE_ARBGROUPS "ins_arbgroups"
#define TSDB_INS_TABLE_CLUSTER "ins_cluster"
#define TSDB_INS_TABLE_DATABASES "ins_databases"

96
include/common/tanal.h Normal file
View File

@ -0,0 +1,96 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_UTIL_ANAL_H_
#define _TD_UTIL_ANAL_H_
#include "os.h"
#include "tdef.h"
#include "thash.h"
#include "tjson.h"
#ifdef __cplusplus
extern "C" {
#endif
#define ANAL_FORECAST_DEFAULT_ROWS 10
#define ANAL_FORECAST_DEFAULT_CONF 95
#define ANAL_FORECAST_DEFAULT_WNCHECK 1
#define ANAL_FORECAST_MAX_ROWS 10000
#define ANAL_ANOMALY_WINDOW_MAX_ROWS 10000
typedef struct {
EAnalAlgoType type;
int32_t anode;
int32_t urlLen;
char *url;
} SAnalUrl;
typedef enum {
ANAL_BUF_TYPE_JSON = 0,
ANAL_BUF_TYPE_JSON_COL = 1,
ANAL_BUF_TYPE_OTHERS,
} EAnalBufType;
typedef enum {
ANAL_HTTP_TYPE_GET = 0,
ANAL_HTTP_TYPE_POST,
} EAnalHttpType;
typedef struct {
TdFilePtr filePtr;
char fileName[TSDB_FILENAME_LEN + 10];
int64_t numOfRows;
} SAnalColBuf;
typedef struct {
EAnalBufType bufType;
TdFilePtr filePtr;
char fileName[TSDB_FILENAME_LEN];
int32_t numOfCols;
SAnalColBuf *pCols;
} SAnalBuf;
int32_t taosAnalInit();
void taosAnalCleanup();
SJson *taosAnalSendReqRetJson(const char *url, EAnalHttpType type, SAnalBuf *pBuf);
int32_t taosAnalGetAlgoUrl(const char *algoName, EAnalAlgoType type, char *url, int32_t urlLen);
bool taosAnalGetOptStr(const char *option, const char *optName, char *optValue, int32_t optMaxLen);
bool taosAnalGetOptInt(const char *option, const char *optName, int64_t *optValue);
int64_t taosAnalGetVersion();
void taosAnalUpdate(int64_t newVer, SHashObj *pHash);
int32_t tsosAnalBufOpen(SAnalBuf *pBuf, int32_t numOfCols);
int32_t taosAnalBufWriteOptStr(SAnalBuf *pBuf, const char *optName, const char *optVal);
int32_t taosAnalBufWriteOptInt(SAnalBuf *pBuf, const char *optName, int64_t optVal);
int32_t taosAnalBufWriteOptFloat(SAnalBuf *pBuf, const char *optName, float optVal);
int32_t taosAnalBufWriteColMeta(SAnalBuf *pBuf, int32_t colIndex, int32_t colType, const char *colName);
int32_t taosAnalBufWriteDataBegin(SAnalBuf *pBuf);
int32_t taosAnalBufWriteColBegin(SAnalBuf *pBuf, int32_t colIndex);
int32_t taosAnalBufWriteColData(SAnalBuf *pBuf, int32_t colIndex, int32_t colType, void *colValue);
int32_t taosAnalBufWriteColEnd(SAnalBuf *pBuf, int32_t colIndex);
int32_t taosAnalBufWriteDataEnd(SAnalBuf *pBuf);
int32_t taosAnalBufClose(SAnalBuf *pBuf);
void taosAnalBufDestroy(SAnalBuf *pBuf);
const char *taosAnalAlgoStr(EAnalAlgoType algoType);
EAnalAlgoType taosAnalAlgoInt(const char *algoName);
const char *taosAnalAlgoUrlStr(EAnalAlgoType algoType);
#ifdef __cplusplus
}
#endif
#endif /*_TD_UTIL_ANAL_H_*/

36
include/common/tglobal.h
View File

@ -42,21 +42,21 @@ typedef enum {
} EEncryptScope;
// cluster
extern char tsFirst[];
extern char tsSecond[];
extern char tsLocalFqdn[];
extern char tsLocalEp[];
extern char tsVersionName[];
extern uint16_t tsServerPort;
extern int32_t tsVersion;
extern int32_t tsStatusInterval;
extern int32_t tsNumOfSupportVnodes;
extern char tsEncryptAlgorithm[];
extern char tsEncryptScope[];
extern EEncryptAlgor tsiEncryptAlgorithm;
extern EEncryptScope tsiEncryptScope;
//extern char tsAuthCode[];
extern char tsEncryptKey[];
extern char tsFirst[];
extern char tsSecond[];
extern char tsLocalFqdn[];
extern char tsLocalEp[];
extern char tsVersionName[];
extern uint16_t tsServerPort;
extern int32_t tsVersion;
extern int32_t tsStatusInterval;
extern int32_t tsNumOfSupportVnodes;
extern char tsEncryptAlgorithm[];
extern char tsEncryptScope[];
extern EEncryptAlgor tsiEncryptAlgorithm;
extern EEncryptScope tsiEncryptScope;
// extern char tsAuthCode[];
extern char tsEncryptKey[];
// common
extern int32_t tsMaxShellConns;
@ -71,6 +71,8 @@ extern int32_t tsTagFilterResCacheSize;
// queue & threads
extern int32_t tsNumOfRpcThreads;
extern int32_t tsNumOfRpcSessions;
extern int32_t tsShareConnLimit;
extern int32_t tsReadTimeout;
extern int32_t tsTimeToGetAvailableConn;
extern int32_t tsKeepAliveIdle;
extern int32_t tsNumOfCommitThreads;
@ -264,8 +266,8 @@ extern bool tsExperimental;
int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd,
const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc);
int32_t taosReadDataFolder(const char *cfgDir, const char **envCmd,
const char *envFile, char *apolloUrl, SArray *pArgs);
int32_t taosReadDataFolder(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl,
SArray *pArgs);
int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs,
bool tsc);
void taosCleanupCfg();

55
include/common/tmsg.h
View File

@ -159,6 +159,8 @@ typedef enum _mgmt_table {
TSDB_MGMT_TABLE_ARBGROUP,
TSDB_MGMT_TABLE_ENCRYPTIONS,
TSDB_MGMT_TABLE_USER_FULL,
TSDB_MGMT_TABLE_ANODE,
TSDB_MGMT_TABLE_ANODE_FULL,
TSDB_MGMT_TABLE_MAX,
} EShowType;
@ -260,6 +262,7 @@ typedef enum ENodeType {
QUERY_NODE_COUNT_WINDOW,
QUERY_NODE_COLUMN_OPTIONS,
QUERY_NODE_TSMA_OPTIONS,
QUERY_NODE_ANOMALY_WINDOW,
// Statement nodes are used in parser and planner module.
QUERY_NODE_SET_OPERATOR = 100,
@ -345,6 +348,9 @@ typedef enum ENodeType {
QUERY_NODE_CREATE_VIEW_STMT,
QUERY_NODE_DROP_VIEW_STMT,
QUERY_NODE_CREATE_SUBTABLE_FROM_FILE_CLAUSE,
QUERY_NODE_CREATE_ANODE_STMT,
QUERY_NODE_DROP_ANODE_STMT,
QUERY_NODE_UPDATE_ANODE_STMT,
// show statement nodes
// see 'sysTableShowAdapter', 'SYSTABLE_SHOW_TYPE_OFFSET'
@ -386,6 +392,8 @@ typedef enum ENodeType {
QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT,
QUERY_NODE_SHOW_ENCRYPTIONS_STMT,
QUERY_NODE_SHOW_TSMAS_STMT,
QUERY_NODE_SHOW_ANODES_STMT,
QUERY_NODE_SHOW_ANODES_FULL_STMT,
QUERY_NODE_CREATE_TSMA_STMT,
QUERY_NODE_SHOW_CREATE_TSMA_STMT,
QUERY_NODE_DROP_TSMA_STMT,
@ -408,6 +416,7 @@ typedef enum ENodeType {
QUERY_NODE_LOGIC_PLAN,
QUERY_NODE_LOGIC_PLAN_GROUP_CACHE,
QUERY_NODE_LOGIC_PLAN_DYN_QUERY_CTRL,
QUERY_NODE_LOGIC_PLAN_FORECAST_FUNC,
// physical plan node
QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN = 1100,
@ -458,6 +467,9 @@ typedef enum ENodeType {
QUERY_NODE_PHYSICAL_PLAN_MERGE_COUNT,
QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT,
QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL,
QUERY_NODE_PHYSICAL_PLAN_MERGE_ANOMALY,
QUERY_NODE_PHYSICAL_PLAN_STREAM_ANOMALY,
QUERY_NODE_PHYSICAL_PLAN_FORECAST_FUNC,
} ENodeType;
typedef struct {
@ -971,6 +983,7 @@ typedef struct SEpSet {
SEp eps[TSDB_MAX_REPLICA];
} SEpSet;
int32_t tEncodeSEpSet(SEncoder* pEncoder, const SEpSet* pEp);
int32_t tDecodeSEpSet(SDecoder* pDecoder, SEpSet* pEp);
int32_t taosEncodeSEpSet(void** buf, const SEpSet* pEp);
@ -1092,6 +1105,22 @@ typedef struct {
int32_t tSerializeRetrieveIpWhite(void* buf, int32_t bufLen, SRetrieveIpWhiteReq* pReq);
int32_t tDeserializeRetrieveIpWhite(void* buf, int32_t bufLen, SRetrieveIpWhiteReq* pReq);
typedef struct {
int32_t dnodeId;
int64_t analVer;
} SRetrieveAnalAlgoReq;
typedef struct {
int64_t ver;
SHashObj* hash; // algoname:algotype -> SAnalUrl
} SRetrieveAnalAlgoRsp;
int32_t tSerializeRetrieveAnalAlgoReq(void* buf, int32_t bufLen, SRetrieveAnalAlgoReq* pReq);
int32_t tDeserializeRetrieveAnalAlgoReq(void* buf, int32_t bufLen, SRetrieveAnalAlgoReq* pReq);
int32_t tSerializeRetrieveAnalAlgoRsp(void* buf, int32_t bufLen, SRetrieveAnalAlgoRsp* pRsp);
int32_t tDeserializeRetrieveAnalAlgoRsp(void* buf, int32_t bufLen, SRetrieveAnalAlgoRsp* pRsp);
void tFreeRetrieveAnalAlgoRsp(SRetrieveAnalAlgoRsp* pRsp);
typedef struct {
int8_t alterType;
int8_t superUser;
@ -1766,6 +1795,7 @@ typedef struct {
SArray* pVloads; // array of SVnodeLoad
int32_t statusSeq;
int64_t ipWhiteVer;
int64_t analVer;
} SStatusReq;
int32_t tSerializeSStatusReq(void* buf, int32_t bufLen, SStatusReq* pReq);
@ -1831,6 +1861,7 @@ typedef struct {
SArray* pDnodeEps; // Array of SDnodeEp
int32_t statusSeq;
int64_t ipWhiteVer;
int64_t analVer;
} SStatusRsp;
int32_t tSerializeSStatusRsp(void* buf, int32_t bufLen, SStatusRsp* pRsp);
@ -2377,6 +2408,30 @@ typedef struct {
int32_t tSerializeSDCreateMnodeReq(void* buf, int32_t bufLen, SDCreateMnodeReq* pReq);
int32_t tDeserializeSDCreateMnodeReq(void* buf, int32_t bufLen, SDCreateMnodeReq* pReq);
typedef struct {
int32_t urlLen;
int32_t sqlLen;
char* url;
char* sql;
} SMCreateAnodeReq;
int32_t tSerializeSMCreateAnodeReq(void* buf, int32_t bufLen, SMCreateAnodeReq* pReq);
int32_t tDeserializeSMCreateAnodeReq(void* buf, int32_t bufLen, SMCreateAnodeReq* pReq);
void tFreeSMCreateAnodeReq(SMCreateAnodeReq* pReq);
typedef struct {
int32_t anodeId;
int32_t sqlLen;
char* sql;
} SMDropAnodeReq, SMUpdateAnodeReq;
int32_t tSerializeSMDropAnodeReq(void* buf, int32_t bufLen, SMDropAnodeReq* pReq);
int32_t tDeserializeSMDropAnodeReq(void* buf, int32_t bufLen, SMDropAnodeReq* pReq);
void tFreeSMDropAnodeReq(SMDropAnodeReq* pReq);
int32_t tSerializeSMUpdateAnodeReq(void* buf, int32_t bufLen, SMUpdateAnodeReq* pReq);
int32_t tDeserializeSMUpdateAnodeReq(void* buf, int32_t bufLen, SMUpdateAnodeReq* pReq);
void tFreeSMUpdateAnodeReq(SMUpdateAnodeReq* pReq);
typedef struct {
int32_t vgId;
int32_t hbSeq;

6
include/common/tmsgdef.h
View File

@ -125,6 +125,11 @@
TD_DEF_MSG_TYPE(TDMT_DND_ALTER_VNODE_TYPE, "dnode-alter-vnode-type", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_DND_CHECK_VNODE_LEARNER_CATCHUP, "dnode-check-vnode-learner-catchup", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_DND_CREATE_ENCRYPT_KEY, "create-encrypt-key", NULL, NULL)
// mnode msg overload
TD_DEF_MSG_TYPE(TDMT_MND_CREATE_ANODE, "create-anode", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_UPDATE_ANODE, "update-anode", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_DROP_ANODE, "drop-anode", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_RETRIEVE_ANAL_ALGO, "retrieve-anal-algo", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_DND_MSG)
TD_NEW_MSG_SEG(TDMT_MND_MSG) // 1<<8
@ -324,6 +329,7 @@
TD_DEF_MSG_TYPE(TDMT_SCH_EXPLAIN, "explain", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_SCH_LINK_BROKEN, "link-broken", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_SCH_TASK_NOTIFY, "task-notify", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_SCH_TASK_RELEASE, "task-release", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_SCH_MSG)

664
include/common/ttokendef.h
View File

@ -69,335 +69,341 @@
#define TK_NK_DOT 51
#define TK_WITH 52
#define TK_ENCRYPT_KEY 53
#define TK_DNODE 54
#define TK_PORT 55
#define TK_DNODES 56
#define TK_RESTORE 57
#define TK_NK_IPTOKEN 58
#define TK_FORCE 59
#define TK_UNSAFE 60
#define TK_CLUSTER 61
#define TK_LOCAL 62
#define TK_QNODE 63
#define TK_BNODE 64
#define TK_SNODE 65
#define TK_MNODE 66
#define TK_VNODE 67
#define TK_DATABASE 68
#define TK_USE 69
#define TK_FLUSH 70
#define TK_TRIM 71
#define TK_S3MIGRATE 72
#define TK_COMPACT 73
#define TK_IF 74
#define TK_NOT 75
#define TK_EXISTS 76
#define TK_BUFFER 77
#define TK_CACHEMODEL 78
#define TK_CACHESIZE 79
#define TK_COMP 80
#define TK_DURATION 81
#define TK_NK_VARIABLE 82
#define TK_MAXROWS 83
#define TK_MINROWS 84
#define TK_KEEP 85
#define TK_PAGES 86
#define TK_PAGESIZE 87
#define TK_TSDB_PAGESIZE 88
#define TK_PRECISION 89
#define TK_REPLICA 90
#define TK_VGROUPS 91
#define TK_SINGLE_STABLE 92
#define TK_RETENTIONS 93
#define TK_SCHEMALESS 94
#define TK_WAL_LEVEL 95
#define TK_WAL_FSYNC_PERIOD 96
#define TK_WAL_RETENTION_PERIOD 97
#define TK_WAL_RETENTION_SIZE 98
#define TK_WAL_ROLL_PERIOD 99
#define TK_WAL_SEGMENT_SIZE 100
#define TK_STT_TRIGGER 101
#define TK_TABLE_PREFIX 102
#define TK_TABLE_SUFFIX 103
#define TK_S3_CHUNKSIZE 104
#define TK_S3_KEEPLOCAL 105
#define TK_S3_COMPACT 106
#define TK_KEEP_TIME_OFFSET 107
#define TK_ENCRYPT_ALGORITHM 108
#define TK_NK_COLON 109
#define TK_BWLIMIT 110
#define TK_START 111
#define TK_TIMESTAMP 112
#define TK_END 113
#define TK_TABLE 114
#define TK_NK_LP 115
#define TK_NK_RP 116
#define TK_USING 117
#define TK_FILE 118
#define TK_STABLE 119
#define TK_COLUMN 120
#define TK_MODIFY 121
#define TK_RENAME 122
#define TK_TAG 123
#define TK_SET 124
#define TK_NK_EQ 125
#define TK_TAGS 126
#define TK_BOOL 127
#define TK_TINYINT 128
#define TK_SMALLINT 129
#define TK_INT 130
#define TK_INTEGER 131
#define TK_BIGINT 132
#define TK_FLOAT 133
#define TK_DOUBLE 134
#define TK_BINARY 135
#define TK_NCHAR 136
#define TK_UNSIGNED 137
#define TK_JSON 138
#define TK_VARCHAR 139
#define TK_MEDIUMBLOB 140
#define TK_BLOB 141
#define TK_VARBINARY 142
#define TK_GEOMETRY 143
#define TK_DECIMAL 144
#define TK_COMMENT 145
#define TK_MAX_DELAY 146
#define TK_WATERMARK 147
#define TK_ROLLUP 148
#define TK_TTL 149
#define TK_SMA 150
#define TK_DELETE_MARK 151
#define TK_FIRST 152
#define TK_LAST 153
#define TK_SHOW 154
#define TK_FULL 155
#define TK_PRIVILEGES 156
#define TK_DATABASES 157
#define TK_TABLES 158
#define TK_STABLES 159
#define TK_MNODES 160
#define TK_QNODES 161
#define TK_ARBGROUPS 162
#define TK_FUNCTIONS 163
#define TK_INDEXES 164
#define TK_ACCOUNTS 165
#define TK_APPS 166
#define TK_CONNECTIONS 167
#define TK_LICENCES 168
#define TK_GRANTS 169
#define TK_LOGS 170
#define TK_MACHINES 171
#define TK_ENCRYPTIONS 172
#define TK_QUERIES 173
#define TK_SCORES 174
#define TK_TOPICS 175
#define TK_VARIABLES 176
#define TK_BNODES 177
#define TK_SNODES 178
#define TK_TRANSACTIONS 179
#define TK_DISTRIBUTED 180
#define TK_CONSUMERS 181
#define TK_SUBSCRIPTIONS 182
#define TK_VNODES 183
#define TK_ALIVE 184
#define TK_VIEWS 185
#define TK_VIEW 186
#define TK_COMPACTS 187
#define TK_NORMAL 188
#define TK_CHILD 189
#define TK_LIKE 190
#define TK_TBNAME 191
#define TK_QTAGS 192
#define TK_AS 193
#define TK_SYSTEM 194
#define TK_TSMA 195
#define TK_INTERVAL 196
#define TK_RECURSIVE 197
#define TK_TSMAS 198
#define TK_FUNCTION 199
#define TK_INDEX 200
#define TK_COUNT 201
#define TK_LAST_ROW 202
#define TK_META 203
#define TK_ONLY 204
#define TK_TOPIC 205
#define TK_CONSUMER 206
#define TK_GROUP 207
#define TK_DESC 208
#define TK_DESCRIBE 209
#define TK_RESET 210
#define TK_QUERY 211
#define TK_CACHE 212
#define TK_EXPLAIN 213
#define TK_ANALYZE 214
#define TK_VERBOSE 215
#define TK_NK_BOOL 216
#define TK_RATIO 217
#define TK_NK_FLOAT 218
#define TK_OUTPUTTYPE 219
#define TK_AGGREGATE 220
#define TK_BUFSIZE 221
#define TK_LANGUAGE 222
#define TK_REPLACE 223
#define TK_STREAM 224
#define TK_INTO 225
#define TK_PAUSE 226
#define TK_RESUME 227
#define TK_PRIMARY 228
#define TK_KEY 229
#define TK_TRIGGER 230
#define TK_AT_ONCE 231
#define TK_WINDOW_CLOSE 232
#define TK_IGNORE 233
#define TK_EXPIRED 234
#define TK_FILL_HISTORY 235
#define TK_UPDATE 236
#define TK_SUBTABLE 237
#define TK_UNTREATED 238
#define TK_KILL 239
#define TK_CONNECTION 240
#define TK_TRANSACTION 241
#define TK_BALANCE 242
#define TK_VGROUP 243
#define TK_LEADER 244
#define TK_MERGE 245
#define TK_REDISTRIBUTE 246
#define TK_SPLIT 247
#define TK_DELETE 248
#define TK_INSERT 249
#define TK_NK_BIN 250
#define TK_NK_HEX 251
#define TK_NULL 252
#define TK_NK_QUESTION 253
#define TK_NK_ALIAS 254
#define TK_NK_ARROW 255
#define TK_ROWTS 256
#define TK_QSTART 257
#define TK_QEND 258
#define TK_QDURATION 259
#define TK_WSTART 260
#define TK_WEND 261
#define TK_WDURATION 262
#define TK_IROWTS 263
#define TK_ISFILLED 264
#define TK_CAST 265
#define TK_POSITION 266
#define TK_IN 267
#define TK_FOR 268
#define TK_NOW 269
#define TK_TODAY 270
#define TK_RAND 271
#define TK_SUBSTR 272
#define TK_SUBSTRING 273
#define TK_BOTH 274
#define TK_TRAILING 275
#define TK_LEADING 276
#define TK_TIMEZONE 277
#define TK_CLIENT_VERSION 278
#define TK_SERVER_VERSION 279
#define TK_SERVER_STATUS 280
#define TK_CURRENT_USER 281
#define TK_PI 282
#define TK_CASE 283
#define TK_WHEN 284
#define TK_THEN 285
#define TK_ELSE 286
#define TK_BETWEEN 287
#define TK_IS 288
#define TK_NK_LT 289
#define TK_NK_GT 290
#define TK_NK_LE 291
#define TK_NK_GE 292
#define TK_NK_NE 293
#define TK_MATCH 294
#define TK_NMATCH 295
#define TK_CONTAINS 296
#define TK_JOIN 297
#define TK_INNER 298
#define TK_LEFT 299
#define TK_RIGHT 300
#define TK_OUTER 301
#define TK_SEMI 302
#define TK_ANTI 303
#define TK_ASOF 304
#define TK_WINDOW 305
#define TK_WINDOW_OFFSET 306
#define TK_JLIMIT 307
#define TK_SELECT 308
#define TK_NK_HINT 309
#define TK_DISTINCT 310
#define TK_WHERE 311
#define TK_PARTITION 312
#define TK_BY 313
#define TK_SESSION 314
#define TK_STATE_WINDOW 315
#define TK_EVENT_WINDOW 316
#define TK_COUNT_WINDOW 317
#define TK_SLIDING 318
#define TK_FILL 319
#define TK_VALUE 320
#define TK_VALUE_F 321
#define TK_NONE 322
#define TK_PREV 323
#define TK_NULL_F 324
#define TK_LINEAR 325
#define TK_NEXT 326
#define TK_HAVING 327
#define TK_RANGE 328
#define TK_EVERY 329
#define TK_ORDER 330
#define TK_SLIMIT 331
#define TK_SOFFSET 332
#define TK_LIMIT 333
#define TK_OFFSET 334
#define TK_ASC 335
#define TK_NULLS 336
#define TK_ABORT 337
#define TK_AFTER 338
#define TK_ATTACH 339
#define TK_BEFORE 340
#define TK_BEGIN 341
#define TK_BITAND 342
#define TK_BITNOT 343
#define TK_BITOR 344
#define TK_BLOCKS 345
#define TK_CHANGE 346
#define TK_COMMA 347
#define TK_CONCAT 348
#define TK_CONFLICT 349
#define TK_COPY 350
#define TK_DEFERRED 351
#define TK_DELIMITERS 352
#define TK_DETACH 353
#define TK_DIVIDE 354
#define TK_DOT 355
#define TK_EACH 356
#define TK_FAIL 357
#define TK_GLOB 358
#define TK_ID 359
#define TK_IMMEDIATE 360
#define TK_IMPORT 361
#define TK_INITIALLY 362
#define TK_INSTEAD 363
#define TK_ISNULL 364
#define TK_MODULES 365
#define TK_NK_BITNOT 366
#define TK_NK_SEMI 367
#define TK_NOTNULL 368
#define TK_OF 369
#define TK_PLUS 370
#define TK_PRIVILEGE 371
#define TK_RAISE 372
#define TK_RESTRICT 373
#define TK_ROW 374
#define TK_STAR 375
#define TK_STATEMENT 376
#define TK_STRICT 377
#define TK_STRING 378
#define TK_TIMES 379
#define TK_VALUES 380
#define TK_VARIABLE 381
#define TK_WAL 382
#define TK_ANODE 54
#define TK_UPDATE 55
#define TK_ANODES 56
#define TK_DNODE 57
#define TK_PORT 58
#define TK_DNODES 59
#define TK_RESTORE 60
#define TK_NK_IPTOKEN 61
#define TK_FORCE 62
#define TK_UNSAFE 63
#define TK_CLUSTER 64
#define TK_LOCAL 65
#define TK_QNODE 66
#define TK_BNODE 67
#define TK_SNODE 68
#define TK_MNODE 69
#define TK_VNODE 70
#define TK_DATABASE 71
#define TK_USE 72
#define TK_FLUSH 73
#define TK_TRIM 74
#define TK_S3MIGRATE 75
#define TK_COMPACT 76
#define TK_IF 77
#define TK_NOT 78
#define TK_EXISTS 79
#define TK_BUFFER 80
#define TK_CACHEMODEL 81
#define TK_CACHESIZE 82
#define TK_COMP 83
#define TK_DURATION 84
#define TK_NK_VARIABLE 85
#define TK_MAXROWS 86
#define TK_MINROWS 87
#define TK_KEEP 88
#define TK_PAGES 89
#define TK_PAGESIZE 90
#define TK_TSDB_PAGESIZE 91
#define TK_PRECISION 92
#define TK_REPLICA 93
#define TK_VGROUPS 94
#define TK_SINGLE_STABLE 95
#define TK_RETENTIONS 96
#define TK_SCHEMALESS 97
#define TK_WAL_LEVEL 98
#define TK_WAL_FSYNC_PERIOD 99
#define TK_WAL_RETENTION_PERIOD 100
#define TK_WAL_RETENTION_SIZE 101
#define TK_WAL_ROLL_PERIOD 102
#define TK_WAL_SEGMENT_SIZE 103
#define TK_STT_TRIGGER 104
#define TK_TABLE_PREFIX 105
#define TK_TABLE_SUFFIX 106
#define TK_S3_CHUNKSIZE 107
#define TK_S3_KEEPLOCAL 108
#define TK_S3_COMPACT 109
#define TK_KEEP_TIME_OFFSET 110
#define TK_ENCRYPT_ALGORITHM 111
#define TK_NK_COLON 112
#define TK_BWLIMIT 113
#define TK_START 114
#define TK_TIMESTAMP 115
#define TK_END 116
#define TK_TABLE 117
#define TK_NK_LP 118
#define TK_NK_RP 119
#define TK_USING 120
#define TK_FILE 121
#define TK_STABLE 122
#define TK_COLUMN 123
#define TK_MODIFY 124
#define TK_RENAME 125
#define TK_TAG 126
#define TK_SET 127
#define TK_NK_EQ 128
#define TK_TAGS 129
#define TK_BOOL 130
#define TK_TINYINT 131
#define TK_SMALLINT 132
#define TK_INT 133
#define TK_INTEGER 134
#define TK_BIGINT 135
#define TK_FLOAT 136
#define TK_DOUBLE 137
#define TK_BINARY 138
#define TK_NCHAR 139
#define TK_UNSIGNED 140
#define TK_JSON 141
#define TK_VARCHAR 142
#define TK_MEDIUMBLOB 143
#define TK_BLOB 144
#define TK_VARBINARY 145
#define TK_GEOMETRY 146
#define TK_DECIMAL 147
#define TK_COMMENT 148
#define TK_MAX_DELAY 149
#define TK_WATERMARK 150
#define TK_ROLLUP 151
#define TK_TTL 152
#define TK_SMA 153
#define TK_DELETE_MARK 154
#define TK_FIRST 155
#define TK_LAST 156
#define TK_SHOW 157
#define TK_FULL 158
#define TK_PRIVILEGES 159
#define TK_DATABASES 160
#define TK_TABLES 161
#define TK_STABLES 162
#define TK_MNODES 163
#define TK_QNODES 164
#define TK_ARBGROUPS 165
#define TK_FUNCTIONS 166
#define TK_INDEXES 167
#define TK_ACCOUNTS 168
#define TK_APPS 169
#define TK_CONNECTIONS 170
#define TK_LICENCES 171
#define TK_GRANTS 172
#define TK_LOGS 173
#define TK_MACHINES 174
#define TK_ENCRYPTIONS 175
#define TK_QUERIES 176
#define TK_SCORES 177
#define TK_TOPICS 178
#define TK_VARIABLES 179
#define TK_BNODES 180
#define TK_SNODES 181
#define TK_TRANSACTIONS 182
#define TK_DISTRIBUTED 183
#define TK_CONSUMERS 184
#define TK_SUBSCRIPTIONS 185
#define TK_VNODES 186
#define TK_ALIVE 187
#define TK_VIEWS 188
#define TK_VIEW 189
#define TK_COMPACTS 190
#define TK_NORMAL 191
#define TK_CHILD 192
#define TK_LIKE 193
#define TK_TBNAME 194
#define TK_QTAGS 195
#define TK_AS 196
#define TK_SYSTEM 197
#define TK_TSMA 198
#define TK_INTERVAL 199
#define TK_RECURSIVE 200
#define TK_TSMAS 201
#define TK_FUNCTION 202
#define TK_INDEX 203
#define TK_COUNT 204
#define TK_LAST_ROW 205
#define TK_META 206
#define TK_ONLY 207
#define TK_TOPIC 208
#define TK_CONSUMER 209
#define TK_GROUP 210
#define TK_DESC 211
#define TK_DESCRIBE 212
#define TK_RESET 213
#define TK_QUERY 214
#define TK_CACHE 215
#define TK_EXPLAIN 216
#define TK_ANALYZE 217
#define TK_VERBOSE 218
#define TK_NK_BOOL 219
#define TK_RATIO 220
#define TK_NK_FLOAT 221
#define TK_OUTPUTTYPE 222
#define TK_AGGREGATE 223
#define TK_BUFSIZE 224
#define TK_LANGUAGE 225
#define TK_REPLACE 226
#define TK_STREAM 227
#define TK_INTO 228
#define TK_PAUSE 229
#define TK_RESUME 230
#define TK_PRIMARY 231
#define TK_KEY 232
#define TK_TRIGGER 233
#define TK_AT_ONCE 234
#define TK_WINDOW_CLOSE 235
#define TK_IGNORE 236
#define TK_EXPIRED 237
#define TK_FILL_HISTORY 238
#define TK_SUBTABLE 239
#define TK_UNTREATED 240
#define TK_KILL 241
#define TK_CONNECTION 242
#define TK_TRANSACTION 243
#define TK_BALANCE 244
#define TK_VGROUP 245
#define TK_LEADER 246
#define TK_MERGE 247
#define TK_REDISTRIBUTE 248
#define TK_SPLIT 249
#define TK_DELETE 250
#define TK_INSERT 251
#define TK_NK_BIN 252
#define TK_NK_HEX 253
#define TK_NULL 254
#define TK_NK_QUESTION 255
#define TK_NK_ALIAS 256
#define TK_NK_ARROW 257
#define TK_ROWTS 258
#define TK_QSTART 259
#define TK_QEND 260
#define TK_QDURATION 261
#define TK_WSTART 262
#define TK_WEND 263
#define TK_WDURATION 264
#define TK_IROWTS 265
#define TK_ISFILLED 266
#define TK_FLOW 267
#define TK_FHIGH 268
#define TK_FROWTS 269
#define TK_CAST 270
#define TK_POSITION 271
#define TK_IN 272
#define TK_FOR 273
#define TK_NOW 274
#define TK_TODAY 275
#define TK_RAND 276
#define TK_SUBSTR 277
#define TK_SUBSTRING 278
#define TK_BOTH 279
#define TK_TRAILING 280
#define TK_LEADING 281
#define TK_TIMEZONE 282
#define TK_CLIENT_VERSION 283
#define TK_SERVER_VERSION 284
#define TK_SERVER_STATUS 285
#define TK_CURRENT_USER 286
#define TK_PI 287
#define TK_CASE 288
#define TK_WHEN 289
#define TK_THEN 290
#define TK_ELSE 291
#define TK_BETWEEN 292
#define TK_IS 293
#define TK_NK_LT 294
#define TK_NK_GT 295
#define TK_NK_LE 296
#define TK_NK_GE 297
#define TK_NK_NE 298
#define TK_MATCH 299
#define TK_NMATCH 300
#define TK_CONTAINS 301
#define TK_JOIN 302
#define TK_INNER 303
#define TK_LEFT 304
#define TK_RIGHT 305
#define TK_OUTER 306
#define TK_SEMI 307
#define TK_ANTI 308
#define TK_ASOF 309
#define TK_WINDOW 310
#define TK_WINDOW_OFFSET 311
#define TK_JLIMIT 312
#define TK_SELECT 313
#define TK_NK_HINT 314
#define TK_DISTINCT 315
#define TK_WHERE 316
#define TK_PARTITION 317
#define TK_BY 318
#define TK_SESSION 319
#define TK_STATE_WINDOW 320
#define TK_EVENT_WINDOW 321
#define TK_COUNT_WINDOW 322
#define TK_ANOMALY_WINDOW 323
#define TK_SLIDING 324
#define TK_FILL 325
#define TK_VALUE 326
#define TK_VALUE_F 327
#define TK_NONE 328
#define TK_PREV 329
#define TK_NULL_F 330
#define TK_LINEAR 331
#define TK_NEXT 332
#define TK_HAVING 333
#define TK_RANGE 334
#define TK_EVERY 335
#define TK_ORDER 336
#define TK_SLIMIT 337
#define TK_SOFFSET 338
#define TK_LIMIT 339
#define TK_OFFSET 340
#define TK_ASC 341
#define TK_NULLS 342
#define TK_ABORT 343
#define TK_AFTER 344
#define TK_ATTACH 345
#define TK_BEFORE 346
#define TK_BEGIN 347
#define TK_BITAND 348
#define TK_BITNOT 349
#define TK_BITOR 350
#define TK_BLOCKS 351
#define TK_CHANGE 352
#define TK_COMMA 353
#define TK_CONCAT 354
#define TK_CONFLICT 355
#define TK_COPY 356
#define TK_DEFERRED 357
#define TK_DELIMITERS 358
#define TK_DETACH 359
#define TK_DIVIDE 360
#define TK_DOT 361
#define TK_EACH 362
#define TK_FAIL 363
#define TK_GLOB 364
#define TK_ID 365
#define TK_IMMEDIATE 366
#define TK_IMPORT 367
#define TK_INITIALLY 368
#define TK_INSTEAD 369
#define TK_ISNULL 370
#define TK_MODULES 371
#define TK_NK_BITNOT 372
#define TK_NK_SEMI 373
#define TK_NOTNULL 374
#define TK_OF 375
#define TK_PLUS 376
#define TK_PRIVILEGE 377
#define TK_RAISE 378
#define TK_RESTRICT 379
#define TK_ROW 380
#define TK_STAR 381
#define TK_STATEMENT 382
#define TK_STRICT 383
#define TK_STRING 384
#define TK_TIMES 385
#define TK_VALUES 386
#define TK_VARIABLE 387
#define TK_WAL 388
#define TK_NK_SPACE 600
#define TK_NK_COMMENT 601

12
include/libs/function/function.h
View File

@ -29,6 +29,7 @@ struct SqlFunctionCtx;
struct SResultRowEntryInfo;
struct SFunctionNode;
struct SExprSupp;
typedef struct SScalarParam SScalarParam;
typedef struct SStreamState SStreamState;
@ -43,6 +44,7 @@ typedef int32_t (*FExecProcess)(struct SqlFunctionCtx *pCtx);
typedef int32_t (*FExecFinalize)(struct SqlFunctionCtx *pCtx, SSDataBlock *pBlock);
typedef int32_t (*FScalarExecProcess)(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
typedef int32_t (*FExecCombine)(struct SqlFunctionCtx *pDestCtx, struct SqlFunctionCtx *pSourceCtx);
typedef int32_t (*FExecDecode)(struct SqlFunctionCtx *pCtx, const char *buf, struct SResultRowEntryInfo *pResultCellInfo, int32_t version);
typedef int32_t (*processFuncByRow)(SArray* pCtx); // array of SqlFunctionCtx
typedef struct SScalarFuncExecFuncs {
@ -57,6 +59,7 @@ typedef struct SFuncExecFuncs {
FExecFinalize finalize;
FExecCombine combine;
FExecCleanUp cleanup;
FExecDecode decode;
processFuncByRow processFuncByRow;
} SFuncExecFuncs;
@ -65,6 +68,8 @@ typedef struct SFuncExecFuncs {
#define TOP_BOTTOM_QUERY_LIMIT 100
#define FUNCTIONS_NAME_MAX_LENGTH 32
#define FUNCTION_RESULT_INFO_VERSION 1
typedef struct SResultRowEntryInfo {
bool initialized : 1; // output buffer has been initialized
bool complete : 1; // query has completed
@ -165,6 +170,11 @@ typedef struct STdbState {
void *txn;
} STdbState;
typedef struct SResultRowStore {
int32_t (*resultRowPut)(struct SExprSupp *pSup, const char* inBuf, size_t inBufSize, char **outBuf, size_t *outBufSize);
int32_t (*resultRowGet)(struct SExprSupp *pSup, const char* inBuf, size_t inBufSize, char **outBuf, size_t *outBufSize);
} SResultRowStore;
struct SStreamState {
STdbState *pTdbState;
struct SStreamFileState *pFileState;
@ -175,6 +185,8 @@ struct SStreamState {
int64_t streamBackendRid;
int8_t dump;
int32_t tsIndex;
SResultRowStore pResultRowStore;
struct SExprSupp *pExprSupp;
};
typedef struct SFunctionStateStore {

6
include/libs/function/functionMgt.h
View File

@ -62,6 +62,7 @@ typedef enum EFunctionType {
FUNCTION_TYPE_UNIQUE,
FUNCTION_TYPE_STATE_COUNT,
FUNCTION_TYPE_STATE_DURATION,
FUNCTION_TYPE_FORECAST,
// math function
FUNCTION_TYPE_ABS = 1000,
@ -149,6 +150,9 @@ typedef enum EFunctionType {
FUNCTION_TYPE_TBUID,
FUNCTION_TYPE_VGID,
FUNCTION_TYPE_VGVER,
FUNCTION_TYPE_FORECAST_LOW,
FUNCTION_TYPE_FORECAST_HIGH,
FUNCTION_TYPE_FORECAST_ROWTS,
// internal function
FUNCTION_TYPE_SELECT_VALUE = 3750,
@ -263,6 +267,7 @@ bool fmIsForbidSysTableFunc(int32_t funcId);
bool fmIsIntervalInterpoFunc(int32_t funcId);
bool fmIsInterpFunc(int32_t funcId);
bool fmIsLastRowFunc(int32_t funcId);
bool fmIsForecastFunc(int32_t funcId);
bool fmIsNotNullOutputFunc(int32_t funcId);
bool fmIsSelectValueFunc(int32_t funcId);
bool fmIsSystemInfoFunc(int32_t funcId);
@ -272,6 +277,7 @@ bool fmIsMultiRowsFunc(int32_t funcId);
bool fmIsKeepOrderFunc(int32_t funcId);
bool fmIsCumulativeFunc(int32_t funcId);
bool fmIsInterpPseudoColumnFunc(int32_t funcId);
bool fmIsForecastPseudoColumnFunc(int32_t funcId);
bool fmIsGroupKeyFunc(int32_t funcId);
bool fmIsBlockDistFunc(int32_t funcId);
bool fmIsIgnoreNullFunc(int32_t funcId);

15
include/libs/nodes/cmdnodes.h
View File

@ -318,6 +318,21 @@ typedef struct SAlterDnodeStmt {
char value[TSDB_DNODE_VALUE_LEN];
} SAlterDnodeStmt;
typedef struct {
ENodeType type;
char url[TSDB_ANAL_ANODE_URL_LEN + 3];
} SCreateAnodeStmt;
typedef struct {
ENodeType type;
int32_t anodeId;
} SDropAnodeStmt;
typedef struct {
ENodeType type;
int32_t anodeId;
} SUpdateAnodeStmt;
typedef struct SShowStmt {
ENodeType type;
SNode* pDbName; // SValueNode

20
include/libs/nodes/plannodes.h
View File

@ -204,6 +204,11 @@ typedef struct SInterpFuncLogicNode {
SNode* pTimeSeries; // SColumnNode
} SInterpFuncLogicNode;
typedef struct SForecastFuncLogicNode {
SLogicNode node;
SNodeList* pFuncs;
} SForecastFuncLogicNode;
typedef struct SGroupCacheLogicNode {
SLogicNode node;
bool grpColsMayBeNull;
@ -275,6 +280,7 @@ typedef enum EWindowType {
WINDOW_TYPE_STATE,
WINDOW_TYPE_EVENT,
WINDOW_TYPE_COUNT,
WINDOW_TYPE_ANOMALY
} EWindowType;
typedef enum EWindowAlgorithm {
@ -315,6 +321,8 @@ typedef struct SWindowLogicNode {
int64_t windowCount;
int64_t windowSliding;
SNodeList* pTsmaSubplans;
SNode* pAnomalyExpr;
char anomalyOpt[TSDB_ANAL_ALGO_OPTION_LEN];
} SWindowLogicNode;
typedef struct SFillLogicNode {
@ -507,6 +515,12 @@ typedef struct SInterpFuncPhysiNode {
SNode* pTimeSeries; // SColumnNode
} SInterpFuncPhysiNode;
typedef struct SForecastFuncPhysiNode {
SPhysiNode node;
SNodeList* pExprs;
SNodeList* pFuncs;
} SForecastFuncPhysiNode;
typedef struct SSortMergeJoinPhysiNode {
SPhysiNode node;
EJoinType joinType;
@ -704,6 +718,12 @@ typedef struct SCountWinodwPhysiNode {
typedef SCountWinodwPhysiNode SStreamCountWinodwPhysiNode;
typedef struct SAnomalyWindowPhysiNode {
SWindowPhysiNode window;
SNode* pAnomalyKey;
char anomalyOpt[TSDB_ANAL_ALGO_OPTION_LEN];
} SAnomalyWindowPhysiNode;
typedef struct SSortPhysiNode {
SPhysiNode node;
SNodeList* pExprs; // these are expression list of order_by_clause and parameter expression of aggregate function

9
include/libs/nodes/querynodes.h
View File

@ -347,6 +347,13 @@ typedef struct SCountWindowNode {
int64_t windowSliding;
} SCountWindowNode;
typedef struct SAnomalyWindowNode {
ENodeType type; // QUERY_NODE_ANOMALY_WINDOW
SNode* pCol; // timestamp primary key
SNode* pExpr;
char anomalyOpt[TSDB_ANAL_ALGO_OPTION_LEN];
} SAnomalyWindowNode;
typedef enum EFillMode {
FILL_MODE_NONE = 1,
FILL_MODE_VALUE,
@ -442,6 +449,8 @@ typedef struct SSelectStmt {
bool hasTailFunc;
bool hasInterpFunc;
bool hasInterpPseudoColFunc;
bool hasForecastFunc;
bool hasForecastPseudoColFunc;
bool hasLastRowFunc;
bool hasLastFunc;
bool hasTimeLineFunc;

1
include/libs/scalar/scalar.h
View File

@ -139,6 +139,7 @@ int32_t mavgScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam
int32_t hllScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t csumScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t diffScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t forecastScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t stateCountScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t stateDurationScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t histogramScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);

15
include/libs/transport/trpc.h
View File

@ -63,6 +63,10 @@ typedef struct SRpcHandleInfo {
int8_t forbiddenIp;
int8_t notFreeAhandle;
int8_t compressed;
int64_t seqNum; // msg seq
int64_t qId; // queryId Get from client, other req's qId = -1;
int32_t refIdMgt;
int32_t msgType;
} SRpcHandleInfo;
typedef struct SRpcMsg {
@ -124,9 +128,13 @@ typedef struct SRpcInit {
int32_t connLimitLock;
int32_t timeToGetConn;
int8_t supportBatch; // 0: no batch, 1. batch
int32_t batchSize;
int8_t notWaitAvaliableConn; // 1: wait to get, 0: no wait
void *parent;
int32_t shareConnLimit;
int8_t shareConn; // 0: no share, 1. share
int8_t notWaitAvaliableConn; // 1: wait to get, 0: no wait
int8_t startReadTimer;
int64_t readTimeout; // s
void *parent;
} SRpcInit;
typedef struct {
@ -144,6 +152,7 @@ typedef struct {
SHashObj *args;
SRpcBrokenlinkVal brokenVal;
void (*freeFunc)(const void *arg);
int64_t st;
} SRpcCtx;
int32_t rpcInit();

17
include/os/osFile.h
View File

@ -130,14 +130,15 @@ int taosSetAutoDelFile(char *path);
bool lastErrorIsFileNotExist();
#ifdef BUILD_WITH_RAND_ERR
#define STUB_RAND_NETWORK_ERR(status) \
do { \
if (tsEnableRandErr && (tsRandErrScope & RAND_ERR_NETWORK)) { \
uint32_t r = taosRand() % tsRandErrDivisor; \
if ((r + 1) <= tsRandErrChance) { \
status = TSDB_CODE_RPC_NETWORK_UNAVAIL; \
} \
} \
#define STUB_RAND_NETWORK_ERR(ret) \
do { \
if (tsEnableRandErr && (tsRandErrScope & RAND_ERR_NETWORK)) { \
uint32_t r = taosRand() % tsRandErrDivisor; \
if ((r + 1) <= tsRandErrChance) { \
ret = TSDB_CODE_RPC_NETWORK_UNAVAIL; \
uError("random network error: %s, %s", tstrerror(ret), __func__); \
} \
} \
while (0)
#else
#define STUB_RAND_NETWORK_ERR(status)

3
include/os/osSocket.h
View File

@ -137,6 +137,7 @@ int32_t taosShutDownSocketRDWR(TdSocketPtr pSocket);
int32_t taosShutDownSocketServerRDWR(TdSocketServerPtr pSocketServer);
int32_t taosSetNonblocking(TdSocketPtr pSocket, int32_t on);
int32_t taosSetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void *optval, int32_t optlen);
int32_t taosSetSockOpt2(int32_t fd);
int32_t taosGetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void *optval, int32_t *optlen);
int32_t taosWriteMsg(TdSocketPtr pSocket, void *ptr, int32_t nbytes);
int32_t taosReadMsg(TdSocketPtr pSocket, void *ptr, int32_t nbytes);
@ -159,7 +160,7 @@ TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, st
int32_t taosGetSocketName(TdSocketPtr pSocket, struct sockaddr *destAddr, int *addrLen);
int32_t taosBlockSIGPIPE();
int32_t taosGetIpv4FromFqdn(const char *fqdn, uint32_t* ip);
int32_t taosGetIpv4FromFqdn(const char *fqdn, uint32_t *ip);
int32_t taosGetFqdn(char *);
void tinet_ntoa(char *ipstr, uint32_t ip);
uint32_t ip2uint(const char *const ip_addr);

10
include/os/osSystem.h
View File

@ -89,9 +89,9 @@ int32_t taosResetTerminalMode();
snprintf(array[size], STACKSIZE, "0x%lx : (%s+0x%lx) [0x%lx]\n", (long)pc, fname, (long)offset, (long)pc); \
} \
if (ignoreNum < size && size > 0) { \
offset = snprintf(buf, bufSize - 1, "obtained %d stack frames\n", (ignoreNum > 0) ? size - ignoreNum : size); \
offset = tsnprintf(buf, bufSize - 1, "obtained %d stack frames\n", (ignoreNum > 0) ? size - ignoreNum : size); \
for (int32_t i = (ignoreNum > 0) ? ignoreNum : 0; i < size; i++) { \
offset += snprintf(buf + offset, bufSize - 1 - offset, "frame:%d, %s\n", (ignoreNum > 0) ? i - ignoreNum : i, \
offset += tsnprintf(buf + offset, bufSize - 1 - offset, "frame:%d, %s\n", (ignoreNum > 0) ? i - ignoreNum : i, \
array[i]); \
} \
} \
@ -140,9 +140,9 @@ int32_t taosResetTerminalMode();
char **strings = backtrace_symbols(array, size); \
int32_t offset = 0; \
if (strings != NULL) { \
offset = snprintf(buf, bufSize - 1, "obtained %d stack frames\n", (ignoreNum > 0) ? size - ignoreNum : size); \
offset = tsnprintf(buf, bufSize - 1, "obtained %d stack frames\n", (ignoreNum > 0) ? size - ignoreNum : size); \
for (int32_t i = (ignoreNum > 0) ? ignoreNum : 0; i < size; i++) { \
offset += snprintf(buf + offset, bufSize - 1 - offset, "frame:%d, %s\n", (ignoreNum > 0) ? i - ignoreNum : i, \
offset += tsnprintf(buf + offset, bufSize - 1 - offset, "frame:%d, %s\n", (ignoreNum > 0) ? i - ignoreNum : i, \
strings[i]); \
} \
} \
@ -193,7 +193,7 @@ int32_t taosResetTerminalMode();
snprintf(buf, bufSize - 1, "obtained %d stack frames\n", (ignoreNum > 0) ? frames - ignoreNum : frames); \
for (i = (ignoreNum > 0) ? ignoreNum : 0; i < frames; i++) { \
SymFromAddr(process, (DWORD64)(stack[i]), 0, symbol); \
offset += snprintf(buf + offset, bufSize - 1 - offset, "frame:%i, %s - 0x%0X\n", \
offset += tsnprintf(buf + offset, bufSize - 1 - offset, "frame:%i, %s - 0x%0X\n", \
(ignoreNum > 0) ? i - ignoreNum : i, symbol->Name, symbol->Address); \
} \
} \

30
include/util/taoserror.h
View File

@ -94,9 +94,9 @@ int32_t taosGetErrSize();
#define TSDB_CODE_HTTP_MODULE_QUIT TAOS_DEF_ERROR_CODE(0, 0x0025)
#define TSDB_CODE_RPC_MODULE_QUIT TAOS_DEF_ERROR_CODE(0, 0x0026)
#define TSDB_CODE_RPC_ASYNC_MODULE_QUIT TAOS_DEF_ERROR_CODE(0, 0x0027)
#define TSDB_CODE_RPC_ASYNC_IN_PROCESS TAOS_DEF_ERROR_CODE(0, 0x0028)
#define TSDB_CODE_RPC_NO_STATE TAOS_DEF_ERROR_CODE(0, 0x0029)
#define TSDB_CODE_RPC_STATE_DROPED TAOS_DEF_ERROR_CODE(0, 0x002A)
//common & util
#define TSDB_CODE_OPS_NOT_SUPPORT TAOS_DEF_ERROR_CODE(0, 0x0100) //
@ -476,6 +476,26 @@ int32_t taosGetErrSize();
#define TSDB_CODE_DNODE_INVALID_MONITOR_PARAS TAOS_DEF_ERROR_CODE(0, 0x0429)
#define TSDB_CODE_MNODE_STOPPED TAOS_DEF_ERROR_CODE(0, 0x042A)
// anode
#define TSDB_CODE_MND_ANODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0430)
#define TSDB_CODE_MND_ANODE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0431)
#define TSDB_CODE_MND_ANODE_TOO_LONG_URL TAOS_DEF_ERROR_CODE(0, 0x0432)
#define TSDB_CODE_MND_ANODE_INVALID_PROTOCOL TAOS_DEF_ERROR_CODE(0, 0x0433)
#define TSDB_CODE_MND_ANODE_INVALID_VERSION TAOS_DEF_ERROR_CODE(0, 0x0434)
#define TSDB_CODE_MND_ANODE_INVALID_ALGO_TYPE TAOS_DEF_ERROR_CODE(0, 0x0435)
#define TSDB_CODE_MND_ANODE_TOO_MANY_ALGO TAOS_DEF_ERROR_CODE(0, 0x0436)
#define TSDB_CODE_MND_ANODE_TOO_LONG_ALGO_NAME TAOS_DEF_ERROR_CODE(0, 0x0437)
#define TSDB_CODE_MND_ANODE_TOO_MANY_ALGO_TYPE TAOS_DEF_ERROR_CODE(0, 0x0438)
// analysis
#define TSDB_CODE_ANAL_URL_RSP_IS_NULL TAOS_DEF_ERROR_CODE(0, 0x0440)
#define TSDB_CODE_ANAL_URL_CANT_ACCESS TAOS_DEF_ERROR_CODE(0, 0x0441)
#define TSDB_CODE_ANAL_ALGO_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x0442)
#define TSDB_CODE_ANAL_ALGO_NOT_LOAD TAOS_DEF_ERROR_CODE(0, 0x0443)
#define TSDB_CODE_ANAL_BUF_INVALID_TYPE TAOS_DEF_ERROR_CODE(0, 0x0444)
#define TSDB_CODE_ANAL_ANODE_RETURN_ERROR TAOS_DEF_ERROR_CODE(0, 0x0445)
#define TSDB_CODE_ANAL_ANODE_TOO_MANY_ROWS TAOS_DEF_ERROR_CODE(0, 0x0446)
// mnode-sma
#define TSDB_CODE_MND_SMA_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0480)
#define TSDB_CODE_MND_SMA_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0481)
@ -867,6 +887,10 @@ int32_t taosGetErrSize();
#define TSDB_CODE_PAR_TAG_NAME_DUPLICATED TAOS_DEF_ERROR_CODE(0, 0x267F)
#define TSDB_CODE_PAR_NOT_ALLOWED_DIFFERENT_BY_ROW_FUNC TAOS_DEF_ERROR_CODE(0, 0x2680)
#define TSDB_CODE_PAR_REGULAR_EXPRESSION_ERROR TAOS_DEF_ERROR_CODE(0, 0x2681)
#define TSDB_CODE_PAR_INVALID_ANOMALY_WIN_TYPE TAOS_DEF_ERROR_CODE(0, 0x2682)
#define TSDB_CODE_PAR_INVALID_ANOMALY_WIN_COL TAOS_DEF_ERROR_CODE(0, 0x2683)
#define TSDB_CODE_PAR_INVALID_ANOMALY_WIN_OPT TAOS_DEF_ERROR_CODE(0, 0x2684)
#define TSDB_CODE_PAR_INVALID_FORECAST_CLAUSE TAOS_DEF_ERROR_CODE(0, 0x2685)
#define TSDB_CODE_PAR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x26FF)
//planner

14
include/util/tdef.h
View File

@ -293,6 +293,12 @@ typedef enum ELogicConditionType {
#define TSDB_SLOW_QUERY_SQL_LEN 512
#define TSDB_SHOW_SUBQUERY_LEN 1000
#define TSDB_LOG_VAR_LEN 32
#define TSDB_ANAL_ANODE_URL_LEN 128
#define TSDB_ANAL_ALGO_NAME_LEN 64
#define TSDB_ANAL_ALGO_TYPE_LEN 24
#define TSDB_ANAL_ALGO_KEY_LEN (TSDB_ANAL_ALGO_NAME_LEN + 9)
#define TSDB_ANAL_ALGO_URL_LEN (TSDB_ANAL_ANODE_URL_LEN + TSDB_ANAL_ALGO_TYPE_LEN + 1)
#define TSDB_ANAL_ALGO_OPTION_LEN 256
#define TSDB_MAX_EP_NUM 10
@ -500,7 +506,7 @@ typedef enum ELogicConditionType {
#ifdef WINDOWS
#define TSDB_MAX_RPC_THREADS 4 // windows pipe only support 4 connections.
#else
#define TSDB_MAX_RPC_THREADS 50
#define TSDB_MAX_RPC_THREADS 20
#endif
#define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type
@ -604,6 +610,12 @@ enum { RAND_ERR_MEMORY = 1, RAND_ERR_FILE = 2, RAND_ERR_NETWORK = 4 };
#define MONITOR_TAG_VALUE_LEN 300
#define MONITOR_METRIC_NAME_LEN 100
typedef enum {
ANAL_ALGO_TYPE_ANOMALY_DETECT = 0,
ANAL_ALGO_TYPE_FORECAST = 1,
ANAL_ALGO_TYPE_END,
} EAnalAlgoType;
#ifdef __cplusplus
}
#endif

3
include/util/tjson.h
View File

@ -68,7 +68,10 @@ int32_t tjsonAddItemToArray(SJson* pJson, SJson* pItem);
SJson* tjsonGetObjectItem(const SJson* pJson, const char* pName);
int32_t tjsonGetObjectName(const SJson* pJson, char** pName);
int32_t tjsonGetObjectValueString(const SJson* pJson, char** pStringValue);
void tjsonGetObjectValueBigInt(const SJson* pJson, int64_t* pVal);
void tjsonGetObjectValueDouble(const SJson* pJson, double* pVal);
int32_t tjsonGetStringValue(const SJson* pJson, const char* pName, char* pVal);
int32_t tjsonGetStringValue2(const SJson* pJson, const char* pName, char* pVal, int32_t maxLen);
int32_t tjsonDupStringValue(const SJson* pJson, const char* pName, char** pVal);
int32_t tjsonGetBigIntValue(const SJson* pJson, const char* pName, int64_t* pVal);
int32_t tjsonGetIntValue(const SJson* pJson, const char* pName, int32_t* pVal);

4
include/util/tlog.h
View File

@ -124,8 +124,8 @@ void taosReleaseCrashLogFile(TdFilePtr pFile, bool truncateFile);
#define uInfo(...) { if (uDebugFlag & DEBUG_INFO) { taosPrintLog("UTL ", DEBUG_INFO, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
#define uDebug(...) { if (uDebugFlag & DEBUG_DEBUG) { taosPrintLog("UTL ", DEBUG_DEBUG, uDebugFlag, __VA_ARGS__); }}
#define uTrace(...) { if (uDebugFlag & DEBUG_TRACE) { taosPrintLog("UTL ", DEBUG_TRACE, uDebugFlag, __VA_ARGS__); }}
#define uDebugL(...) { if (uDebugFlag & DEBUG_DEBUG) { taosPrintLongString("UTL ", DEBUG_DEBUG, uDebugFlag, __VA_ARGS__); }}
#define uInfoL(...) { if (uDebugFlag & DEBUG_INFO) { taosPrintLongString("UTL ", DEBUG_INFO, uDebugFlag, __VA_ARGS__); }}
#define uDebugL(...){ if (uDebugFlag & DEBUG_DEBUG) { taosPrintLongString("UTL ", DEBUG_DEBUG, uDebugFlag, __VA_ARGS__); }}
#define uInfoL(...) { if (uDebugFlag & DEBUG_INFO) { taosPrintLongString("UTL ", DEBUG_INFO, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
#define pError(...) { taosPrintLog("APP ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }
#define pPrint(...) { taosPrintLog("APP ", DEBUG_INFO, 255, __VA_ARGS__); }

3
source/client/src/clientEnv.c
View File

@ -370,7 +370,10 @@ int32_t openTransporter(const char *user, const char *auth, int32_t numOfThread,
connLimitNum = TMAX(connLimitNum, 10);
connLimitNum = TMIN(connLimitNum, 1000);
rpcInit.connLimitNum = connLimitNum;
rpcInit.shareConnLimit = tsShareConnLimit;
rpcInit.timeToGetConn = tsTimeToGetAvailableConn;
rpcInit.startReadTimer = 1;
rpcInit.readTimeout = tsReadTimeout;
int32_t code = taosVersionStrToInt(version, &(rpcInit.compatibilityVer));
if (TSDB_CODE_SUCCESS != code) {

29
source/client/src/clientImpl.c
View File

@ -410,7 +410,6 @@ int32_t asyncExecDdlQuery(SRequestObj* pRequest, SQuery* pQuery) {
SAppInstInfo* pAppInfo = getAppInfo(pRequest);
SMsgSendInfo* pSendMsg = buildMsgInfoImpl(pRequest);
// int64_t transporterId = 0;
int32_t code = asyncSendMsgToServer(pAppInfo->pTransporter, &pMsgInfo->epSet, NULL, pSendMsg);
if (code) {
doRequestCallback(pRequest, code);
@ -1921,19 +1920,19 @@ TAOS* taos_connect_auth(const char* ip, const char* user, const char* auth, cons
return NULL;
}
//TAOS* taos_connect_l(const char* ip, int ipLen, const char* user, int userLen, const char* pass, int passLen,
// const char* db, int dbLen, uint16_t port) {
// char ipStr[TSDB_EP_LEN] = {0};
// char dbStr[TSDB_DB_NAME_LEN] = {0};
// char userStr[TSDB_USER_LEN] = {0};
// char passStr[TSDB_PASSWORD_LEN] = {0};
// TAOS* taos_connect_l(const char* ip, int ipLen, const char* user, int userLen, const char* pass, int passLen,
// const char* db, int dbLen, uint16_t port) {
// char ipStr[TSDB_EP_LEN] = {0};
// char dbStr[TSDB_DB_NAME_LEN] = {0};
// char userStr[TSDB_USER_LEN] = {0};
// char passStr[TSDB_PASSWORD_LEN] = {0};
//
// tstrncpy(ipStr, ip, TMIN(TSDB_EP_LEN - 1, ipLen));
// tstrncpy(userStr, user, TMIN(TSDB_USER_LEN - 1, userLen));
// tstrncpy(passStr, pass, TMIN(TSDB_PASSWORD_LEN - 1, passLen));
// tstrncpy(dbStr, db, TMIN(TSDB_DB_NAME_LEN - 1, dbLen));
// return taos_connect(ipStr, userStr, passStr, dbStr, port);
//}
// tstrncpy(ipStr, ip, TMIN(TSDB_EP_LEN - 1, ipLen));
// tstrncpy(userStr, user, TMIN(TSDB_USER_LEN - 1, userLen));
// tstrncpy(passStr, pass, TMIN(TSDB_PASSWORD_LEN - 1, passLen));
// tstrncpy(dbStr, db, TMIN(TSDB_DB_NAME_LEN - 1, dbLen));
// return taos_connect(ipStr, userStr, passStr, dbStr, port);
// }
void doSetOneRowPtr(SReqResultInfo* pResultInfo) {
for (int32_t i = 0; i < pResultInfo->numOfCols; ++i) {
@ -2301,7 +2300,8 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int
(void)snprintf(varDataVal(dst), TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE, "%.9lf", jsonVd);
varDataSetLen(dst, strlen(varDataVal(dst)));
} else if (jsonInnerType == TSDB_DATA_TYPE_BOOL) {
(void)snprintf(varDataVal(dst), TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE, "%s", (*((char*)jsonInnerData) == 1) ? "true" : "false");
(void)snprintf(varDataVal(dst), TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE, "%s",
(*((char*)jsonInnerData) == 1) ? "true" : "false");
varDataSetLen(dst, strlen(varDataVal(dst)));
} else {
tscError("doConvertJson error: invalid type:%d", jsonInnerType);
@ -2570,6 +2570,7 @@ TSDB_SERVER_STATUS taos_check_server_status(const char* fqdn, int port, char* de
connLimitNum = TMIN(connLimitNum, 500);
rpcInit.connLimitNum = connLimitNum;
rpcInit.timeToGetConn = tsTimeToGetAvailableConn;
rpcInit.readTimeout = tsReadTimeout;
if (TSDB_CODE_SUCCESS != taosVersionStrToInt(version, &(rpcInit.compatibilityVer))) {
tscError("faild to convert taos version from str to int, errcode:%s", terrstr());
goto _OVER;

26
source/client/src/clientMain.c
View File

@ -492,53 +492,53 @@ int taos_print_row_with_size(char *str, uint32_t size, TAOS_ROW row, TAOS_FIELD
}
if (row[i] == NULL) {
len += snprintf(str + len, size - len, "%s", TSDB_DATA_NULL_STR);
len += tsnprintf(str + len, size - len, "%s", TSDB_DATA_NULL_STR);
continue;
}
switch (fields[i].type) {
case TSDB_DATA_TYPE_TINYINT:
len += snprintf(str + len, size - len, "%d", *((int8_t *)row[i]));
len += tsnprintf(str + len, size - len, "%d", *((int8_t *)row[i]));
break;
case TSDB_DATA_TYPE_UTINYINT:
len += snprintf(str + len, size - len, "%u", *((uint8_t *)row[i]));
len += tsnprintf(str + len, size - len, "%u", *((uint8_t *)row[i]));
break;
case TSDB_DATA_TYPE_SMALLINT:
len += snprintf(str + len, size - len, "%d", *((int16_t *)row[i]));
len += tsnprintf(str + len, size - len, "%d", *((int16_t *)row[i]));
break;
case TSDB_DATA_TYPE_USMALLINT:
len += snprintf(str + len, size - len, "%u", *((uint16_t *)row[i]));
len += tsnprintf(str + len, size - len, "%u", *((uint16_t *)row[i]));
break;
case TSDB_DATA_TYPE_INT:
len += snprintf(str + len, size - len, "%d", *((int32_t *)row[i]));
len += tsnprintf(str + len, size - len, "%d", *((int32_t *)row[i]));
break;
case TSDB_DATA_TYPE_UINT:
len += snprintf(str + len, size - len, "%u", *((uint32_t *)row[i]));
len += tsnprintf(str + len, size - len, "%u", *((uint32_t *)row[i]));
break;
case TSDB_DATA_TYPE_BIGINT:
len += snprintf(str + len, size - len, "%" PRId64, *((int64_t *)row[i]));
len += tsnprintf(str + len, size - len, "%" PRId64, *((int64_t *)row[i]));
break;
case TSDB_DATA_TYPE_UBIGINT:
len += snprintf(str + len, size - len, "%" PRIu64, *((uint64_t *)row[i]));
len += tsnprintf(str + len, size - len, "%" PRIu64, *((uint64_t *)row[i]));
break;
case TSDB_DATA_TYPE_FLOAT: {
float fv = 0;
fv = GET_FLOAT_VAL(row[i]);
len += snprintf(str + len, size - len, "%f", fv);
len += tsnprintf(str + len, size - len, "%f", fv);
} break;
case TSDB_DATA_TYPE_DOUBLE: {
double dv = 0;
dv = GET_DOUBLE_VAL(row[i]);
len += snprintf(str + len, size - len, "%lf", dv);
len += tsnprintf(str + len, size - len, "%lf", dv);
} break;
case TSDB_DATA_TYPE_VARBINARY: {
@ -576,11 +576,11 @@ int taos_print_row_with_size(char *str, uint32_t size, TAOS_ROW row, TAOS_FIELD
} break;
case TSDB_DATA_TYPE_TIMESTAMP:
len += snprintf(str + len, size - len, "%" PRId64, *((int64_t *)row[i]));
len += tsnprintf(str + len, size - len, "%" PRId64, *((int64_t *)row[i]));
break;
case TSDB_DATA_TYPE_BOOL:
len += snprintf(str + len, size - len, "%d", *((int8_t *)row[i]));
len += tsnprintf(str + len, size - len, "%d", *((int8_t *)row[i]));
default:
break;
}

16
source/common/src/systable.c
View File

@ -399,6 +399,20 @@ static const SSysDbTableSchema userCompactsDetailSchema[] = {
{.name = "start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
};
static const SSysDbTableSchema anodesSchema[] = {
{.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
{.name = "url", .bytes = TSDB_ANAL_ANODE_URL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
{.name = "status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true},
{.name = "update_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true},
};
static const SSysDbTableSchema anodesFullSchema[] = {
{.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
{.name = "type", .bytes = TSDB_ANAL_ALGO_TYPE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
{.name = "algo", .bytes = TSDB_ANAL_ALGO_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
};
static const SSysDbTableSchema tsmaSchema[] = {
{.name = "tsma_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
@ -473,6 +487,8 @@ static const SSysTableMeta infosMeta[] = {
{TSDB_INS_TABLE_ARBGROUPS, arbGroupsSchema, tListLen(arbGroupsSchema), true},
{TSDB_INS_TABLE_ENCRYPTIONS, encryptionsSchema, tListLen(encryptionsSchema), true},
{TSDB_INS_TABLE_TSMAS, tsmaSchema, tListLen(tsmaSchema), false},
{TSDB_INS_TABLE_ANODES, anodesSchema, tListLen(anodesSchema), true},
{TSDB_INS_TABLE_ANODES_FULL, anodesFullSchema, tListLen(anodesFullSchema), true},
};
static const SSysDbTableSchema connectionsSchema[] = {

41
source/common/src/tglobal.c
View File

@ -56,6 +56,8 @@ int32_t tsShellActivityTimer = 3; // second
// queue & threads
int32_t tsNumOfRpcThreads = 1;
int32_t tsNumOfRpcSessions = 30000;
int32_t tsShareConnLimit = 8;
int32_t tsReadTimeout = 900;
int32_t tsTimeToGetAvailableConn = 500000;
int32_t tsKeepAliveIdle = 60;
@ -362,7 +364,7 @@ static int32_t taosSplitS3Cfg(SConfig *pCfg, const char *name, char gVarible[TSD
char *strDup = NULL;
if ((strDup = taosStrdup(pItem->str)) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
code = terrno;
goto _exit;
}
@ -610,12 +612,18 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "randErrorScope", tsRandErrScope, 0, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_BOTH));
tsNumOfRpcThreads = tsNumOfCores / 2;
tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 2, TSDB_MAX_RPC_THREADS);
tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 1, TSDB_MAX_RPC_THREADS);
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfRpcThreads", tsNumOfRpcThreads, 1, 1024, CFG_SCOPE_BOTH, CFG_DYN_NONE));
tsNumOfRpcSessions = TRANGE(tsNumOfRpcSessions, 100, 100000);
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfRpcSessions", tsNumOfRpcSessions, 1, 100000, CFG_SCOPE_BOTH, CFG_DYN_NONE));
tsShareConnLimit = TRANGE(tsShareConnLimit, 1, 512);
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "shareConnLimit", tsShareConnLimit, 1, 512, CFG_SCOPE_BOTH, CFG_DYN_NONE));
tsReadTimeout = TRANGE(tsReadTimeout, 64, 24 * 3600 * 7);
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "readTimeout", tsReadTimeout, 64, 24 * 3600 * 7, CFG_SCOPE_BOTH, CFG_DYN_NONE));
tsTimeToGetAvailableConn = TRANGE(tsTimeToGetAvailableConn, 20, 10000000);
TAOS_CHECK_RETURN(
cfgAddInt32(pCfg, "timeToGetAvailableConn", tsTimeToGetAvailableConn, 20, 1000000, CFG_SCOPE_BOTH, CFG_DYN_NONE));
@ -871,8 +879,7 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) {
pItem = cfgGetItem(pCfg, "numOfRpcThreads");
if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
tsNumOfRpcThreads = numOfCores / 2;
tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 2, TSDB_MAX_RPC_THREADS);
tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 1, TSDB_MAX_RPC_THREADS);
pItem->i32 = tsNumOfRpcThreads;
pItem->stype = stype;
}
@ -884,6 +891,20 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) {
pItem->stype = stype;
}
pItem = cfgGetItem(pCfg, "shareConnLimit");
if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
tsShareConnLimit = TRANGE(tsShareConnLimit, 1, 512);
pItem->i32 = tsShareConnLimit;
pItem->stype = stype;
}
pItem = cfgGetItem(pCfg, "readTimeout");
if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
tsReadTimeout = TRANGE(tsReadTimeout, 64, 24 * 3600 * 7);
pItem->i32 = tsReadTimeout;
pItem->stype = stype;
}
pItem = cfgGetItem(pCfg, "timeToGetAvailableConn");
if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
tsTimeToGetAvailableConn = TRANGE(tsTimeToGetAvailableConn, 20, 1000000);
@ -1261,6 +1282,12 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "numOfRpcSessions");
tsNumOfRpcSessions = pItem->i32;
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "shareConnLimit");
tsShareConnLimit = pItem->i32;
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "readTimeout");
tsReadTimeout = pItem->i32;
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "timeToGetAvailableConn");
tsTimeToGetAvailableConn = pItem->i32;
@ -1362,6 +1389,12 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "numOfRpcSessions");
tsNumOfRpcSessions = pItem->i32;
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "shareConnLimit");
tsShareConnLimit = pItem->i32;
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "readTimeout");
tsReadTimeout = pItem->i32;
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "timeToGetAvailableConn");
tsTimeToGetAvailableConn = pItem->i32;

263
source/common/src/tmsg.c
View File

@ -40,6 +40,7 @@
#define TD_MSG_RANGE_CODE_
#include "tmsgdef.h"
#include "tanal.h"
#include "tcol.h"
#include "tlog.h"
@ -1453,6 +1454,7 @@ int32_t tSerializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) {
}
TAOS_CHECK_EXIT(tEncodeI64(&encoder, pReq->ipWhiteVer));
TAOS_CHECK_EXIT(tEncodeI64(&encoder, pReq->analVer));
TAOS_CHECK_EXIT(tSerializeSMonitorParas(&encoder, &pReq->clusterCfg.monitorParas));
tEndEncode(&encoder);
@ -1576,6 +1578,10 @@ int32_t tDeserializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) {
TAOS_CHECK_EXIT(tDecodeI64(&decoder, &pReq->ipWhiteVer));
}
if (!tDecodeIsEnd(&decoder)) {
TAOS_CHECK_EXIT(tDecodeI64(&decoder, &pReq->analVer));
}
if (!tDecodeIsEnd(&decoder)) {
TAOS_CHECK_EXIT(tDeserializeSMonitorParas(&decoder, &pReq->clusterCfg.monitorParas));
}
@ -1652,6 +1658,7 @@ int32_t tSerializeSStatusRsp(void *buf, int32_t bufLen, SStatusRsp *pRsp) {
TAOS_CHECK_EXIT(tEncodeI32(&encoder, pRsp->statusSeq));
TAOS_CHECK_EXIT(tEncodeI64(&encoder, pRsp->ipWhiteVer));
TAOS_CHECK_EXIT(tEncodeI64(&encoder, pRsp->analVer));
tEndEncode(&encoder);
_exit:
@ -1704,6 +1711,10 @@ int32_t tDeserializeSStatusRsp(void *buf, int32_t bufLen, SStatusRsp *pRsp) {
TAOS_CHECK_EXIT(tDecodeI64(&decoder, &pRsp->ipWhiteVer));
}
if (!tDecodeIsEnd(&decoder)) {
TAOS_CHECK_EXIT(tDecodeI64(&decoder, &pRsp->analVer));
}
tEndDecode(&decoder);
_exit:
tDecoderClear(&decoder);
@ -2045,6 +2056,156 @@ _exit:
return code;
}
int32_t tSerializeRetrieveAnalAlgoReq(void *buf, int32_t bufLen, SRetrieveAnalAlgoReq *pReq) {
SEncoder encoder = {0};
int32_t code = 0;
int32_t lino;
int32_t tlen;
tEncoderInit(&encoder, buf, bufLen);
TAOS_CHECK_EXIT(tStartEncode(&encoder));
TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->dnodeId));
TAOS_CHECK_EXIT(tEncodeI64(&encoder, pReq->analVer));
tEndEncode(&encoder);
_exit:
if (code) {
tlen = code;
} else {
tlen = encoder.pos;
}
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeRetrieveAnalAlgoReq(void *buf, int32_t bufLen, SRetrieveAnalAlgoReq *pReq) {
SDecoder decoder = {0};
int32_t code = 0;
int32_t lino;
tDecoderInit(&decoder, buf, bufLen);
TAOS_CHECK_EXIT(tStartDecode(&decoder));
TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->dnodeId));
TAOS_CHECK_EXIT(tDecodeI64(&decoder, &pReq->analVer));
tEndDecode(&decoder);
_exit:
tDecoderClear(&decoder);
return code;
}
int32_t tSerializeRetrieveAnalAlgoRsp(void *buf, int32_t bufLen, SRetrieveAnalAlgoRsp *pRsp) {
SEncoder encoder = {0};
int32_t code = 0;
int32_t lino;
int32_t tlen;
tEncoderInit(&encoder, buf, bufLen);
int32_t numOfAlgos = 0;
void *pIter = taosHashIterate(pRsp->hash, NULL);
while (pIter != NULL) {
SAnalUrl *pUrl = pIter;
size_t nameLen = 0;
const char *name = taosHashGetKey(pIter, &nameLen);
if (nameLen > 0 && nameLen <= TSDB_ANAL_ALGO_KEY_LEN && pUrl->urlLen > 0) {
numOfAlgos++;
}
pIter = taosHashIterate(pRsp->hash, pIter);
}
TAOS_CHECK_EXIT(tStartEncode(&encoder));
TAOS_CHECK_EXIT(tEncodeI64(&encoder, pRsp->ver));
TAOS_CHECK_EXIT(tEncodeI32(&encoder, numOfAlgos));
pIter = taosHashIterate(pRsp->hash, NULL);
while (pIter != NULL) {
SAnalUrl *pUrl = pIter;
size_t nameLen = 0;
const char *name = taosHashGetKey(pIter, &nameLen);
if (nameLen > 0 && pUrl->urlLen > 0) {
TAOS_CHECK_EXIT(tEncodeI32(&encoder, nameLen));
TAOS_CHECK_EXIT(tEncodeBinary(&encoder, (const uint8_t *)name, nameLen));
TAOS_CHECK_EXIT(tEncodeI32(&encoder, pUrl->anode));
TAOS_CHECK_EXIT(tEncodeI32(&encoder, pUrl->type));
TAOS_CHECK_EXIT(tEncodeI32(&encoder, pUrl->urlLen));
TAOS_CHECK_EXIT(tEncodeBinary(&encoder, (const uint8_t *)pUrl->url, pUrl->urlLen));
}
pIter = taosHashIterate(pRsp->hash, pIter);
}
tEndEncode(&encoder);
_exit:
if (code) {
tlen = code;
} else {
tlen = encoder.pos;
}
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeRetrieveAnalAlgoRsp(void *buf, int32_t bufLen, SRetrieveAnalAlgoRsp *pRsp) {
if (pRsp->hash == NULL) {
pRsp->hash = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK);
if (pRsp->hash == NULL) {
terrno = TSDB_CODE_OUT_OF_BUFFER;
return terrno;
}
}
SDecoder decoder = {0};
int32_t code = 0;
int32_t lino;
tDecoderInit(&decoder, buf, bufLen);
int32_t numOfAlgos = 0;
int32_t nameLen;
int32_t type;
char name[TSDB_ANAL_ALGO_KEY_LEN];
SAnalUrl url = {0};
TAOS_CHECK_EXIT(tStartDecode(&decoder));
TAOS_CHECK_EXIT(tDecodeI64(&decoder, &pRsp->ver));
TAOS_CHECK_EXIT(tDecodeI32(&decoder, &numOfAlgos));
for (int32_t f = 0; f < numOfAlgos; ++f) {
TAOS_CHECK_EXIT(tDecodeI32(&decoder, &nameLen));
if (nameLen > 0 && nameLen <= TSDB_ANAL_ALGO_NAME_LEN) {
TAOS_CHECK_EXIT(tDecodeCStrTo(&decoder, name));
}
TAOS_CHECK_EXIT(tDecodeI32(&decoder, &url.anode));
TAOS_CHECK_EXIT(tDecodeI32(&decoder, &type));
url.type = (EAnalAlgoType)type;
TAOS_CHECK_EXIT(tDecodeI32(&decoder, &url.urlLen));
if (url.urlLen > 0) {
TAOS_CHECK_EXIT(tDecodeBinaryAlloc(&decoder, (void **)&url.url, NULL) < 0);
}
TAOS_CHECK_EXIT(taosHashPut(pRsp->hash, name, nameLen, &url, sizeof(SAnalUrl)));
}
tEndDecode(&decoder);
_exit:
tDecoderClear(&decoder);
return code;
}
void tFreeRetrieveAnalAlgoRsp(SRetrieveAnalAlgoRsp *pRsp) {
void *pIter = taosHashIterate(pRsp->hash, NULL);
while (pIter != NULL) {
SAnalUrl *pUrl = (SAnalUrl *)pIter;
taosMemoryFree(pUrl->url);
pIter = taosHashIterate(pRsp->hash, pIter);
}
taosHashCleanup(pRsp->hash);
pRsp->hash = NULL;
}
void tFreeSCreateUserReq(SCreateUserReq *pReq) {
FREESQL();
taosMemoryFreeClear(pReq->pIpRanges);
@ -2962,6 +3123,108 @@ _exit:
return code;
}
int32_t tSerializeSMCreateAnodeReq(void *buf, int32_t bufLen, SMCreateAnodeReq *pReq) {
SEncoder encoder = {0};
int32_t code = 0;
int32_t lino;
int32_t tlen;
tEncoderInit(&encoder, buf, bufLen);
TAOS_CHECK_EXIT(tStartEncode(&encoder));
TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->urlLen));
if (pReq->urlLen > 0) {
TAOS_CHECK_EXIT(tEncodeBinary(&encoder, (const uint8_t *)pReq->url, pReq->urlLen));
}
ENCODESQL();
tEndEncode(&encoder);
_exit:
if (code) {
tlen = code;
} else {
tlen = encoder.pos;
}
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSMCreateAnodeReq(void *buf, int32_t bufLen, SMCreateAnodeReq *pReq) {
SDecoder decoder = {0};
int32_t code = 0;
int32_t lino;
tDecoderInit(&decoder, buf, bufLen);
TAOS_CHECK_EXIT(tStartDecode(&decoder));
TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->urlLen));
if (pReq->urlLen > 0) {
TAOS_CHECK_EXIT(tDecodeBinaryAlloc(&decoder, (void **)&pReq->url, NULL));
}
DECODESQL();
tEndDecode(&decoder);
_exit:
tDecoderClear(&decoder);
return code;
}
void tFreeSMCreateAnodeReq(SMCreateAnodeReq *pReq) {
taosMemoryFreeClear(pReq->url);
FREESQL();
}
int32_t tSerializeSMDropAnodeReq(void *buf, int32_t bufLen, SMDropAnodeReq *pReq) {
SEncoder encoder = {0};
int32_t code = 0;
int32_t lino;
int32_t tlen;
tEncoderInit(&encoder, buf, bufLen);
TAOS_CHECK_EXIT(tStartEncode(&encoder));
TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->anodeId));
ENCODESQL();
tEndEncode(&encoder);
_exit:
if (code) {
tlen = code;
} else {
tlen = encoder.pos;
}
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSMDropAnodeReq(void *buf, int32_t bufLen, SMDropAnodeReq *pReq) {
SDecoder decoder = {0};
int32_t code = 0;
int32_t lino;
tDecoderInit(&decoder, buf, bufLen);
TAOS_CHECK_EXIT(tStartDecode(&decoder));
TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->anodeId));
DECODESQL();
tEndDecode(&decoder);
_exit:
tDecoderClear(&decoder);
return code;
}
void tFreeSMDropAnodeReq(SMDropAnodeReq *pReq) { FREESQL(); }
int32_t tSerializeSMUpdateAnodeReq(void *buf, int32_t bufLen, SMUpdateAnodeReq *pReq) {
return tSerializeSMDropAnodeReq(buf, bufLen, pReq);
}
int32_t tDeserializeSMUpdateAnodeReq(void *buf, int32_t bufLen, SMUpdateAnodeReq *pReq) {
return tDeserializeSMDropAnodeReq(buf, bufLen, pReq);
}
void tFreeSMUpdateAnodeReq(SMUpdateAnodeReq *pReq) { tFreeSMDropAnodeReq(pReq); }
int32_t tSerializeSCreateDnodeReq(void *buf, int32_t bufLen, SCreateDnodeReq *pReq) {
SEncoder encoder = {0};
int32_t code = 0;

1
source/dnode/mgmt/exe/dmMain.c
View File

@ -183,6 +183,7 @@ static void dmSetSignalHandle() {
}
#endif
}
extern bool generateNewMeta;
static int32_t dmParseArgs(int32_t argc, char const *argv[]) {

4
source/dnode/mgmt/mgmt_dnode/inc/dmInt.h
View File

@ -28,6 +28,7 @@ typedef struct SDnodeMgmt {
const char *path;
const char *name;
TdThread statusThread;
TdThread statusInfoThread;
TdThread notifyThread;
TdThread monitorThread;
TdThread auditThread;
@ -49,6 +50,7 @@ typedef struct SDnodeMgmt {
// dmHandle.c
SArray *dmGetMsgHandles();
void dmSendStatusReq(SDnodeMgmt *pMgmt);
void dmUpdateStatusInfo(SDnodeMgmt *pMgmt);
void dmSendNotifyReq(SDnodeMgmt *pMgmt, SNotifyReq *pReq);
int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t dmProcessAuthRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg);
@ -62,7 +64,9 @@ int32_t dmProcessCreateEncryptKeyReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg);
// dmWorker.c
int32_t dmPutNodeMsgToMgmtQueue(SDnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t dmStartStatusThread(SDnodeMgmt *pMgmt);
int32_t dmStartStatusInfoThread(SDnodeMgmt *pMgmt);
void dmStopStatusThread(SDnodeMgmt *pMgmt);
void dmStopStatusInfoThread(SDnodeMgmt *pMgmt);
int32_t dmStartNotifyThread(SDnodeMgmt *pMgmt);
void dmStopNotifyThread(SDnodeMgmt *pMgmt);
int32_t dmStartMonitorThread(SDnodeMgmt *pMgmt);

86
source/dnode/mgmt/mgmt_dnode/src/dmHandle.c
View File

@ -18,10 +18,13 @@
#include "dmInt.h"
#include "monitor.h"
#include "systable.h"
#include "tanal.h"
#include "tchecksum.h"
extern SConfig *tsCfg;
SMonVloadInfo tsVinfo = {0};
static void dmUpdateDnodeCfg(SDnodeMgmt *pMgmt, SDnodeCfg *pCfg) {
int32_t code = 0;
if (pMgmt->pData->dnodeId == 0 || pMgmt->pData->clusterId == 0) {
@ -71,7 +74,7 @@ static void dmMayShouldUpdateIpWhiteList(SDnodeMgmt *pMgmt, int64_t ver) {
SRpcMsg rpcMsg = {.pCont = pHead,
.contLen = contLen,
.msgType = TDMT_MND_RETRIEVE_IP_WHITE,
.info.ahandle = (void *)0x9527,
.info.ahandle = 0,
.info.notFreeAhandle = 1,
.info.refId = 0,
.info.noResp = 0,
@ -86,6 +89,46 @@ static void dmMayShouldUpdateIpWhiteList(SDnodeMgmt *pMgmt, int64_t ver) {
}
}
static void dmMayShouldUpdateAnalFunc(SDnodeMgmt *pMgmt, int64_t newVer) {
int32_t code = 0;
int64_t oldVer = taosAnalGetVersion();
if (oldVer == newVer) return;
dDebug("analysis on dnode ver:%" PRId64 ", status ver:%" PRId64, oldVer, newVer);
SRetrieveAnalAlgoReq req = {.dnodeId = pMgmt->pData->dnodeId, .analVer = oldVer};
int32_t contLen = tSerializeRetrieveAnalAlgoReq(NULL, 0, &req);
if (contLen < 0) {
dError("failed to serialize analysis function ver request since %s", tstrerror(contLen));
return;
}
void *pHead = rpcMallocCont(contLen);
contLen = tSerializeRetrieveAnalAlgoReq(pHead, contLen, &req);
if (contLen < 0) {
rpcFreeCont(pHead);
dError("failed to serialize analysis function ver request since %s", tstrerror(contLen));
return;
}
SRpcMsg rpcMsg = {
.pCont = pHead,
.contLen = contLen,
.msgType = TDMT_MND_RETRIEVE_ANAL_ALGO,
.info.ahandle = (void *)0x9527,
.info.refId = 0,
.info.noResp = 0,
.info.handle = 0,
};
SEpSet epset = {0};
(void)dmGetMnodeEpSet(pMgmt->pData, &epset);
code = rpcSendRequest(pMgmt->msgCb.clientRpc, &epset, &rpcMsg, NULL);
if (code != 0) {
dError("failed to send retrieve analysis func ver request since %s", tstrerror(code));
}
}
static void dmProcessStatusRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) {
const STraceId *trace = &pRsp->info.traceId;
dGTrace("status rsp received from mnode, statusSeq:%d code:0x%x", pMgmt->statusSeq, pRsp->code);
@ -113,6 +156,7 @@ static void dmProcessStatusRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) {
dmUpdateEps(pMgmt->pData, statusRsp.pDnodeEps);
}
dmMayShouldUpdateIpWhiteList(pMgmt, statusRsp.ipWhiteVer);
dmMayShouldUpdateAnalFunc(pMgmt, statusRsp.analVer);
}
tFreeSStatusRsp(&statusRsp);
}
@ -163,9 +207,16 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) {
(void)taosThreadRwlockUnlock(&pMgmt->pData->lock);
dDebug("send status req to mnode, statusSeq:%d, begin to get vnode loads", pMgmt->statusSeq);
SMonVloadInfo vinfo = {0};
(*pMgmt->getVnodeLoadsFp)(&vinfo);
req.pVloads = vinfo.pVloads;
if (taosThreadMutexLock(&pMgmt->pData->statusInfolock) != 0) {
dError("failed to lock status info lock");
return;
}
req.pVloads = tsVinfo.pVloads;
tsVinfo.pVloads = NULL;
if (taosThreadMutexUnlock(&pMgmt->pData->statusInfolock) != 0) {
dError("failed to unlock status info lock");
return;
}
dDebug("send status req to mnode, statusSeq:%d, begin to get mnode loads", pMgmt->statusSeq);
SMonMloadInfo minfo = {0};
@ -178,6 +229,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) {
pMgmt->statusSeq++;
req.statusSeq = pMgmt->statusSeq;
req.ipWhiteVer = pMgmt->pData->ipWhiteVer;
req.analVer = taosAnalGetVersion();
int32_t contLen = tSerializeSStatusReq(NULL, 0, &req);
if (contLen < 0) {
@ -197,7 +249,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) {
SRpcMsg rpcMsg = {.pCont = pHead,
.contLen = contLen,
.msgType = TDMT_MND_STATUS,
.info.ahandle = (void *)0x9527,
.info.ahandle = 0,
.info.notFreeAhandle = 1,
.info.refId = 0,
.info.noResp = 0,
@ -231,6 +283,28 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) {
dmProcessStatusRsp(pMgmt, &rpcRsp);
}
void dmUpdateStatusInfo(SDnodeMgmt *pMgmt) {
SMonVloadInfo vinfo = {0};
dDebug("begin to get vnode loads");
(*pMgmt->getVnodeLoadsFp)(&vinfo);
dDebug("begin to lock status info");
if (taosThreadMutexLock(&pMgmt->pData->statusInfolock) != 0) {
dError("failed to lock status info lock");
return;
}
if (tsVinfo.pVloads == NULL) {
tsVinfo.pVloads = vinfo.pVloads;
vinfo.pVloads = NULL;
} else {
taosArrayDestroy(vinfo.pVloads);
vinfo.pVloads = NULL;
}
if (taosThreadMutexUnlock(&pMgmt->pData->statusInfolock) != 0) {
dError("failed to unlock status info lock");
return;
}
}
void dmSendNotifyReq(SDnodeMgmt *pMgmt, SNotifyReq *pReq) {
int32_t contLen = tSerializeSNotifyReq(NULL, 0, pReq);
if (contLen < 0) {
@ -248,7 +322,7 @@ void dmSendNotifyReq(SDnodeMgmt *pMgmt, SNotifyReq *pReq) {
SRpcMsg rpcMsg = {.pCont = pHead,
.contLen = contLen,
.msgType = TDMT_MND_NOTIFY,
.info.ahandle = (void *)0x9527,
.info.ahandle = 0,
.info.notFreeAhandle = 1,
.info.refId = 0,
.info.noResp = 1,

9
source/dnode/mgmt/mgmt_dnode/src/dmInt.c
View File

@ -16,12 +16,16 @@
#define _DEFAULT_SOURCE
#include "dmInt.h"
#include "libs/function/tudf.h"
#include "tanal.h"
static int32_t dmStartMgmt(SDnodeMgmt *pMgmt) {
int32_t code = 0;
if ((code = dmStartStatusThread(pMgmt)) != 0) {
return code;
}
if ((code = dmStartStatusInfoThread(pMgmt)) != 0) {
return code;
}
#if defined(TD_ENTERPRISE)
if ((code = dmStartNotifyThread(pMgmt)) != 0) {
return code;
@ -44,6 +48,7 @@ static void dmStopMgmt(SDnodeMgmt *pMgmt) {
dmStopMonitorThread(pMgmt);
dmStopAuditThread(pMgmt);
dmStopStatusThread(pMgmt);
dmStopStatusInfoThread(pMgmt);
#if defined(TD_ENTERPRISE)
dmStopNotifyThread(pMgmt);
#endif
@ -80,6 +85,10 @@ static int32_t dmOpenMgmt(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
dError("failed to start udfd since %s", tstrerror(code));
}
if ((code = taosAnalInit()) != 0) {
dError("failed to init analysis env since %s", tstrerror(code));
}
pOutput->pMgmt = pMgmt;
return 0;
}

66
source/dnode/mgmt/mgmt_dnode/src/dmWorker.c
View File

@ -47,6 +47,49 @@ static void *dmStatusThreadFp(void *param) {
return NULL;
}
extern SMonVloadInfo tsVinfo;
static void *dmStatusInfoThreadFp(void *param) {
SDnodeMgmt *pMgmt = param;
int64_t lastTime = taosGetTimestampMs();
setThreadName("dnode-status-info");
int32_t upTimeCount = 0;
int64_t upTime = 0;
while (1) {
taosMsleep(200);
if (pMgmt->pData->dropped || pMgmt->pData->stopped) break;
int64_t curTime = taosGetTimestampMs();
if (curTime < lastTime) lastTime = curTime;
float interval = (curTime - lastTime) / 1000.0f;
if (interval >= tsStatusInterval) {
dmUpdateStatusInfo(pMgmt);
lastTime = curTime;
if ((upTimeCount = ((upTimeCount + 1) & 63)) == 0) {
upTime = taosGetOsUptime() - tsDndStartOsUptime;
tsDndUpTime = TMAX(tsDndUpTime, upTime);
}
}
}
dDebug("begin to lock status info when thread exit");
if (taosThreadMutexLock(&pMgmt->pData->statusInfolock) != 0) {
dError("failed to lock status info lock");
return NULL;
}
if (tsVinfo.pVloads != NULL) {
taosArrayDestroy(tsVinfo.pVloads);
tsVinfo.pVloads = NULL;
}
if (taosThreadMutexUnlock(&pMgmt->pData->statusInfolock) != 0) {
dError("failed to unlock status info lock");
return NULL;
}
return NULL;
}
SDmNotifyHandle dmNotifyHdl = {.state = 0};
#define TIMESERIES_STASH_NUM 5
static void *dmNotifyThreadFp(void *param) {
@ -280,6 +323,22 @@ int32_t dmStartStatusThread(SDnodeMgmt *pMgmt) {
return 0;
}
int32_t dmStartStatusInfoThread(SDnodeMgmt *pMgmt) {
int32_t code = 0;
TdThreadAttr thAttr;
(void)taosThreadAttrInit(&thAttr);
(void)taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
if (taosThreadCreate(&pMgmt->statusInfoThread, &thAttr, dmStatusInfoThreadFp, pMgmt) != 0) {
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to create status Info thread since %s", tstrerror(code));
return code;
}
(void)taosThreadAttrDestroy(&thAttr);
tmsgReportStartup("dnode-status-info", "initialized");
return 0;
}
void dmStopStatusThread(SDnodeMgmt *pMgmt) {
if (taosCheckPthreadValid(pMgmt->statusThread)) {
(void)taosThreadJoin(pMgmt->statusThread, NULL);
@ -287,6 +346,13 @@ void dmStopStatusThread(SDnodeMgmt *pMgmt) {
}
}
void dmStopStatusInfoThread(SDnodeMgmt *pMgmt) {
if (taosCheckPthreadValid(pMgmt->statusInfoThread)) {
(void)taosThreadJoin(pMgmt->statusInfoThread, NULL);
taosThreadClear(&pMgmt->statusInfoThread);
}
}
int32_t dmStartNotifyThread(SDnodeMgmt *pMgmt) {
int32_t code = 0;
TdThreadAttr thAttr;

6
source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
View File

@ -141,6 +141,9 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_MND_DNODE_LIST, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_SNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_SNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_ANODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_UPDATE_ANODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_ANODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_USE_DB, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
@ -180,6 +183,7 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_VND_FETCH_TTL_EXPIRED_TBS_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TABLE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_RETRIEVE_ANAL_ALGO, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_RETRIEVE_IP_WHITE, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_GET_USER_WHITELIST, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_GET_INDEX, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
@ -208,7 +212,7 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_VIEW, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_VIEW_META, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STATIS, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_COMPACT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_COMPACT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_CONFIG_CLUSTER, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_COMPACT_PROGRESS_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_ENCRYPT_KEY, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;

18
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -415,14 +415,24 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
goto _OVER;
}
// taosThreadMutexLock(&pMgmt->createLock);
code = taosThreadMutexLock(&pMgmt->createLock);
if (code != 0) {
dError("vgId:%d, failed to lock since %s", req.vgId, tstrerror(code));
goto _OVER;
}
code = vmWriteVnodeListToFile(pMgmt);
if (code != 0) {
code = terrno != 0 ? terrno : code;
// taosThreadMutexUnlock(&pMgmt->createLock);
int32_t ret = taosThreadMutexUnlock(&pMgmt->createLock);
if (ret != 0) {
dError("vgId:%d, failed to unlock since %s", req.vgId, tstrerror(ret));
}
goto _OVER;
}
// taosThreadMutexUnlock(&pMgmt->createLock);
int32_t ret = taosThreadMutexUnlock(&pMgmt->createLock);
if (ret != 0) {
dError("vgId:%d, failed to unlock since %s", req.vgId, tstrerror(ret));
}
_OVER:
if (code != 0) {
@ -1037,7 +1047,7 @@ SArray *vmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_TRIM, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_S3MIGRATE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE, vmPutMsgToMultiMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_VNODE_TYPE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_CHECK_VNODE_LEARNER_CATCHUP, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;

2
source/dnode/mgmt/node_mgmt/src/dmEnv.c
View File

@ -21,6 +21,7 @@
#include "tgrant.h"
#include "tcompare.h"
#include "tcs.h"
#include "tanal.h"
// clang-format on
#define DM_INIT_AUDIT() \
@ -208,6 +209,7 @@ void dmCleanup() {
dError("failed to close udfc");
}
udfStopUdfd();
taosAnalCleanup();
taosStopCacheRefreshWorker();
(void)dmDiskClose();
DestroyRegexCache();

1
source/dnode/mgmt/node_mgmt/src/dmMgmt.c
View File

@ -214,6 +214,7 @@ int32_t dmInitVars(SDnode *pDnode) {
}
(void)taosThreadRwlockInit(&pData->lock, NULL);
(void)taosThreadMutexInit(&pData->statusInfolock, NULL);
(void)taosThreadMutexInit(&pDnode->mutex, NULL);
return 0;
}

31
source/dnode/mgmt/node_mgmt/src/dmTransport.c
View File

@ -17,6 +17,7 @@
#include "dmMgmt.h"
#include "qworker.h"
#include "tversion.h"
#include "tanal.h"
static inline void dmSendRsp(SRpcMsg *pMsg) {
if (rpcSendResponse(pMsg) != 0) {
@ -106,6 +107,16 @@ static bool dmIsForbiddenIp(int8_t forbidden, char *user, uint32_t clientIp) {
}
}
static void dmUpdateAnalFunc(SDnodeData *pData, void *pTrans, SRpcMsg *pRpc) {
SRetrieveAnalAlgoRsp rsp = {0};
if (tDeserializeRetrieveAnalAlgoRsp(pRpc->pCont, pRpc->contLen, &rsp) == 0) {
taosAnalUpdate(rsp.ver, rsp.hash);
rsp.hash = NULL;
}
tFreeRetrieveAnalAlgoRsp(&rsp);
rpcFreeCont(pRpc->pCont);
}
static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
SDnodeTrans *pTrans = &pDnode->trans;
int32_t code = -1;
@ -154,6 +165,9 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
case TDMT_MND_RETRIEVE_IP_WHITE_RSP:
dmUpdateRpcIpWhite(&pDnode->data, pTrans->serverRpc, pRpc);
return;
case TDMT_MND_RETRIEVE_ANAL_ALGO_RSP:
dmUpdateAnalFunc(&pDnode->data, pTrans->serverRpc, pRpc);
return;
default:
break;
}
@ -253,7 +267,7 @@ _OVER:
}
if (IsReq(pRpc)) {
SRpcMsg rsp = {.code = code, .info = pRpc->info};
SRpcMsg rsp = {.code = code, .info = pRpc->info, .msgType = pRpc->msgType + 1};
if (code == TSDB_CODE_MNODE_NOT_FOUND) {
dmBuildMnodeRedirectRsp(pDnode, &rsp);
}
@ -404,9 +418,12 @@ int32_t dmInitClient(SDnode *pDnode) {
rpcInit.connLimitNum = connLimitNum;
rpcInit.connLimitLock = 1;
rpcInit.supportBatch = 1;
rpcInit.batchSize = 8 * 1024;
rpcInit.shareConnLimit = tsShareConnLimit * 2;
rpcInit.shareConn = 1;
rpcInit.timeToGetConn = tsTimeToGetAvailableConn;
rpcInit.notWaitAvaliableConn = 0;
rpcInit.startReadTimer = 1;
rpcInit.readTimeout = tsReadTimeout;
if (taosVersionStrToInt(version, &(rpcInit.compatibilityVer)) != 0) {
dError("failed to convert version string:%s to int", version);
@ -452,8 +469,10 @@ int32_t dmInitStatusClient(SDnode *pDnode) {
rpcInit.connLimitNum = connLimitNum;
rpcInit.connLimitLock = 1;
rpcInit.supportBatch = 1;
rpcInit.batchSize = 8 * 1024;
rpcInit.shareConnLimit = tsShareConnLimit * 2;
rpcInit.timeToGetConn = tsTimeToGetAvailableConn;
rpcInit.startReadTimer = 0;
rpcInit.readTimeout = 0;
if (taosVersionStrToInt(version, &(rpcInit.compatibilityVer)) != 0) {
dError("failed to convert version string:%s to int", version);
@ -500,8 +519,11 @@ int32_t dmInitSyncClient(SDnode *pDnode) {
rpcInit.connLimitNum = connLimitNum;
rpcInit.connLimitLock = 1;
rpcInit.supportBatch = 1;
rpcInit.batchSize = 8 * 1024;
rpcInit.shareConnLimit = tsShareConnLimit * 8;
rpcInit.timeToGetConn = tsTimeToGetAvailableConn;
rpcInit.startReadTimer = 1;
rpcInit.readTimeout = tsReadTimeout;
if (taosVersionStrToInt(version, &(rpcInit.compatibilityVer)) != 0) {
dError("failed to convert version string:%s to int", version);
}
@ -555,6 +577,7 @@ int32_t dmInitServer(SDnode *pDnode) {
rpcInit.idleTime = tsShellActivityTimer * 1000;
rpcInit.parent = pDnode;
rpcInit.compressSize = tsCompressMsgSize;
rpcInit.shareConnLimit = tsShareConnLimit * 16;
if (taosVersionStrToInt(version, &(rpcInit.compatibilityVer)) != 0) {
dError("failed to convert version string:%s to int", version);

1
source/dnode/mgmt/node_util/inc/dmUtil.h
View File

@ -144,6 +144,7 @@ typedef struct {
char machineId[TSDB_MACHINE_ID_LEN + 1];
EEncryptAlgor encryptAlgorigthm;
EEncryptScope encryptScope;
TdThreadMutex statusInfolock;
} SDnodeData;
typedef struct {

3
source/dnode/mnode/impl/CMakeLists.txt
View File

@ -15,6 +15,9 @@ IF (TD_ENTERPRISE)
add_definitions(-DUSE_COS)
ENDIF()
IF(${BUILD_WITH_ANALYSIS})
add_definitions(-DUSE_ANAL)
ENDIF()
ENDIF ()
add_library(mnode STATIC ${MNODE_SRC})

32
source/dnode/mnode/impl/inc/mndAnode.h Normal file
View File

@ -0,0 +1,32 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_MND_ANODE_H_
#define _TD_MND_ANODE_H_
#include "mndInt.h"
#ifdef __cplusplus
extern "C" {
#endif
int32_t mndInitAnode(SMnode *pMnode);
void mndCleanupAnode(SMnode *pMnode);
#ifdef __cplusplus
}
#endif
#endif /*_TD_MND_ANODE_H_*/

21
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -78,6 +78,9 @@ typedef enum {
MND_OPER_DROP_VIEW,
MND_OPER_CONFIG_CLUSTER,
MND_OPER_BALANCE_VGROUP_LEADER,
MND_OPER_CREATE_ANODE,
MND_OPER_UPDATE_ANODE,
MND_OPER_DROP_ANODE
} EOperType;
typedef enum {
@ -232,6 +235,24 @@ typedef struct {
char machineId[TSDB_MACHINE_ID_LEN + 1];
} SDnodeObj;
typedef struct {
int32_t nameLen;
char* name;
} SAnodeAlgo;
typedef struct {
int32_t id;
int64_t createdTime;
int64_t updateTime;
int32_t version;
int32_t urlLen;
int32_t numOfAlgos;
int32_t status;
SRWLatch lock;
char* url;
SArray** algos;
} SAnodeObj;
typedef struct {
int32_t id;
int64_t createdTime;

903
source/dnode/mnode/impl/src/mndAnode.c Normal file
View File

@ -0,0 +1,903 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "mndAnode.h"
#include "audit.h"
#include "mndDnode.h"
#include "mndPrivilege.h"
#include "mndShow.h"
#include "mndTrans.h"
#include "mndUser.h"
#include "tanal.h"
#include "tjson.h"
#ifdef USE_ANAL
#define TSDB_ANODE_VER_NUMBER 1
#define TSDB_ANODE_RESERVE_SIZE 64
static SSdbRaw *mndAnodeActionEncode(SAnodeObj *pObj);
static SSdbRow *mndAnodeActionDecode(SSdbRaw *pRaw);
static int32_t mndAnodeActionInsert(SSdb *pSdb, SAnodeObj *pObj);
static int32_t mndAnodeActionUpdate(SSdb *pSdb, SAnodeObj *pOld, SAnodeObj *pNew);
static int32_t mndAnodeActionDelete(SSdb *pSdb, SAnodeObj *pObj);
static int32_t mndProcessCreateAnodeReq(SRpcMsg *pReq);
static int32_t mndProcessUpdateAnodeReq(SRpcMsg *pReq);
static int32_t mndProcessDropAnodeReq(SRpcMsg *pReq);
static int32_t mndProcessAnalAlgoReq(SRpcMsg *pReq);
static int32_t mndRetrieveAnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
static void mndCancelGetNextAnode(SMnode *pMnode, void *pIter);
static int32_t mndRetrieveAnodesFull(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
static void mndCancelGetNextAnodeFull(SMnode *pMnode, void *pIter);
static int32_t mndGetAnodeAlgoList(const char *url, SAnodeObj *pObj);
static int32_t mndGetAnodeStatus(SAnodeObj *pObj, char *status, int32_t statusLen);
int32_t mndInitAnode(SMnode *pMnode) {
SSdbTable table = {
.sdbType = SDB_ANODE,
.keyType = SDB_KEY_INT32,
.encodeFp = (SdbEncodeFp)mndAnodeActionEncode,
.decodeFp = (SdbDecodeFp)mndAnodeActionDecode,
.insertFp = (SdbInsertFp)mndAnodeActionInsert,
.updateFp = (SdbUpdateFp)mndAnodeActionUpdate,
.deleteFp = (SdbDeleteFp)mndAnodeActionDelete,
};
mndSetMsgHandle(pMnode, TDMT_MND_CREATE_ANODE, mndProcessCreateAnodeReq);
mndSetMsgHandle(pMnode, TDMT_MND_UPDATE_ANODE, mndProcessUpdateAnodeReq);
mndSetMsgHandle(pMnode, TDMT_MND_DROP_ANODE, mndProcessDropAnodeReq);
mndSetMsgHandle(pMnode, TDMT_MND_RETRIEVE_ANAL_ALGO, mndProcessAnalAlgoReq);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_ANODE, mndRetrieveAnodes);
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_ANODE, mndCancelGetNextAnode);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_ANODE_FULL, mndRetrieveAnodesFull);
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_ANODE_FULL, mndCancelGetNextAnodeFull);
return sdbSetTable(pMnode->pSdb, table);
}
void mndCleanupAnode(SMnode *pMnode) {}
SAnodeObj *mndAcquireAnode(SMnode *pMnode, int32_t anodeId) {
SAnodeObj *pObj = sdbAcquire(pMnode->pSdb, SDB_ANODE, &anodeId);
if (pObj == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) {
terrno = TSDB_CODE_MND_ANODE_NOT_EXIST;
}
return pObj;
}
void mndReleaseAnode(SMnode *pMnode, SAnodeObj *pObj) {
SSdb *pSdb = pMnode->pSdb;
sdbRelease(pSdb, pObj);
}
static SSdbRaw *mndAnodeActionEncode(SAnodeObj *pObj) {
int32_t code = 0;
int32_t lino = 0;
terrno = TSDB_CODE_OUT_OF_MEMORY;
int32_t rawDataLen = sizeof(SAnodeObj) + TSDB_ANODE_RESERVE_SIZE + pObj->urlLen;
for (int32_t t = 0; t < pObj->numOfAlgos; ++t) {
SArray *algos = pObj->algos[t];
for (int32_t a = 0; a < (int32_t)taosArrayGetSize(algos); ++a) {
SAnodeAlgo *algo = taosArrayGet(algos, a);
rawDataLen += (2 * sizeof(int32_t) + algo->nameLen);
}
rawDataLen += sizeof(int32_t);
}
SSdbRaw *pRaw = sdbAllocRaw(SDB_ANODE, TSDB_ANODE_VER_NUMBER, rawDataLen);
if (pRaw == NULL) goto _OVER;
int32_t dataPos = 0;
SDB_SET_INT32(pRaw, dataPos, pObj->id, _OVER)
SDB_SET_INT64(pRaw, dataPos, pObj->createdTime, _OVER)
SDB_SET_INT64(pRaw, dataPos, pObj->updateTime, _OVER)
SDB_SET_INT32(pRaw, dataPos, pObj->version, _OVER)
SDB_SET_INT32(pRaw, dataPos, pObj->urlLen, _OVER)
SDB_SET_BINARY(pRaw, dataPos, pObj->url, pObj->urlLen, _OVER)
SDB_SET_INT32(pRaw, dataPos, pObj->numOfAlgos, _OVER)
for (int32_t i = 0; i < pObj->numOfAlgos; ++i) {
SArray *algos = pObj->algos[i];
SDB_SET_INT32(pRaw, dataPos, (int32_t)taosArrayGetSize(algos), _OVER)
for (int32_t j = 0; j < (int32_t)taosArrayGetSize(algos); ++j) {
SAnodeAlgo *algo = taosArrayGet(algos, j);
SDB_SET_INT32(pRaw, dataPos, algo->nameLen, _OVER)
SDB_SET_BINARY(pRaw, dataPos, algo->name, algo->nameLen, _OVER)
SDB_SET_INT32(pRaw, dataPos, 0, _OVER) // reserved
}
}
SDB_SET_RESERVE(pRaw, dataPos, TSDB_ANODE_RESERVE_SIZE, _OVER)
terrno = 0;
_OVER:
if (terrno != 0) {
mError("anode:%d, failed to encode to raw:%p since %s", pObj->id, pRaw, terrstr());
sdbFreeRaw(pRaw);
return NULL;
}
mTrace("anode:%d, encode to raw:%p, row:%p", pObj->id, pRaw, pObj);
return pRaw;
}
static SSdbRow *mndAnodeActionDecode(SSdbRaw *pRaw) {
int32_t code = 0;
int32_t lino = 0;
terrno = TSDB_CODE_OUT_OF_MEMORY;
SSdbRow *pRow = NULL;
SAnodeObj *pObj = NULL;
int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
if (sver != TSDB_ANODE_VER_NUMBER) {
terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
goto _OVER;
}
pRow = sdbAllocRow(sizeof(SAnodeObj));
if (pRow == NULL) goto _OVER;
pObj = sdbGetRowObj(pRow);
if (pObj == NULL) goto _OVER;
int32_t dataPos = 0;
SDB_GET_INT32(pRaw, dataPos, &pObj->id, _OVER)
SDB_GET_INT64(pRaw, dataPos, &pObj->createdTime, _OVER)
SDB_GET_INT64(pRaw, dataPos, &pObj->updateTime, _OVER)
SDB_GET_INT32(pRaw, dataPos, &pObj->version, _OVER)
SDB_GET_INT32(pRaw, dataPos, &pObj->urlLen, _OVER)
if (pObj->urlLen > 0) {
pObj->url = taosMemoryCalloc(pObj->urlLen, 1);
if (pObj->url == NULL) goto _OVER;
SDB_GET_BINARY(pRaw, dataPos, pObj->url, pObj->urlLen, _OVER)
}
SDB_GET_INT32(pRaw, dataPos, &pObj->numOfAlgos, _OVER)
if (pObj->numOfAlgos > 0) {
pObj->algos = taosMemoryCalloc(pObj->numOfAlgos, sizeof(SArray *));
if (pObj->algos == NULL) {
goto _OVER;
}
}
for (int32_t i = 0; i < pObj->numOfAlgos; ++i) {
int32_t numOfAlgos = 0;
SDB_GET_INT32(pRaw, dataPos, &numOfAlgos, _OVER)
pObj->algos[i] = taosArrayInit(2, sizeof(SAnodeAlgo));
if (pObj->algos[i] == NULL) goto _OVER;
for (int32_t j = 0; j < numOfAlgos; ++j) {
SAnodeAlgo algoObj = {0};
int32_t reserved = 0;
SDB_GET_INT32(pRaw, dataPos, &algoObj.nameLen, _OVER)
if (algoObj.nameLen > 0) {
algoObj.name = taosMemoryCalloc(algoObj.nameLen, 1);
if (algoObj.name == NULL) goto _OVER;
}
SDB_GET_BINARY(pRaw, dataPos, algoObj.name, algoObj.nameLen, _OVER)
SDB_GET_INT32(pRaw, dataPos, &reserved, _OVER);
if (taosArrayPush(pObj->algos[i], &algoObj) == NULL) goto _OVER;
}
}
SDB_GET_RESERVE(pRaw, dataPos, TSDB_ANODE_RESERVE_SIZE, _OVER)
terrno = 0;
_OVER:
if (terrno != 0) {
mError("anode:%d, failed to decode from raw:%p since %s", pObj == NULL ? 0 : pObj->id, pRaw, terrstr());
if (pObj != NULL) {
taosMemoryFreeClear(pObj->url);
}
taosMemoryFreeClear(pRow);
return NULL;
}
mTrace("anode:%d, decode from raw:%p, row:%p", pObj->id, pRaw, pObj);
return pRow;
}
static void mndFreeAnode(SAnodeObj *pObj) {
taosMemoryFreeClear(pObj->url);
for (int32_t i = 0; i < pObj->numOfAlgos; ++i) {
SArray *algos = pObj->algos[i];
for (int32_t j = 0; j < (int32_t)taosArrayGetSize(algos); ++j) {
SAnodeAlgo *algo = taosArrayGet(algos, j);
taosMemoryFreeClear(algo->name);
}
taosArrayDestroy(algos);
}
taosMemoryFreeClear(pObj->algos);
}
static int32_t mndAnodeActionInsert(SSdb *pSdb, SAnodeObj *pObj) {
mTrace("anode:%d, perform insert action, row:%p", pObj->id, pObj);
return 0;
}
static int32_t mndAnodeActionDelete(SSdb *pSdb, SAnodeObj *pObj) {
mTrace("anode:%d, perform delete action, row:%p", pObj->id, pObj);
mndFreeAnode(pObj);
return 0;
}
static int32_t mndAnodeActionUpdate(SSdb *pSdb, SAnodeObj *pOld, SAnodeObj *pNew) {
mTrace("anode:%d, perform update action, old row:%p new row:%p", pOld->id, pOld, pNew);
taosWLockLatch(&pOld->lock);
int32_t numOfAlgos = pNew->numOfAlgos;
void *algos = pNew->algos;
pNew->numOfAlgos = pOld->numOfAlgos;
pNew->algos = pOld->algos;
pOld->numOfAlgos = numOfAlgos;
pOld->algos = algos;
pOld->updateTime = pNew->updateTime;
pOld->version = pNew->version;
taosWUnLockLatch(&pOld->lock);
return 0;
}
static int32_t mndSetCreateAnodeRedoLogs(STrans *pTrans, SAnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndAnodeActionEncode(pObj);
if (pRedoRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendRedolog(pTrans, pRedoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING));
TAOS_RETURN(code);
}
static int32_t mndSetCreateAnodeUndoLogs(STrans *pTrans, SAnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pUndoRaw = mndAnodeActionEncode(pObj);
if (pUndoRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendUndolog(pTrans, pUndoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED));
TAOS_RETURN(code);
}
static int32_t mndSetCreateAnodeCommitLogs(STrans *pTrans, SAnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndAnodeActionEncode(pObj);
if (pCommitRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY));
TAOS_RETURN(code);
}
static int32_t mndCreateAnode(SMnode *pMnode, SRpcMsg *pReq, SMCreateAnodeReq *pCreate) {
int32_t code = -1;
STrans *pTrans = NULL;
SAnodeObj anodeObj = {0};
anodeObj.id = sdbGetMaxId(pMnode->pSdb, SDB_ANODE);
anodeObj.createdTime = taosGetTimestampMs();
anodeObj.updateTime = anodeObj.createdTime;
anodeObj.version = 0;
anodeObj.urlLen = pCreate->urlLen;
if (anodeObj.urlLen > TSDB_ANAL_ANODE_URL_LEN) {
code = TSDB_CODE_MND_ANODE_TOO_LONG_URL;
goto _OVER;
}
anodeObj.url = taosMemoryCalloc(1, pCreate->urlLen);
if (anodeObj.url == NULL) goto _OVER;
(void)memcpy(anodeObj.url, pCreate->url, pCreate->urlLen);
code = mndGetAnodeAlgoList(anodeObj.url, &anodeObj);
if (code != 0) goto _OVER;
pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-anode");
if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mndTransSetSerial(pTrans);
mInfo("trans:%d, used to create anode:%s as anode:%d", pTrans->id, pCreate->url, anodeObj.id);
TAOS_CHECK_GOTO(mndSetCreateAnodeRedoLogs(pTrans, &anodeObj), NULL, _OVER);
TAOS_CHECK_GOTO(mndSetCreateAnodeUndoLogs(pTrans, &anodeObj), NULL, _OVER);
TAOS_CHECK_GOTO(mndSetCreateAnodeCommitLogs(pTrans, &anodeObj), NULL, _OVER);
TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0;
_OVER:
mndFreeAnode(&anodeObj);
mndTransDrop(pTrans);
TAOS_RETURN(code);
}
static SAnodeObj *mndAcquireAnodeByURL(SMnode *pMnode, char *url) {
SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL;
while (1) {
SAnodeObj *pAnode = NULL;
pIter = sdbFetch(pSdb, SDB_ANODE, pIter, (void **)&pAnode);
if (pIter == NULL) break;
if (strcasecmp(url, pAnode->url) == 0) {
sdbCancelFetch(pSdb, pIter);
return pAnode;
}
sdbRelease(pSdb, pAnode);
}
terrno = TSDB_CODE_MND_ANODE_NOT_EXIST;
return NULL;
}
static int32_t mndProcessCreateAnodeReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
int32_t code = -1;
SAnodeObj *pObj = NULL;
SMCreateAnodeReq createReq = {0};
TAOS_CHECK_GOTO(tDeserializeSMCreateAnodeReq(pReq->pCont, pReq->contLen, &createReq), NULL, _OVER);
mInfo("anode:%s, start to create", createReq.url);
TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_ANODE), NULL, _OVER);
pObj = mndAcquireAnodeByURL(pMnode, createReq.url);
if (pObj != NULL) {
code = TSDB_CODE_MND_ANODE_ALREADY_EXIST;
goto _OVER;
}
code = mndCreateAnode(pMnode, pReq, &createReq);
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
_OVER:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("anode:%s, failed to create since %s", createReq.url, tstrerror(code));
}
mndReleaseAnode(pMnode, pObj);
tFreeSMCreateAnodeReq(&createReq);
TAOS_RETURN(code);
}
static int32_t mndUpdateAnode(SMnode *pMnode, SAnodeObj *pAnode, SRpcMsg *pReq) {
mInfo("anode:%d, start to update", pAnode->id);
int32_t code = -1;
STrans *pTrans = NULL;
SAnodeObj anodeObj = {0};
anodeObj.id = pAnode->id;
anodeObj.updateTime = taosGetTimestampMs();
code = mndGetAnodeAlgoList(pAnode->url, &anodeObj);
if (code != 0) goto _OVER;
pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "update-anode");
if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mInfo("trans:%d, used to update anode:%d", pTrans->id, anodeObj.id);
TAOS_CHECK_GOTO(mndSetCreateAnodeCommitLogs(pTrans, &anodeObj), NULL, _OVER);
TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0;
_OVER:
mndFreeAnode(&anodeObj);
mndTransDrop(pTrans);
TAOS_RETURN(code);
}
static int32_t mndUpdateAllAnodes(SMnode *pMnode, SRpcMsg *pReq) {
mInfo("update all anodes");
SSdb *pSdb = pMnode->pSdb;
int32_t code = 0;
int32_t rows = 0;
int32_t numOfRows = sdbGetSize(pSdb, SDB_ANODE);
void *pIter = NULL;
while (1) {
SAnodeObj *pObj = NULL;
ESdbStatus objStatus = 0;
pIter = sdbFetchAll(pSdb, SDB_ANODE, pIter, (void **)&pObj, &objStatus, true);
if (pIter == NULL) break;
rows++;
void *transReq = NULL;
if (rows == numOfRows) transReq = pReq;
code = mndUpdateAnode(pMnode, pObj, transReq);
sdbRelease(pSdb, pObj);
if (code != 0) break;
}
if (code == 0 && rows == numOfRows) {
code = TSDB_CODE_ACTION_IN_PROGRESS;
}
return code;
}
static int32_t mndProcessUpdateAnodeReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
int32_t code = -1;
SAnodeObj *pObj = NULL;
SMUpdateAnodeReq updateReq = {0};
TAOS_CHECK_GOTO(tDeserializeSMUpdateAnodeReq(pReq->pCont, pReq->contLen, &updateReq), NULL, _OVER);
TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_UPDATE_ANODE), NULL, _OVER);
if (updateReq.anodeId == -1) {
code = mndUpdateAllAnodes(pMnode, pReq);
} else {
pObj = mndAcquireAnode(pMnode, updateReq.anodeId);
if (pObj == NULL) {
code = TSDB_CODE_MND_ANODE_NOT_EXIST;
goto _OVER;
}
code = mndUpdateAnode(pMnode, pObj, pReq);
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
}
_OVER:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
if (updateReq.anodeId != -1) {
mError("anode:%d, failed to update since %s", updateReq.anodeId, tstrerror(code));
}
}
mndReleaseAnode(pMnode, pObj);
tFreeSMUpdateAnodeReq(&updateReq);
TAOS_RETURN(code);
}
static int32_t mndSetDropAnodeRedoLogs(STrans *pTrans, SAnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndAnodeActionEncode(pObj);
if (pRedoRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendRedolog(pTrans, pRedoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING));
TAOS_RETURN(code);
}
static int32_t mndSetDropAnodeCommitLogs(STrans *pTrans, SAnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndAnodeActionEncode(pObj);
if (pCommitRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED));
TAOS_RETURN(code);
}
static int32_t mndSetDropAnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SAnodeObj *pObj, bool force) {
if (pObj == NULL) return 0;
TAOS_CHECK_RETURN(mndSetDropAnodeRedoLogs(pTrans, pObj));
TAOS_CHECK_RETURN(mndSetDropAnodeCommitLogs(pTrans, pObj));
return 0;
}
static int32_t mndDropAnode(SMnode *pMnode, SRpcMsg *pReq, SAnodeObj *pObj) {
int32_t code = -1;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, "drop-anode");
if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mndTransSetSerial(pTrans);
mInfo("trans:%d, used to drop anode:%d", pTrans->id, pObj->id);
TAOS_CHECK_GOTO(mndSetDropAnodeInfoToTrans(pMnode, pTrans, pObj, false), NULL, _OVER);
TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0;
_OVER:
mndTransDrop(pTrans);
TAOS_RETURN(code);
}
static int32_t mndProcessDropAnodeReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
int32_t code = -1;
SAnodeObj *pObj = NULL;
SMDropAnodeReq dropReq = {0};
TAOS_CHECK_GOTO(tDeserializeSMDropAnodeReq(pReq->pCont, pReq->contLen, &dropReq), NULL, _OVER);
mInfo("anode:%d, start to drop", dropReq.anodeId);
TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_ANODE), NULL, _OVER);
if (dropReq.anodeId <= 0) {
code = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
pObj = mndAcquireAnode(pMnode, dropReq.anodeId);
if (pObj == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
code = mndDropAnode(pMnode, pReq, pObj);
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
_OVER:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("anode:%d, failed to drop since %s", dropReq.anodeId, tstrerror(code));
}
mndReleaseAnode(pMnode, pObj);
tFreeSMDropAnodeReq(&dropReq);
TAOS_RETURN(code);
}
static int32_t mndRetrieveAnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb;
int32_t numOfRows = 0;
int32_t cols = 0;
SAnodeObj *pObj = NULL;
char buf[TSDB_ANAL_ANODE_URL_LEN + VARSTR_HEADER_SIZE];
char status[64];
int32_t code = 0;
while (numOfRows < rows) {
pShow->pIter = sdbFetch(pSdb, SDB_ANODE, pShow->pIter, (void **)&pObj);
if (pShow->pIter == NULL) break;
cols = 0;
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
code = colDataSetVal(pColInfo, numOfRows, (const char *)&pObj->id, false);
if (code != 0) goto _end;
STR_WITH_MAXSIZE_TO_VARSTR(buf, pObj->url, pShow->pMeta->pSchemas[cols].bytes);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
code = colDataSetVal(pColInfo, numOfRows, (const char *)buf, false);
if (code != 0) goto _end;
status[0] = 0;
if (mndGetAnodeStatus(pObj, status, 64) == 0) {
STR_TO_VARSTR(buf, status);
} else {
STR_TO_VARSTR(buf, "offline");
}
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
code = colDataSetVal(pColInfo, numOfRows, buf, false);
if (code != 0) goto _end;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
code = colDataSetVal(pColInfo, numOfRows, (const char *)&pObj->createdTime, false);
if (code != 0) goto _end;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
code = colDataSetVal(pColInfo, numOfRows, (const char *)&pObj->updateTime, false);
if (code != 0) goto _end;
numOfRows++;
sdbRelease(pSdb, pObj);
}
_end:
if (code != 0) sdbRelease(pSdb, pObj);
pShow->numOfRows += numOfRows;
return numOfRows;
}
static void mndCancelGetNextAnode(SMnode *pMnode, void *pIter) {
SSdb *pSdb = pMnode->pSdb;
sdbCancelFetchByType(pSdb, pIter, SDB_ANODE);
}
static int32_t mndRetrieveAnodesFull(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb;
int32_t numOfRows = 0;
int32_t cols = 0;
SAnodeObj *pObj = NULL;
char buf[TSDB_ANAL_ALGO_NAME_LEN + VARSTR_HEADER_SIZE];
int32_t code = 0;
while (numOfRows < rows) {
pShow->pIter = sdbFetch(pSdb, SDB_ANODE, pShow->pIter, (void **)&pObj);
if (pShow->pIter == NULL) break;
for (int32_t t = 0; t < pObj->numOfAlgos; ++t) {
SArray *algos = pObj->algos[t];
for (int32_t a = 0; a < taosArrayGetSize(algos); ++a) {
SAnodeAlgo *algo = taosArrayGet(algos, a);
cols = 0;
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
code = colDataSetVal(pColInfo, numOfRows, (const char *)&pObj->id, false);
if (code != 0) goto _end;
STR_TO_VARSTR(buf, taosAnalAlgoStr(t));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
code = colDataSetVal(pColInfo, numOfRows, buf, false);
if (code != 0) goto _end;
STR_TO_VARSTR(buf, algo->name);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
code = colDataSetVal(pColInfo, numOfRows, buf, false);
if (code != 0) goto _end;
numOfRows++;
}
}
sdbRelease(pSdb, pObj);
}
_end:
if (code != 0) sdbRelease(pSdb, pObj);
pShow->numOfRows += numOfRows;
return numOfRows;
}
static void mndCancelGetNextAnodeFull(SMnode *pMnode, void *pIter) {
SSdb *pSdb = pMnode->pSdb;
sdbCancelFetchByType(pSdb, pIter, SDB_ANODE);
}
static int32_t mndDecodeAlgoList(SJson *pJson, SAnodeObj *pObj) {
int32_t code = 0;
int32_t protocol = 0;
double tmp = 0;
char buf[TSDB_ANAL_ALGO_NAME_LEN + 1] = {0};
code = tjsonGetDoubleValue(pJson, "protocol", &tmp);
if (code < 0) return TSDB_CODE_INVALID_JSON_FORMAT;
protocol = (int32_t)(tmp * 1000);
if (protocol != 100 && protocol != 1000) return TSDB_CODE_MND_ANODE_INVALID_PROTOCOL;
code = tjsonGetDoubleValue(pJson, "version", &tmp);
pObj->version = (int32_t)(tmp * 1000);
#if 0
if (code < 0) return TSDB_CODE_INVALID_JSON_FORMAT;
if (pObj->version <= 0) return TSDB_CODE_MND_ANODE_INVALID_VERSION;
#endif
SJson *details = tjsonGetObjectItem(pJson, "details");
if (details == NULL) return TSDB_CODE_INVALID_JSON_FORMAT;
int32_t numOfDetails = tjsonGetArraySize(details);
pObj->algos = taosMemoryCalloc(ANAL_ALGO_TYPE_END, sizeof(SArray *));
if (pObj->algos == NULL) return TSDB_CODE_OUT_OF_MEMORY;
pObj->numOfAlgos = ANAL_ALGO_TYPE_END;
for (int32_t i = 0; i < ANAL_ALGO_TYPE_END; ++i) {
pObj->algos[i] = taosArrayInit(4, sizeof(SAnodeAlgo));
if (pObj->algos[i] == NULL) return TSDB_CODE_OUT_OF_MEMORY;
}
for (int32_t d = 0; d < numOfDetails; ++d) {
SJson *detail = tjsonGetArrayItem(details, d);
if (detail == NULL) return TSDB_CODE_INVALID_JSON_FORMAT;
code = tjsonGetStringValue2(detail, "type", buf, sizeof(buf));
if (code < 0) return TSDB_CODE_INVALID_JSON_FORMAT;
EAnalAlgoType type = taosAnalAlgoInt(buf);
if (type < 0 || type >= ANAL_ALGO_TYPE_END) return TSDB_CODE_MND_ANODE_INVALID_ALGO_TYPE;
SJson *algos = tjsonGetObjectItem(detail, "algo");
if (algos == NULL) return TSDB_CODE_INVALID_JSON_FORMAT;
int32_t numOfAlgos = tjsonGetArraySize(algos);
for (int32_t a = 0; a < numOfAlgos; ++a) {
SJson *algo = tjsonGetArrayItem(algos, a);
if (algo == NULL) return TSDB_CODE_INVALID_JSON_FORMAT;
code = tjsonGetStringValue2(algo, "name", buf, sizeof(buf));
if (code < 0) return TSDB_CODE_MND_ANODE_TOO_LONG_ALGO_NAME;
SAnodeAlgo algoObj = {0};
algoObj.nameLen = strlen(buf) + 1;
if (algoObj.nameLen <= 1) return TSDB_CODE_INVALID_JSON_FORMAT;
algoObj.name = taosMemoryCalloc(algoObj.nameLen, 1);
tstrncpy(algoObj.name, buf, algoObj.nameLen);
if (taosArrayPush(pObj->algos[type], &algoObj) == NULL) return TSDB_CODE_OUT_OF_MEMORY;
}
}
return 0;
}
static int32_t mndGetAnodeAlgoList(const char *url, SAnodeObj *pObj) {
char anodeUrl[TSDB_ANAL_ANODE_URL_LEN + 1] = {0};
snprintf(anodeUrl, TSDB_ANAL_ANODE_URL_LEN, "%s/%s", url, "list");
SJson *pJson = taosAnalSendReqRetJson(anodeUrl, ANAL_HTTP_TYPE_GET, NULL);
if (pJson == NULL) return terrno;
int32_t code = mndDecodeAlgoList(pJson, pObj);
if (pJson != NULL) tjsonDelete(pJson);
TAOS_RETURN(code);
}
static int32_t mndGetAnodeStatus(SAnodeObj *pObj, char *status, int32_t statusLen) {
int32_t code = 0;
int32_t protocol = 0;
double tmp = 0;
char anodeUrl[TSDB_ANAL_ANODE_URL_LEN + 1] = {0};
snprintf(anodeUrl, TSDB_ANAL_ANODE_URL_LEN, "%s/%s", pObj->url, "status");
SJson *pJson = taosAnalSendReqRetJson(anodeUrl, ANAL_HTTP_TYPE_GET, NULL);
if (pJson == NULL) return terrno;
code = tjsonGetDoubleValue(pJson, "protocol", &tmp);
if (code < 0) {
code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER;
}
protocol = (int32_t)(tmp * 1000);
if (protocol != 100 && protocol != 1000) {
code = TSDB_CODE_MND_ANODE_INVALID_PROTOCOL;
goto _OVER;
}
code = tjsonGetStringValue2(pJson, "status", status, statusLen);
if (code < 0) {
code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER;
}
if (strlen(status) == 0) {
code = TSDB_CODE_MND_ANODE_INVALID_PROTOCOL;
goto _OVER;
}
_OVER:
if (pJson != NULL) tjsonDelete(pJson);
TAOS_RETURN(code);
}
static int32_t mndProcessAnalAlgoReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb;
int32_t code = -1;
SAnodeObj *pObj = NULL;
SAnalUrl url;
int32_t nameLen;
char name[TSDB_ANAL_ALGO_KEY_LEN];
SRetrieveAnalAlgoReq req = {0};
SRetrieveAnalAlgoRsp rsp = {0};
TAOS_CHECK_GOTO(tDeserializeRetrieveAnalAlgoReq(pReq->pCont, pReq->contLen, &req), NULL, _OVER);
rsp.ver = sdbGetTableVer(pSdb, SDB_ANODE);
if (req.analVer != rsp.ver) {
mInfo("dnode:%d, update analysis old ver:%" PRId64 " to new ver:%" PRId64, req.dnodeId, req.analVer, rsp.ver);
rsp.hash = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK);
if (rsp.hash == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
void *pIter = NULL;
while (1) {
SAnodeObj *pAnode = NULL;
pIter = sdbFetch(pSdb, SDB_ANODE, pIter, (void **)&pAnode);
if (pIter == NULL) break;
url.anode = pAnode->id;
for (int32_t t = 0; t < pAnode->numOfAlgos; ++t) {
SArray *algos = pAnode->algos[t];
url.type = t;
for (int32_t a = 0; a < taosArrayGetSize(algos); ++a) {
SAnodeAlgo *algo = taosArrayGet(algos, a);
nameLen = 1 + tsnprintf(name, sizeof(name) - 1, "%d:%s", url.type, algo->name);
SAnalUrl *pOldUrl = taosHashAcquire(rsp.hash, name, nameLen);
if (pOldUrl == NULL || (pOldUrl != NULL && pOldUrl->anode < url.anode)) {
if (pOldUrl != NULL) {
taosMemoryFreeClear(pOldUrl->url);
if (taosHashRemove(rsp.hash, name, nameLen) != 0) {
sdbRelease(pSdb, pAnode);
goto _OVER;
}
}
url.url = taosMemoryMalloc(TSDB_ANAL_ANODE_URL_LEN + TSDB_ANAL_ALGO_TYPE_LEN + 1);
if (url.url == NULL) {
sdbRelease(pSdb, pAnode);
goto _OVER;
}
url.urlLen = 1 + tsnprintf(url.url, TSDB_ANAL_ANODE_URL_LEN + TSDB_ANAL_ALGO_TYPE_LEN, "%s/%s", pAnode->url,
taosAnalAlgoUrlStr(url.type));
if (taosHashPut(rsp.hash, name, nameLen, &url, sizeof(SAnalUrl)) != 0) {
taosMemoryFree(url.url);
sdbRelease(pSdb, pAnode);
goto _OVER;
}
}
}
sdbRelease(pSdb, pAnode);
}
}
}
int32_t contLen = tSerializeRetrieveAnalAlgoRsp(NULL, 0, &rsp);
void *pHead = rpcMallocCont(contLen);
(void)tSerializeRetrieveAnalAlgoRsp(pHead, contLen, &rsp);
pReq->info.rspLen = contLen;
pReq->info.rsp = pHead;
_OVER:
tFreeRetrieveAnalAlgoRsp(&rsp);
TAOS_RETURN(code);
}
#else
static int32_t mndProcessUnsupportReq(SRpcMsg *pReq) { return TSDB_CODE_OPS_NOT_SUPPORT; }
static int32_t mndRetrieveUnsupport(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
return TSDB_CODE_OPS_NOT_SUPPORT;
}
int32_t mndInitAnode(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_MND_CREATE_ANODE, mndProcessUnsupportReq);
mndSetMsgHandle(pMnode, TDMT_MND_UPDATE_ANODE, mndProcessUnsupportReq);
mndSetMsgHandle(pMnode, TDMT_MND_DROP_ANODE, mndProcessUnsupportReq);
mndSetMsgHandle(pMnode, TDMT_MND_RETRIEVE_ANAL_ALGO, mndProcessUnsupportReq);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_ANODE, mndRetrieveUnsupport);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_ANODE_FULL, mndRetrieveUnsupport);
return 0;
}
void mndCleanupAnode(SMnode *pMnode) {}
#endif

5
source/dnode/mnode/impl/src/mndDnode.c
View File

@ -730,6 +730,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
pMnode->ipWhiteVer = mndGetIpWhiteVer(pMnode);
int64_t analVer = sdbGetTableVer(pMnode->pSdb, SDB_ANODE);
int64_t dnodeVer = sdbGetTableVer(pMnode->pSdb, SDB_DNODE) + sdbGetTableVer(pMnode->pSdb, SDB_MNODE);
int64_t curMs = taosGetTimestampMs();
bool online = mndIsDnodeOnline(pDnode, curMs);
@ -738,7 +739,8 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
bool supportVnodesChanged = pDnode->numOfSupportVnodes != statusReq.numOfSupportVnodes;
bool encryptKeyChanged = pDnode->encryptionKeyChksum != statusReq.clusterCfg.encryptionKeyChksum;
bool enableWhiteListChanged = statusReq.clusterCfg.enableWhiteList != (tsEnableWhiteList ? 1 : 0);
bool needCheck = !online || dnodeChanged || reboot || supportVnodesChanged ||
bool analVerChanged = (analVer != statusReq.analVer);
bool needCheck = !online || dnodeChanged || reboot || supportVnodesChanged || analVerChanged ||
pMnode->ipWhiteVer != statusReq.ipWhiteVer || encryptKeyChanged || enableWhiteListChanged;
const STraceId *trace = &pReq->info.traceId;
mGTrace("dnode:%d, status received, accessTimes:%d check:%d online:%d reboot:%d changed:%d statusSeq:%d", pDnode->id,
@ -862,6 +864,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
SStatusRsp statusRsp = {0};
statusRsp.statusSeq++;
statusRsp.analVer = analVer;
statusRsp.dnodeVer = dnodeVer;
statusRsp.dnodeCfg.dnodeId = pDnode->id;
statusRsp.dnodeCfg.clusterId = pMnode->clusterId;

4
source/dnode/mnode/impl/src/mndMain.c
View File

@ -16,6 +16,7 @@
#define _DEFAULT_SOURCE
#include "mndAcct.h"
#include "mndArbGroup.h"
#include "mndAnode.h"
#include "mndCluster.h"
#include "mndCompact.h"
#include "mndCompactDetail.h"
@ -236,7 +237,7 @@ static void mndPullupGrant(SMnode *pMnode) {
.pCont = pReq,
.contLen = contLen,
.info.notFreeAhandle = 1,
.info.ahandle = (void *)0x9527};
.info.ahandle = 0};
// TODO check return value
if (tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg) < 0) {
mError("failed to put into write-queue since %s, line:%d", terrstr(), __LINE__);
@ -608,6 +609,7 @@ static int32_t mndInitSteps(SMnode *pMnode) {
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-mnode", mndInitMnode, mndCleanupMnode));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-qnode", mndInitQnode, mndCleanupQnode));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-snode", mndInitSnode, mndCleanupSnode));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-anode", mndInitAnode, mndCleanupAnode));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-arbgroup", mndInitArbGroup, mndCleanupArbGroup));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-dnode", mndInitDnode, mndCleanupDnode));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-user", mndInitUser, mndCleanupUser));

31
source/dnode/mnode/impl/src/mndQuery.c
View File

@ -30,9 +30,9 @@ void mndPostProcessQueryMsg(SRpcMsg *pMsg) {
(void)qWorkerAbortPreprocessQueryMsg(pMnode->pQuery, pMsg);
}
int32_t mndProcessQueryMsg(SRpcMsg *pMsg, SQueueInfo* pInfo) {
int32_t code = -1;
SMnode *pMnode = pMsg->info.node;
int32_t mndProcessQueryMsg(SRpcMsg *pMsg, SQueueInfo *pInfo) {
int32_t code = -1;
SMnode *pMnode = pMsg->info.node;
SReadHandle handle = {.mnd = pMnode, .pMsgCb = &pMnode->msgCb, .pWorkerCb = pInfo->workerCb};
@ -67,26 +67,25 @@ int32_t mndProcessQueryMsg(SRpcMsg *pMsg, SQueueInfo* pInfo) {
return code;
}
static FORCE_INLINE void mnodeFreeSBatchRspMsg(void* p) {
static FORCE_INLINE void mnodeFreeSBatchRspMsg(void *p) {
if (NULL == p) {
return;
}
SBatchRspMsg* pRsp = (SBatchRspMsg*)p;
SBatchRspMsg *pRsp = (SBatchRspMsg *)p;
rpcFreeCont(pRsp->msg);
}
int32_t mndProcessBatchMetaMsg(SRpcMsg *pMsg) {
int32_t code = 0;
int32_t rspSize = 0;
SBatchReq batchReq = {0};
SBatchMsg req = {0};
int32_t code = 0;
int32_t rspSize = 0;
SBatchReq batchReq = {0};
SBatchMsg req = {0};
SBatchRspMsg rsp = {0};
SBatchRsp batchRsp = {0};
SRpcMsg reqMsg = *pMsg;
void *pRsp = NULL;
SMnode *pMnode = pMsg->info.node;
SBatchRsp batchRsp = {0};
SRpcMsg reqMsg = *pMsg;
void *pRsp = NULL;
SMnode *pMnode = pMsg->info.node;
if ((code = tDeserializeSBatchReq(pMsg->pCont, pMsg->contLen, &batchReq)) != 0) {
code = TSDB_CODE_OUT_OF_MEMORY;
@ -94,7 +93,7 @@ int32_t mndProcessBatchMetaMsg(SRpcMsg *pMsg) {
goto _exit;
}
int32_t msgNum = taosArrayGetSize(batchReq.pMsgs);
int32_t msgNum = taosArrayGetSize(batchReq.pMsgs);
if (msgNum >= MAX_META_MSG_IN_BATCH) {
code = TSDB_CODE_INVALID_MSG;
mError("too many msgs %d in mnode batch meta req", msgNum);
@ -108,7 +107,7 @@ int32_t mndProcessBatchMetaMsg(SRpcMsg *pMsg) {
}
for (int32_t i = 0; i < msgNum; ++i) {
SBatchMsg* req = taosArrayGet(batchReq.pMsgs, i);
SBatchMsg *req = taosArrayGet(batchReq.pMsgs, i);
reqMsg.msgType = req->msgType;
reqMsg.pCont = req->msg;

4
source/dnode/mnode/impl/src/mndShow.c
View File

@ -68,6 +68,10 @@ static int32_t convertToRetrieveType(char *name, int32_t len) {
type = TSDB_MGMT_TABLE_QNODE;
} else if (strncasecmp(name, TSDB_INS_TABLE_SNODES, len) == 0) {
type = TSDB_MGMT_TABLE_SNODE;
} else if (strncasecmp(name, TSDB_INS_TABLE_ANODES, len) == 0) {
type = TSDB_MGMT_TABLE_ANODE;
} else if (strncasecmp(name, TSDB_INS_TABLE_ANODES_FULL, len) == 0) {
type = TSDB_MGMT_TABLE_ANODE_FULL;
} else if (strncasecmp(name, TSDB_INS_TABLE_ARBGROUPS, len) == 0) {
type = TSDB_MGMT_TABLE_ARBGROUP;
} else if (strncasecmp(name, TSDB_INS_TABLE_CLUSTER, len) == 0) {

2
source/dnode/mnode/impl/src/mndSma.c
View File

@ -2252,7 +2252,7 @@ static int32_t mndRetrieveTSMA(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlo
if (nodeType(pFunc) == QUERY_NODE_FUNCTION) {
SFunctionNode *pFuncNode = (SFunctionNode *)pFunc;
if (!fmIsTSMASupportedFunc(pFuncNode->funcId)) continue;
len += snprintf(start, TSDB_MAX_SAVED_SQL_LEN - len, "%s%s", start != buf + VARSTR_HEADER_SIZE ? "," : "",
len += tsnprintf(start, TSDB_MAX_SAVED_SQL_LEN - len, "%s%s", start != buf + VARSTR_HEADER_SIZE ? "," : "",
((SExprNode *)pFunc)->userAlias);
if (len >= TSDB_MAX_SAVED_SQL_LEN) {
len = TSDB_MAX_SAVED_SQL_LEN;

27
source/dnode/mnode/impl/src/mndTrans.c
View File

@ -474,6 +474,21 @@ static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans) {
// pTrans->startFunc = 0;
}
if (pTrans->stage == TRN_STAGE_COMMIT) {
pTrans->stage = TRN_STAGE_COMMIT_ACTION;
mInfo("trans:%d, stage from commit to commitAction since perform update action", pTrans->id);
}
if (pTrans->stage == TRN_STAGE_ROLLBACK) {
pTrans->stage = TRN_STAGE_UNDO_ACTION;
mInfo("trans:%d, stage from rollback to undoAction since perform update action", pTrans->id);
}
if (pTrans->stage == TRN_STAGE_PRE_FINISH) {
pTrans->stage = TRN_STAGE_FINISH;
mInfo("trans:%d, stage from pre-finish to finished since perform update action", pTrans->id);
}
return 0;
}
@ -563,17 +578,17 @@ static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pOld, STrans *pNew) {
if (pOld->stage == TRN_STAGE_COMMIT) {
pOld->stage = TRN_STAGE_COMMIT_ACTION;
mTrace("trans:%d, stage from commit to commitAction since perform update action", pNew->id);
mInfo("trans:%d, stage from commit to commitAction since perform update action", pNew->id);
}
if (pOld->stage == TRN_STAGE_ROLLBACK) {
pOld->stage = TRN_STAGE_UNDO_ACTION;
mTrace("trans:%d, stage from rollback to undoAction since perform update action", pNew->id);
mInfo("trans:%d, stage from rollback to undoAction since perform update action", pNew->id);
}
if (pOld->stage == TRN_STAGE_PRE_FINISH) {
pOld->stage = TRN_STAGE_FINISH;
mTrace("trans:%d, stage from pre-finish to finished since perform update action", pNew->id);
mInfo("trans:%d, stage from pre-finish to finished since perform update action", pNew->id);
}
return 0;
@ -1295,7 +1310,7 @@ static void mndTransResetActions(SMnode *pMnode, STrans *pTrans, SArray *pArray)
}
}
// execute at bottom half
// execute in sync context
static int32_t mndTransWriteSingleLog(SMnode *pMnode, STrans *pTrans, STransAction *pAction, bool topHalf) {
if (pAction->rawWritten) return 0;
if (topHalf) {
@ -1321,7 +1336,7 @@ static int32_t mndTransWriteSingleLog(SMnode *pMnode, STrans *pTrans, STransActi
TAOS_RETURN(code);
}
// execute at top half
// execute in trans context
static int32_t mndTransSendSingleMsg(SMnode *pMnode, STrans *pTrans, STransAction *pAction, bool topHalf) {
if (pAction->msgSent) return 0;
if (mndCannotExecuteTransAction(pMnode, topHalf)) {
@ -1701,6 +1716,7 @@ static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans, bool
return continueExec;
}
// in trans context
static bool mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans, bool topHalf) {
if (mndCannotExecuteTransAction(pMnode, topHalf)) return false;
@ -1775,6 +1791,7 @@ static bool mndTransPerformUndoActionStage(SMnode *pMnode, STrans *pTrans, bool
return continueExec;
}
// in trans context
static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans, bool topHalf) {
if (mndCannotExecuteTransAction(pMnode, topHalf)) return false;

3
source/dnode/mnode/sdb/inc/sdb.h
View File

@ -161,7 +161,8 @@ typedef enum {
SDB_COMPACT_DETAIL = 25,
SDB_GRANT = 26, // grant log
SDB_ARBGROUP = 27,
SDB_MAX = 28
SDB_ANODE = 28,
SDB_MAX = 29
} ESdbType;
typedef struct SSdbRaw {

57
source/dnode/mnode/sdb/src/sdbFile.c
View File

@ -25,6 +25,9 @@
#define SDB_RESERVE_SIZE 512
#define SDB_FILE_VER 1
#define SDB_TABLE_SIZE_EXTRA SDB_MAX
#define SDB_RESERVE_SIZE_EXTRA (512 - (SDB_TABLE_SIZE_EXTRA - SDB_TABLE_SIZE) * 2 * sizeof(int64_t))
static int32_t sdbDeployData(SSdb *pSdb) {
int32_t code = 0;
mInfo("start to deploy sdb");
@ -154,7 +157,38 @@ static int32_t sdbReadFileHead(SSdb *pSdb, TdFilePtr pFile) {
}
}
char reserve[SDB_RESERVE_SIZE] = {0};
// for sdb compatibility
for (int32_t i = SDB_TABLE_SIZE; i < SDB_TABLE_SIZE_EXTRA; ++i) {
int64_t maxId = 0;
ret = taosReadFile(pFile, &maxId, sizeof(int64_t));
if (ret < 0) {
code = TAOS_SYSTEM_ERROR(errno);
TAOS_RETURN(code);
}
if (ret != sizeof(int64_t)) {
code = TSDB_CODE_FILE_CORRUPTED;
TAOS_RETURN(code);
}
if (i < SDB_MAX) {
pSdb->maxId[i] = maxId;
}
int64_t ver = 0;
ret = taosReadFile(pFile, &ver, sizeof(int64_t));
if (ret < 0) {
code = TAOS_SYSTEM_ERROR(errno);
TAOS_RETURN(code);
}
if (ret != sizeof(int64_t)) {
code = TSDB_CODE_FILE_CORRUPTED;
TAOS_RETURN(code);
}
if (i < SDB_MAX) {
pSdb->tableVer[i] = ver;
}
}
char reserve[SDB_RESERVE_SIZE_EXTRA] = {0};
ret = taosReadFile(pFile, reserve, sizeof(reserve));
if (ret < 0) {
return terrno;
@ -207,7 +241,26 @@ static int32_t sdbWriteFileHead(SSdb *pSdb, TdFilePtr pFile) {
}
}
char reserve[SDB_RESERVE_SIZE] = {0};
// for sdb compatibility
for (int32_t i = SDB_TABLE_SIZE; i < SDB_TABLE_SIZE_EXTRA; ++i) {
int64_t maxId = 0;
if (i < SDB_MAX) {
maxId = pSdb->maxId[i];
}
if (taosWriteFile(pFile, &maxId, sizeof(int64_t)) != sizeof(int64_t)) {
return terrno;
}
int64_t ver = 0;
if (i < SDB_MAX) {
ver = pSdb->tableVer[i];
}
if (taosWriteFile(pFile, &ver, sizeof(int64_t)) != sizeof(int64_t)) {
return terrno;
}
}
char reserve[SDB_RESERVE_SIZE_EXTRA] = {0};
if (taosWriteFile(pFile, reserve, sizeof(reserve)) != sizeof(reserve)) {
return terrno;
}

2
source/dnode/mnode/sdb/src/sdbHash.c
View File

@ -74,6 +74,8 @@ const char *sdbTableName(ESdbType type) {
return "grant";
case SDB_ARBGROUP:
return "arb_group";
case SDB_ANODE:
return "anode";
default:
return "undefine";
}

10
source/libs/command/inc/commandInt.h
View File

@ -188,17 +188,17 @@ do { \
#define EXPLAIN_ROW_NEW(level, ...) \
do { \
if (isVerboseLine) { \
tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE - VARSTR_HEADER_SIZE, "%*s", (level) * 3 + 3, ""); \
tlen = tsnprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE - VARSTR_HEADER_SIZE, "%*s", (level) * 3 + 3, ""); \
} else { \
tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE - VARSTR_HEADER_SIZE, "%*s%s", (level) * 3, "", "-> "); \
tlen = tsnprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE - VARSTR_HEADER_SIZE, "%*s%s", (level) * 3, "", "-> "); \
} \
tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - VARSTR_HEADER_SIZE - tlen, __VA_ARGS__); \
tlen += tsnprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - VARSTR_HEADER_SIZE - tlen, __VA_ARGS__); \
} while (0)
#define EXPLAIN_ROW_APPEND(...) tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - VARSTR_HEADER_SIZE - tlen, __VA_ARGS__)
#define EXPLAIN_ROW_APPEND(...) tlen += tsnprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - VARSTR_HEADER_SIZE - tlen, __VA_ARGS__)
#define EXPLAIN_ROW_END() do { varDataSetLen(tbuf, tlen); tlen += VARSTR_HEADER_SIZE; isVerboseLine = true; } while (0)
#define EXPLAIN_SUM_ROW_NEW(...) tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE - VARSTR_HEADER_SIZE, __VA_ARGS__)
#define EXPLAIN_SUM_ROW_NEW(...) tlen = tsnprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE - VARSTR_HEADER_SIZE, __VA_ARGS__)
#define EXPLAIN_SUM_ROW_END() do { varDataSetLen(tbuf, tlen); tlen += VARSTR_HEADER_SIZE; } while (0)
#define EXPLAIN_ROW_APPEND_LIMIT_IMPL(_pLimit, sl) do { \

2
source/libs/command/src/command.c
View File

@ -515,7 +515,7 @@ void appendColumnFields(char* buf, int32_t* len, STableCfg* pCfg) {
TSDB_DATA_TYPE_GEOMETRY == pSchema->type) {
typeLen += tsnprintf(type + typeLen, LTYPE_LEN - typeLen, "(%d)", (int32_t)(pSchema->bytes - VARSTR_HEADER_SIZE));
} else if (TSDB_DATA_TYPE_NCHAR == pSchema->type) {
typeLen += snprintf(type + typeLen, LTYPE_LEN - typeLen, "(%d)",
typeLen += tsnprintf(type + typeLen, LTYPE_LEN - typeLen, "(%d)",
(int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE));
}

4
source/libs/executor/CMakeLists.txt
View File

@ -5,6 +5,10 @@ if(${TD_DARWIN})
target_compile_options(executor PRIVATE -Wno-error=deprecated-non-prototype)
endif(${TD_DARWIN})
IF(${BUILD_WITH_ANALYSIS})
add_definitions(-DUSE_ANAL)
ENDIF()
target_link_libraries(executor
PRIVATE os util common function parser planner qcom scalar nodes index wal tdb geometry
)

4
source/libs/executor/inc/executil.h
View File

@ -48,6 +48,7 @@ typedef struct SGroupResInfo {
} SGroupResInfo;
typedef struct SResultRow {
int32_t version;
int32_t pageId; // pageId & rowId is the position of current result in disk-based output buffer
int32_t offset : 29; // row index in buffer page
bool startInterp; // the time window start timestamp has done the interpolation already.
@ -152,6 +153,9 @@ static FORCE_INLINE SResultRow* getResultRowByPos(SDiskbasedBuf* pBuf, SResultRo
return pRow;
}
int32_t getResultRowFromBuf(struct SExprSupp *pSup, const char* inBuf, size_t inBufSize, char **outBuf, size_t *outBufSize);
int32_t putResultRowToBuf(struct SExprSupp *pSup, const char* inBuf, size_t inBufSize, char **outBuf, size_t *outBufSize);
int32_t initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, int32_t order);
void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo);

4
source/libs/executor/inc/operator.h
View File

@ -133,6 +133,8 @@ int32_t createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStreamPart
int32_t createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, SExecTaskInfo* pTaskInfo, SOperatorInfo** pInfo);
int32_t createForecastOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, SExecTaskInfo* pTaskInfo, SOperatorInfo** pInfo);
int32_t createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SSortMergeJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo, SOperatorInfo** pInfo);
int32_t createHashJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SHashJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo, SOperatorInfo** pInfo);
@ -159,6 +161,8 @@ int32_t createCountwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* phy
int32_t createGroupCacheOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SGroupCachePhysiNode* pPhyciNode, SExecTaskInfo* pTaskInfo, SOperatorInfo** pInfo);
int32_t createAnomalywindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* physiNode, SExecTaskInfo* pTaskInfo, SOperatorInfo** pInfo);
int32_t createDynQueryCtrlOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SDynQueryCtrlPhysiNode* pPhyciNode, SExecTaskInfo* pTaskInfo, SOperatorInfo** pInfo);
// clang-format on

636
source/libs/executor/src/anomalywindowoperator.c Normal file
View File

@ -0,0 +1,636 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "executorInt.h"
#include "filter.h"
#include "function.h"
#include "functionMgt.h"
#include "operator.h"
#include "querytask.h"
#include "tanal.h"
#include "tcommon.h"
#include "tcompare.h"
#include "tdatablock.h"
#include "tjson.h"
#include "ttime.h"
#ifdef USE_ANAL
typedef struct {
SArray* blocks; // SSDataBlock*
SArray* windows; // STimeWindow
uint64_t groupId;
int64_t cachedRows;
int32_t curWinIndex;
STimeWindow curWin;
SResultRow* pResultRow;
} SAnomalyWindowSupp;
typedef struct {
SOptrBasicInfo binfo;
SAggSupporter aggSup;
SExprSupp scalarSup;
int32_t tsSlotId;
STimeWindowAggSupp twAggSup;
char algoName[TSDB_ANAL_ALGO_NAME_LEN];
char algoUrl[TSDB_ANAL_ALGO_URL_LEN];
char anomalyOpt[TSDB_ANAL_ALGO_OPTION_LEN];
SAnomalyWindowSupp anomalySup;
SWindowRowsSup anomalyWinRowSup;
SColumn anomalyCol;
SStateKeys anomalyKey;
} SAnomalyWindowOperatorInfo;
static void anomalyDestroyOperatorInfo(void* param);
static int32_t anomalyAggregateNext(SOperatorInfo* pOperator, SSDataBlock** ppRes);
static void anomalyAggregateBlocks(SOperatorInfo* pOperator);
static int32_t anomalyCacheBlock(SAnomalyWindowOperatorInfo* pInfo, SSDataBlock* pBlock);
int32_t createAnomalywindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* physiNode, SExecTaskInfo* pTaskInfo,
SOperatorInfo** pOptrInfo) {
QRY_PARAM_CHECK(pOptrInfo);
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SAnomalyWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SAnomalyWindowOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
SAnomalyWindowPhysiNode* pAnomalyNode = (SAnomalyWindowPhysiNode*)physiNode;
SColumnNode* pColNode = (SColumnNode*)(pAnomalyNode->pAnomalyKey);
if (pInfo == NULL || pOperator == NULL) {
code = terrno;
goto _error;
}
if (!taosAnalGetOptStr(pAnomalyNode->anomalyOpt, "algo", pInfo->algoName, sizeof(pInfo->algoName))) {
qError("failed to get anomaly_window algorithm name from %s", pAnomalyNode->anomalyOpt);
code = TSDB_CODE_ANAL_ALGO_NOT_FOUND;
goto _error;
}
if (taosAnalGetAlgoUrl(pInfo->algoName, ANAL_ALGO_TYPE_ANOMALY_DETECT, pInfo->algoUrl, sizeof(pInfo->algoUrl)) != 0) {
qError("failed to get anomaly_window algorithm url from %s", pInfo->algoName);
code = TSDB_CODE_ANAL_ALGO_NOT_LOAD;
goto _error;
}
pOperator->exprSupp.hasWindowOrGroup = true;
pInfo->tsSlotId = ((SColumnNode*)pAnomalyNode->window.pTspk)->slotId;
strncpy(pInfo->anomalyOpt, pAnomalyNode->anomalyOpt, sizeof(pInfo->anomalyOpt));
if (pAnomalyNode->window.pExprs != NULL) {
int32_t numOfScalarExpr = 0;
SExprInfo* pScalarExprInfo = NULL;
code = createExprInfo(pAnomalyNode->window.pExprs, NULL, &pScalarExprInfo, &numOfScalarExpr);
QUERY_CHECK_CODE(code, lino, _error);
code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr, &pTaskInfo->storageAPI.functionStore);
QUERY_CHECK_CODE(code, lino, _error);
}
size_t keyBufSize = 0;
int32_t num = 0;
SExprInfo* pExprInfo = NULL;
code = createExprInfo(pAnomalyNode->window.pFuncs, NULL, &pExprInfo, &num);
QUERY_CHECK_CODE(code, lino, _error);
initResultSizeInfo(&pOperator->resultInfo, 4096);
code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str,
pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore);
QUERY_CHECK_CODE(code, lino, _error);
SSDataBlock* pResBlock = createDataBlockFromDescNode(pAnomalyNode->window.node.pOutputDataBlockDesc);
QUERY_CHECK_NULL(pResBlock, code, lino, _error, terrno);
initBasicInfo(&pInfo->binfo, pResBlock);
code = blockDataEnsureCapacity(pResBlock, pOperator->resultInfo.capacity);
QUERY_CHECK_CODE(code, lino, _error);
initResultRowInfo(&pInfo->binfo.resultRowInfo);
pInfo->binfo.inputTsOrder = pAnomalyNode->window.node.inputTsOrder;
pInfo->binfo.outputTsOrder = pAnomalyNode->window.node.outputTsOrder;
pInfo->anomalyCol = extractColumnFromColumnNode(pColNode);
pInfo->anomalyKey.type = pInfo->anomalyCol.type;
pInfo->anomalyKey.bytes = pInfo->anomalyCol.bytes;
pInfo->anomalyKey.pData = taosMemoryCalloc(1, pInfo->anomalyCol.bytes);
if (pInfo->anomalyKey.pData == NULL) {
goto _error;
}
int32_t itemSize = sizeof(int32_t) + pInfo->aggSup.resultRowSize + pInfo->anomalyKey.bytes;
pInfo->anomalySup.pResultRow = taosMemoryCalloc(1, itemSize);
pInfo->anomalySup.blocks = taosArrayInit(16, sizeof(SSDataBlock*));
pInfo->anomalySup.windows = taosArrayInit(16, sizeof(STimeWindow));
if (pInfo->anomalySup.windows == NULL || pInfo->anomalySup.blocks == NULL || pInfo->anomalySup.pResultRow == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _error;
}
code = filterInitFromNode((SNode*)pAnomalyNode->window.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
QUERY_CHECK_CODE(code, lino, _error);
code = initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
QUERY_CHECK_CODE(code, lino, _error);
setOperatorInfo(pOperator, "AnomalyWindowOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_ANOMALY, true, OP_NOT_OPENED,
pInfo, pTaskInfo);
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, anomalyAggregateNext, NULL, anomalyDestroyOperatorInfo,
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
code = appendDownstream(pOperator, &downstream, 1);
QUERY_CHECK_CODE(code, lino, _error);
*pOptrInfo = pOperator;
qDebug("anomaly_window operator is created, algo:%s url:%s opt:%s", pInfo->algoName, pInfo->algoUrl,
pInfo->anomalyOpt);
return TSDB_CODE_SUCCESS;
_error:
if (pInfo != NULL) {
anomalyDestroyOperatorInfo(pInfo);
}
destroyOperatorAndDownstreams(pOperator, &downstream, 1);
pTaskInfo->code = code;
qError("failed to create anomaly_window operator, algo:%s code:0x%x", pInfo->algoName, code);
return code;
}
static int32_t anomalyAggregateNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SAnomalyWindowOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SOptrBasicInfo* pBInfo = &pInfo->binfo;
SAnomalyWindowSupp* pSupp = &pInfo->anomalySup;
SSDataBlock* pRes = pInfo->binfo.pRes;
int64_t st = taosGetTimestampUs();
int32_t numOfBlocks = taosArrayGetSize(pSupp->blocks);
blockDataCleanup(pRes);
while (1) {
SSDataBlock* pBlock = getNextBlockFromDownstream(pOperator, 0);
if (pBlock == NULL) {
break;
}
if (pSupp->groupId == 0 || pSupp->groupId == pBlock->info.id.groupId) {
pSupp->groupId = pBlock->info.id.groupId;
numOfBlocks++;
pSupp->cachedRows += pBlock->info.rows;
qDebug("group:%" PRId64 ", blocks:%d, rows:%" PRId64 ", total rows:%" PRId64, pSupp->groupId, numOfBlocks,
pBlock->info.rows, pSupp->cachedRows);
code = anomalyCacheBlock(pInfo, pBlock);
QUERY_CHECK_CODE(code, lino, _end);
} else {
qDebug("group:%" PRId64 ", read finish for new group coming, blocks:%d", pSupp->groupId, numOfBlocks);
anomalyAggregateBlocks(pOperator);
pSupp->groupId = pBlock->info.id.groupId;
numOfBlocks = 1;
pSupp->cachedRows = pBlock->info.rows;
qDebug("group:%" PRId64 ", new group, rows:%" PRId64 ", total rows:%" PRId64, pSupp->groupId,
pBlock->info.rows, pSupp->cachedRows);
code = anomalyCacheBlock(pInfo, pBlock);
QUERY_CHECK_CODE(code, lino, _end);
}
if (pRes->info.rows > 0) {
(*ppRes) = pRes;
qDebug("group:%" PRId64 ", return to upstream, blocks:%d", pRes->info.id.groupId, numOfBlocks);
return code;
}
}
if (numOfBlocks > 0) {
qDebug("group:%" PRId64 ", read finish, blocks:%d", pInfo->anomalySup.groupId, numOfBlocks);
anomalyAggregateBlocks(pOperator);
}
int64_t cost = taosGetTimestampUs() - st;
qDebug("all groups finished, cost:%" PRId64 "us", cost);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
pTaskInfo->code = code;
T_LONG_JMP(pTaskInfo->env, code);
}
(*ppRes) = (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes;
return code;
}
static void anomalyDestroyOperatorInfo(void* param) {
SAnomalyWindowOperatorInfo* pInfo = (SAnomalyWindowOperatorInfo*)param;
if (pInfo == NULL) return;
qDebug("anomaly_window operator is destroyed, algo:%s", pInfo->algoName);
cleanupBasicInfo(&pInfo->binfo);
cleanupAggSup(&pInfo->aggSup);
cleanupExprSupp(&pInfo->scalarSup);
colDataDestroy(&pInfo->twAggSup.timeWindowData);
for (int32_t i = 0; i < taosArrayGetSize(pInfo->anomalySup.blocks); ++i) {
SSDataBlock* pBlock = taosArrayGetP(pInfo->anomalySup.blocks, i);
blockDataDestroy(pBlock);
}
taosArrayDestroy(pInfo->anomalySup.blocks);
taosArrayDestroy(pInfo->anomalySup.windows);
taosMemoryFreeClear(pInfo->anomalySup.pResultRow);
taosMemoryFreeClear(pInfo->anomalyKey.pData);
taosMemoryFreeClear(param);
}
static int32_t anomalyCacheBlock(SAnomalyWindowOperatorInfo* pInfo, SSDataBlock* pSrc) {
if (pInfo->anomalySup.cachedRows > ANAL_ANOMALY_WINDOW_MAX_ROWS) {
return TSDB_CODE_ANAL_ANODE_TOO_MANY_ROWS;
}
SSDataBlock* pDst = NULL;
int32_t code = createOneDataBlock(pSrc, true, &pDst);
if (code != 0) return code;
if (pDst == NULL) return TSDB_CODE_OUT_OF_MEMORY;
if (taosArrayPush(pInfo->anomalySup.blocks, &pDst) == NULL) return TSDB_CODE_OUT_OF_MEMORY;
return 0;
}
static int32_t anomalyFindWindow(SAnomalyWindowSupp* pSupp, TSKEY key) {
for (int32_t i = pSupp->curWinIndex; i < taosArrayGetSize(pSupp->windows); ++i) {
STimeWindow* pWindow = taosArrayGet(pSupp->windows, i);
if (key >= pWindow->skey && key < pWindow->ekey) {
pSupp->curWin = *pWindow;
pSupp->curWinIndex = i;
return 0;
}
}
return -1;
}
static int32_t anomalyParseJson(SJson* pJson, SArray* pWindows) {
int32_t code = 0;
int32_t rows = 0;
STimeWindow win = {0};
taosArrayClear(pWindows);
tjsonGetInt32ValueFromDouble(pJson, "rows", rows, code);
if (code < 0) return TSDB_CODE_INVALID_JSON_FORMAT;
if (rows <= 0) return 0;
SJson* res = tjsonGetObjectItem(pJson, "res");
if (res == NULL) return TSDB_CODE_INVALID_JSON_FORMAT;
int32_t ressize = tjsonGetArraySize(res);
if (ressize != rows) return TSDB_CODE_INVALID_JSON_FORMAT;
for (int32_t i = 0; i < rows; ++i) {
SJson* row = tjsonGetArrayItem(res, i);
if (row == NULL) return TSDB_CODE_INVALID_JSON_FORMAT;
int32_t colsize = tjsonGetArraySize(row);
if (colsize != 2) return TSDB_CODE_INVALID_JSON_FORMAT;
SJson* start = tjsonGetArrayItem(row, 0);
SJson* end = tjsonGetArrayItem(row, 1);
if (start == NULL || end == NULL) return TSDB_CODE_INVALID_JSON_FORMAT;
tjsonGetObjectValueBigInt(start, &win.skey);
tjsonGetObjectValueBigInt(end, &win.ekey);
if (win.skey >= win.ekey) {
win.ekey = win.skey + 1;
}
if (taosArrayPush(pWindows, &win) == NULL) return TSDB_CODE_OUT_OF_BUFFER;
}
int32_t numOfWins = taosArrayGetSize(pWindows);
qDebug("anomaly window recevied, total:%d", numOfWins);
for (int32_t i = 0; i < numOfWins; ++i) {
STimeWindow* pWindow = taosArrayGet(pWindows, i);
qDebug("anomaly win:%d [%" PRId64 ", %" PRId64 ")", i, pWindow->skey, pWindow->ekey);
}
return 0;
}
static int32_t anomalyAnalysisWindow(SOperatorInfo* pOperator) {
SAnomalyWindowOperatorInfo* pInfo = pOperator->info;
SAnomalyWindowSupp* pSupp = &pInfo->anomalySup;
SJson* pJson = NULL;
SAnalBuf analBuf = {.bufType = ANAL_BUF_TYPE_JSON};
char dataBuf[64] = {0};
int32_t code = 0;
int64_t ts = 0;
// int64_t ts = taosGetTimestampMs();
snprintf(analBuf.fileName, sizeof(analBuf.fileName), "%s/tdengine-anomaly-%" PRId64 "-%" PRId64, tsTempDir, ts,
pSupp->groupId);
code = tsosAnalBufOpen(&analBuf, 2);
if (code != 0) goto _OVER;
const char* prec = TSDB_TIME_PRECISION_MILLI_STR;
if (pInfo->anomalyCol.precision == TSDB_TIME_PRECISION_MICRO) prec = TSDB_TIME_PRECISION_MICRO_STR;
if (pInfo->anomalyCol.precision == TSDB_TIME_PRECISION_NANO) prec = TSDB_TIME_PRECISION_NANO_STR;
code = taosAnalBufWriteColMeta(&analBuf, 0, TSDB_DATA_TYPE_TIMESTAMP, "ts");
if (code != 0) goto _OVER;
code = taosAnalBufWriteColMeta(&analBuf, 1, pInfo->anomalyCol.type, "val");
if (code != 0) goto _OVER;
code = taosAnalBufWriteDataBegin(&analBuf);
if (code != 0) goto _OVER;
int32_t numOfBlocks = (int32_t)taosArrayGetSize(pSupp->blocks);
// timestamp
code = taosAnalBufWriteColBegin(&analBuf, 0);
if (code != 0) goto _OVER;
for (int32_t i = 0; i < numOfBlocks; ++i) {
SSDataBlock* pBlock = taosArrayGetP(pSupp->blocks, i);
if (pBlock == NULL) break;
SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId);
if (pTsCol == NULL) break;
for (int32_t j = 0; j < pBlock->info.rows; ++j) {
code = taosAnalBufWriteColData(&analBuf, 0, TSDB_DATA_TYPE_TIMESTAMP, &((TSKEY*)pTsCol->pData)[j]);
if (code != 0) goto _OVER;
}
}
code = taosAnalBufWriteColEnd(&analBuf, 0);
if (code != 0) goto _OVER;
// data
code = taosAnalBufWriteColBegin(&analBuf, 1);
if (code != 0) goto _OVER;
for (int32_t i = 0; i < numOfBlocks; ++i) {
SSDataBlock* pBlock = taosArrayGetP(pSupp->blocks, i);
if (pBlock == NULL) break;
SColumnInfoData* pValCol = taosArrayGet(pBlock->pDataBlock, pInfo->anomalyCol.slotId);
if (pValCol == NULL) break;
for (int32_t j = 0; j < pBlock->info.rows; ++j) {
code = taosAnalBufWriteColData(&analBuf, 1, pValCol->info.type, colDataGetData(pValCol, j));
if (code != 0) goto _OVER;
if (code != 0) goto _OVER;
}
}
code = taosAnalBufWriteColEnd(&analBuf, 1);
if (code != 0) goto _OVER;
code = taosAnalBufWriteDataEnd(&analBuf);
if (code != 0) goto _OVER;
code = taosAnalBufWriteOptStr(&analBuf, "option", pInfo->anomalyOpt);
if (code != 0) goto _OVER;
code = taosAnalBufWriteOptStr(&analBuf, "algo", pInfo->algoName);
if (code != 0) goto _OVER;
code = taosAnalBufWriteOptStr(&analBuf, "prec", prec);
if (code != 0) goto _OVER;
int64_t wncheck = ANAL_FORECAST_DEFAULT_WNCHECK;
bool hasWncheck = taosAnalGetOptInt(pInfo->anomalyOpt, "wncheck", &wncheck);
if (!hasWncheck) {
qDebug("anomaly_window wncheck not found from %s, use default:%" PRId64, pInfo->anomalyOpt, wncheck);
}
code = taosAnalBufWriteOptInt(&analBuf, "wncheck", wncheck);
if (code != 0) goto _OVER;
code = taosAnalBufClose(&analBuf);
if (code != 0) goto _OVER;
pJson = taosAnalSendReqRetJson(pInfo->algoUrl, ANAL_HTTP_TYPE_POST, &analBuf);
if (pJson == NULL) {
code = terrno;
goto _OVER;
}
code = anomalyParseJson(pJson, pSupp->windows);
if (code != 0) goto _OVER;
_OVER:
if (code != 0) {
qError("failed to analysis window since %s", tstrerror(code));
}
taosAnalBufDestroy(&analBuf);
if (pJson != NULL) tjsonDelete(pJson);
return code;
}
static int32_t anomalyAggregateRows(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
SAnomalyWindowOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SExprSupp* pExprSup = &pOperator->exprSupp;
SAnomalyWindowSupp* pSupp = &pInfo->anomalySup;
SWindowRowsSup* pRowSup = &pInfo->anomalyWinRowSup;
SResultRow* pResRow = pSupp->pResultRow;
int32_t numOfOutput = pOperator->exprSupp.numOfExprs;
int32_t code = setResultRowInitCtx(pResRow, pExprSup->pCtx, pExprSup->numOfExprs, pExprSup->rowEntryInfoOffset);
if (code == 0) {
updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pSupp->curWin, 0);
code = applyAggFunctionOnPartialTuples(pTaskInfo, pExprSup->pCtx, &pInfo->twAggSup.timeWindowData,
pRowSup->startRowIndex, pRowSup->numOfRows, pBlock->info.rows, numOfOutput);
}
return code;
}
static int32_t anomalyBuildResult(SOperatorInfo* pOperator) {
SAnomalyWindowOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SExprSupp* pExprSup = &pOperator->exprSupp;
SSDataBlock* pRes = pInfo->binfo.pRes;
SResultRow* pResRow = pInfo->anomalySup.pResultRow;
doUpdateNumOfRows(pExprSup->pCtx, pResRow, pExprSup->numOfExprs, pExprSup->rowEntryInfoOffset);
int32_t code = copyResultrowToDataBlock(pExprSup->pExprInfo, pExprSup->numOfExprs, pResRow, pExprSup->pCtx, pRes,
pExprSup->rowEntryInfoOffset, pTaskInfo);
if (code == 0) {
pRes->info.rows += pResRow->numOfRows;
}
clearResultRowInitFlag(pExprSup->pCtx, pExprSup->numOfExprs);
return code;
}
static void anomalyAggregateBlocks(SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SAnomalyWindowOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SExprSupp* pExprSup = &pOperator->exprSupp;
SSDataBlock* pRes = pInfo->binfo.pRes;
SAnomalyWindowSupp* pSupp = &pInfo->anomalySup;
SWindowRowsSup* pRowSup = &pInfo->anomalyWinRowSup;
SResultRow* pResRow = pSupp->pResultRow;
int32_t numOfOutput = pOperator->exprSupp.numOfExprs;
int32_t rowsInWin = 0;
int32_t rowsInBlock = 0;
const int64_t gid = pSupp->groupId;
const int32_t order = pInfo->binfo.inputTsOrder;
int32_t numOfBlocks = (int32_t)taosArrayGetSize(pSupp->blocks);
if (numOfBlocks == 0) goto _OVER;
qDebug("group:%" PRId64 ", aggregate blocks, blocks:%d", pSupp->groupId, numOfBlocks);
pRes->info.id.groupId = pSupp->groupId;
code = anomalyAnalysisWindow(pOperator);
QUERY_CHECK_CODE(code, lino, _OVER);
int32_t numOfWins = taosArrayGetSize(pSupp->windows);
qDebug("group:%" PRId64 ", wins:%d, rows:%" PRId64, pSupp->groupId, numOfWins, pSupp->cachedRows);
for (int32_t w = 0; w < numOfWins; ++w) {
STimeWindow* pWindow = taosArrayGet(pSupp->windows, w);
if (w == 0) {
pSupp->curWin = *pWindow;
pRowSup->win.skey = pSupp->curWin.skey;
}
qDebug("group:%" PRId64 ", win:%d [%" PRId64 ", %" PRId64 ")", pSupp->groupId, w, pWindow->skey, pWindow->ekey);
}
if (numOfWins <= 0) goto _OVER;
if (numOfWins > pRes->info.capacity) {
code = blockDataEnsureCapacity(pRes, numOfWins);
QUERY_CHECK_CODE(code, lino, _OVER);
}
for (int32_t b = 0; b < numOfBlocks; ++b) {
SSDataBlock* pBlock = taosArrayGetP(pSupp->blocks, b);
if (pBlock == NULL) break;
pRes->info.scanFlag = pBlock->info.scanFlag;
code = setInputDataBlock(pExprSup, pBlock, order, MAIN_SCAN, true);
if (code != 0) break;
code = blockDataUpdateTsWindow(pBlock, pInfo->tsSlotId);
if (code != 0) break;
// there is an scalar expression that needs to be calculated right before apply the group aggregation.
if (pInfo->scalarSup.pExprInfo != NULL) {
code = projectApplyFunctions(pInfo->scalarSup.pExprInfo, pBlock, pBlock, pInfo->scalarSup.pCtx,
pInfo->scalarSup.numOfExprs, NULL);
if (code != 0) break;
}
SColumnInfoData* pValCol = taosArrayGet(pBlock->pDataBlock, pInfo->anomalyCol.slotId);
if (pValCol == NULL) break;
SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId);
if (pTsCol == NULL) break;
TSKEY* tsList = (TSKEY*)pTsCol->pData;
bool lastBlock = (b == numOfBlocks - 1);
qTrace("group:%" PRId64 ", block:%d win:%d, riwin:%d riblock:%d, rows:%" PRId64, pSupp->groupId, b,
pSupp->curWinIndex, rowsInWin, rowsInBlock, pBlock->info.rows);
for (int32_t r = 0; r < pBlock->info.rows; ++r) {
TSKEY key = tsList[r];
bool keyInWin = (key >= pSupp->curWin.skey && key < pSupp->curWin.ekey);
bool lastRow = (r == pBlock->info.rows - 1);
if (keyInWin) {
if (r < 5) {
qTrace("group:%" PRId64 ", block:%d win:%d, row:%d ts:%" PRId64 ", riwin:%d riblock:%d", pSupp->groupId, b,
pSupp->curWinIndex, r, key, rowsInWin, rowsInBlock);
}
if (rowsInBlock == 0) {
doKeepNewWindowStartInfo(pRowSup, tsList, r, gid);
}
doKeepTuple(pRowSup, tsList[r], gid);
rowsInBlock++;
rowsInWin++;
} else {
if (rowsInBlock > 0) {
qTrace("group:%" PRId64 ", block:%d win:%d, row:%d ts:%" PRId64 ", riwin:%d riblock:%d, agg", pSupp->groupId,
b, pSupp->curWinIndex, r, key, rowsInWin, rowsInBlock);
code = anomalyAggregateRows(pOperator, pBlock);
QUERY_CHECK_CODE(code, lino, _OVER);
rowsInBlock = 0;
}
if (rowsInWin > 0) {
qTrace("group:%" PRId64 ", block:%d win:%d, row:%d ts:%" PRId64 ", riwin:%d riblock:%d, build result",
pSupp->groupId, b, pSupp->curWinIndex, r, key, rowsInWin, rowsInBlock);
code = anomalyBuildResult(pOperator);
QUERY_CHECK_CODE(code, lino, _OVER);
rowsInWin = 0;
}
if (anomalyFindWindow(pSupp, tsList[r]) == 0) {
qTrace("group:%" PRId64 ", block:%d win:%d, row:%d ts:%" PRId64 ", riwin:%d riblock:%d, new window detect",
pSupp->groupId, b, pSupp->curWinIndex, r, key, rowsInWin, rowsInBlock);
doKeepNewWindowStartInfo(pRowSup, tsList, r, gid);
doKeepTuple(pRowSup, tsList[r], gid);
rowsInBlock = 1;
rowsInWin = 1;
} else {
qTrace("group:%" PRId64 ", block:%d win:%d, row:%d ts:%" PRId64 ", riwin:%d riblock:%d, window not found",
pSupp->groupId, b, pSupp->curWinIndex, r, key, rowsInWin, rowsInBlock);
rowsInBlock = 0;
rowsInWin = 0;
}
}
if (lastRow && rowsInBlock > 0) {
qTrace("group:%" PRId64 ", block:%d win:%d, row:%d ts:%" PRId64 ", riwin:%d riblock:%d, agg since lastrow",
pSupp->groupId, b, pSupp->curWinIndex, r, key, rowsInWin, rowsInBlock);
code = anomalyAggregateRows(pOperator, pBlock);
QUERY_CHECK_CODE(code, lino, _OVER);
rowsInBlock = 0;
}
}
if (lastBlock && rowsInWin > 0) {
qTrace("group:%" PRId64 ", block:%d win:%d, riwin:%d riblock:%d, build result since lastblock", pSupp->groupId, b,
pSupp->curWinIndex, rowsInWin, rowsInBlock);
code = anomalyBuildResult(pOperator);
QUERY_CHECK_CODE(code, lino, _OVER);
rowsInWin = 0;
}
}
code = doFilter(pRes, pOperator->exprSupp.pFilterInfo, NULL);
QUERY_CHECK_CODE(code, lino, _OVER);
_OVER:
for (int32_t i = 0; i < numOfBlocks; ++i) {
SSDataBlock* pBlock = taosArrayGetP(pSupp->blocks, i);
qDebug("%s, clear block, pBlock:%p pBlock->pDataBlock:%p", __func__, pBlock, pBlock->pDataBlock);
blockDataDestroy(pBlock);
}
taosArrayClear(pSupp->blocks);
taosArrayClear(pSupp->windows);
pSupp->cachedRows = 0;
pSupp->curWin.ekey = 0;
pSupp->curWin.skey = 0;
pSupp->curWinIndex = 0;
}
#else
int32_t createAnomalywindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* physiNode, SExecTaskInfo* pTaskInfo,
SOperatorInfo** pOptrInfo) {
return TSDB_CODE_OPS_NOT_SUPPORT;
}
void destroyForecastInfo(void* param) {}
#endif

12
source/libs/executor/src/exchangeoperator.c
View File

@ -226,7 +226,7 @@ static SSDataBlock* doLoadRemoteDataImpl(SOperatorInfo* pOperator) {
concurrentlyLoadRemoteDataImpl(pOperator, pExchangeInfo, pTaskInfo);
}
if (TSDB_CODE_SUCCESS != pOperator->pTaskInfo->code) {
qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(pOperator->pTaskInfo->code));
T_LONG_JMP(pTaskInfo->env, pOperator->pTaskInfo->code);
}
if (taosArrayGetSize(pExchangeInfo->pResultBlockList) == 0) {
@ -530,6 +530,16 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) {
int32_t index = pWrapper->sourceIndex;
SSourceDataInfo* pSourceDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, index);
int64_t* pRpcHandle = taosArrayGet(pExchangeInfo->pFetchRpcHandles, index);
if (pRpcHandle != NULL) {
int32_t ret = asyncFreeConnById(pExchangeInfo->pTransporter, *pRpcHandle);
if (ret != 0) {
qDebug("failed to free rpc handle, code:%s, %p", tstrerror(ret), pExchangeInfo);
}
*pRpcHandle = -1;
}
if (!pSourceDataInfo) {
return terrno;
}

109
source/libs/executor/src/executil.c
View File

@ -88,11 +88,116 @@ size_t getResultRowSize(SqlFunctionCtx* pCtx, int32_t numOfOutput) {
rowSize += pCtx[i].resDataInfo.interBufSize;
}
rowSize += (numOfOutput * sizeof(bool));
// expand rowSize to mark if col is null for top/bottom result(saveTupleData)
return rowSize;
}
// Convert buf read from rocksdb to result row
int32_t getResultRowFromBuf(SExprSupp *pSup, const char* inBuf, size_t inBufSize, char **outBuf, size_t *outBufSize) {
if (inBuf == NULL || pSup == NULL) {
qError("invalid input parameters, inBuf:%p, pSup:%p", inBuf, pSup);
return TSDB_CODE_INVALID_PARA;
}
SqlFunctionCtx *pCtx = pSup->pCtx;
int32_t *offset = pSup->rowEntryInfoOffset;
SResultRow *pResultRow = NULL;
size_t processedSize = 0;
int32_t code = TSDB_CODE_SUCCESS;
// calculate the size of output buffer
*outBufSize = getResultRowSize(pCtx, pSup->numOfExprs);
*outBuf = taosMemoryMalloc(*outBufSize);
if (*outBuf == NULL) {
qError("failed to allocate memory for output buffer, size:%zu", *outBufSize);
return terrno;
}
pResultRow = (SResultRow*)*outBuf;
(void)memcpy(pResultRow, inBuf, sizeof(SResultRow));
inBuf += sizeof(SResultRow);
processedSize += sizeof(SResultRow);
for (int32_t i = 0; i < pSup->numOfExprs; ++i) {
int32_t len = *(int32_t*)inBuf;
inBuf += sizeof(int32_t);
processedSize += sizeof(int32_t);
if (pResultRow->version != FUNCTION_RESULT_INFO_VERSION && pCtx->fpSet.decode) {
code = pCtx->fpSet.decode(&pCtx[i], inBuf, getResultEntryInfo(pResultRow, i, offset), pResultRow->version);
if (code != TSDB_CODE_SUCCESS) {
qError("failed to decode result row, code:%d", code);
return code;
}
} else {
(void)memcpy(getResultEntryInfo(pResultRow, i, offset), inBuf, len);
}
inBuf += len;
processedSize += len;
}
if (processedSize < inBufSize) {
// stream stores extra data after result row
size_t leftLen = inBufSize - processedSize;
TAOS_MEMORY_REALLOC(*outBuf, *outBufSize + leftLen);
if (*outBuf == NULL) {
qError("failed to reallocate memory for output buffer, size:%zu", *outBufSize + leftLen);
return terrno;
}
(void)memcpy(*outBuf + *outBufSize, inBuf, leftLen);
inBuf += leftLen;
processedSize += leftLen;
*outBufSize += leftLen;
}
return TSDB_CODE_SUCCESS;
}
// Convert result row to buf for rocksdb
int32_t putResultRowToBuf(SExprSupp *pSup, const char* inBuf, size_t inBufSize, char **outBuf, size_t *outBufSize) {
if (pSup == NULL || inBuf == NULL || outBuf == NULL || outBufSize == NULL) {
qError("invalid input parameters, inBuf:%p, pSup:%p, outBufSize:%p, outBuf:%p", inBuf, pSup, outBufSize, outBuf);
return TSDB_CODE_INVALID_PARA;
}
SqlFunctionCtx *pCtx = pSup->pCtx;
int32_t *offset = pSup->rowEntryInfoOffset;
SResultRow *pResultRow = (SResultRow*)inBuf;
size_t rowSize = getResultRowSize(pCtx, pSup->numOfExprs);
if (rowSize > inBufSize) {
qError("invalid input buffer size, rowSize:%zu, inBufSize:%zu", rowSize, inBufSize);
return TSDB_CODE_INVALID_PARA;
}
// calculate the size of output buffer
*outBufSize = rowSize + sizeof(int32_t) * pSup->numOfExprs;
if (rowSize < inBufSize) {
*outBufSize += inBufSize - rowSize;
}
*outBuf = taosMemoryMalloc(*outBufSize);
if (*outBuf == NULL) {
qError("failed to allocate memory for output buffer, size:%zu", *outBufSize);
return terrno;
}
char *pBuf = *outBuf;
pResultRow->version = FUNCTION_RESULT_INFO_VERSION;
(void)memcpy(pBuf, pResultRow, sizeof(SResultRow));
pBuf += sizeof(SResultRow);
for (int32_t i = 0; i < pSup->numOfExprs; ++i) {
size_t len = sizeof(SResultRowEntryInfo) + pCtx[i].resDataInfo.interBufSize;
*(int32_t *) pBuf = (int32_t)len;
pBuf += sizeof(int32_t);
(void)memcpy(pBuf, getResultEntryInfo(pResultRow, i, offset), len);
pBuf += len;
}
if (rowSize < inBufSize) {
// stream stores extra data after result row
size_t leftLen = inBufSize - rowSize;
(void)memcpy(pBuf, inBuf + rowSize, leftLen);
pBuf += leftLen;
}
return TSDB_CODE_SUCCESS;
}
static void freeEx(void* p) { taosMemoryFree(*(void**)p); }
void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo) {

659
source/libs/executor/src/forecastoperator.c Normal file
View File

@ -0,0 +1,659 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "executorInt.h"
#include "filter.h"
#include "function.h"
#include "functionMgt.h"
#include "operator.h"
#include "querytask.h"
#include "storageapi.h"
#include "tanal.h"
#include "tcommon.h"
#include "tcompare.h"
#include "tdatablock.h"
#include "tfill.h"
#include "ttime.h"
#ifdef USE_ANAL
typedef struct {
char algoName[TSDB_ANAL_ALGO_NAME_LEN];
char algoUrl[TSDB_ANAL_ALGO_URL_LEN];
char algoOpt[TSDB_ANAL_ALGO_OPTION_LEN];
int64_t maxTs;
int64_t minTs;
int64_t numOfRows;
uint64_t groupId;
int64_t optRows;
int64_t cachedRows;
int32_t numOfBlocks;
int16_t resTsSlot;
int16_t resValSlot;
int16_t resLowSlot;
int16_t resHighSlot;
int16_t inputTsSlot;
int16_t inputValSlot;
int8_t inputValType;
int8_t inputPrecision;
SAnalBuf analBuf;
} SForecastSupp;
typedef struct SForecastOperatorInfo {
SSDataBlock* pRes;
SExprSupp scalarSup; // scalar calculation
SForecastSupp forecastSupp;
} SForecastOperatorInfo;
static void destroyForecastInfo(void* param);
static FORCE_INLINE int32_t forecastEnsureBlockCapacity(SSDataBlock* pBlock, int32_t newRowsNum) {
if (pBlock->info.rows < pBlock->info.capacity) {
return TSDB_CODE_SUCCESS;
}
int32_t code = blockDataEnsureCapacity(pBlock, newRowsNum);
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
return code;
}
return TSDB_CODE_SUCCESS;
}
static int32_t forecastCacheBlock(SForecastSupp* pSupp, SSDataBlock* pBlock) {
if (pSupp->cachedRows > ANAL_FORECAST_MAX_ROWS) {
return TSDB_CODE_ANAL_ANODE_TOO_MANY_ROWS;
}
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SAnalBuf* pBuf = &pSupp->analBuf;
qDebug("block:%d, %p rows:%" PRId64, pSupp->numOfBlocks, pBlock, pBlock->info.rows);
pSupp->numOfBlocks++;
for (int32_t j = 0; j < pBlock->info.rows; ++j) {
SColumnInfoData* pValCol = taosArrayGet(pBlock->pDataBlock, pSupp->inputValSlot);
SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, pSupp->inputTsSlot);
if (pTsCol == NULL || pValCol == NULL) break;
int64_t ts = ((TSKEY*)pTsCol->pData)[j];
char* val = colDataGetData(pValCol, j);
int16_t valType = pValCol->info.type;
pSupp->minTs = MIN(pSupp->minTs, ts);
pSupp->maxTs = MAX(pSupp->maxTs, ts);
pSupp->numOfRows++;
code = taosAnalBufWriteColData(pBuf, 0, TSDB_DATA_TYPE_TIMESTAMP, &ts);
if (TSDB_CODE_SUCCESS != code) return code;
code = taosAnalBufWriteColData(pBuf, 1, valType, val);
if (TSDB_CODE_SUCCESS != code) return code;
}
return 0;
}
static int32_t forecastCloseBuf(SForecastSupp* pSupp) {
SAnalBuf* pBuf = &pSupp->analBuf;
int32_t code = 0;
for (int32_t i = 0; i < 2; ++i) {
code = taosAnalBufWriteColEnd(pBuf, i);
if (code != 0) return code;
}
code = taosAnalBufWriteDataEnd(pBuf);
if (code != 0) return code;
code = taosAnalBufWriteOptStr(pBuf, "option", pSupp->algoOpt);
if (code != 0) return code;
code = taosAnalBufWriteOptStr(pBuf, "algo", pSupp->algoName);
if (code != 0) return code;
const char* prec = TSDB_TIME_PRECISION_MILLI_STR;
if (pSupp->inputPrecision == TSDB_TIME_PRECISION_MICRO) prec = TSDB_TIME_PRECISION_MICRO_STR;
if (pSupp->inputPrecision == TSDB_TIME_PRECISION_NANO) prec = TSDB_TIME_PRECISION_NANO_STR;
code = taosAnalBufWriteOptStr(pBuf, "prec", prec);
if (code != 0) return code;
int64_t wncheck = ANAL_FORECAST_DEFAULT_WNCHECK;
bool hasWncheck = taosAnalGetOptInt(pSupp->algoOpt, "wncheck", &wncheck);
if (!hasWncheck) {
qDebug("forecast wncheck not found from %s, use default:%" PRId64, pSupp->algoOpt, wncheck);
}
code = taosAnalBufWriteOptInt(pBuf, "wncheck", wncheck);
if (code != 0) return code;
bool noConf = (pSupp->resHighSlot == -1 && pSupp->resLowSlot == -1);
code = taosAnalBufWriteOptInt(pBuf, "return_conf", !noConf);
if (code != 0) return code;
pSupp->optRows = ANAL_FORECAST_DEFAULT_ROWS;
bool hasRows = taosAnalGetOptInt(pSupp->algoOpt, "rows", &pSupp->optRows);
if (!hasRows) {
qDebug("forecast rows not found from %s, use default:%" PRId64, pSupp->algoOpt, pSupp->optRows);
}
code = taosAnalBufWriteOptInt(pBuf, "forecast_rows", pSupp->optRows);
if (code != 0) return code;
int64_t conf = ANAL_FORECAST_DEFAULT_CONF;
bool hasConf = taosAnalGetOptInt(pSupp->algoOpt, "conf", &conf);
if (!hasConf) {
qDebug("forecast conf not found from %s, use default:%" PRId64, pSupp->algoOpt, conf);
}
code = taosAnalBufWriteOptInt(pBuf, "conf", conf);
if (code != 0) return code;
int32_t len = strlen(pSupp->algoOpt);
int64_t every = (pSupp->maxTs - pSupp->minTs) / (pSupp->numOfRows - 1);
int64_t start = pSupp->maxTs + every;
bool hasStart = taosAnalGetOptInt(pSupp->algoOpt, "start", &start);
if (!hasStart) {
qDebug("forecast start not found from %s, use %" PRId64, pSupp->algoOpt, start);
}
code = taosAnalBufWriteOptInt(pBuf, "start", start);
if (code != 0) return code;
bool hasEvery = taosAnalGetOptInt(pSupp->algoOpt, "every", &every);
if (!hasEvery) {
qDebug("forecast every not found from %s, use %" PRId64, pSupp->algoOpt, every);
}
code = taosAnalBufWriteOptInt(pBuf, "every", every);
if (code != 0) return code;
code = taosAnalBufClose(pBuf);
return code;
}
static int32_t forecastAnalysis(SForecastSupp* pSupp, SSDataBlock* pBlock) {
SAnalBuf* pBuf = &pSupp->analBuf;
int32_t resCurRow = pBlock->info.rows;
int8_t tmpI8;
int16_t tmpI16;
int32_t tmpI32;
int64_t tmpI64;
float tmpFloat;
double tmpDouble;
int32_t code = 0;
SColumnInfoData* pResValCol = taosArrayGet(pBlock->pDataBlock, pSupp->resValSlot);
if (NULL == pResValCol) return TSDB_CODE_OUT_OF_RANGE;
SColumnInfoData* pResTsCol = (pSupp->resTsSlot != -1 ? taosArrayGet(pBlock->pDataBlock, pSupp->resTsSlot) : NULL);
SColumnInfoData* pResLowCol = (pSupp->resLowSlot != -1 ? taosArrayGet(pBlock->pDataBlock, pSupp->resLowSlot) : NULL);
SColumnInfoData* pResHighCol =
(pSupp->resHighSlot != -1 ? taosArrayGet(pBlock->pDataBlock, pSupp->resHighSlot) : NULL);
SJson* pJson = taosAnalSendReqRetJson(pSupp->algoUrl, ANAL_HTTP_TYPE_POST, pBuf);
if (pJson == NULL) return terrno;
int32_t rows = 0;
tjsonGetInt32ValueFromDouble(pJson, "rows", rows, code);
if (code < 0) goto _OVER;
if (rows <= 0) goto _OVER;
SJson* res = tjsonGetObjectItem(pJson, "res");
if (res == NULL) goto _OVER;
int32_t ressize = tjsonGetArraySize(res);
bool returnConf = (pSupp->resHighSlot != -1 || pSupp->resLowSlot != -1);
if (returnConf) {
if (ressize != 4) goto _OVER;
} else if (ressize != 2) {
goto _OVER;
}
if (pResTsCol != NULL) {
resCurRow = pBlock->info.rows;
SJson* tsJsonArray = tjsonGetArrayItem(res, 0);
if (tsJsonArray == NULL) goto _OVER;
int32_t tsSize = tjsonGetArraySize(tsJsonArray);
if (tsSize != rows) goto _OVER;
for (int32_t i = 0; i < tsSize; ++i) {
SJson* tsJson = tjsonGetArrayItem(tsJsonArray, i);
tjsonGetObjectValueBigInt(tsJson, &tmpI64);
colDataSetInt64(pResTsCol, resCurRow, &tmpI64);
resCurRow++;
}
}
if (pResLowCol != NULL) {
resCurRow = pBlock->info.rows;
SJson* lowJsonArray = tjsonGetArrayItem(res, 2);
if (lowJsonArray == NULL) goto _OVER;
int32_t lowSize = tjsonGetArraySize(lowJsonArray);
if (lowSize != rows) goto _OVER;
for (int32_t i = 0; i < lowSize; ++i) {
SJson* lowJson = tjsonGetArrayItem(lowJsonArray, i);
tjsonGetObjectValueDouble(lowJson, &tmpDouble);
tmpFloat = (float)tmpDouble;
colDataSetFloat(pResLowCol, resCurRow, &tmpFloat);
resCurRow++;
}
}
if (pResHighCol != NULL) {
resCurRow = pBlock->info.rows;
SJson* highJsonArray = tjsonGetArrayItem(res, 3);
if (highJsonArray == NULL) goto _OVER;
int32_t highSize = tjsonGetArraySize(highJsonArray);
if (highSize != rows) goto _OVER;
for (int32_t i = 0; i < highSize; ++i) {
SJson* highJson = tjsonGetArrayItem(highJsonArray, i);
tjsonGetObjectValueDouble(highJson, &tmpDouble);
tmpFloat = (float)tmpDouble;
colDataSetFloat(pResHighCol, resCurRow, &tmpFloat);
resCurRow++;
}
}
resCurRow = pBlock->info.rows;
SJson* valJsonArray = tjsonGetArrayItem(res, 1);
if (valJsonArray == NULL) goto _OVER;
int32_t valSize = tjsonGetArraySize(valJsonArray);
if (valSize != rows) goto _OVER;
for (int32_t i = 0; i < valSize; ++i) {
SJson* valJson = tjsonGetArrayItem(valJsonArray, i);
tjsonGetObjectValueDouble(valJson, &tmpDouble);
switch (pSupp->inputValType) {
case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_UTINYINT:
case TSDB_DATA_TYPE_TINYINT: {
tmpI8 = (int8_t)tmpDouble;
colDataSetInt8(pResValCol, resCurRow, &tmpI8);
break;
}
case TSDB_DATA_TYPE_USMALLINT:
case TSDB_DATA_TYPE_SMALLINT: {
tmpI16 = (int16_t)tmpDouble;
colDataSetInt16(pResValCol, resCurRow, &tmpI16);
break;
}
case TSDB_DATA_TYPE_INT:
case TSDB_DATA_TYPE_UINT: {
tmpI32 = (int32_t)tmpDouble;
colDataSetInt32(pResValCol, resCurRow, &tmpI32);
break;
}
case TSDB_DATA_TYPE_TIMESTAMP:
case TSDB_DATA_TYPE_UBIGINT:
case TSDB_DATA_TYPE_BIGINT: {
tmpI64 = (int64_t)tmpDouble;
colDataSetInt64(pResValCol, resCurRow, &tmpI64);
break;
}
case TSDB_DATA_TYPE_FLOAT: {
tmpFloat = (float)tmpDouble;
colDataSetFloat(pResValCol, resCurRow, &tmpFloat);
break;
}
case TSDB_DATA_TYPE_DOUBLE: {
colDataSetDouble(pResValCol, resCurRow, &tmpDouble);
break;
}
default:
code = TSDB_CODE_FUNC_FUNTION_PARA_TYPE;
goto _OVER;
}
resCurRow++;
}
// for (int32_t i = rows; i < pSupp->optRows; ++i) {
// colDataSetNNULL(pResValCol, rows, (pSupp->optRows - rows));
// if (pResTsCol != NULL) {
// colDataSetNNULL(pResTsCol, rows, (pSupp->optRows - rows));
// }
// if (pResLowCol != NULL) {
// colDataSetNNULL(pResLowCol, rows, (pSupp->optRows - rows));
// }
// if (pResHighCol != NULL) {
// colDataSetNNULL(pResHighCol, rows, (pSupp->optRows - rows));
// }
// }
// if (rows == pSupp->optRows) {
// pResValCol->hasNull = false;
// }
pBlock->info.rows += rows;
if (pJson != NULL) tjsonDelete(pJson);
return 0;
_OVER:
if (pJson != NULL) tjsonDelete(pJson);
if (code == 0) {
code = TSDB_CODE_INVALID_JSON_FORMAT;
}
qError("failed to perform forecast finalize since %s", tstrerror(code));
return TSDB_CODE_INVALID_JSON_FORMAT;
}
static int32_t forecastAggregateBlocks(SForecastSupp* pSupp, SSDataBlock* pResBlock) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SAnalBuf* pBuf = &pSupp->analBuf;
code = forecastCloseBuf(pSupp);
QUERY_CHECK_CODE(code, lino, _end);
code = forecastEnsureBlockCapacity(pResBlock, 1);
QUERY_CHECK_CODE(code, lino, _end);
code = forecastAnalysis(pSupp, pResBlock);
QUERY_CHECK_CODE(code, lino, _end);
uInfo("block:%d, forecast finalize", pSupp->numOfBlocks);
_end:
pSupp->numOfBlocks = 0;
taosAnalBufDestroy(&pSupp->analBuf);
return code;
}
static int32_t forecastNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SForecastOperatorInfo* pInfo = pOperator->info;
SSDataBlock* pResBlock = pInfo->pRes;
SForecastSupp* pSupp = &pInfo->forecastSupp;
SAnalBuf* pBuf = &pSupp->analBuf;
int64_t st = taosGetTimestampUs();
int32_t numOfBlocks = pSupp->numOfBlocks;
blockDataCleanup(pResBlock);
while (1) {
SSDataBlock* pBlock = getNextBlockFromDownstream(pOperator, 0);
if (pBlock == NULL) {
break;
}
if (pSupp->groupId == 0 || pSupp->groupId == pBlock->info.id.groupId) {
pSupp->groupId = pBlock->info.id.groupId;
numOfBlocks++;
pSupp->cachedRows += pBlock->info.rows;
qDebug("group:%" PRId64 ", blocks:%d, rows:%" PRId64 ", total rows:%" PRId64, pSupp->groupId, numOfBlocks,
pBlock->info.rows, pSupp->cachedRows);
code = forecastCacheBlock(pSupp, pBlock);
QUERY_CHECK_CODE(code, lino, _end);
} else {
qDebug("group:%" PRId64 ", read finish for new group coming, blocks:%d", pSupp->groupId, numOfBlocks);
code = forecastAggregateBlocks(pSupp, pResBlock);
QUERY_CHECK_CODE(code, lino, _end);
pSupp->groupId = pBlock->info.id.groupId;
numOfBlocks = 1;
pSupp->cachedRows = pBlock->info.rows;
qDebug("group:%" PRId64 ", new group, rows:%" PRId64 ", total rows:%" PRId64, pSupp->groupId, pBlock->info.rows,
pSupp->cachedRows);
code = forecastCacheBlock(pSupp, pBlock);
QUERY_CHECK_CODE(code, lino, _end);
}
if (pResBlock->info.rows > 0) {
(*ppRes) = pResBlock;
qDebug("group:%" PRId64 ", return to upstream, blocks:%d", pResBlock->info.id.groupId, numOfBlocks);
return code;
}
}
if (numOfBlocks > 0) {
qDebug("group:%" PRId64 ", read finish, blocks:%d", pSupp->groupId, numOfBlocks);
code = forecastAggregateBlocks(pSupp, pResBlock);
QUERY_CHECK_CODE(code, lino, _end);
}
int64_t cost = taosGetTimestampUs() - st;
qDebug("all groups finished, cost:%" PRId64 "us", cost);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
pTaskInfo->code = code;
T_LONG_JMP(pTaskInfo->env, code);
}
(*ppRes) = (pResBlock->info.rows == 0) ? NULL : pResBlock;
return code;
}
static int32_t forecastParseOutput(SForecastSupp* pSupp, SExprSupp* pExprSup) {
pSupp->resLowSlot = -1;
pSupp->resHighSlot = -1;
pSupp->resTsSlot = -1;
pSupp->resValSlot = -1;
for (int32_t j = 0; j < pExprSup->numOfExprs; ++j) {
SExprInfo* pExprInfo = &pExprSup->pExprInfo[j];
int32_t dstSlot = pExprInfo->base.resSchema.slotId;
if (pExprInfo->pExpr->_function.functionType == FUNCTION_TYPE_FORECAST) {
pSupp->resValSlot = dstSlot;
} else if (pExprInfo->pExpr->_function.functionType == FUNCTION_TYPE_FORECAST_ROWTS) {
pSupp->resTsSlot = dstSlot;
} else if (pExprInfo->pExpr->_function.functionType == FUNCTION_TYPE_FORECAST_LOW) {
pSupp->resLowSlot = dstSlot;
} else if (pExprInfo->pExpr->_function.functionType == FUNCTION_TYPE_FORECAST_HIGH) {
pSupp->resHighSlot = dstSlot;
} else {
}
}
return 0;
}
static int32_t forecastParseInput(SForecastSupp* pSupp, SNodeList* pFuncs) {
SNode* pNode = NULL;
pSupp->inputTsSlot = -1;
pSupp->inputValSlot = -1;
pSupp->inputValType = -1;
pSupp->inputPrecision = -1;
FOREACH(pNode, pFuncs) {
if ((nodeType(pNode) == QUERY_NODE_TARGET) && (nodeType(((STargetNode*)pNode)->pExpr) == QUERY_NODE_FUNCTION)) {
SFunctionNode* pFunc = (SFunctionNode*)((STargetNode*)pNode)->pExpr;
int32_t numOfParam = LIST_LENGTH(pFunc->pParameterList);
if (pFunc->funcType == FUNCTION_TYPE_FORECAST) {
if (numOfParam == 3) {
SNode* p1 = nodesListGetNode(pFunc->pParameterList, 0);
SNode* p2 = nodesListGetNode(pFunc->pParameterList, 1);
SNode* p3 = nodesListGetNode(pFunc->pParameterList, 2);
if (p1 == NULL || p2 == NULL || p3 == NULL) return TSDB_CODE_PLAN_INTERNAL_ERROR;
if (p1->type != QUERY_NODE_COLUMN) return TSDB_CODE_PLAN_INTERNAL_ERROR;
if (p2->type != QUERY_NODE_VALUE) return TSDB_CODE_PLAN_INTERNAL_ERROR;
if (p3->type != QUERY_NODE_COLUMN) return TSDB_CODE_PLAN_INTERNAL_ERROR;
SColumnNode* pValNode = (SColumnNode*)p1;
SValueNode* pOptNode = (SValueNode*)p2;
SColumnNode* pTsNode = (SColumnNode*)p3;
pSupp->inputTsSlot = pTsNode->slotId;
pSupp->inputPrecision = pTsNode->node.resType.precision;
pSupp->inputValSlot = pValNode->slotId;
pSupp->inputValType = pValNode->node.resType.type;
tstrncpy(pSupp->algoOpt, pOptNode->literal, sizeof(pSupp->algoOpt));
} else if (numOfParam == 2) {
SNode* p1 = nodesListGetNode(pFunc->pParameterList, 0);
SNode* p2 = nodesListGetNode(pFunc->pParameterList, 1);
if (p1 == NULL || p2 == NULL) return TSDB_CODE_PLAN_INTERNAL_ERROR;
if (p1->type != QUERY_NODE_COLUMN) return TSDB_CODE_PLAN_INTERNAL_ERROR;
if (p2->type != QUERY_NODE_COLUMN) return TSDB_CODE_PLAN_INTERNAL_ERROR;
SColumnNode* pValNode = (SColumnNode*)p1;
SColumnNode* pTsNode = (SColumnNode*)p2;
pSupp->inputTsSlot = pTsNode->slotId;
pSupp->inputPrecision = pTsNode->node.resType.precision;
pSupp->inputValSlot = pValNode->slotId;
pSupp->inputValType = pValNode->node.resType.type;
tstrncpy(pSupp->algoOpt, "algo=arima", TSDB_ANAL_ALGO_OPTION_LEN);
} else {
return TSDB_CODE_PLAN_INTERNAL_ERROR;
}
}
}
}
return 0;
}
static int32_t forecastParseAlgo(SForecastSupp* pSupp) {
pSupp->maxTs = 0;
pSupp->minTs = INT64_MAX;
pSupp->numOfRows = 0;
if (!taosAnalGetOptStr(pSupp->algoOpt, "algo", pSupp->algoName, sizeof(pSupp->algoName))) {
qError("failed to get forecast algorithm name from %s", pSupp->algoOpt);
return TSDB_CODE_ANAL_ALGO_NOT_FOUND;
}
if (taosAnalGetAlgoUrl(pSupp->algoName, ANAL_ALGO_TYPE_FORECAST, pSupp->algoUrl, sizeof(pSupp->algoUrl)) != 0) {
qError("failed to get forecast algorithm url from %s", pSupp->algoName);
return TSDB_CODE_ANAL_ALGO_NOT_LOAD;
}
return 0;
}
static int32_t forecastCreateBuf(SForecastSupp* pSupp) {
SAnalBuf* pBuf = &pSupp->analBuf;
int64_t ts = 0; // taosGetTimestampMs();
pBuf->bufType = ANAL_BUF_TYPE_JSON_COL;
snprintf(pBuf->fileName, sizeof(pBuf->fileName), "%s/tdengine-forecast-%" PRId64, tsTempDir, ts);
int32_t code = tsosAnalBufOpen(pBuf, 2);
if (code != 0) goto _OVER;
code = taosAnalBufWriteColMeta(pBuf, 0, TSDB_DATA_TYPE_TIMESTAMP, "ts");
if (code != 0) goto _OVER;
code = taosAnalBufWriteColMeta(pBuf, 1, pSupp->inputValType, "val");
if (code != 0) goto _OVER;
code = taosAnalBufWriteDataBegin(pBuf);
if (code != 0) goto _OVER;
for (int32_t i = 0; i < 2; ++i) {
code = taosAnalBufWriteColBegin(pBuf, i);
if (code != 0) goto _OVER;
}
_OVER:
if (code != 0) {
(void)taosAnalBufClose(pBuf);
taosAnalBufDestroy(pBuf);
}
return code;
}
int32_t createForecastOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo,
SOperatorInfo** pOptrInfo) {
QRY_PARAM_CHECK(pOptrInfo);
int32_t code = 0;
int32_t lino = 0;
SForecastOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SForecastOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pOperator == NULL || pInfo == NULL) {
code = terrno;
goto _error;
}
SForecastSupp* pSupp = &pInfo->forecastSupp;
SForecastFuncPhysiNode* pForecastPhyNode = (SForecastFuncPhysiNode*)pPhyNode;
SExprSupp* pExprSup = &pOperator->exprSupp;
int32_t numOfExprs = 0;
SExprInfo* pExprInfo = NULL;
code = createExprInfo(pForecastPhyNode->pFuncs, NULL, &pExprInfo, &numOfExprs);
QUERY_CHECK_CODE(code, lino, _error);
code = initExprSupp(pExprSup, pExprInfo, numOfExprs, &pTaskInfo->storageAPI.functionStore);
QUERY_CHECK_CODE(code, lino, _error);
if (pForecastPhyNode->pExprs != NULL) {
int32_t num = 0;
SExprInfo* pScalarExprInfo = NULL;
code = createExprInfo(pForecastPhyNode->pExprs, NULL, &pScalarExprInfo, &num);
QUERY_CHECK_CODE(code, lino, _error);
code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, num, &pTaskInfo->storageAPI.functionStore);
QUERY_CHECK_CODE(code, lino, _error);
}
code = filterInitFromNode((SNode*)pForecastPhyNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
QUERY_CHECK_CODE(code, lino, _error);
code = forecastParseInput(pSupp, pForecastPhyNode->pFuncs);
QUERY_CHECK_CODE(code, lino, _error);
code = forecastParseOutput(pSupp, pExprSup);
QUERY_CHECK_CODE(code, lino, _error);
code = forecastParseAlgo(pSupp);
QUERY_CHECK_CODE(code, lino, _error);
code = forecastCreateBuf(pSupp);
QUERY_CHECK_CODE(code, lino, _error);
initResultSizeInfo(&pOperator->resultInfo, 4096);
pInfo->pRes = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);
QUERY_CHECK_NULL(pInfo->pRes, code, lino, _error, terrno);
setOperatorInfo(pOperator, "ForecastOperator", QUERY_NODE_PHYSICAL_PLAN_FORECAST_FUNC, false, OP_NOT_OPENED, pInfo,
pTaskInfo);
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, forecastNext, NULL, destroyForecastInfo, optrDefaultBufFn,
NULL, optrDefaultGetNextExtFn, NULL);
code = blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
QUERY_CHECK_CODE(code, lino, _error);
code = appendDownstream(pOperator, &downstream, 1);
QUERY_CHECK_CODE(code, lino, _error);
*pOptrInfo = pOperator;
qDebug("forecast env is initialized, option:%s", pSupp->algoOpt);
return TSDB_CODE_SUCCESS;
_error:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
if (pInfo != NULL) destroyForecastInfo(pInfo);
destroyOperatorAndDownstreams(pOperator, &downstream, 1);
pTaskInfo->code = code;
return code;
}
static void destroyForecastInfo(void* param) {
SForecastOperatorInfo* pInfo = (SForecastOperatorInfo*)param;
blockDataDestroy(pInfo->pRes);
pInfo->pRes = NULL;
cleanupExprSupp(&pInfo->scalarSup);
taosAnalBufDestroy(&pInfo->forecastSupp.analBuf);
taosMemoryFreeClear(param);
}
#else
int32_t createForecastOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo,
SOperatorInfo** pOptrInfo) {
return TSDB_CODE_OPS_NOT_SUPPORT;
}
#endif

4
source/libs/executor/src/groupcacheoperator.c
View File

@ -84,9 +84,9 @@ static void logGroupCacheExecInfo(SGroupCacheOperatorInfo* pGrpCacheOperator) {
if (NULL == buf) {
return;
}
int32_t offset = snprintf(buf, bufSize, "groupCache exec info, downstreamBlkNum:");
int32_t offset = tsnprintf(buf, bufSize, "groupCache exec info, downstreamBlkNum:");
for (int32_t i = 0; i < pGrpCacheOperator->downstreamNum; ++i) {
offset += snprintf(buf + offset, bufSize, " %" PRId64 , pGrpCacheOperator->execInfo.pDownstreamBlkNum[i]);
offset += tsnprintf(buf + offset, bufSize, " %" PRId64 , pGrpCacheOperator->execInfo.pDownstreamBlkNum[i]);
}
qDebug("%s", buf);
taosMemoryFree(buf);

4
source/libs/executor/src/operator.c
View File

@ -619,6 +619,8 @@ int32_t createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHand
code = createIndefinitOutputOperatorInfo(ops[0], pPhyNode, pTaskInfo, &pOptr);
} else if (QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC == type) {
code = createTimeSliceOperatorInfo(ops[0], pPhyNode, pTaskInfo, &pOptr);
} else if (QUERY_NODE_PHYSICAL_PLAN_FORECAST_FUNC == type) {
code = createForecastOperatorInfo(ops[0], pPhyNode, pTaskInfo, &pOptr);
} else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT == type) {
code = createEventwindowOperatorInfo(ops[0], pPhyNode, pTaskInfo, &pOptr);
} else if (QUERY_NODE_PHYSICAL_PLAN_GROUP_CACHE == type) {
@ -629,6 +631,8 @@ int32_t createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHand
code = createStreamCountAggOperatorInfo(ops[0], pPhyNode, pTaskInfo, pHandle, &pOptr);
} else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_COUNT == type) {
code = createCountwindowOperatorInfo(ops[0], pPhyNode, pTaskInfo, &pOptr);
} else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_ANOMALY == type) {
code = createAnomalywindowOperatorInfo(ops[0], pPhyNode, pTaskInfo, &pOptr);
} else {
code = TSDB_CODE_INVALID_PARA;
pTaskInfo->code = code;

18
source/libs/executor/src/streamtimewindowoperator.c
View File

@ -2006,6 +2006,12 @@ int32_t createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiN
pInfo->stateStore = pTaskInfo->storageAPI.stateStore;
int32_t funResSize = getMaxFunResSize(&pOperator->exprSupp, numOfCols);
pInfo->pState->pFileState = NULL;
// used for backward compatibility of function's result info
pInfo->pState->pResultRowStore.resultRowGet = getResultRowFromBuf;
pInfo->pState->pResultRowStore.resultRowPut = putResultRowToBuf;
pInfo->pState->pExprSupp = &pOperator->exprSupp;
code =
pAPI->stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize,
compareTs, pInfo->pState, pInfo->twAggSup.deleteMark, GET_TASKID(pTaskInfo),
@ -2223,6 +2229,12 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SExprSupp* pExpSup, in
pSup->stateStore.streamStateSetNumber(pSup->pState, -1, tsIndex);
int32_t funResSize = getMaxFunResSize(pExpSup, numOfOutput);
pSup->pState->pFileState = NULL;
// used for backward compatibility of function's result info
pSup->pState->pResultRowStore.resultRowGet = getResultRowFromBuf;
pSup->pState->pResultRowStore.resultRowPut = putResultRowToBuf;
pSup->pState->pExprSupp = pExpSup;
code = pSup->stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SSessionKey), pSup->resultRowSize, funResSize,
sesionTs, pSup->pState, pTwAggSup->deleteMark, taskIdStr,
pHandle->checkpointId, STREAM_STATE_BUFF_SORT, &pSup->pState->pFileState);
@ -5385,6 +5397,12 @@ int32_t createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
pInfo->stateStore = pTaskInfo->storageAPI.stateStore;
pInfo->pState->pFileState = NULL;
// used for backward compatibility of function's result info
pInfo->pState->pResultRowStore.resultRowGet = getResultRowFromBuf;
pInfo->pState->pResultRowStore.resultRowPut = putResultRowToBuf;
pInfo->pState->pExprSupp = &pOperator->exprSupp;
code = pTaskInfo->storageAPI.stateStore.streamFileStateInit(
tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize, compareTs, pInfo->pState,
pInfo->twAggSup.deleteMark, GET_TASKID(pTaskInfo), pHandle->checkpointId, STREAM_STATE_BUFF_HASH,

2
source/libs/function/inc/builtinsimpl.h
View File

@ -116,6 +116,8 @@ int32_t diffFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo);
int32_t diffFunction(SqlFunctionCtx* pCtx);
int32_t diffFunctionByRow(SArray* pCtx);
bool getForecastConfEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv);
bool getDerivativeFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
int32_t derivativeFuncSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo);
int32_t derivativeFunction(SqlFunctionCtx* pCtx);

1
source/libs/function/inc/functionMgtInt.h
View File

@ -58,6 +58,7 @@ extern "C" {
#define FUNC_MGT_TSMA_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(29)
#define FUNC_MGT_COUNT_LIKE_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(30) // funcs that should also return 0 when no rows found
#define FUNC_MGT_PROCESS_BY_ROW FUNC_MGT_FUNC_CLASSIFICATION_MASK(31)
#define FUNC_MGT_FORECAST_PC_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(32)
#define FUNC_MGT_TEST_MASK(val, mask) (((val) & (mask)) != 0)

84
source/libs/function/src/builtins.c
View File

@ -19,6 +19,7 @@
#include "geomFunc.h"
#include "querynodes.h"
#include "scalar.h"
#include "tanal.h"
#include "taoserror.h"
#include "ttime.h"
@ -2078,6 +2079,47 @@ static int32_t translateMode(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
return translateUniqueMode(pFunc, pErrBuf, len, false);
}
static int32_t translateForecast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
if (2 != numOfParams && 1 != numOfParams) {
return invaildFuncParaNumErrMsg(pErrBuf, len, "FORECAST require 1 or 2 parameters");
}
uint8_t valType = getSDataTypeFromNode(nodesListGetNode(pFunc->pParameterList, 0))->type;
if (!IS_MATHABLE_TYPE(valType)) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, "FORECAST only support mathable column");
}
if (numOfParams == 2) {
uint8_t optionType = getSDataTypeFromNode(nodesListGetNode(pFunc->pParameterList, 1))->type;
if (TSDB_DATA_TYPE_BINARY != optionType) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, "FORECAST option should be varchar");
}
SNode* pOption = nodesListGetNode(pFunc->pParameterList, 1);
if (QUERY_NODE_VALUE != nodeType(pOption)) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, "FORECAST option should be value");
}
SValueNode* pValue = (SValueNode*)pOption;
if (!taosAnalGetOptStr(pValue->literal, "algo", NULL, 0) != 0) {
return invaildFuncParaValueErrMsg(pErrBuf, len, "FORECAST option should include algo field");
}
pValue->notReserved = true;
}
pFunc->node.resType = (SDataType){.bytes = tDataTypes[valType].bytes, .type = valType};
return TSDB_CODE_SUCCESS;
}
static int32_t translateForecastConf(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_FLOAT].bytes, .type = TSDB_DATA_TYPE_FLOAT};
return TSDB_CODE_SUCCESS;
}
static EFuncReturnRows forecastEstReturnRows(SFunctionNode* pFunc) { return FUNC_RETURN_ROWS_N; }
static int32_t translateDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
if (numOfParams > 2) {
@ -4797,6 +4839,48 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.sprocessFunc = randFunction,
.finalizeFunc = NULL
},
{
.name = "forecast",
.type = FUNCTION_TYPE_FORECAST,
.classification = FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC |
FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_SYSTABLE_FUNC | FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_PRIMARY_KEY_FUNC,
.translateFunc = translateForecast,
.getEnvFunc = getSelectivityFuncEnv,
.initFunc = functionSetup,
.processFunc = NULL,
.finalizeFunc = NULL,
.estimateReturnRowsFunc = forecastEstReturnRows,
},
{
.name = "_frowts",
.type = FUNCTION_TYPE_FORECAST_ROWTS,
.classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_FORECAST_PC_FUNC | FUNC_MGT_KEEP_ORDER_FUNC,
.translateFunc = translateTimePseudoColumn,
.getEnvFunc = getTimePseudoFuncEnv,
.initFunc = NULL,
.sprocessFunc = NULL,
.finalizeFunc = NULL
},
{
.name = "_flow",
.type = FUNCTION_TYPE_FORECAST_LOW,
.classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_FORECAST_PC_FUNC | FUNC_MGT_KEEP_ORDER_FUNC,
.translateFunc = translateForecastConf,
.getEnvFunc = getForecastConfEnv,
.initFunc = NULL,
.sprocessFunc = NULL,
.finalizeFunc = NULL
},
{
.name = "_fhigh",
.type = FUNCTION_TYPE_FORECAST_HIGH,
.classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_FORECAST_PC_FUNC | FUNC_MGT_KEEP_ORDER_FUNC,
.translateFunc = translateForecastConf,
.getEnvFunc = getForecastConfEnv,
.initFunc = NULL,
.sprocessFunc = NULL,
.finalizeFunc = NULL
},
};
// clang-format on

26
source/libs/function/src/builtinsimpl.c
View File

@ -19,6 +19,7 @@
#include "functionResInfoInt.h"
#include "query.h"
#include "querynodes.h"
#include "tanal.h"
#include "tcompare.h"
#include "tdatablock.h"
#include "tdigest.h"
@ -3342,6 +3343,11 @@ bool funcInputGetNextRowIndex(SInputColumnInfoData* pInput, int32_t from, bool f
}
}
bool getForecastConfEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
pEnv->calcMemSize = sizeof(float);
return true;
}
int32_t diffResultIsNull(SqlFunctionCtx* pCtx, SFuncInputRow* pRow){
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
SDiffInfo* pDiffInfo = GET_ROWCELL_INTERBUF(pResInfo);
@ -4734,10 +4740,10 @@ int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
int32_t len;
char buf[512] = {0};
if (!pInfo->normalized) {
len = snprintf(varDataVal(buf), sizeof(buf) - VARSTR_HEADER_SIZE, "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%" PRId64 "}",
len = tsnprintf(varDataVal(buf), sizeof(buf) - VARSTR_HEADER_SIZE, "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%" PRId64 "}",
pInfo->bins[i].lower, pInfo->bins[i].upper, pInfo->bins[i].count);
} else {
len = snprintf(varDataVal(buf), sizeof(buf) - VARSTR_HEADER_SIZE, "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%lf}", pInfo->bins[i].lower,
len = tsnprintf(varDataVal(buf), sizeof(buf) - VARSTR_HEADER_SIZE, "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%lf}", pInfo->bins[i].lower,
pInfo->bins[i].upper, pInfo->bins[i].percentage);
}
varDataSetLen(buf, len);
@ -6365,7 +6371,7 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
compRatio = pData->totalSize * 100 / (double)totalRawSize;
}
int32_t len = snprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE,
int32_t len = tsnprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE,
"Total_Blocks=[%d] Total_Size=[%.2f KiB] Average_size=[%.2f KiB] Compression_Ratio=[%.2f %c]",
pData->numOfBlocks, pData->totalSize / 1024.0, averageSize / 1024.0, compRatio, '%');
@ -6380,7 +6386,7 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
avgRows = pData->totalRows / pData->numOfBlocks;
}
len = snprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "Block_Rows=[%" PRId64 "] MinRows=[%d] MaxRows=[%d] AvgRows=[%" PRId64 "]",
len = tsnprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "Block_Rows=[%" PRId64 "] MinRows=[%d] MaxRows=[%d] AvgRows=[%" PRId64 "]",
pData->totalRows, pData->minRows, pData->maxRows, avgRows);
varDataSetLen(st, len);
code = colDataSetVal(pColInfo, row++, st, false);
@ -6388,14 +6394,14 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
return code;
}
len = snprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "Inmem_Rows=[%d] Stt_Rows=[%d] ", pData->numOfInmemRows, pData->numOfSttRows);
len = tsnprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "Inmem_Rows=[%d] Stt_Rows=[%d] ", pData->numOfInmemRows, pData->numOfSttRows);
varDataSetLen(st, len);
code = colDataSetVal(pColInfo, row++, st, false);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
len = snprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "Total_Tables=[%d] Total_Filesets=[%d] Total_Vgroups=[%d]", pData->numOfTables,
len = tsnprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "Total_Tables=[%d] Total_Filesets=[%d] Total_Vgroups=[%d]", pData->numOfTables,
pData->numOfFiles, pData->numOfVgroups);
varDataSetLen(st, len);
@ -6404,7 +6410,7 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
return code;
}
len = snprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE,
len = tsnprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE,
"--------------------------------------------------------------------------------");
varDataSetLen(st, len);
code = colDataSetVal(pColInfo, row++, st, false);
@ -6431,7 +6437,7 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
int32_t bucketRange = ceil(((double) (pData->defMaxRows - pData->defMinRows)) / numOfBuckets);
for (int32_t i = 0; i < tListLen(pData->blockRowsHisto); ++i) {
len = snprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "%04d |", pData->defMinRows + bucketRange * (i + 1));
len = tsnprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "%04d |", pData->defMinRows + bucketRange * (i + 1));
int32_t num = 0;
if (pData->blockRowsHisto[i] > 0) {
@ -6439,13 +6445,13 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
}
for (int32_t j = 0; j < num; ++j) {
int32_t x = snprintf(varDataVal(st) + len, sizeof(st) - VARSTR_HEADER_SIZE - len, "%c", '|');
int32_t x = tsnprintf(varDataVal(st) + len, sizeof(st) - VARSTR_HEADER_SIZE - len, "%c", '|');
len += x;
}
if (pData->blockRowsHisto[i] > 0) {
double v = pData->blockRowsHisto[i] * 100.0 / pData->numOfBlocks;
len += snprintf(varDataVal(st) + len, sizeof(st) - VARSTR_HEADER_SIZE - len, " %d (%.2f%c)", pData->blockRowsHisto[i], v, '%');
len += tsnprintf(varDataVal(st) + len, sizeof(st) - VARSTR_HEADER_SIZE - len, " %d (%.2f%c)", pData->blockRowsHisto[i], v, '%');
}
varDataSetLen(st, len);

9
source/libs/function/src/functionMgt.c
View File

@ -232,6 +232,15 @@ bool fmIsInterpFunc(int32_t funcId) {
bool fmIsInterpPseudoColumnFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_INTERP_PC_FUNC); }
bool fmIsForecastFunc(int32_t funcId) {
if (funcId < 0 || funcId >= funcMgtBuiltinsNum) {
return false;
}
return FUNCTION_TYPE_FORECAST == funcMgtBuiltins[funcId].type;
}
bool fmIsForecastPseudoColumnFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_FORECAST_PC_FUNC); }
bool fmIsLastRowFunc(int32_t funcId) {
if (funcId < 0 || funcId >= funcMgtBuiltinsNum) {
return false;

21
source/libs/nodes/src/nodesCloneFuncs.c
View File

@ -368,6 +368,13 @@ static int32_t countWindowNodeCopy(const SCountWindowNode* pSrc, SCountWindowNod
return TSDB_CODE_SUCCESS;
}
static int32_t anomalyWindowNodeCopy(const SAnomalyWindowNode* pSrc, SAnomalyWindowNode* pDst) {
CLONE_NODE_FIELD(pCol);
CLONE_NODE_FIELD(pExpr);
COPY_CHAR_ARRAY_FIELD(anomalyOpt);
return TSDB_CODE_SUCCESS;
}
static int32_t sessionWindowNodeCopy(const SSessionWindowNode* pSrc, SSessionWindowNode* pDst) {
CLONE_NODE_FIELD_EX(pCol, SColumnNode*);
CLONE_NODE_FIELD_EX(pGap, SValueNode*);
@ -622,6 +629,8 @@ static int32_t logicWindowCopy(const SWindowLogicNode* pSrc, SWindowLogicNode* p
COPY_SCALAR_FIELD(windowAlgo);
COPY_SCALAR_FIELD(windowCount);
COPY_SCALAR_FIELD(windowSliding);
CLONE_NODE_FIELD(pAnomalyExpr);
COPY_CHAR_ARRAY_FIELD(anomalyOpt);
return TSDB_CODE_SUCCESS;
}
@ -674,6 +683,12 @@ static int32_t logicInterpFuncCopy(const SInterpFuncLogicNode* pSrc, SInterpFunc
return TSDB_CODE_SUCCESS;
}
static int32_t logicForecastFuncCopy(const SForecastFuncLogicNode* pSrc, SForecastFuncLogicNode* pDst) {
COPY_BASE_OBJECT_FIELD(node, logicNodeCopy);
CLONE_NODE_LIST_FIELD(pFuncs);
return TSDB_CODE_SUCCESS;
}
static int32_t logicGroupCacheCopy(const SGroupCacheLogicNode* pSrc, SGroupCacheLogicNode* pDst) {
COPY_BASE_OBJECT_FIELD(node, logicNodeCopy);
COPY_SCALAR_FIELD(grpColsMayBeNull);
@ -937,6 +952,9 @@ int32_t nodesCloneNode(const SNode* pNode, SNode** ppNode) {
case QUERY_NODE_COUNT_WINDOW:
code = countWindowNodeCopy((const SCountWindowNode*)pNode, (SCountWindowNode*)pDst);
break;
case QUERY_NODE_ANOMALY_WINDOW:
code = anomalyWindowNodeCopy((const SAnomalyWindowNode*)pNode, (SAnomalyWindowNode*)pDst);
break;
case QUERY_NODE_SESSION_WINDOW:
code = sessionWindowNodeCopy((const SSessionWindowNode*)pNode, (SSessionWindowNode*)pDst);
break;
@ -1021,6 +1039,9 @@ int32_t nodesCloneNode(const SNode* pNode, SNode** ppNode) {
case QUERY_NODE_LOGIC_PLAN_INTERP_FUNC:
code = logicInterpFuncCopy((const SInterpFuncLogicNode*)pNode, (SInterpFuncLogicNode*)pDst);
break;
case QUERY_NODE_LOGIC_PLAN_FORECAST_FUNC:
code = logicForecastFuncCopy((const SForecastFuncLogicNode*)pNode, (SForecastFuncLogicNode*)pDst);
break;
case QUERY_NODE_LOGIC_PLAN_GROUP_CACHE:
code = logicGroupCacheCopy((const SGroupCacheLogicNode*)pNode, (SGroupCacheLogicNode*)pDst);
break;

206
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -97,6 +97,8 @@ const char* nodesNodeName(ENodeType type) {
return "WindowOffset";
case QUERY_NODE_COUNT_WINDOW:
return "CountWindow";
case QUERY_NODE_ANOMALY_WINDOW:
return "AnomalyWindow";
case QUERY_NODE_SET_OPERATOR:
return "SetOperator";
case QUERY_NODE_SELECT_STMT:
@ -153,6 +155,12 @@ const char* nodesNodeName(ENodeType type) {
return "CreateQnodeStmt";
case QUERY_NODE_DROP_QNODE_STMT:
return "DropQnodeStmt";
case QUERY_NODE_CREATE_ANODE_STMT:
return "CreateAnodeStmt";
case QUERY_NODE_DROP_ANODE_STMT:
return "DropAnodeStmt";
case QUERY_NODE_UPDATE_ANODE_STMT:
return "UpdateAnodeStmt";
case QUERY_NODE_CREATE_SNODE_STMT:
return "CreateSnodeStmt";
case QUERY_NODE_DROP_SNODE_STMT:
@ -213,6 +221,10 @@ const char* nodesNodeName(ENodeType type) {
return "ShowModulesStmt";
case QUERY_NODE_SHOW_QNODES_STMT:
return "ShowQnodesStmt";
case QUERY_NODE_SHOW_ANODES_STMT:
return "ShowAnodesStmt";
case QUERY_NODE_SHOW_ANODES_FULL_STMT:
return "ShowAnodesFullStmt";
case QUERY_NODE_SHOW_SNODES_STMT:
return "ShowSnodesStmt";
case QUERY_NODE_SHOW_BNODES_STMT:
@ -328,6 +340,8 @@ const char* nodesNodeName(ENodeType type) {
return "LogicIndefRowsFunc";
case QUERY_NODE_LOGIC_PLAN_INTERP_FUNC:
return "LogicInterpFunc";
case QUERY_NODE_LOGIC_PLAN_FORECAST_FUNC:
return "LogicForecastFunc";
case QUERY_NODE_LOGIC_PLAN_GROUP_CACHE:
return "LogicGroupCache";
case QUERY_NODE_LOGIC_PLAN_DYN_QUERY_CTRL:
@ -362,6 +376,10 @@ const char* nodesNodeName(ENodeType type) {
return "PhysiMergeCountWindow";
case QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT:
return "PhysiStreamCountWindow";
case QUERY_NODE_PHYSICAL_PLAN_MERGE_ANOMALY:
return "PhysiMergeAnomalyWindow";
case QUERY_NODE_PHYSICAL_PLAN_STREAM_ANOMALY:
return "PhysiStreamAnomalyWindow";
case QUERY_NODE_PHYSICAL_PLAN_PROJECT:
return "PhysiProject";
case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN:
@ -413,6 +431,8 @@ const char* nodesNodeName(ENodeType type) {
return "PhysiIndefRowsFunc";
case QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC:
return "PhysiInterpFunc";
case QUERY_NODE_PHYSICAL_PLAN_FORECAST_FUNC:
return "PhysiForecastFunc";
case QUERY_NODE_PHYSICAL_PLAN_DISPATCH:
return "PhysiDispatch";
case QUERY_NODE_PHYSICAL_PLAN_INSERT:
@ -1260,6 +1280,30 @@ static int32_t jsonToLogicInterpFuncNode(const SJson* pJson, void* pObj) {
return code;
}
static const char* jkForecastFuncLogicPlanFuncs = "Funcs";
static int32_t logicForecastFuncNodeToJson(const void* pObj, SJson* pJson) {
const SForecastFuncLogicNode* pNode = (const SForecastFuncLogicNode*)pObj;
int32_t code = logicPlanNodeToJson(pObj, pJson);
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkForecastFuncLogicPlanFuncs, pNode->pFuncs);
}
return code;
}
static int32_t jsonToLogicForecastFuncNode(const SJson* pJson, void* pObj) {
SForecastFuncLogicNode* pNode = (SForecastFuncLogicNode*)pObj;
int32_t code = jsonToLogicPlanNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkForecastFuncLogicPlanFuncs, &pNode->pFuncs);
}
return code;
}
static const char* jkGroupCacheLogicPlanGrpColsMayBeNull = "GroupColsMayBeNull";
static const char* jkGroupCacheLogicPlanGroupByUid = "GroupByUid";
static const char* jkGroupCacheLogicPlanGlobalGroup = "GlobalGroup";
@ -3011,6 +3055,36 @@ static int32_t jsonToPhysiCountWindowNode(const SJson* pJson, void* pObj) {
return code;
}
static const char* jkAnomalyWindowPhysiPlanAnomalyKey = "AnomalyKey";
static const char* jkAnomalyWindowPhysiPlanAnomalyOption = "AnomalyOpt";
static int32_t physiAnomalyWindowNodeToJson(const void* pObj, SJson* pJson) {
const SAnomalyWindowPhysiNode* pNode = (const SAnomalyWindowPhysiNode*)pObj;
int32_t code = physiWindowNodeToJson(pObj, pJson);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkAnomalyWindowPhysiPlanAnomalyKey, nodeToJson, pNode->pAnomalyKey);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddStringToObject(pJson, jkAnomalyWindowPhysiPlanAnomalyOption, pNode->anomalyOpt);
}
return code;
}
static int32_t jsonToPhysiAnomalyWindowNode(const SJson* pJson, void* pObj) {
SAnomalyWindowPhysiNode* pNode = (SAnomalyWindowPhysiNode*)pObj;
int32_t code = jsonToPhysiWindowNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkAnomalyWindowPhysiPlanAnomalyKey, &pNode->pAnomalyKey);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetStringValue(pJson, jkAnomalyWindowPhysiPlanAnomalyOption, pNode->anomalyOpt);
}
return code;
}
static const char* jkPartitionPhysiPlanExprs = "Exprs";
static const char* jkPartitionPhysiPlanPartitionKeys = "PartitionKeys";
static const char* jkPartitionPhysiPlanTargets = "Targets";
@ -3198,6 +3272,37 @@ static int32_t jsonToPhysiInterpFuncNode(const SJson* pJson, void* pObj) {
return code;
}
static const char* jkForecastFuncPhysiPlanExprs = "Exprs";
static const char* jkForecastFuncPhysiPlanFuncs = "Funcs";
static int32_t physiForecastFuncNodeToJson(const void* pObj, SJson* pJson) {
const SForecastFuncPhysiNode* pNode = (const SForecastFuncPhysiNode*)pObj;
int32_t code = physicPlanNodeToJson(pObj, pJson);
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkForecastFuncPhysiPlanExprs, pNode->pExprs);
}
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkForecastFuncPhysiPlanFuncs, pNode->pFuncs);
}
return code;
}
static int32_t jsonToPhysiForecastFuncNode(const SJson* pJson, void* pObj) {
SForecastFuncPhysiNode* pNode = (SForecastFuncPhysiNode*)pObj;
int32_t code = jsonToPhysicPlanNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkForecastFuncPhysiPlanExprs, &pNode->pExprs);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkForecastFuncPhysiPlanFuncs, &pNode->pFuncs);
}
return code;
}
static const char* jkDataSinkInputDataBlockDesc = "InputDataBlockDesc";
static int32_t physicDataSinkNodeToJson(const void* pObj, SJson* pJson) {
@ -4765,6 +4870,36 @@ static int32_t jsonToCountWindowNode(const SJson* pJson, void* pObj) {
return code;
}
static const char* jkAnomalyWindowTsPrimaryKey = "AnomalyTsPrimaryKey";
static const char* jkAnomalyWindowExpr = "AnomalyWindowExpr";
static const char* jkAnomalyWindowOption = "AnomalyWindowOpt";
static int32_t anomalyWindowNodeToJson(const void* pObj, SJson* pJson) {
const SAnomalyWindowNode* pNode = (const SAnomalyWindowNode*)pObj;
int32_t code = tjsonAddObject(pJson, jkAnomalyWindowTsPrimaryKey, nodeToJson, pNode->pCol);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkAnomalyWindowExpr, nodeToJson, pNode->pExpr);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddStringToObject(pJson, jkAnomalyWindowOption, pNode->anomalyOpt);
}
return code;
}
static int32_t jsonToAnomalyWindowNode(const SJson* pJson, void* pObj) {
SAnomalyWindowNode* pNode = (SAnomalyWindowNode*)pObj;
int32_t code = jsonToNodeObject(pJson, jkAnomalyWindowTsPrimaryKey, &pNode->pCol);
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkAnomalyWindowExpr, (SNode**)&pNode->pExpr);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetStringValue(pJson, jkAnomalyWindowOption, pNode->anomalyOpt);
}
return code;
}
static const char* jkIntervalWindowInterval = "Interval";
static const char* jkIntervalWindowOffset = "Offset";
static const char* jkIntervalWindowSliding = "Sliding";
@ -6451,6 +6586,39 @@ static int32_t dropQnodeStmtToJson(const void* pObj, SJson* pJson) { return drop
static int32_t jsonToDropQnodeStmt(const SJson* pJson, void* pObj) { return jsonToDropComponentNodeStmt(pJson, pObj); }
static const char* jkCreateAnodeStmtUrl = "Url";
static const char* jkUpdateDropANodeStmtId = "AnodeId";
static int32_t createAnodeStmtToJson(const void* pObj, SJson* pJson) {
const SCreateAnodeStmt* pNode = (const SCreateAnodeStmt*)pObj;
return tjsonAddStringToObject(pJson, jkCreateAnodeStmtUrl, pNode->url);
}
static int32_t jsonToCreateAnodeStmt(const SJson* pJson, void* pObj) {
SCreateAnodeStmt* pNode = (SCreateAnodeStmt*)pObj;
return tjsonGetStringValue(pJson, jkCreateAnodeStmtUrl, pNode->url);
}
static int32_t updateAnodeStmtToJson(const void* pObj, SJson* pJson) {
const SUpdateAnodeStmt* pNode = (const SUpdateAnodeStmt*)pObj;
return tjsonAddIntegerToObject(pJson, jkUpdateDropANodeStmtId, pNode->anodeId);
}
static int32_t jsonToUpdateAnodeStmt(const SJson* pJson, void* pObj) {
SUpdateAnodeStmt* pNode = (SUpdateAnodeStmt*)pObj;
return tjsonGetIntValue(pJson, jkUpdateDropANodeStmtId, &pNode->anodeId);
}
static int32_t dropAnodeStmtToJson(const void* pObj, SJson* pJson) {
const SDropAnodeStmt* pNode = (const SDropAnodeStmt*)pObj;
return tjsonAddIntegerToObject(pJson, jkUpdateDropANodeStmtId, pNode->anodeId);
}
static int32_t jsonToDropAnodeStmt(const SJson* pJson, void* pObj) {
SDropAnodeStmt* pNode = (SDropAnodeStmt*)pObj;
return tjsonGetIntValue(pJson, jkUpdateDropANodeStmtId, &pNode->anodeId);
}
static int32_t createSnodeStmtToJson(const void* pObj, SJson* pJson) {
return createComponentNodeStmtToJson(pObj, pJson);
}
@ -7016,6 +7184,14 @@ static int32_t showQnodesStmtToJson(const void* pObj, SJson* pJson) { return sho
static int32_t jsonToShowQnodesStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); }
static int32_t showAnodesStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); }
static int32_t jsonToShowAnodesStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); }
static int32_t showAnodesFullStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); }
static int32_t jsonToShowAnodesFullStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); }
static int32_t showArbGroupsStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); }
static int32_t jsonToShowArbGroupsStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); }
@ -7552,6 +7728,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
return windowOffsetNodeToJson(pObj, pJson);
case QUERY_NODE_COUNT_WINDOW:
return countWindowNodeToJson(pObj, pJson);
case QUERY_NODE_ANOMALY_WINDOW:
return anomalyWindowNodeToJson(pObj, pJson);
case QUERY_NODE_SET_OPERATOR:
return setOperatorToJson(pObj, pJson);
case QUERY_NODE_SELECT_STMT:
@ -7604,6 +7782,12 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
return createQnodeStmtToJson(pObj, pJson);
case QUERY_NODE_DROP_QNODE_STMT:
return dropQnodeStmtToJson(pObj, pJson);
case QUERY_NODE_CREATE_ANODE_STMT:
return createAnodeStmtToJson(pObj, pJson);
case QUERY_NODE_DROP_ANODE_STMT:
return dropAnodeStmtToJson(pObj, pJson);
case QUERY_NODE_UPDATE_ANODE_STMT:
return updateAnodeStmtToJson(pObj, pJson);
case QUERY_NODE_CREATE_SNODE_STMT:
return createSnodeStmtToJson(pObj, pJson);
case QUERY_NODE_DROP_SNODE_STMT:
@ -7654,6 +7838,10 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
return showMnodesStmtToJson(pObj, pJson);
case QUERY_NODE_SHOW_QNODES_STMT:
return showQnodesStmtToJson(pObj, pJson);
case QUERY_NODE_SHOW_ANODES_STMT:
return showAnodesStmtToJson(pObj, pJson);
case QUERY_NODE_SHOW_ANODES_FULL_STMT:
return showAnodesFullStmtToJson(pObj, pJson);
case QUERY_NODE_SHOW_ARBGROUPS_STMT:
return showArbGroupsStmtToJson(pObj, pJson);
case QUERY_NODE_SHOW_CLUSTER_STMT:
@ -7743,6 +7931,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
return logicIndefRowsFuncNodeToJson(pObj, pJson);
case QUERY_NODE_LOGIC_PLAN_INTERP_FUNC:
return logicInterpFuncNodeToJson(pObj, pJson);
case QUERY_NODE_LOGIC_PLAN_FORECAST_FUNC:
return logicForecastFuncNodeToJson(pObj, pJson);
case QUERY_NODE_LOGIC_PLAN_GROUP_CACHE:
return logicGroupCacheNodeToJson(pObj, pJson);
case QUERY_NODE_LOGIC_PLAN_DYN_QUERY_CTRL:
@ -7803,6 +7993,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
case QUERY_NODE_PHYSICAL_PLAN_MERGE_COUNT:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT:
return physiCountWindowNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_MERGE_ANOMALY:
return physiAnomalyWindowNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_PARTITION:
return physiPartitionNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION:
@ -7811,6 +8003,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
return physiIndefRowsFuncNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC:
return physiInterpFuncNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_FORECAST_FUNC:
return physiForecastFuncNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_DISPATCH:
return physiDispatchNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_INSERT:
@ -7902,6 +8096,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
return jsonToWindowOffsetNode(pJson, pObj);
case QUERY_NODE_COUNT_WINDOW:
return jsonToCountWindowNode(pJson, pObj);
case QUERY_NODE_ANOMALY_WINDOW:
return jsonToAnomalyWindowNode(pJson, pObj);
case QUERY_NODE_SET_OPERATOR:
return jsonToSetOperator(pJson, pObj);
case QUERY_NODE_SELECT_STMT:
@ -8004,6 +8200,10 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
return jsonToShowMnodesStmt(pJson, pObj);
case QUERY_NODE_SHOW_QNODES_STMT:
return jsonToShowQnodesStmt(pJson, pObj);
case QUERY_NODE_SHOW_ANODES_STMT:
return jsonToShowAnodesStmt(pJson, pObj);
case QUERY_NODE_SHOW_ANODES_FULL_STMT:
return jsonToShowAnodesFullStmt(pJson, pObj);
case QUERY_NODE_SHOW_ARBGROUPS_STMT:
return jsonToShowArbGroupsStmt(pJson, pObj);
case QUERY_NODE_SHOW_CLUSTER_STMT:
@ -8101,6 +8301,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
return jsonToLogicIndefRowsFuncNode(pJson, pObj);
case QUERY_NODE_LOGIC_PLAN_INTERP_FUNC:
return jsonToLogicInterpFuncNode(pJson, pObj);
case QUERY_NODE_LOGIC_PLAN_FORECAST_FUNC:
return jsonToLogicForecastFuncNode(pJson, pObj);
case QUERY_NODE_LOGIC_PLAN_GROUP_CACHE:
return jsonToLogicGroupCacheNode(pJson, pObj);
case QUERY_NODE_LOGIC_PLAN_DYN_QUERY_CTRL:
@ -8161,6 +8363,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
case QUERY_NODE_PHYSICAL_PLAN_MERGE_COUNT:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT:
return jsonToPhysiCountWindowNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_MERGE_ANOMALY:
return jsonToPhysiAnomalyWindowNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_PARTITION:
return jsonToPhysiPartitionNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION:
@ -8169,6 +8373,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
return jsonToPhysiIndefRowsFuncNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC:
return jsonToPhysiInterpFuncNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_FORECAST_FUNC:
return jsonToPhysiForecastFuncNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_DISPATCH:
return jsonToPhysiDispatchNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT:

96
source/libs/nodes/src/nodesMsgFuncs.c
View File

@ -3539,6 +3539,46 @@ static int32_t msgToPhysiCountWindowNode(STlvDecoder* pDecoder, void* pObj) {
return code;
}
enum { PHY_ANOMALY_CODE_WINDOW = 1, PHY_ANOMALY_CODE_KEY, PHY_ANOMALY_CODE_WINDOW_OPTION };
static int32_t physiAnomalyWindowNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
const SAnomalyWindowPhysiNode* pNode = (const SAnomalyWindowPhysiNode*)pObj;
int32_t code = tlvEncodeObj(pEncoder, PHY_ANOMALY_CODE_WINDOW, physiWindowNodeToMsg, &pNode->window);
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_ANOMALY_CODE_KEY, nodeToMsg, pNode->pAnomalyKey);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeCStr(pEncoder, PHY_ANOMALY_CODE_WINDOW_OPTION, pNode->anomalyOpt);
}
return code;
}
static int32_t msgToPhysiAnomalyWindowNode(STlvDecoder* pDecoder, void* pObj) {
SAnomalyWindowPhysiNode* pNode = (SAnomalyWindowPhysiNode*)pObj;
int32_t code = TSDB_CODE_SUCCESS;
STlv* pTlv = NULL;
tlvForEach(pDecoder, pTlv, code) {
switch (pTlv->type) {
case PHY_ANOMALY_CODE_WINDOW:
code = tlvDecodeObjFromTlv(pTlv, msgToPhysiWindowNode, &pNode->window);
break;
case PHY_ANOMALY_CODE_KEY:
code = msgToNodeFromTlv(pTlv, (void**)&pNode->pAnomalyKey);
break;
case PHY_ANOMALY_CODE_WINDOW_OPTION:
code = tlvDecodeCStr(pTlv, pNode->anomalyOpt, sizeof(pNode->anomalyOpt));
break;
default:
break;
}
}
return code;
}
enum {
PHY_PARTITION_CODE_BASE_NODE = 1,
PHY_PARTITION_CODE_EXPR,
@ -3770,6 +3810,50 @@ static int32_t msgToPhysiInterpFuncNode(STlvDecoder* pDecoder, void* pObj) {
return code;
}
enum {
PHY_FORECAST_FUNC_CODE_BASE_NODE = 1,
PHY_FORECAST_FUNC_CODE_EXPR,
PHY_FORECAST_FUNC_CODE_FUNCS,
};
static int32_t physiForecastFuncNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
const SForecastFuncPhysiNode* pNode = (const SForecastFuncPhysiNode*)pObj;
int32_t code = tlvEncodeObj(pEncoder, PHY_FORECAST_FUNC_CODE_BASE_NODE, physiNodeToMsg, &pNode->node);
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_FORECAST_FUNC_CODE_EXPR, nodeListToMsg, pNode->pExprs);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_FORECAST_FUNC_CODE_FUNCS, nodeListToMsg, pNode->pFuncs);
}
return code;
}
static int32_t msgToPhysiForecastFuncNode(STlvDecoder* pDecoder, void* pObj) {
SForecastFuncPhysiNode* pNode = (SForecastFuncPhysiNode*)pObj;
int32_t code = TSDB_CODE_SUCCESS;
STlv* pTlv = NULL;
tlvForEach(pDecoder, pTlv, code) {
switch (pTlv->type) {
case PHY_FORECAST_FUNC_CODE_BASE_NODE:
code = tlvDecodeObjFromTlv(pTlv, msgToPhysiNode, &pNode->node);
break;
case PHY_FORECAST_FUNC_CODE_EXPR:
code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pExprs);
break;
case PHY_FORECAST_FUNC_CODE_FUNCS:
code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pFuncs);
break;
default:
break;
}
}
return code;
}
enum { PHY_DATA_SINK_CODE_INPUT_DESC = 1 };
static int32_t physicDataSinkNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
@ -4536,6 +4620,9 @@ static int32_t specificNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
case QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT:
code = physiCountWindowNodeToMsg(pObj, pEncoder);
break;
case QUERY_NODE_PHYSICAL_PLAN_MERGE_ANOMALY:
code = physiAnomalyWindowNodeToMsg(pObj, pEncoder);
break;
case QUERY_NODE_PHYSICAL_PLAN_PARTITION:
code = physiPartitionNodeToMsg(pObj, pEncoder);
break;
@ -4548,6 +4635,9 @@ static int32_t specificNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
case QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC:
code = physiInterpFuncNodeToMsg(pObj, pEncoder);
break;
case QUERY_NODE_PHYSICAL_PLAN_FORECAST_FUNC:
code = physiForecastFuncNodeToMsg(pObj, pEncoder);
break;
case QUERY_NODE_PHYSICAL_PLAN_DISPATCH:
code = physiDispatchNodeToMsg(pObj, pEncoder);
break;
@ -4698,6 +4788,9 @@ static int32_t msgToSpecificNode(STlvDecoder* pDecoder, void* pObj) {
case QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT:
code = msgToPhysiCountWindowNode(pDecoder, pObj);
break;
case QUERY_NODE_PHYSICAL_PLAN_MERGE_ANOMALY:
code = msgToPhysiAnomalyWindowNode(pDecoder, pObj);
break;
case QUERY_NODE_PHYSICAL_PLAN_PARTITION:
code = msgToPhysiPartitionNode(pDecoder, pObj);
break;
@ -4710,6 +4803,9 @@ static int32_t msgToSpecificNode(STlvDecoder* pDecoder, void* pObj) {
case QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC:
code = msgToPhysiInterpFuncNode(pDecoder, pObj);
break;
case QUERY_NODE_PHYSICAL_PLAN_FORECAST_FUNC:
code = msgToPhysiForecastFuncNode(pDecoder, pObj);
break;
case QUERY_NODE_PHYSICAL_PLAN_DISPATCH:
code = msgToPhysiDispatchNode(pDecoder, pObj);
break;

16
source/libs/nodes/src/nodesTraverseFuncs.c
View File

@ -181,6 +181,14 @@ static EDealRes dispatchExpr(SNode* pNode, ETraversalOrder order, FNodeWalker wa
res = walkExpr(pEvent->pCol, order, walker, pContext);
break;
}
case QUERY_NODE_ANOMALY_WINDOW: {
SAnomalyWindowNode* pAnomaly = (SAnomalyWindowNode*)pNode;
res = walkExpr(pAnomaly->pExpr, order, walker, pContext);
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkExpr(pAnomaly->pCol, order, walker, pContext);
}
break;
}
default:
break;
}
@ -392,6 +400,14 @@ static EDealRes rewriteExpr(SNode** pRawNode, ETraversalOrder order, FNodeRewrit
res = rewriteExpr(&pEvent->pCol, order, rewriter, pContext);
break;
}
case QUERY_NODE_ANOMALY_WINDOW: {
SAnomalyWindowNode* pAnomaly = (SAnomalyWindowNode*)pNode;
res = rewriteExpr(&pAnomaly->pExpr, order, rewriter, pContext);
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = rewriteExpr(&pAnomaly->pCol, order, rewriter, pContext);
}
break;
}
default:
break;
}

44
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -419,6 +419,8 @@ int32_t nodesMakeNode(ENodeType type, SNode** ppNodeOut) {
code = makeNode(type, sizeof(SEventWindowNode), &pNode); break;
case QUERY_NODE_COUNT_WINDOW:
code = makeNode(type, sizeof(SCountWindowNode), &pNode); break;
case QUERY_NODE_ANOMALY_WINDOW:
code = makeNode(type, sizeof(SAnomalyWindowNode), &pNode); break;
case QUERY_NODE_HINT:
code = makeNode(type, sizeof(SHintNode), &pNode); break;
case QUERY_NODE_VIEW:
@ -474,6 +476,12 @@ int32_t nodesMakeNode(ENodeType type, SNode** ppNodeOut) {
code = makeNode(type, sizeof(SDropDnodeStmt), &pNode); break;
case QUERY_NODE_ALTER_DNODE_STMT:
code = makeNode(type, sizeof(SAlterDnodeStmt), &pNode); break;
case QUERY_NODE_CREATE_ANODE_STMT:
code = makeNode(type, sizeof(SCreateAnodeStmt), &pNode); break;
case QUERY_NODE_DROP_ANODE_STMT:
code = makeNode(type, sizeof(SDropAnodeStmt), &pNode); break;
case QUERY_NODE_UPDATE_ANODE_STMT:
code = makeNode(type, sizeof(SUpdateAnodeStmt), &pNode); break;
case QUERY_NODE_CREATE_INDEX_STMT:
code = makeNode(type, sizeof(SCreateIndexStmt), &pNode); break;
case QUERY_NODE_DROP_INDEX_STMT:
@ -540,6 +548,8 @@ int32_t nodesMakeNode(ENodeType type, SNode** ppNodeOut) {
case QUERY_NODE_SHOW_MNODES_STMT:
case QUERY_NODE_SHOW_MODULES_STMT:
case QUERY_NODE_SHOW_QNODES_STMT:
case QUERY_NODE_SHOW_ANODES_STMT:
case QUERY_NODE_SHOW_ANODES_FULL_STMT:
case QUERY_NODE_SHOW_SNODES_STMT:
case QUERY_NODE_SHOW_BNODES_STMT:
case QUERY_NODE_SHOW_ARBGROUPS_STMT:
@ -647,6 +657,8 @@ int32_t nodesMakeNode(ENodeType type, SNode** ppNodeOut) {
code = makeNode(type, sizeof(SIndefRowsFuncLogicNode), &pNode); break;
case QUERY_NODE_LOGIC_PLAN_INTERP_FUNC:
code = makeNode(type, sizeof(SInterpFuncLogicNode), &pNode); break;
case QUERY_NODE_LOGIC_PLAN_FORECAST_FUNC:
code = makeNode(type, sizeof(SForecastFuncLogicNode), &pNode); break;
case QUERY_NODE_LOGIC_PLAN_GROUP_CACHE:
code = makeNode(type, sizeof(SGroupCacheLogicNode), &pNode); break;
case QUERY_NODE_LOGIC_PLAN_DYN_QUERY_CTRL:
@ -722,6 +734,8 @@ int32_t nodesMakeNode(ENodeType type, SNode** ppNodeOut) {
code = makeNode(type, sizeof(SStreamEventWinodwPhysiNode), &pNode); break;
case QUERY_NODE_PHYSICAL_PLAN_MERGE_COUNT:
code = makeNode(type, sizeof(SCountWinodwPhysiNode), &pNode); break;
case QUERY_NODE_PHYSICAL_PLAN_MERGE_ANOMALY:
code = makeNode(type, sizeof(SAnomalyWindowPhysiNode), &pNode); break;
case QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT:
code = makeNode(type, sizeof(SStreamCountWinodwPhysiNode), &pNode); break;
case QUERY_NODE_PHYSICAL_PLAN_PARTITION:
@ -732,6 +746,8 @@ int32_t nodesMakeNode(ENodeType type, SNode** ppNodeOut) {
code = makeNode(type, sizeof(SIndefRowsFuncPhysiNode), &pNode); break;
case QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC:
code = makeNode(type, sizeof(SInterpFuncLogicNode), &pNode); break;
case QUERY_NODE_PHYSICAL_PLAN_FORECAST_FUNC:
code = makeNode(type, sizeof(SForecastFuncLogicNode), &pNode); break;
case QUERY_NODE_PHYSICAL_PLAN_DISPATCH:
code = makeNode(type, sizeof(SDataDispatcherNode), &pNode); break;
case QUERY_NODE_PHYSICAL_PLAN_INSERT:
@ -1019,6 +1035,11 @@ void nodesDestroyNode(SNode* pNode) {
nodesDestroyNode(pEvent->pCol);
break;
}
case QUERY_NODE_ANOMALY_WINDOW: {
SAnomalyWindowNode* pAnomaly = (SAnomalyWindowNode*)pNode;
nodesDestroyNode(pAnomaly->pCol);
break;
}
case QUERY_NODE_HINT: {
SHintNode* pHint = (SHintNode*)pNode;
destroyHintValue(pHint->option, pHint->value);
@ -1167,6 +1188,9 @@ void nodesDestroyNode(SNode* pNode) {
case QUERY_NODE_CREATE_DNODE_STMT: // no pointer field
case QUERY_NODE_DROP_DNODE_STMT: // no pointer field
case QUERY_NODE_ALTER_DNODE_STMT: // no pointer field
case QUERY_NODE_CREATE_ANODE_STMT: // no pointer field
case QUERY_NODE_UPDATE_ANODE_STMT: // no pointer field
case QUERY_NODE_DROP_ANODE_STMT: // no pointer field
break;
case QUERY_NODE_CREATE_INDEX_STMT: {
SCreateIndexStmt* pStmt = (SCreateIndexStmt*)pNode;
@ -1252,6 +1276,8 @@ void nodesDestroyNode(SNode* pNode) {
case QUERY_NODE_SHOW_MNODES_STMT:
case QUERY_NODE_SHOW_MODULES_STMT:
case QUERY_NODE_SHOW_QNODES_STMT:
case QUERY_NODE_SHOW_ANODES_STMT:
case QUERY_NODE_SHOW_ANODES_FULL_STMT:
case QUERY_NODE_SHOW_SNODES_STMT:
case QUERY_NODE_SHOW_BNODES_STMT:
case QUERY_NODE_SHOW_ARBGROUPS_STMT:
@ -1500,6 +1526,12 @@ void nodesDestroyNode(SNode* pNode) {
nodesDestroyNode(pLogicNode->pTimeSeries);
break;
}
case QUERY_NODE_LOGIC_PLAN_FORECAST_FUNC: {
SForecastFuncLogicNode* pLogicNode = (SForecastFuncLogicNode*)pNode;
destroyLogicNode((SLogicNode*)pLogicNode);
nodesDestroyList(pLogicNode->pFuncs);
break;
}
case QUERY_NODE_LOGIC_PLAN_GROUP_CACHE: {
SGroupCacheLogicNode* pLogicNode = (SGroupCacheLogicNode*)pNode;
destroyLogicNode((SLogicNode*)pLogicNode);
@ -1663,6 +1695,11 @@ void nodesDestroyNode(SNode* pNode) {
destroyWinodwPhysiNode((SWindowPhysiNode*)pPhyNode);
break;
}
case QUERY_NODE_PHYSICAL_PLAN_MERGE_ANOMALY: {
SAnomalyWindowPhysiNode* pPhyNode = (SAnomalyWindowPhysiNode*)pNode;
destroyWinodwPhysiNode((SWindowPhysiNode*)pPhyNode);
break;
}
case QUERY_NODE_PHYSICAL_PLAN_PARTITION: {
destroyPartitionPhysiNode((SPartitionPhysiNode*)pNode);
break;
@ -1690,6 +1727,13 @@ void nodesDestroyNode(SNode* pNode) {
nodesDestroyNode(pPhyNode->pTimeSeries);
break;
}
case QUERY_NODE_PHYSICAL_PLAN_FORECAST_FUNC: {
SForecastFuncPhysiNode* pPhyNode = (SForecastFuncPhysiNode*)pNode;
destroyPhysiNode((SPhysiNode*)pPhyNode);
nodesDestroyList(pPhyNode->pExprs);
nodesDestroyList(pPhyNode->pFuncs);
break;
}
case QUERY_NODE_PHYSICAL_PLAN_DISPATCH:
destroyDataSinkNode((SDataSinkNode*)pNode);
break;

4
source/libs/parser/inc/parAst.h
View File

@ -154,6 +154,7 @@ SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, SNode*
SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pExpr);
SNode* createEventWindowNode(SAstCreateContext* pCxt, SNode* pStartCond, SNode* pEndCond);
SNode* createCountWindowNode(SAstCreateContext* pCxt, const SToken* pCountToken, const SToken* pSlidingToken);
SNode* createAnomalyWindowNode(SAstCreateContext* pCxt, SNode* pExpr, const SToken* pFuncOpt);
SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding,
SNode* pFill);
SNode* createWindowOffsetNode(SAstCreateContext* pCxt, SNode* pStartOffset, SNode* pEndOffset);
@ -251,6 +252,9 @@ SNode* createDropUserStmt(SAstCreateContext* pCxt, SToken* pUserName);
SNode* createCreateDnodeStmt(SAstCreateContext* pCxt, const SToken* pFqdn, const SToken* pPort);
SNode* createDropDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, bool force, bool unsafe);
SNode* createAlterDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, const SToken* pConfig, const SToken* pValue);
SNode* createCreateAnodeStmt(SAstCreateContext* pCxt, const SToken* pUrl);
SNode* createDropAnodeStmt(SAstCreateContext* pCxt, const SToken* pAnode);
SNode* createUpdateAnodeStmt(SAstCreateContext* pCxt, const SToken* pAnode, bool updateAll);
SNode* createEncryptKeyStmt(SAstCreateContext* pCxt, const SToken* pValue);
SNode* createRealTableNodeForIndexName(SAstCreateContext* pCxt, SToken* pDbName, SToken* pIndexName);
SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, bool ignoreExists, SNode* pIndexName,

15
source/libs/parser/inc/sql.y
View File

@ -157,6 +157,12 @@ with_clause_opt(A) ::= WITH search_condition(B).
/************************************************ create encrypt_key *********************************************/
cmd ::= CREATE ENCRYPT_KEY NK_STRING(A). { pCxt->pRootNode = createEncryptKeyStmt(pCxt, &A); }
/************************************************ create drop update anode ***************************************/
cmd ::= CREATE ANODE NK_STRING(A). { pCxt->pRootNode = createCreateAnodeStmt(pCxt, &A); }
cmd ::= UPDATE ANODE NK_INTEGER(A). { pCxt->pRootNode = createUpdateAnodeStmt(pCxt, &A, false); }
cmd ::= UPDATE ALL ANODES. { pCxt->pRootNode = createUpdateAnodeStmt(pCxt, NULL, true); }
cmd ::= DROP ANODE NK_INTEGER(A). { pCxt->pRootNode = createDropAnodeStmt(pCxt, &A); }
/************************************************ create/drop/alter/restore dnode *********************************************/
cmd ::= CREATE DNODE dnode_endpoint(A). { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &A, NULL); }
cmd ::= CREATE DNODE dnode_endpoint(A) PORT NK_INTEGER(B). { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &A, &B); }
@ -524,6 +530,8 @@ cmd ::= SHOW db_name_cond_opt(A) VGROUPS.
cmd ::= SHOW MNODES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_MNODES_STMT); }
//cmd ::= SHOW MODULES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_MODULES_STMT); }
cmd ::= SHOW QNODES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QNODES_STMT); }
cmd ::= SHOW ANODES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_ANODES_STMT); }
cmd ::= SHOW ANODES FULL. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_ANODES_FULL_STMT); }
cmd ::= SHOW ARBGROUPS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_ARBGROUPS_STMT); }
cmd ::= SHOW FUNCTIONS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_FUNCTIONS_STMT); }
cmd ::= SHOW INDEXES FROM table_name_cond(A) from_db_opt(B). { pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, B, A, OP_TYPE_EQUAL); }
@ -1187,6 +1195,9 @@ pseudo_column(A) ::= WDURATION(B).
pseudo_column(A) ::= IROWTS(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }
pseudo_column(A) ::= ISFILLED(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }
pseudo_column(A) ::= QTAGS(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }
pseudo_column(A) ::= FLOW(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }
pseudo_column(A) ::= FHIGH(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }
pseudo_column(A) ::= FROWTS(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }
function_expression(A) ::= function_name(B) NK_LP expression_list(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, C)); }
function_expression(A) ::= star_func(B) NK_LP star_func_para_list(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, C)); }
@ -1505,6 +1516,10 @@ twindow_clause_opt(A) ::=
COUNT_WINDOW NK_LP NK_INTEGER(B) NK_RP. { A = createCountWindowNode(pCxt, &B, &B); }
twindow_clause_opt(A) ::=
COUNT_WINDOW NK_LP NK_INTEGER(B) NK_COMMA NK_INTEGER(C) NK_RP. { A = createCountWindowNode(pCxt, &B, &C); }
twindow_clause_opt(A) ::=
ANOMALY_WINDOW NK_LP expr_or_subquery(B) NK_RP. { A = createAnomalyWindowNode(pCxt, releaseRawExprNode(pCxt, B), NULL); }
twindow_clause_opt(A) ::=
ANOMALY_WINDOW NK_LP expr_or_subquery(B) NK_COMMA NK_STRING(C) NK_RP. { A = createAnomalyWindowNode(pCxt, releaseRawExprNode(pCxt, B), &C); }
sliding_opt(A) ::= . { A = NULL; }
sliding_opt(A) ::= SLIDING NK_LP interval_sliding_duration_literal(B) NK_RP. { A = releaseRawExprNode(pCxt, B); }

Some files were not shown because too many files have changed in this diff Show More