diff --git a/cmake/cmake.define b/cmake/cmake.define index 9fae397363..7bcd400fa6 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -169,11 +169,48 @@ ELSE () SET(COMPILER_SUPPORT_AVX512VL false) ELSE() CHECK_C_COMPILER_FLAG("-mfma" COMPILER_SUPPORT_FMA) - CHECK_C_COMPILER_FLAG("-mavx" COMPILER_SUPPORT_AVX) - CHECK_C_COMPILER_FLAG("-mavx2" COMPILER_SUPPORT_AVX2) CHECK_C_COMPILER_FLAG("-mavx512f" COMPILER_SUPPORT_AVX512F) CHECK_C_COMPILER_FLAG("-mavx512vbmi" COMPILER_SUPPORT_AVX512BMI) CHECK_C_COMPILER_FLAG("-mavx512vl" COMPILER_SUPPORT_AVX512VL) + + INCLUDE(CheckCSourceRuns) + SET(CMAKE_REQUIRED_FLAGS "-mavx") + check_c_source_runs(" + #include + int main() { + __m256d a, b, c; + double buf[4] = {0}; + a = _mm256_loadu_pd(buf); + b = _mm256_loadu_pd(buf); + c = _mm256_add_pd(a, b); + _mm256_storeu_pd(buf, c); + for (int i = 0; i < sizeof(buf) / sizeof(buf[0]); ++i) { + if (buf[i] != 0) { + return 1; + } + } + return 0; + } + " COMPILER_SUPPORT_AVX) + + SET(CMAKE_REQUIRED_FLAGS "-mavx2") + check_c_source_runs(" + #include + int main() { + __m256i a, b, c; + int buf[8] = {0}; + a = _mm256_loadu_si256((__m256i *)buf); + b = _mm256_loadu_si256((__m256i *)buf); + c = _mm256_and_si256(a, b); + _mm256_storeu_si256((__m256i *)buf, c); + for (int i = 0; i < sizeof(buf) / sizeof(buf[0]); ++i) { + if (buf[i] != 0) { + return 1; + } + } + return 0; + } + " COMPILER_SUPPORT_AVX2) ENDIF() IF (COMPILER_SUPPORT_SSE42) diff --git a/cmake/stub_CMakeLists.txt.in b/cmake/stub_CMakeLists.txt.in index 4c7a2fb002..a3ded8bbaa 100644 --- a/cmake/stub_CMakeLists.txt.in +++ b/cmake/stub_CMakeLists.txt.in @@ -9,6 +9,5 @@ ExternalProject_Add(stub CONFIGURE_COMMAND "" BUILD_COMMAND "" INSTALL_COMMAND "" - TEST_COMMAND "" - GIT_SHALLOW true + TEST_COMMAND "" ) diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index ef6ed4af1d..13826a1a74 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG 3.0 + GIT_TAG main SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 9a6a5329ae..9bbda8309f 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 3.0 + GIT_TAG main SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/docs/examples/JDBC/JDBCDemo/pom.xml b/docs/examples/JDBC/JDBCDemo/pom.xml index 315b147cce..4b3e1ab675 100644 --- a/docs/examples/JDBC/JDBCDemo/pom.xml +++ b/docs/examples/JDBC/JDBCDemo/pom.xml @@ -19,7 +19,7 @@ com.taosdata.jdbc taos-jdbcdriver - 3.3.3 + 3.4.0 org.locationtech.jts diff --git a/docs/examples/JDBC/SpringJdbcTemplate/readme.md b/docs/examples/JDBC/SpringJdbcTemplate/readme.md index f59bcdbeb5..0e9812385a 100644 --- a/docs/examples/JDBC/SpringJdbcTemplate/readme.md +++ b/docs/examples/JDBC/SpringJdbcTemplate/readme.md @@ -1,11 +1,11 @@ ## TDengine Spring JDBC Template Demo -`Spring JDBC Template` 简化了原生 JDBC Connection 获取释放等操作，使得操作数据库更加方便。 +`Spring JDBC Template` simplifies the operations of acquiring and releasing native JDBC Connections, making database operations more convenient. -### 配置 +### Configuration -修改 `src/main/resources/applicationContext.xml` 文件中 TDengine 的配置信息： +Modify the TDengine configuration in the `src/main/resources/applicationContext.xml` file: ```xml @@ -20,13 +20,15 @@ ``` -### 打包运行 +### Package and run + +Navigate to the `TDengine/tests/examples/JDBC/SpringJdbcTemplate` directory and execute the following commands to generate an executable jar file. -进入 `TDengine/tests/examples/JDBC/SpringJdbcTemplate` 目录下，执行以下命令可以生成可执行 jar 包。 ```shell mvn clean package ``` -打包成功之后，进入 `target/` 目录下，执行以下命令就可运行测试： +After successfully packaging, navigate to the `target/` directory and execute the following commands to run the tests: + ```shell java -jar target/SpringJdbcTemplate-1.0-SNAPSHOT-jar-with-dependencies.jar -``` \ No newline at end of file +``` diff --git a/docs/examples/JDBC/connectionPools/pom.xml b/docs/examples/JDBC/connectionPools/pom.xml index 1791bfe8bc..e3ef30d2f8 100644 --- a/docs/examples/JDBC/connectionPools/pom.xml +++ b/docs/examples/JDBC/connectionPools/pom.xml @@ -18,7 +18,7 @@ com.taosdata.jdbc taos-jdbcdriver - 3.3.3 + 3.4.0 diff --git a/docs/examples/JDBC/connectionPools/src/main/resources/log4j.properties b/docs/examples/JDBC/connectionPools/src/main/resources/log4j.properties index 1299357be3..a7f4d3d492 100644 --- a/docs/examples/JDBC/connectionPools/src/main/resources/log4j.properties +++ b/docs/examples/JDBC/connectionPools/src/main/resources/log4j.properties @@ -1,21 +1,21 @@ -### 设置### +### Settings### log4j.rootLogger=debug,stdout,DebugLog,ErrorLog -### 输出信息到控制抬 ### +### Output information to the console ### log4j.appender.stdout=org.apache.log4j.ConsoleAppender log4j.appender.stdout.Target=System.out log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=[%-5p] %d{yyyy-MM-dd HH:mm:ss,SSS} method:%l%n%m%n -### 输出DEBUG 级别以上的日志到=logs/debug.log +### Output logs of DEBUG level and above to logs/debug.log log4j.appender.DebugLog=org.apache.log4j.DailyRollingFileAppender log4j.appender.DebugLog.File=logs/debug.log log4j.appender.DebugLog.Append=true log4j.appender.DebugLog.Threshold=DEBUG log4j.appender.DebugLog.layout=org.apache.log4j.PatternLayout log4j.appender.DebugLog.layout.ConversionPattern=%-d{yyyy-MM-dd HH:mm:ss} [ %t:%r ] - [ %p ] %m%n -### 输出ERROR 级别以上的日志到=logs/error.log +### Output logs of ERROR level and above to logs/error.log log4j.appender.ErrorLog=org.apache.log4j.DailyRollingFileAppender log4j.appender.ErrorLog.File=logs/error.log log4j.appender.ErrorLog.Append=true log4j.appender.ErrorLog.Threshold=ERROR log4j.appender.ErrorLog.layout=org.apache.log4j.PatternLayout -log4j.appender.ErrorLog.layout.ConversionPattern=%-d{yyyy-MM-dd HH:mm:ss} [ %t:%r ] - [ %p ] %m%n \ No newline at end of file +log4j.appender.ErrorLog.layout.ConversionPattern=%-d{yyyy-MM-dd HH:mm:ss} [ %t:%r ] - [ %p ] %m%n diff --git a/docs/examples/JDBC/connectionPools/src/main/resources/proxool.xml b/docs/examples/JDBC/connectionPools/src/main/resources/proxool.xml index 0e2ac6368a..dcef51ffb5 100644 --- a/docs/examples/JDBC/connectionPools/src/main/resources/proxool.xml +++ b/docs/examples/JDBC/connectionPools/src/main/resources/proxool.xml @@ -1,27 +1,28 @@ + ds - + jdbc:TAOS-RS://127.0.0.1:6041/log - + com.taosdata.jdbc.rs.RestfulDriver - + - + 100 - + 100 - + 1 - + 5 - + 30000 - + select server_version() - \ No newline at end of file + diff --git a/docs/examples/JDBC/consumer-demo/pom.xml b/docs/examples/JDBC/consumer-demo/pom.xml index dcabfc1249..709f87d9c1 100644 --- a/docs/examples/JDBC/consumer-demo/pom.xml +++ b/docs/examples/JDBC/consumer-demo/pom.xml @@ -17,7 +17,7 @@ com.taosdata.jdbc taos-jdbcdriver - 3.3.3 + 3.4.0 com.google.guava diff --git a/docs/examples/JDBC/consumer-demo/src/main/java/com/taosdata/Worker.java b/docs/examples/JDBC/consumer-demo/src/main/java/com/taosdata/Worker.java index f6e21cd729..6823ca5b4d 100644 --- a/docs/examples/JDBC/consumer-demo/src/main/java/com/taosdata/Worker.java +++ b/docs/examples/JDBC/consumer-demo/src/main/java/com/taosdata/Worker.java @@ -35,17 +35,18 @@ public class Worker implements Runnable { public void run() { while (!Thread.interrupted()) { try { - // 控制请求频率 + // Control request rate if (semaphore.tryAcquire()) { ConsumerRecords records = consumer.poll(Duration.ofMillis(sleepTime)); pool.submit(() -> { RateLimiter limiter = RateLimiter.create(rate); try { for (ConsumerRecord record : records) { - // 流量控制 + // Traffic control limiter.acquire(); - // 业务处理数据 - System.out.println("[" + LocalDateTime.now() + "] Thread id:" + Thread.currentThread().getId() + " -> " + record.value()); + // Business data processing + System.out.println("[" + LocalDateTime.now() + "] Thread id:" + + Thread.currentThread().getId() + " -> " + record.value()); } } finally { semaphore.release(); diff --git a/docs/examples/JDBC/mybatisplus-demo/readme b/docs/examples/JDBC/mybatisplus-demo/readme index b31b6c34bf..a4816d7631 100644 --- a/docs/examples/JDBC/mybatisplus-demo/readme +++ b/docs/examples/JDBC/mybatisplus-demo/readme @@ -1,14 +1,14 @@ -# 使用说明 +# Instructions -## 创建使用db +## Create and use the database ```shell $ taos > create database mp_test ``` -## 执行测试用例 +## Execute test cases ```shell $ mvn clean test -``` \ No newline at end of file +``` diff --git a/docs/examples/JDBC/springbootdemo/readme.md b/docs/examples/JDBC/springbootdemo/readme.md index 625d43e4ed..d11bb33c83 100644 --- a/docs/examples/JDBC/springbootdemo/readme.md +++ b/docs/examples/JDBC/springbootdemo/readme.md @@ -1,6 +1,6 @@ ## TDengine SpringBoot + Mybatis Demo -## 需要提前创建 test 数据库 +## Need to create a test database in advance ``` $ taos -s 'create database if not exists test' @@ -8,7 +8,7 @@ $ taos -s 'create database if not exists test' $ curl http://localhost:8080/weather/init ``` -### 配置 application.properties +### Configure application.properties ```properties # datasource config spring.datasource.driver-class-name=com.taosdata.jdbc.TSDBDriver @@ -38,9 +38,9 @@ mybatis.mapper-locations=classpath:mapper/*.xml logging.level.com.taosdata.jdbc.springbootdemo.dao=debug ``` -### 主要功能 +### Main functions -* 创建数据库和表 +* Create databases and tables ```xml @@ -52,14 +52,14 @@ logging.level.com.taosdata.jdbc.springbootdemo.dao=debug ``` -* 插入单条记录 +* Insert a single record ```xml insert into test.weather (ts, temperature, humidity) values (now, #{temperature,jdbcType=INTEGER}, #{humidity,jdbcType=FLOAT}) ``` -* 插入多条记录 +* Insert multiple records ```xml @@ -69,7 +69,7 @@ logging.level.com.taosdata.jdbc.springbootdemo.dao=debug ``` -* 分页查询 +* Pagination query ```xml diff --git a/docs/examples/JDBC/taosdemo/pom.xml b/docs/examples/JDBC/taosdemo/pom.xml index ffe159ea49..ab5912aa9e 100644 --- a/docs/examples/JDBC/taosdemo/pom.xml +++ b/docs/examples/JDBC/taosdemo/pom.xml @@ -67,7 +67,7 @@ com.taosdata.jdbc taos-jdbcdriver - 3.3.3 + 3.4.0 diff --git a/docs/examples/JDBC/taosdemo/readme.md b/docs/examples/JDBC/taosdemo/readme.md index 986eef8a05..141391d1f6 100644 --- a/docs/examples/JDBC/taosdemo/readme.md +++ b/docs/examples/JDBC/taosdemo/readme.md @@ -1,11 +1,14 @@ ``` cd tests/examples/JDBC/taosdemo mvn clean package -Dmaven.test.skip=true -# 先建表，再插入的 +# Create tables first, then insert data java -jar target/taosdemo-2.0.1-jar-with-dependencies.jar -host -database -doCreateTable true -superTableSQL "create table weather(ts timestamp, f1 int) tags(t1 nchar(4))" -numOfTables 1000 -numOfRowsPerTable 100000000 -numOfThreadsForInsert 10 -numOfTablesPerSQL 10 -numOfValuesPerSQL 100 -# 不建表，直接插入的 +# Insert data directly without creating tables java -jar target/taosdemo-2.0.1-jar-with-dependencies.jar -host -database -doCreateTable false -superTableSQL "create table weather(ts timestamp, f1 int) tags(t1 nchar(4))" -numOfTables 1000 -numOfRowsPerTable 100000000 -numOfThreadsForInsert 10 -numOfTablesPerSQL 10 -numOfValuesPerSQL 100 ``` 如果发生错误 Exception in thread "main" java.lang.UnsatisfiedLinkError: no taos in java.library.path 请检查是否安装 TDengine 客户端安装包或编译 TDengine 安装。如果确定已经安装过还出现这个错误，可以在命令行 java 后加 -Djava.library.path=/usr/lib 来指定寻找共享库的路径。 + + +If you encounter the error Exception in thread "main" `java.lang.UnsatisfiedLinkError: no taos in java.library.path`, please check whether the TDengine client package is installed or TDengine is compiled and installed. If you are sure it is installed and still encounter this error, you can add `-Djava.library.path=/usr/lib` after the `java` command to specify the path to the shared library. diff --git a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/TaosDemoApplication.java b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/TaosDemoApplication.java index 6854054703..40d44475b8 100644 --- a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/TaosDemoApplication.java +++ b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/TaosDemoApplication.java @@ -24,14 +24,14 @@ public class TaosDemoApplication { private static final Logger logger = LogManager.getLogger(TaosDemoApplication.class); public static void main(String[] args) throws IOException { - // 读配置参数 + // Read configuration parameters JdbcTaosdemoConfig config = new JdbcTaosdemoConfig(args); boolean isHelp = Arrays.asList(args).contains("--help"); if (isHelp || config.host == null || config.host.isEmpty()) { JdbcTaosdemoConfig.printHelp(); System.exit(0); } - // 初始化 + // final DataSource dataSource = DataSourceFactory.getInstance(config.host, config.port, config.user, config.password); if (config.executeSql != null && !config.executeSql.isEmpty() @@ -50,7 +50,7 @@ public class TaosDemoApplication { final SuperTableService superTableService = new SuperTableService(dataSource); final SubTableService subTableService = new SubTableService(dataSource); - // 创建数据库 + // create database long start = System.currentTimeMillis(); Map databaseParam = new HashMap<>(); databaseParam.put("database", config.database); @@ -81,13 +81,13 @@ public class TaosDemoApplication { config.prefixOfFields, config.numOfTags, config.prefixOfTags); } /**********************************************************************************/ - // 建表 + // create table start = System.currentTimeMillis(); if (config.doCreateTable) { superTableService.drop(superTableMeta.getDatabase(), superTableMeta.getName()); superTableService.create(superTableMeta); if (!config.autoCreateTable) { - // 批量建子表 + // create sub tables in batch subTableService.createSubTable(superTableMeta, config.numOfTables, config.prefixOfTable, config.numOfThreadsForCreate); } @@ -95,7 +95,7 @@ public class TaosDemoApplication { end = System.currentTimeMillis(); logger.info(">>> create table time cost : " + (end - start) + " ms."); /**********************************************************************************/ - // 插入 + // insert data long tableSize = config.numOfTables; int threadSize = config.numOfThreadsForInsert; long startTime = getProperStartTime(config.startTime, config.days); @@ -111,10 +111,9 @@ public class TaosDemoApplication { end = System.currentTimeMillis(); logger.info("insert " + affectedRows + " rows, time cost: " + (end - start) + " ms"); /**********************************************************************************/ - // 查询 /**********************************************************************************/ - // 删除表 + // drop table if (config.dropTable) { superTableService.drop(config.database, config.superTable); } diff --git a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SubTableMapper.java b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SubTableMapper.java index e0ddd220c1..013d24eb87 100644 --- a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SubTableMapper.java +++ b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SubTableMapper.java @@ -9,21 +9,22 @@ import java.util.List; @Repository public interface SubTableMapper { - // 创建：子表 + // Create: SubTable void createUsingSuperTable(SubTableMeta subTableMeta); - // 插入：一张子表多个values + // Insert: Multiple records into one SubTable int insertOneTableMultiValues(SubTableValue subTableValue); - // 插入：一张子表多个values, 自动建表 + // Insert: Multiple records into one SubTable, auto create SubTables int insertOneTableMultiValuesUsingSuperTable(SubTableValue subTableValue); - // 插入：多张表多个values + // Insert: Multiple records into multiple SubTable int insertMultiTableMultiValues(List tables); - // 插入：多张表多个values，自动建表 + // Insert: Multiple records into multiple SubTable, auto create SubTables int insertMultiTableMultiValuesUsingSuperTable(List tables); - // + // -} \ No newline at end of file +} diff --git a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SuperTableMapper.java b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SuperTableMapper.java index 9f8cec9e8f..15cafd04fb 100644 --- a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SuperTableMapper.java +++ b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SuperTableMapper.java @@ -6,24 +6,26 @@ import org.springframework.stereotype.Repository; @Repository public interface SuperTableMapper { - // 创建超级表 create table if not exists xxx.xxx (f1 type1, f2 type2, ... ) tags( t1 type1, t2 type2 ...) + // Create super table: create table if not exists xxx.xxx (f1 type1, f2 type2, + // ... ) tags( t1 type1, t2 type2 ...) void createSuperTable(SuperTableMeta tableMetadata); - // 删除超级表 drop table if exists xxx; + // Drop super table: drop table if exists xxx; void dropSuperTable(String database, String name); - // + // - // + // - // + // - // + // - // + // - // - - // + // + // } diff --git a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/TableMapper.java b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/TableMapper.java index 32d1875e4d..c0f75d2204 100644 --- a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/TableMapper.java +++ b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/TableMapper.java @@ -9,19 +9,18 @@ import java.util.List; @Repository public interface TableMapper { - // 创建：普通表 + // Create: Normal table void create(TableMeta tableMeta); - // 插入：一张表多个value + // Insert: Multiple records into one table int insertOneTableMultiValues(TableValue values); - // 插入: 一张表多个value，指定的列 + // Insert: Multiple records into one table, specified columns int insertOneTableMultiValuesWithColumns(TableValue values); - // 插入：多个表多个value + // Insert: Multiple records into multiple tables int insertMultiTableMultiValues(List tables); - // 插入：多个表多个value, 指定的列 + // Insert: Multiple records into multiple tables, specified columns int insertMultiTableMultiValuesWithColumns(List tables); - -} \ No newline at end of file +} diff --git a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/DatabaseService.java b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/DatabaseService.java index 3c8e962406..68ddd78323 100644 --- a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/DatabaseService.java +++ b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/DatabaseService.java @@ -14,12 +14,12 @@ public class DatabaseService { this.databaseMapper = new DatabaseMapperImpl(dataSource); } - // 建库，指定 name + // Create database with specified name public void createDatabase(String database) { databaseMapper.createDatabase(database); } - // 建库，指定参数 keep,days,replica等 + // Create database with specified parameters such as keep, days, replica, etc. public void createDatabase(Map map) { if (map.isEmpty()) return; diff --git a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SubTableService.java b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SubTableService.java index b0a79dea78..690b02f065 100644 --- a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SubTableService.java +++ b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SubTableService.java @@ -27,7 +27,8 @@ public class SubTableService extends AbstractService { this.mapper = new SubTableMapperImpl(datasource); } - public void createSubTable(SuperTableMeta superTableMeta, long numOfTables, String prefixOfTable, int numOfThreadsForCreate) { + public void createSubTable(SuperTableMeta superTableMeta, long numOfTables, String prefixOfTable, + int numOfThreadsForCreate) { ExecutorService executor = Executors.newFixedThreadPool(numOfThreadsForCreate); for (long i = 0; i < numOfTables; i++) { long tableIndex = i; @@ -35,54 +36,58 @@ public class SubTableService extends AbstractService { } executor.shutdown(); try { - executor.awaitTermination(Long.MAX_VALUE,TimeUnit.NANOSECONDS); + executor.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); } catch (InterruptedException e) { e.printStackTrace(); } } public void createSubTable(SuperTableMeta superTableMeta, String tableName) { - // 构造数据 + // Construct data SubTableMeta meta = SubTableMetaGenerator.generate(superTableMeta, tableName); createSubTable(meta); } - // 创建一张子表，可以指定database，supertable，tablename，tag值 + // Create a sub-table, specifying database, super table, table name, and tag + // values public void createSubTable(SubTableMeta subTableMeta) { mapper.createUsingSuperTable(subTableMeta); } /*************************************************************************************************************************/ - // 插入：多线程，多表 + // Insert: Multi-threaded, multiple tables public int insert(List subTableValues, int threadSize, int frequency) { ExecutorService executor = Executors.newFixedThreadPool(threadSize); Future future = executor.submit(() -> insert(subTableValues)); executor.shutdown(); - //TODO：frequency + // TODO：frequency return getAffectRows(future); } - // 插入：单表，insert into xxx values(),()... + // Insert: Single table, insert into xxx values(),()... public int insert(SubTableValue subTableValue) { return mapper.insertOneTableMultiValues(subTableValue); } - // 插入: 多表，insert into xxx values(),()... xxx values(),()... + // Insert: Multiple tables, insert into xxx values(),()... xxx values(),()... public int insert(List subTableValues) { return mapper.insertMultiTableMultiValues(subTableValues); } - // 插入：单表，自动建表, insert into xxx using xxx tags(...) values(),()... + // Insert: Single table, auto-create table, insert into xxx using xxx tags(...) + // values(),()... public int insertAutoCreateTable(SubTableValue subTableValue) { return mapper.insertOneTableMultiValuesUsingSuperTable(subTableValue); } - // 插入：多表，自动建表, insert into xxx using XXX tags(...) values(),()... xxx using XXX tags(...) values(),()... + // Insert: Multiple tables, auto-create tables, insert into xxx using XXX + // tags(...) values(),()... xxx using XXX tags(...) values(),()... public int insertAutoCreateTable(List subTableValues) { return mapper.insertMultiTableMultiValuesUsingSuperTable(subTableValues); } - public int insertMultiThreads(SuperTableMeta superTableMeta, int threadSize, long tableSize, long startTime, long gap, JdbcTaosdemoConfig config) { + public int insertMultiThreads(SuperTableMeta superTableMeta, int threadSize, long tableSize, long startTime, + long gap, JdbcTaosdemoConfig config) { List taskList = new ArrayList<>(); List threads = IntStream.range(0, threadSize) .mapToObj(i -> { @@ -94,8 +99,7 @@ public class SubTableService extends AbstractService { startTime, config.timeGap, config.numOfRowsPerTable, config.numOfTablesPerSQL, config.numOfValuesPerSQL, config.order, config.rate, config.range, - config.prefixOfTable, config.autoCreateTable) - ); + config.prefixOfTable, config.autoCreateTable)); taskList.add(task); return new Thread(task, "InsertThread-" + i); }).collect(Collectors.toList()); @@ -126,7 +130,7 @@ public class SubTableService extends AbstractService { private class InsertTask implements Callable { private final long startTableInd; // included - private final long endTableInd; // excluded + private final long endTableInd; // excluded private final long startTime; private final long timeGap; private final long numOfRowsPerTable; @@ -140,10 +144,10 @@ public class SubTableService extends AbstractService { private final boolean autoCreateTable; public InsertTask(SuperTableMeta superTableMeta, long startTableInd, long endTableInd, - long startTime, long timeGap, - long numOfRowsPerTable, long numOfTablesPerSQL, long numOfValuesPerSQL, - int order, int rate, long range, - String prefixOfTable, boolean autoCreateTable) { + long startTime, long timeGap, + long numOfRowsPerTable, long numOfTablesPerSQL, long numOfValuesPerSQL, + int order, int rate, long range, + String prefixOfTable, boolean autoCreateTable) { this.superTableMeta = superTableMeta; this.startTableInd = startTableInd; this.endTableInd = endTableInd; @@ -159,7 +163,6 @@ public class SubTableService extends AbstractService { this.autoCreateTable = autoCreateTable; } - @Override public Integer call() { @@ -171,23 +174,27 @@ public class SubTableService extends AbstractService { int affectRows = 0; // row - for (long rowCnt = 0; rowCnt < numOfRowsPerTable; ) { + for (long rowCnt = 0; rowCnt < numOfRowsPerTable;) { long rowSize = numOfValuesPerSQL; if (rowCnt + rowSize > numOfRowsPerTable) { rowSize = numOfRowsPerTable - rowCnt; } - //table - for (long tableCnt = startTableInd; tableCnt < endTableInd; ) { + // table + for (long tableCnt = startTableInd; tableCnt < endTableInd;) { long tableSize = numOfTablesPerSQL; if (tableCnt + tableSize > endTableInd) { tableSize = endTableInd - tableCnt; } long startTime = this.startTime + rowCnt * timeGap; -// System.out.println(Thread.currentThread().getName() + " >>> " + "rowCnt: " + rowCnt + ", rowSize: " + rowSize + ", " + "tableCnt: " + tableCnt + ",tableSize: " + tableSize + ", " + "startTime: " + startTime + ",timeGap: " + timeGap + ""); + // System.out.println(Thread.currentThread().getName() + " >>> " + "rowCnt: " + + // rowCnt + ", rowSize: " + rowSize + ", " + "tableCnt: " + tableCnt + + // ",tableSize: " + tableSize + ", " + "startTime: " + startTime + ",timeGap: " + // + timeGap + ""); /***********************************************/ - // 生成数据 - List data = SubTableValueGenerator.generate(superTableMeta, prefixOfTable, tableCnt, tableSize, rowSize, startTime, timeGap); - // 乱序 + // Construct data + List data = SubTableValueGenerator.generate(superTableMeta, prefixOfTable, tableCnt, + tableSize, rowSize, startTime, timeGap); + // disorder if (order != 0) SubTableValueGenerator.disrupt(data, rate, range); // insert @@ -205,5 +212,4 @@ public class SubTableService extends AbstractService { } } - } diff --git a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SuperTableService.java b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SuperTableService.java index b91348e2d0..47798e0c4e 100644 --- a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SuperTableService.java +++ b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SuperTableService.java @@ -14,7 +14,7 @@ public class SuperTableService { this.superTableMapper = new SuperTableMapperImpl(dataSource); } - // 创建超级表，指定每个field的名称和类型，每个tag的名称和类型 + // Create super table, specifying the name and type of each field and each tag public void create(SuperTableMeta superTableMeta) { superTableMapper.createSuperTable(superTableMeta); } diff --git a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/TableService.java b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/TableService.java index 2504fdb0b4..2bfb963b4a 100644 --- a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/TableService.java +++ b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/TableService.java @@ -11,15 +11,14 @@ public class TableService extends AbstractService { private TableMapper tableMapper; - //创建一张表 + // Create a table public void create(TableMeta tableMeta) { tableMapper.create(tableMeta); } - //创建多张表 + // Create multiple tables public void create(List tables) { tables.stream().forEach(this::create); } - } diff --git a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/FieldValueGenerator.java b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/FieldValueGenerator.java index 73cd981a46..b3ce35231c 100644 --- a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/FieldValueGenerator.java +++ b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/FieldValueGenerator.java @@ -11,7 +11,8 @@ public class FieldValueGenerator { public static Random random = new Random(System.currentTimeMillis()); - // 生成start到end的时间序列，时间戳为顺序，不含有乱序，field的value为随机生成 + // Generate a time series from start to end, timestamps are in order without + // disorder, field values are randomly generated public static List generate(long start, long end, long timeGap, List fieldMetaList) { List values = new ArrayList<>(); @@ -29,9 +30,12 @@ public class FieldValueGenerator { return values; } - // 生成start到end的时间序列，时间戳为顺序，含有乱序，rate为乱序的比例，range为乱序前跳范围，field的value为随机生成 + // Generate a time series from start to end, timestamps are in order but include + // disorder, rate is the proportion of disorder, range is the jump range before + // disorder, field values are randomly generated public static List disrupt(List values, int rate, long range) { - long timeGap = (long) (values.get(1).getFields().get(0).getValue()) - (long) (values.get(0).getFields().get(0).getValue()); + long timeGap = (long) (values.get(1).getFields().get(0).getValue()) + - (long) (values.get(0).getFields().get(0).getValue()); int bugSize = values.size() * rate / 100; Set bugIndSet = new HashSet<>(); while (bugIndSet.size() < bugSize) { diff --git a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/SubTableMetaGenerator.java b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/SubTableMetaGenerator.java index 88e3c0d26a..7bdd72ec3b 100644 --- a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/SubTableMetaGenerator.java +++ b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/SubTableMetaGenerator.java @@ -9,7 +9,7 @@ import java.util.List; public class SubTableMetaGenerator { - // 创建tableSize张子表，使用tablePrefix作为子表名的前缀，使用superTableMeta的元数据 + // Create tableSize sub-tables, using tablePrefix as the prefix for sub-table names, and using the metadata from superTableMeta // create table xxx using XXX tags(XXX) public static List generate(SuperTableMeta superTableMeta, int tableSize, String tablePrefix) { List subTableMetaList = new ArrayList<>(); diff --git a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/SuperTableMetaGenerator.java b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/SuperTableMetaGenerator.java index 05aefd01ac..383b492c5a 100644 --- a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/SuperTableMetaGenerator.java +++ b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/SuperTableMetaGenerator.java @@ -10,10 +10,11 @@ import java.util.List; public class SuperTableMetaGenerator { - // 创建超级表，使用指定SQL语句 + // Create super table using the specified SQL statement public static SuperTableMeta generate(String superTableSQL) { SuperTableMeta tableMeta = new SuperTableMeta(); - // for example : create table superTable (ts timestamp, temperature float, humidity int) tags(location nchar(64), groupId int) + // for example : create table superTable (ts timestamp, temperature float, + // humidity int) tags(location nchar(64), groupId int) superTableSQL = superTableSQL.trim().toLowerCase(); if (!superTableSQL.startsWith("create")) throw new RuntimeException("invalid create super table SQL"); @@ -54,8 +55,9 @@ public class SuperTableMetaGenerator { return tableMeta; } - // 创建超级表,指定field和tag的个数 - public static SuperTableMeta generate(String database, String name, int fieldSize, String fieldPrefix, int tagSize, String tagPrefix) { + // Create super table with specified number of fields and tags + public static SuperTableMeta generate(String database, String name, int fieldSize, String fieldPrefix, int tagSize, + String tagPrefix) { if (fieldSize < 2 || tagSize < 1) { throw new RuntimeException("create super table but fieldSize less than 2 or tagSize less than 1"); } @@ -66,7 +68,8 @@ public class SuperTableMetaGenerator { List fields = new ArrayList<>(); fields.add(new FieldMeta("ts", "timestamp")); for (int i = 1; i <= fieldSize; i++) { - fields.add(new FieldMeta(fieldPrefix + "" + i, TaosConstants.DATA_TYPES[i % TaosConstants.DATA_TYPES.length])); + fields.add( + new FieldMeta(fieldPrefix + "" + i, TaosConstants.DATA_TYPES[i % TaosConstants.DATA_TYPES.length])); } tableMetadata.setFields(fields); // tags diff --git a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/TagValueGenerator.java b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/TagValueGenerator.java index b8024fea45..f7b18ca7cf 100644 --- a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/TagValueGenerator.java +++ b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/TagValueGenerator.java @@ -9,7 +9,7 @@ import java.util.List; public class TagValueGenerator { - // 创建标签值：使用tagMetas + // Create tag values using tagMetas public static List generate(List tagMetas) { List tagValues = new ArrayList<>(); for (int i = 0; i < tagMetas.size(); i++) { diff --git a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/utils/TimeStampUtil.java b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/utils/TimeStampUtil.java index 9cfce16d82..53748169ae 100644 --- a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/utils/TimeStampUtil.java +++ b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/utils/TimeStampUtil.java @@ -41,17 +41,17 @@ public class TimeStampUtil { if (start == 0) start = now - size * timeGap; - // 如果size小于1异常 + // If size is less than 1, throw an exception if (size < 1) throw new IllegalArgumentException("size less than 1."); - // 如果timeGap为1，已经超长，需要前移start + // If timeGap is 1 and it exceeds the limit, move start forward if (start + size > now) { start = now - size; return new TimeTuple(start, now, 1); } long end = start + (long) (timeGap * size); if (end > now) { - //压缩timeGap + // Compress timeGap end = now; double gap = (end - start) / (size * 1.0f); if (gap < 1.0f) { diff --git a/docs/examples/JDBC/taosdemo/src/main/resources/log4j.properties b/docs/examples/JDBC/taosdemo/src/main/resources/log4j.properties index 352545854d..40b1478a24 100644 --- a/docs/examples/JDBC/taosdemo/src/main/resources/log4j.properties +++ b/docs/examples/JDBC/taosdemo/src/main/resources/log4j.properties @@ -1,21 +1,21 @@ -### 设置### +### Settings ### log4j.rootLogger=info,stdout -### 输出信息到控制抬 ### +### Output information to the console ### log4j.appender.stdout=org.apache.log4j.ConsoleAppender log4j.appender.stdout.Target=System.out log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=[%-5p] %d{yyyy-MM-dd HH:mm:ss,SSS} method:%l%n%m%n -### 输出DEBUG 级别以上的日志到=logs/debug.log +### Output logs of DEBUG level and above to logs/debug.log ### log4j.appender.DebugLog=org.apache.log4j.DailyRollingFileAppender log4j.appender.DebugLog.File=logs/debug.log log4j.appender.DebugLog.Append=true log4j.appender.DebugLog.Threshold=DEBUG log4j.appender.DebugLog.layout=org.apache.log4j.PatternLayout log4j.appender.DebugLog.layout.ConversionPattern=%-d{yyyy-MM-dd HH:mm:ss} [ %t:%r ] - [ %p ] %m%n -### 输出ERROR 级别以上的日志到=logs/error.log +### Output logs of ERROR level and above to logs/error.log ### log4j.appender.ErrorLog=org.apache.log4j.DailyRollingFileAppender log4j.appender.ErrorLog.File=logs/error.log log4j.appender.ErrorLog.Append=true log4j.appender.ErrorLog.Threshold=ERROR log4j.appender.ErrorLog.layout=org.apache.log4j.PatternLayout -log4j.appender.ErrorLog.layout.ConversionPattern=%-d{yyyy-MM-dd HH:mm:ss} [ %t:%r ] - [ %p ] %m%n \ No newline at end of file +log4j.appender.ErrorLog.layout.ConversionPattern=%-d{yyyy-MM-dd HH:mm:ss} [ %t:%r ] - [ %p ] %m%n diff --git a/docs/examples/java/pom.xml b/docs/examples/java/pom.xml index f23b73177e..e1a9504249 100644 --- a/docs/examples/java/pom.xml +++ b/docs/examples/java/pom.xml @@ -22,7 +22,7 @@ com.taosdata.jdbc taos-jdbcdriver - 3.3.3 + 3.4.0 diff --git a/docs/examples/java/src/main/java/com/taos/example/SchemalessWsTest.java b/docs/examples/java/src/main/java/com/taos/example/SchemalessWsTest.java index 08f66c2227..0f35e38f57 100644 --- a/docs/examples/java/src/main/java/com/taos/example/SchemalessWsTest.java +++ b/docs/examples/java/src/main/java/com/taos/example/SchemalessWsTest.java @@ -17,8 +17,8 @@ public class SchemalessWsTest { private static final String jsonDemo = "{\"metric\": \"metric_json\",\"timestamp\": 1626846400,\"value\": 10.3, \"tags\": {\"groupid\": 2, \"location\": \"California.SanFrancisco\", \"id\": \"d1001\"}}"; public static void main(String[] args) throws SQLException { - final String url = "jdbc:TAOS-RS://" + host + ":6041?user=root&password=taosdata&batchfetch=true"; - try(Connection connection = DriverManager.getConnection(url)){ + final String url = "jdbc:TAOS-WS://" + host + ":6041?user=root&password=taosdata"; + try (Connection connection = DriverManager.getConnection(url)) { init(connection); AbstractConnection conn = connection.unwrap(AbstractConnection.class); diff --git a/docs/examples/java/src/main/java/com/taos/example/WSConnectExample.java b/docs/examples/java/src/main/java/com/taos/example/WSConnectExample.java index afe74ace83..052af71a83 100644 --- a/docs/examples/java/src/main/java/com/taos/example/WSConnectExample.java +++ b/docs/examples/java/src/main/java/com/taos/example/WSConnectExample.java @@ -12,9 +12,9 @@ public class WSConnectExample { public static void main(String[] args) throws Exception { // use // String jdbcUrl = - // "jdbc:TAOS-RS://localhost:6041/dbName?user=root&password=taosdata&batchfetch=true"; + // "jdbc:TAOS-WS://localhost:6041/dbName?user=root&password=taosdata"; // if you want to connect a specified database named "dbName". - String jdbcUrl = "jdbc:TAOS-RS://localhost:6041?user=root&password=taosdata&batchfetch=true"; + String jdbcUrl = "jdbc:TAOS-WS://localhost:6041?user=root&password=taosdata"; Properties connProps = new Properties(); connProps.setProperty(TSDBDriver.PROPERTY_KEY_ENABLE_AUTO_RECONNECT, "true"); connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); diff --git a/docs/examples/java/src/main/java/com/taos/example/WSParameterBindingBasicDemo.java b/docs/examples/java/src/main/java/com/taos/example/WSParameterBindingBasicDemo.java index eab8df06b9..1353ebbddc 100644 --- a/docs/examples/java/src/main/java/com/taos/example/WSParameterBindingBasicDemo.java +++ b/docs/examples/java/src/main/java/com/taos/example/WSParameterBindingBasicDemo.java @@ -15,7 +15,7 @@ public class WSParameterBindingBasicDemo { public static void main(String[] args) throws SQLException { - String jdbcUrl = "jdbc:TAOS-RS://" + host + ":6041/?batchfetch=true"; + String jdbcUrl = "jdbc:TAOS-WS://" + host + ":6041"; try (Connection conn = DriverManager.getConnection(jdbcUrl, "root", "taosdata")) { init(conn); @@ -40,7 +40,7 @@ public class WSParameterBindingBasicDemo { pstmt.setFloat(4, random.nextFloat()); pstmt.addBatch(); } - int [] exeResult = pstmt.executeBatch(); + int[] exeResult = pstmt.executeBatch(); // you can check exeResult here System.out.println("Successfully inserted " + exeResult.length + " rows to power.meters."); } @@ -60,7 +60,8 @@ public class WSParameterBindingBasicDemo { try (Statement stmt = conn.createStatement()) { stmt.execute("CREATE DATABASE IF NOT EXISTS power"); stmt.execute("USE power"); - stmt.execute("CREATE STABLE IF NOT EXISTS power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (groupId INT, location BINARY(24))"); + stmt.execute( + "CREATE STABLE IF NOT EXISTS power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (groupId INT, location BINARY(24))"); } } } diff --git a/docs/examples/java/src/main/java/com/taos/example/WSParameterBindingFullDemo.java b/docs/examples/java/src/main/java/com/taos/example/WSParameterBindingFullDemo.java index f23fb187f4..7eaccb3db2 100644 --- a/docs/examples/java/src/main/java/com/taos/example/WSParameterBindingFullDemo.java +++ b/docs/examples/java/src/main/java/com/taos/example/WSParameterBindingFullDemo.java @@ -40,7 +40,7 @@ public class WSParameterBindingFullDemo { public static void main(String[] args) throws SQLException { - String jdbcUrl = "jdbc:TAOS-RS://" + host + ":6041/?batchfetch=true"; + String jdbcUrl = "jdbc:TAOS-WS://" + host + ":6041/"; try (Connection conn = DriverManager.getConnection(jdbcUrl, "root", "taosdata")) { @@ -51,8 +51,10 @@ public class WSParameterBindingFullDemo { stmtAll(conn); } catch (SQLException ex) { - // handle any errors, please refer to the JDBC specifications for detailed exceptions info - System.out.println("Failed to insert data using stmt, ErrCode:" + ex.getErrorCode() + "; ErrMessage: " + ex.getMessage()); + // handle any errors, please refer to the JDBC specifications for detailed + // exceptions info + System.out.println("Failed to insert data using stmt, ErrCode:" + ex.getErrorCode() + "; ErrMessage: " + + ex.getMessage()); throw ex; } catch (Exception ex) { System.out.println("Failed to insert data using stmt, ErrMessage: " + ex.getMessage()); @@ -104,30 +106,29 @@ public class WSParameterBindingFullDemo { pstmt.setTagBoolean(3, true); pstmt.setTagString(4, "binary_value"); pstmt.setTagNString(5, "nchar_value"); - pstmt.setTagVarbinary(6, new byte[]{(byte) 0x98, (byte) 0xf4, 0x6e}); - pstmt.setTagGeometry(7, new byte[]{ + pstmt.setTagVarbinary(6, new byte[] { (byte) 0x98, (byte) 0xf4, 0x6e }); + pstmt.setTagGeometry(7, new byte[] { 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x59, 0x40, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x59, 0x40}); + 0x00, 0x00, 0x00, 0x59, 0x40 }); long current = System.currentTimeMillis(); - pstmt.setTimestamp(1, new Timestamp(current)); pstmt.setInt(2, 1); pstmt.setDouble(3, 1.1); pstmt.setBoolean(4, true); pstmt.setString(5, "binary_value"); pstmt.setNString(6, "nchar_value"); - pstmt.setVarbinary(7, new byte[]{(byte) 0x98, (byte) 0xf4, 0x6e}); - pstmt.setGeometry(8, new byte[]{ + pstmt.setVarbinary(7, new byte[] { (byte) 0x98, (byte) 0xf4, 0x6e }); + pstmt.setGeometry(8, new byte[] { 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x59, 0x40, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x59, 0x40}); + 0x00, 0x00, 0x00, 0x59, 0x40 }); pstmt.addBatch(); pstmt.executeBatch(); System.out.println("Successfully inserted rows to example_all_type_stmt.ntb"); diff --git a/docs/zh/01-index.md b/docs/zh/01-index.md index df6c327866..32ea117fbb 100644 --- a/docs/zh/01-index.md +++ b/docs/zh/01-index.md @@ -6,7 +6,7 @@ slug: / TDengine 是一款[开源](https://www.taosdata.com/tdengine/open_source_time-series_database)、[高性能](https://www.taosdata.com/fast)、[云原生](https://www.taosdata.com/tdengine/cloud_native_time-series_database)的时序数据库（Time Series Database, TSDB）, 它专为物联网、车联网、工业互联网、金融、IT 运维等场景优化设计。同时它还带有内建的缓存、流式计算、数据订阅等系统功能，能大幅减少系统设计的复杂度，降低研发和运营成本，是一款极简的时序数据处理平台。本文档是 TDengine 的用户手册，主要是介绍 TDengine 的基本概念、安装、使用、功能、开发接口、运营维护、TDengine 内核设计等等，它主要是面向架构师、开发工程师与系统管理员的。如果你对时序数据的基本概念、价值以及其所能带来的业务价值尚不了解，请参考[时序数据基础](./concept) -TDengine 充分利用了时序数据的特点，提出了“一个数据采集点一张表”与“超级表”的概念，设计了创新的存储引擎，让数据的写入、查询和存储效率都得到极大的提升。为正确理解并使用 TDengine，无论如何，请您仔细阅读[快速入门](./basic)一章。 +TDengine 充分利用了时序数据的特点，提出了“一个数据采集点一张表”与“超级表”的概念，设计了创新的存储引擎，让数据的写入、查询和存储效率都得到极大的提升。为正确理解并使用 TDengine，无论如何，请您仔细阅读[数据模型](./basic/model)一章。 如果你是开发工程师，请一定仔细阅读[开发指南](./develop)一章，该部分对数据库连接、建模、插入数据、查询、流式计算、缓存、数据订阅、用户自定义函数等功能都做了详细介绍，并配有各种编程语言的示例代码。大部分情况下，你只要复制粘贴示例代码，针对自己的应用稍作改动，就能跑起来。对 REST API、各种编程语言的连接器（Connector）想做更多详细了解的话，请看[连接器](./reference/connector)一章。 diff --git a/docs/zh/07-develop/01-connect/_connect_rust.mdx b/docs/zh/07-develop/01-connect/_connect_rust.mdx index 0e65e8f920..d88a3335ca 100644 --- a/docs/zh/07-develop/01-connect/_connect_rust.mdx +++ b/docs/zh/07-develop/01-connect/_connect_rust.mdx @@ -3,6 +3,6 @@ ``` :::note -对于 Rust 连接器， 连接方式的不同只体现在使用的特性不同。如果启用了 "ws" 特性，那么只有 Websocket 的实现会被编译进来。 +对于 Rust 连接器， 连接方式的不同只体现在使用的特性不同。如果启用了 "ws" 特性，那么只有 WebSocket 的实现会被编译进来。 ::: diff --git a/docs/zh/07-develop/01-connect/index.md b/docs/zh/07-develop/01-connect/index.md index 1dfb95d169..bd26bea46d 100644 --- a/docs/zh/07-develop/01-connect/index.md +++ b/docs/zh/07-develop/01-connect/index.md @@ -28,7 +28,7 @@ TDengine 提供了丰富的应用程序开发接口，为了便于用户快速 1. 通过客户端驱动程序 taosc 直接与服务端程序 taosd 建立连接，这种连接方式下文中简称 “原生连接”。 2. 通过 taosAdapter 组件提供的 REST API 建立与 taosd 的连接，这种连接方式下文中简称 “REST 连接” -3. 通过 taosAdapter 组件提供的 Websocket API 建立与 taosd 的连接，这种连接方式下文中简称 “Websocket 连接” +3. 通过 taosAdapter 组件提供的 WebSocket API 建立与 taosd 的连接，这种连接方式下文中简称 “WebSocket 连接” ![TDengine connection type](connection-type-zh.webp) @@ -38,9 +38,9 @@ TDengine 提供了丰富的应用程序开发接口，为了便于用户快速 关键不同点在于： 1. 使用 原生连接，需要保证客户端的驱动程序 taosc 和服务端的 TDengine 版本配套。 -2. 使用 REST 连接，用户无需安装客户端驱动程序 taosc，具有跨平台易用的优势，但是无法体验数据订阅和二进制数据类型等功能。另外与 原生连接 和 Websocket 连接相比，REST连接的性能最低。REST 接口是无状态的。在使用 REST 连接时，需要在 SQL 中指定表、超级表的数据库名称。 -3. 使用 Websocket 连接，用户也无需安装客户端驱动程序 taosc。 -4. 连接云服务实例，必须使用 REST 连接 或 Websocket 连接。 +2. 使用 REST 连接，用户无需安装客户端驱动程序 taosc，具有跨平台易用的优势，但是无法体验数据订阅和二进制数据类型等功能。另外与 原生连接 和 WebSocket 连接相比，REST连接的性能最低。REST 接口是无状态的。在使用 REST 连接时，需要在 SQL 中指定表、超级表的数据库名称。 +3. 使用 WebSocket 连接，用户也无需安装客户端驱动程序 taosc。 +4. 连接云服务实例，必须使用 REST 连接 或 WebSocket 连接。 **推荐使用 WebSocket 连接** @@ -126,7 +126,7 @@ TDengine 提供了丰富的应用程序开发接口，为了便于用户快速 ```bash pip3 install taos-ws-py ``` - :::note 此安装包为 Websocket 连接器 + :::note 此安装包为 WebSocket 连接器 - 同时安装 `taospy` 和 `taos-ws-py` ```bash pip3 install taospy[ws] @@ -182,7 +182,7 @@ taos = { version = "*"} ``` :::info -Rust 连接器通过不同的特性区分不同的连接方式。默认同时支持原生连接和 Websocket 连接，如果仅需要建立 Websocket 连接，可设置 `ws` 特性： +Rust 连接器通过不同的特性区分不同的连接方式。默认同时支持原生连接和 WebSocket 连接，如果仅需要建立 WebSocket 连接，可设置 `ws` 特性： ```toml taos = { version = "*", default-features = false, features = ["ws"] } @@ -201,7 +201,7 @@ taos = { version = "*", default-features = false, features = ["ws"] } ``` npm install @tdengine/websocket ``` - :::note Node.js 目前只支持 Websocket 连接 + :::note Node.js 目前只支持 WebSocket 连接 - **安装验证** - 新建安装验证目录，例如：`~/tdengine-test`，下载 GitHub 上 [nodejsChecker.js 源代码](https://github.com/taosdata/TDengine/tree/main/docs/examples/node/websocketexample/nodejsChecker.js)到本地。 - 在命令行中执行以下命令。 @@ -271,12 +271,10 @@ dotnet add package TDengine.Connector Java 连接器建立连接的参数有 URL 和 Properties。 TDengine 的 JDBC URL 规范格式为： - `jdbc:[TAOS|TAOS-RS]://[host_name]:[port]/[database_name]?[user={user}|&password={password}|&charset={charset}|&cfgdir={config_dir}|&locale={locale}|&timezone={timezone}|&batchfetch={batchfetch}]` + `jdbc:[TAOS|TAOS-WS|TAOS-RS]://[host_name]:[port]/[database_name]?[user={user}|&password={password}|&charset={charset}|&cfgdir={config_dir}|&locale={locale}|&timezone={timezone}|&batchfetch={batchfetch}]` URL 和 Properties 的详细参数说明和如何使用详见 [url 规范](../../reference/connector/java/#url-规范) - **注**：REST 连接中增加 `batchfetch` 参数并设置为 true，将开启 WebSocket 连接。 - Python 连接器使用 `connect()` 方法来建立连接，下面是连接参数的具体说明： @@ -387,8 +385,8 @@ DSN 的详细说明和如何使用详见 [连接功能](../../reference/connecto - `reconnectIntervalMs`：重连间隔毫秒时间，默认为 2000。 -**Websocket 连接** -C/C++ 语言连接器 Websocket 连接方式使用 `ws_connect()` 函数用于建立与 TDengine 数据库的连接。其参数为 DSN 描述字符串，其基本结构如下： +**WebSocket 连接** +C/C++ 语言连接器 WebSocket 连接方式使用 `ws_connect()` 函数用于建立与 TDengine 数据库的连接。其参数为 DSN 描述字符串，其基本结构如下： ```text [+]://[[:@]:][/][?=[&=]] @@ -417,8 +415,8 @@ C/C++ 语言连接器原生连接方式使用 `taos_connect()` 函数用于建 -### Websocket 连接 -下面是各语言连接器建立 Websocket 连接代码样例。演示了如何使用 Websocket 连接方式连接到 TDengine 数据库，并对连接设定一些参数。整个过程主要涉及到数据库连接的建立和异常处理。 +### WebSocket 连接 +下面是各语言连接器建立 WebSocket 连接代码样例。演示了如何使用 WebSocket 连接方式连接到 TDengine 数据库，并对连接设定一些参数。整个过程主要涉及到数据库连接的建立和异常处理。 diff --git a/docs/zh/07-develop/02-sql.md b/docs/zh/07-develop/02-sql.md index 5461c975dd..b4274045fc 100644 --- a/docs/zh/07-develop/02-sql.md +++ b/docs/zh/07-develop/02-sql.md @@ -33,7 +33,7 @@ REST API：直接调用 `taosadapter` 提供的 REST API 接口，进行数据 -```python title="Websocket 连接" +```python title="WebSocket 连接" {{#include docs/examples/python/create_db_ws.py}} ``` @@ -69,7 +69,7 @@ REST API：直接调用 `taosadapter` 提供的 REST API 接口，进行数据 -```c title="Websocket 连接" +```c title="WebSocket 连接" {{#include docs/examples/c-ws/create_db_demo.c:create_db_and_table}} ``` @@ -114,7 +114,7 @@ NOW 为系统内部函数，默认为客户端所在计算机当前时间。 NOW -```python title="Websocket 连接" +```python title="WebSocket 连接" {{#include docs/examples/python/insert_ws.py}} ``` @@ -151,7 +151,7 @@ NOW 为系统内部函数，默认为客户端所在计算机当前时间。 NOW -```c title="Websocket 连接" +```c title="WebSocket 连接" {{#include docs/examples/c-ws/insert_data_demo.c:insert_data}} ``` @@ -189,7 +189,7 @@ curl --location -uroot:taosdata 'http://127.0.0.1:6041/rest/sql' \ -```python title="Websocket 连接" +```python title="WebSocket 连接" {{#include docs/examples/python/query_ws.py}} ``` @@ -230,7 +230,7 @@ rust 连接器还支持使用 **serde** 进行反序列化行为结构体的结 -```c title="Websocket 连接" +```c title="WebSocket 连接" {{#include docs/examples/c-ws/query_data_demo.c:query_data}} ``` @@ -273,7 +273,7 @@ reqId 可用于请求链路追踪，reqId 就像分布式系统中的 traceId -```python title="Websocket 连接" +```python title="WebSocket 连接" {{#include docs/examples/python/reqid_ws.py}} ``` @@ -310,7 +310,7 @@ reqId 可用于请求链路追踪，reqId 就像分布式系统中的 traceId -```c "Websocket 连接" +```c "WebSocket 连接" {{#include docs/examples/c-ws/with_reqid_demo.c:with_reqid}} ``` diff --git a/docs/zh/07-develop/04-schemaless.md b/docs/zh/07-develop/04-schemaless.md index a865b58b28..bf10b41736 100644 --- a/docs/zh/07-develop/04-schemaless.md +++ b/docs/zh/07-develop/04-schemaless.md @@ -191,7 +191,7 @@ st,t1=3,t2=4,t3=t3 c1=3i64,c6="passit" 1626006833640000000 ::: -### Websocket 连接 +### WebSocket 连接 diff --git a/docs/zh/07-develop/05-stmt.md b/docs/zh/07-develop/05-stmt.md index 624600ba4d..74b44ba8e6 100644 --- a/docs/zh/07-develop/05-stmt.md +++ b/docs/zh/07-develop/05-stmt.md @@ -23,7 +23,7 @@ import TabItem from "@theme/TabItem"; - 执行批量插入操作，将这些数据行插入到对应的子表中。 3. 最后打印实际插入表中的行数。 -## Websocket 连接 +## WebSocket 连接 ```java diff --git a/docs/zh/07-develop/07-tmq.md b/docs/zh/07-develop/07-tmq.md index c668203259..a91a764c67 100644 --- a/docs/zh/07-develop/07-tmq.md +++ b/docs/zh/07-develop/07-tmq.md @@ -94,7 +94,7 @@ Rust 连接器创建消费者的参数为 DSN， 可以设置的参数列表请 -- Websocket 连接: 因为使用 dsn，不需要 `td.connect.ip`，`td.connect.port`，`td.connect.user` 和 `td.connect.pass` 四个配置项，其余同通用配置项。 +- WebSocket 连接: 因为使用 dsn，不需要 `td.connect.ip`，`td.connect.port`，`td.connect.user` 和 `td.connect.pass` 四个配置项，其余同通用配置项。 - 原生连接: 同通用基础配置项。 @@ -103,8 +103,8 @@ Rust 连接器创建消费者的参数为 DSN， 可以设置的参数列表请 -### Websocket 连接 -介绍各语言连接器使用 Websocket 连接方式创建消费者。指定连接的服务器地址，设置自动提交，从最新消息开始消费，指定 `group.id` 和 `client.id` 等信息。有的语言的连接器还支持反序列化参数。 +### WebSocket 连接 +介绍各语言连接器使用 WebSocket 连接方式创建消费者。指定连接的服务器地址，设置自动提交，从最新消息开始消费，指定 `group.id` 和 `client.id` 等信息。有的语言的连接器还支持反序列化参数。 @@ -234,7 +234,7 @@ Rust 连接器创建消费者的参数为 DSN， 可以设置的参数列表请 ## 订阅消费数据 消费者订阅主题后，可以开始接收并处理这些主题中的消息。订阅消费数据的示例代码如下： -### Websocket 连接 +### WebSocket 连接 @@ -403,7 +403,7 @@ Rust 连接器创建消费者的参数为 DSN， 可以设置的参数列表请 ## 指定订阅的 Offset 消费者可以指定从特定 Offset 开始读取分区中的消息，这允许消费者重读消息或跳过已处理的消息。下面展示各语言连接器如何指定订阅的 Offset。 -### Websocket 连接 +### WebSocket 连接 @@ -549,7 +549,7 @@ Rust 连接器创建消费者的参数为 DSN， 可以设置的参数列表请 **注意**：手工提交消费进度前确保消息正常处理完成，否则处理出错的消息不会被再次消费。自动提交是在本次 `poll` 消息时可能会提交上次消息的消费进度，因此请确保消息处理完毕再进行下一次 `poll` 或消息获取。 -### Websocket 连接 +### WebSocket 连接 @@ -663,7 +663,7 @@ Rust 连接器创建消费者的参数为 DSN， 可以设置的参数列表请 ## 取消订阅和关闭消费 消费者可以取消对主题的订阅，停止接收消息。当消费者不再需要时，应该关闭消费者实例，以释放资源和断开与 TDengine 服务器的连接。 -### Websocket 连接 +### WebSocket 连接 @@ -766,7 +766,7 @@ Rust 连接器创建消费者的参数为 DSN， 可以设置的参数列表请 ## 完整示例 -### Websocket 连接 +### WebSocket 连接
diff --git a/docs/zh/08-operation/18-dual.md b/docs/zh/08-operation/18-dual.md index c7871a8e1e..caddb7ab3b 100644 --- a/docs/zh/08-operation/18-dual.md +++ b/docs/zh/08-operation/18-dual.md @@ -30,9 +30,8 @@ toc_max_heading_level: 4 目前只有 Java 连接器在 WebSocket 连接模式下支持双活，其配置示例如下 ```java -url = "jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata"; +url = "jdbc:TAOS-WS://" + host + ":6041/?user=root&password=taosdata"; Properties properties = new Properties(); -properties.setProperty(TSDBDriver.PROPERTY_KEY_BATCH_LOAD, "true"); properties.setProperty(TSDBDriver.PROPERTY_KEY_SLAVE_CLUSTER_HOST, "192.168.1.11"); properties.setProperty(TSDBDriver.PROPERTY_KEY_SLAVE_CLUSTER_PORT, "6041"); properties.setProperty(TSDBDriver.PROPERTY_KEY_ENABLE_AUTO_RECONNECT, "true"); @@ -43,13 +42,13 @@ connection = DriverManager.getConnection(url, properties); 其中的配置属性及含义如下表 -| 属性名 | 含义 | -| ----------------- | ------------------ | -| PROPERTY_KEY_SLAVE_CLUSTER_HOST | 第二节点的主机名或者 ip，默认空 | -| PROPERTY_KEY_SLAVE_CLUSTER_PORT | 第二节点的端口号，默认空 | -| PROPERTY_KEY_ENABLE_AUTO_RECONNECT | 是否启用自动重连。仅在使用 Websocket 连接时生效。true: 启用，false: 不启用。默认为 false。双活场景下请设置为 true | -| PROPERTY_KEY_RECONNECT_INTERVAL_MS | 重连的时间间隔，单位毫秒：默认 2000 毫秒，也就是 2 秒；最小值为 0， 表示立即重试；最大值不做限制 | -| PROPERTY_KEY_RECONNECT_RETRY_COUNT | 每节点最多重试次数：默认值为 3；最小值为 0，表示不进行重试；最大值不做限制 | +| 属性名 | 含义 | +| ---------------------------------- | ----------------------------------------------------------------------------------------------------------------- | +| PROPERTY_KEY_SLAVE_CLUSTER_HOST | 第二节点的主机名或者 ip，默认空 | +| PROPERTY_KEY_SLAVE_CLUSTER_PORT | 第二节点的端口号，默认空 | +| PROPERTY_KEY_ENABLE_AUTO_RECONNECT | 是否启用自动重连。仅在使用 WebSocket 连接时生效。true: 启用，false: 不启用。默认为 false。双活场景下请设置为 true | +| PROPERTY_KEY_RECONNECT_INTERVAL_MS | 重连的时间间隔，单位毫秒：默认 2000 毫秒，也就是 2 秒；最小值为 0， 表示立即重试；最大值不做限制 | +| PROPERTY_KEY_RECONNECT_RETRY_COUNT | 每节点最多重试次数：默认值为 3；最小值为 0，表示不进行重试；最大值不做限制 | ### 约束条件 diff --git a/docs/zh/14-reference/01-components/03-taosadapter.md b/docs/zh/14-reference/01-components/03-taosadapter.md index a8e8fc3418..7d69cab598 100644 --- a/docs/zh/14-reference/01-components/03-taosadapter.md +++ b/docs/zh/14-reference/01-components/03-taosadapter.md @@ -306,7 +306,7 @@ http 返回内容： ## taosAdapter 监控指标 -taosAdapter 采集 REST/Websocket 相关请求的监控指标。将监控指标上报给 taosKeeper，这些监控指标会被 taosKeeper 写入监控数据库，默认是 `log` 库，可以在 taoskeeper 配置文件中修改。以下是这些监控指标的详细介绍。 +taosAdapter 采集 REST/WebSocket 相关请求的监控指标。将监控指标上报给 taosKeeper，这些监控指标会被 taosKeeper 写入监控数据库，默认是 `log` 库，可以在 taoskeeper 配置文件中修改。以下是这些监控指标的详细介绍。 #### adapter\_requests 表 @@ -331,7 +331,7 @@ taosAdapter 采集 REST/Websocket 相关请求的监控指标。将监控指标 | query\_in\_process | INT UNSIGNED | | 正在处理查询请求数 | | write\_in\_process | INT UNSIGNED | | 正在处理写入请求数 | | endpoint | VARCHAR | | 请求端点 | -| req\_type | NCHAR | TAG | 请求类型：0 为 REST，1 为 Websocket | +| req\_type | NCHAR | TAG | 请求类型：0 为 REST，1 为 WebSocket | ## 结果返回条数限制 diff --git a/docs/zh/14-reference/01-components/07-explorer.md b/docs/zh/14-reference/01-components/07-explorer.md index c63bc703e2..499fb3697c 100644 --- a/docs/zh/14-reference/01-components/07-explorer.md +++ b/docs/zh/14-reference/01-components/07-explorer.md @@ -4,11 +4,11 @@ sidebar_label: taosExplorer toc_max_heading_level: 4 --- -taosExplorer 是一个为用户提供 TDengine 实例的可视化管理交互工具的 web 服务。本节主要讲述其安装和部署。它的各项功能都是基于简单易上手的图形界面，可以直接尝试，如果有需要也可以考高级功能和运维指南中的相关内容。为了确保访问 taosExplorer 的最佳体验，请使用 Chrome 79 及以上版本，或 Edge 79 及以上版本。 +taosExplorer 是一个为用户提供 TDengine 实例的可视化管理交互工具的 web 服务，虽然它没有开源，但随开源版安装包免费提供。本节主要讲述其安装和部署。它的各项功能都是基于简单易上手的图形界面，可以直接尝试，如果有需要也可以参考高级功能和运维指南中的相关内容。为了确保访问 taosExplorer 的最佳体验，请使用 Chrome 79 及以上版本，或 Edge 79 及以上版本。 ## 安装 -taosEexplorer 无需单独安装，从 TDengine 3.3.0.0 版本开始，它随着 TDengine Enterprise Server 安装包一起发布，安装完成后，就可以看到 `taos-explorer` 服务。 +taosEexplorer 无需单独安装，从 TDengine 3.3.0.0 版本开始，它随着 TDengine 安装包一起发布，安装完成后，就可以看到 `taos-explorer` 服务。如果按照 GitHub 里步骤自己编译 TDengine 源代码生成的安装包不包含 taosExplorer。 ## 配置 diff --git a/docs/zh/14-reference/02-tools/10-taosbenchmark.md b/docs/zh/14-reference/02-tools/10-taosbenchmark.md index 3f15d6b8e3..3c43c58915 100644 --- a/docs/zh/14-reference/02-tools/10-taosbenchmark.md +++ b/docs/zh/14-reference/02-tools/10-taosbenchmark.md @@ -477,7 +477,9 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\) | 13 | BOOL | bool | 14 | NCHAR | nchar | 15 | VARCHAR | varchar -| 15 | JSON | json +| 16 | VARBINARY | varbinary +| 17 | GEOMETRY | geometry +| 18 | JSON | json 注意：taosBenchmark 配置文件中数据类型必须小写方可识别 diff --git a/docs/zh/14-reference/05-connector/10-cpp.mdx b/docs/zh/14-reference/05-connector/10-cpp.mdx index c618601fb9..7164baad2a 100644 --- a/docs/zh/14-reference/05-connector/10-cpp.mdx +++ b/docs/zh/14-reference/05-connector/10-cpp.mdx @@ -5,14 +5,14 @@ toc_max_heading_level: 4 --- C/C++ 开发人员可以使用 TDengine 的客户端驱动，即 C/C++连接器 （以下都用 TDengine 客户端驱动表示），开发自己的应用来连接 TDengine 集群完成数据存储、查询以及其他功能。TDengine 客户端驱动的 API 类似于 MySQL 的 C API。应用程序使用时，需要包含 TDengine 头文件，里面列出了提供的 API 的函数原型；应用程序还要链接到所在平台上对应的动态库。 -TDengine 的客户端驱动提供了 taosws 和 taos 两个动态库，分别支持 Websocket 连接和原生连接。 Websocket 连接和原生连接的区别是 Websocket 连接方式不要求客户端和服务端版本完全匹配，而原生连接要求，在性能上 Websocket 连接方式也接近于原生连接，一般我们推荐使用 Websocket 连接方式。 +TDengine 的客户端驱动提供了 taosws 和 taos 两个动态库，分别支持 WebSocket 连接和原生连接。 WebSocket 连接和原生连接的区别是 WebSocket 连接方式不要求客户端和服务端版本完全匹配，而原生连接要求，在性能上 WebSocket 连接方式也接近于原生连接，一般我们推荐使用 WebSocket 连接方式。 下面我们分开介绍两种连接方式的使用方法。 -## Websocket 连接方式 +## WebSocket 连接方式 -Websocket 连接方式需要使用 taosws.h 头文件和 taosws 动态库。 +WebSocket 连接方式需要使用 taosws.h 头文件和 taosws 动态库。 ```c #include @@ -44,7 +44,7 @@ TDengine 客户端驱动的动态库位于： ### 错误码 在 C 接口的设计中，错误码采用整数类型表示，每个错误码都对应一个特定的错误状态。如未特别说明，当 API 的返回值是整数时，_0_ 代表成功，其它是代表失败原因的错误码，当返回值是指针时， _NULL_ 表示失败。 -Websocket 连接方式单独的错误码在 `taosws.h` 中， +WebSocket 连接方式单独的错误码在 `taosws.h` 中， | 错误码 | 错误描述 | 可能的出错场景或者可能的原因 | 建议用户采取的措施 | @@ -82,7 +82,7 @@ WebSocket 连接方式错误码只保留了原生连接错误码的后两个字 #### DSN -C/C++ Websocket 连接器通过 DSN 连接描述字符串来表示连接信息。 +C/C++ WebSocket 连接器通过 DSN 连接描述字符串来表示连接信息。 DSN 描述字符串基本结构如下： ```text @@ -96,16 +96,16 @@ DSN 描述字符串基本结构如下： - **driver**: 必须指定驱动名以便连接器选择何种方式创建连接，支持如下驱动名： - **taos**: 默认驱动，支持 SQL 执行，参数绑定，无模式写入。 - **tmq**: 使用 TMQ 订阅数据。 -- **protocol**: 显示指定以何种方式建立连接，例如：`taos+ws://localhost:6041` 指定以 Websocket 方式建立连接。 - - **http/ws**: 使用 Websocket 协议。 - - **https/wss**: 在 Websocket 连接方式下显示启用 SSL/TLS 协议。 +- **protocol**: 显示指定以何种方式建立连接，例如：`taos+ws://localhost:6041` 指定以 WebSocket 方式建立连接。 + - **http/ws**: 使用 WebSocket 协议。 + - **https/wss**: 在 WebSocket 连接方式下显示启用 SSL/TLS 协议。 - **username/password**: 用于创建连接的用户名及密码。 -- **host/port**: 指定创建连接的服务器及端口，当不指定服务器地址及端口时 Websocket 连接默认为 `localhost:6041` 。 +- **host/port**: 指定创建连接的服务器及端口，当不指定服务器地址及端口时 WebSocket 连接默认为 `localhost:6041` 。 - **database**: 指定默认连接的数据库名，可选参数。 - **params**：其他可选参数。 -一个完整的 DSN 描述字符串示例如下：`taos+ws://localhost:6041/test`， 表示使用 Websocket（`ws`）方式通过 `6041` 端口连接服务器 `localhost`，并指定默认数据库为 `test`。 +一个完整的 DSN 描述字符串示例如下：`taos+ws://localhost:6041/test`， 表示使用 WebSocket（`ws`）方式通过 `6041` 端口连接服务器 `localhost`，并指定默认数据库为 `test`。 #### 基础 API diff --git a/docs/zh/14-reference/05-connector/14-java.mdx b/docs/zh/14-reference/05-connector/14-java.mdx index ba4cb38afd..0a167dd5ee 100644 --- a/docs/zh/14-reference/05-connector/14-java.mdx +++ b/docs/zh/14-reference/05-connector/14-java.mdx @@ -33,14 +33,15 @@ REST 连接支持所有能运行 Java 的平台。 | taos-jdbcdriver 版本 | 主要变化 | TDengine 版本 | | :------------------: | :----------------------------------------------------------------------------------------------------------------------------------------------------: | :----------------: | -| 3.3.3 | 1. 解决了 Websocket statement 关闭导致的内存泄漏 | - | -| 3.3.2 | 1. 优化 Websocket 连接下的参数绑定性能；2. 优化了对 mybatis 的支持 | - | -| 3.3.0 | 1. 优化 Websocket 连接下的数据传输性能；2. 支持跳过 SSL 验证，默认关闭 | 3.3.2.0 及更高版本 | +| 3.4.0 | 1. 使用 jackson 库替换 fastjson 库；2. WebSocket 采用独立协议标识；3. 优化后台拉取线程使用，避免用户误用导致超时。 | - | +| 3.3.3 | 1. 解决了 WebSocket statement 关闭导致的内存泄漏 | - | +| 3.3.2 | 1. 优化 WebSocket 连接下的参数绑定性能；2. 优化了对 mybatis 的支持 | - | +| 3.3.0 | 1. 优化 WebSocket 连接下的数据传输性能；2. 支持跳过 SSL 验证，默认关闭 | 3.3.2.0 及更高版本 | | 3.2.11 | 解决了 Native 连接关闭结果集 bug | - | -| 3.2.10 | 1. REST/WebSocket 连接支持传输中的数据压缩；2. Websocket 自动重连机制，默认关闭；3. Connection 类提供无模式写入的方法；4. 优化了原生连接的数据拉取性能；5. 修复了一些已知问题；6.元数据获取函数可以返回支持的函数列表。 | - | -| 3.2.9 | 解决了 Websocket prepareStatement 关闭 bug | - | -| 3.2.8 | 优化了自动提交, 解决了 websocket 手动提交 bug, 优化 Websocket prepareStatement 使用一个连接, 元数据支持视图 | - | -| 3.2.7 | 支持 VARBINARY 和 GEOMETRY 类型，增加 native 连接的时区设置支持。增加 websocket 自动重连功能。 | 3.2.0.0 及更高版本 | +| 3.2.10 | 1. REST/WebSocket 连接支持传输中的数据压缩；2. WebSocket 自动重连机制，默认关闭；3. Connection 类提供无模式写入的方法；4. 优化了原生连接的数据拉取性能；5. 修复了一些已知问题；6.元数据获取函数可以返回支持的函数列表。 | - | +| 3.2.9 | 解决了 WebSocket prepareStatement 关闭 bug | - | +| 3.2.8 | 优化了自动提交, 解决了 WebSocket 手动提交 bug, 优化 WebSocket prepareStatement 使用一个连接, 元数据支持视图 | - | +| 3.2.7 | 支持 VARBINARY 和 GEOMETRY 类型，增加 native 连接的时区设置支持。增加 WebSocket 自动重连功能。 | 3.2.0.0 及更高版本 | | 3.2.5 | 数据订阅增加 committed()、assignment() 方法 | 3.1.0.3 及更高版本 | | 3.2.4 | 数据订阅在 WebSocket 连接下增加 enable.auto.commit 参数，以及 unsubscribe() 方法。 | - | | 3.2.3 | 修复 ResultSet 在一些情况数据解析失败 | - | @@ -195,17 +196,14 @@ WKB规范请参考[Well-Known Binary (WKB)](https://libgeos.org/specifications/w ## API 参考 ### JDBC 驱动 -taos-jdbcdriver 实现了 JDBC 标准的 Driver 接口，提供了两个实现类：RestfulDriver 和 TSDBDriver。 -Websocket 和 REST 连接使用驱动类 `com.taosdata.jdbc.rs.RestfulDriver`。原生连接使用驱动类 `com.taosdata.jdbc.TSDBDriver`。 - +taos-jdbcdriver 实现了 JDBC 标准的 Driver 接口，提供了 3 个实现类。 +- WebSocket 连接使用驱动类 `com.taosdata.jdbc.ws.WebSocketDriver`。 +- 原生连接使用驱动类 `com.taosdata.jdbc.TSDBDriver`。 +- REST 连接使用驱动类 `com.taosdata.jdbc.rs.RestfulDriver`。 #### URL 规范 TDengine 的 JDBC URL 规范格式为： -`jdbc:[TAOS|TAOS-RS]://[host_name]:[port]/[database_name]?[user={user}|&password={password}|&charset={charset}|&cfgdir={config_dir}|&locale={locale}|&timezone={timezone}|&batchfetch={batchfetch}]` - -对于建立连接，原生连接与 REST 连接有细微不同。 Websocket 和 REST 连接使用驱动类 `com.taosdata.jdbc.rs.RestfulDriver`。原生连接使用驱动类 `com.taosdata.jdbc.TSDBDriver`。 - -**注**：REST 连接中增加 `batchfetch` 参数并设置为 true，将开启 WebSocket 连接。 +`jdbc:[TAOS|TAOS-WS|TAOS-RS]://[host_name]:[port]/[database_name]?[user={user}|&password={password}|&charset={charset}|&cfgdir={config_dir}|&locale={locale}|&timezone={timezone}|&batchfetch={batchfetch}]` **原生连接** `jdbc:TAOS://taosdemo.com:6030/power?user=root&password=taosdata`，使用了 JDBC 原生连接的 TSDBDriver，建立了到 hostname 为 taosdemo.com，端口为 6030（TDengine 的默认端口），数据库名为 power 的连接。这个 URL @@ -234,23 +232,38 @@ TDengine 中，只要保证 firstEp 和 secondEp 中一个节点有效，就可 > **注意**：这里的配置文件指的是调用 JDBC Connector 的应用程序所在机器上的配置文件，Linux OS 上默认值 /etc/taos/taos.cfg ，Windows OS 上默认值 C://TDengine/cfg/taos.cfg。 +**WebSocket 连接** +使用 JDBC WebSocket 连接，不需要依赖客户端驱动。与 JDBC 原生连接相比，仅需要： -**Websocket 和 REST 连接** -使用 JDBC Websocket 或 REST 连接，不需要依赖客户端驱动。与 JDBC 原生连接相比，仅需要： +1. driverClass 指定为“com.taosdata.jdbc.ws.WebSocketDriver”； +2. jdbcUrl 以“jdbc:TAOS-WS://”开头； +3. 使用 6041 作为连接端口。 + +对于 WebSocket 连接，url 中的配置参数如下： +- user：登录 TDengine 用户名，默认值 'root'。 +- password：用户登录密码，默认值 'taosdata'。 +- charset: 当开启批量拉取数据时，指定解析字符串数据的字符集。 +- batchErrorIgnore：true：在执行 Statement 的 executeBatch 时，如果中间有一条 SQL 执行失败，继续执行下面的 SQL 了。false：不再执行失败 SQL 后的任何语句。默认值为：false。 +- httpConnectTimeout: 连接超时时间，单位 ms， 默认值为 60000。 +- messageWaitTimeout: 消息超时时间, 单位 ms， 默认值为 60000。 +- useSSL: 连接中是否使用 SSL。 + +**注意**：部分配置项（比如：locale、timezone）在 WebSocket 连接中不生效。 + +**REST 连接** +使用 JDBC REST 连接，不需要依赖客户端驱动。与 JDBC 原生连接相比，仅需要： 1. driverClass 指定为“com.taosdata.jdbc.rs.RestfulDriver”； 2. jdbcUrl 以“jdbc:TAOS-RS://”开头； 3. 使用 6041 作为连接端口。 -对于 Websocket 和 REST 连接，url 中的配置参数如下： +对于 REST 连接，url 中的配置参数如下： - user：登录 TDengine 用户名，默认值 'root'。 - password：用户登录密码，默认值 'taosdata'。 -- batchfetch: true：在执行查询时批量拉取结果集；false：逐行拉取结果集。默认值为：false。逐行拉取结果集使用 HTTP 方式进行数据传输。JDBC REST 连接支持批量拉取数据功能。taos-jdbcdriver 与 TDengine 之间通过 WebSocket 连接进行数据传输。相较于 HTTP，WebSocket 可以使 JDBC REST 连接支持大数据量查询，并提升查询性能。 - charset: 当开启批量拉取数据时，指定解析字符串数据的字符集。 - batchErrorIgnore：true：在执行 Statement 的 executeBatch 时，如果中间有一条 SQL 执行失败，继续执行下面的 SQL 了。false：不再执行失败 SQL 后的任何语句。默认值为：false。 - httpConnectTimeout: 连接超时时间，单位 ms， 默认值为 60000。 -- httpSocketTimeout: socket 超时时间，单位 ms，默认值为 60000。仅在 batchfetch 设置为 false 时生效。 -- messageWaitTimeout: 消息超时时间, 单位 ms， 默认值为 60000。 仅在 batchfetch 设置为 true 时生效。 +- httpSocketTimeout: socket 超时时间，单位 ms，默认值为 60000。 - useSSL: 连接中是否使用 SSL。 - httpPoolSize: REST 并发请求大小，默认 20。 @@ -272,7 +285,7 @@ TDengine 中，只要保证 firstEp 和 secondEp 中一个节点有效，就可 properties 中的配置参数如下： - TSDBDriver.PROPERTY_KEY_USER：登录 TDengine 用户名，默认值 'root'。 - TSDBDriver.PROPERTY_KEY_PASSWORD：用户登录密码，默认值 'taosdata'。 -- TSDBDriver.PROPERTY_KEY_BATCH_LOAD: true：在执行查询时批量拉取结果集；false：逐行拉取结果集。默认值为：false。 +- TSDBDriver.PROPERTY_KEY_BATCH_LOAD: true：在执行查询时批量拉取结果集；false：逐行拉取结果集。默认值为：false。因历史原因使用 REST 连接时，若设置此参数为 true 会变成 WebSocket 连接。 - TSDBDriver.PROPERTY_KEY_BATCH_ERROR_IGNORE：true：在执行 Statement 的 executeBatch 时，如果中间有一条 SQL 执行失败，继续执行下面的 sq 了。false：不再执行失败 SQL 后的任何语句。默认值为：false。 - TSDBDriver.PROPERTY_KEY_CONFIG_DIR：仅在使用 JDBC 原生连接时生效。客户端配置文件目录路径，Linux OS 上默认值 `/etc/taos`，Windows OS 上默认值 `C:/TDengine/cfg`。 - TSDBDriver.PROPERTY_KEY_CHARSET：客户端使用的字符集，默认值为系统字符集。 @@ -280,16 +293,16 @@ properties 中的配置参数如下： - TSDBDriver.PROPERTY_KEY_TIME_ZONE：仅在使用 JDBC 原生连接时生效。 客户端使用的时区，默认值为系统当前时区。因为历史的原因，我们只支持POSIX标准的部分规范，如UTC-8(代表中国上上海), GMT-8，Asia/Shanghai 这几种形式。 - TSDBDriver.HTTP_CONNECT_TIMEOUT: 连接超时时间，单位 ms， 默认值为 60000。仅在 REST 连接时生效。 - TSDBDriver.HTTP_SOCKET_TIMEOUT: socket 超时时间，单位 ms，默认值为 60000。仅在 REST 连接且 batchfetch 设置为 false 时生效。 -- TSDBDriver.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT: 消息超时时间, 单位 ms， 默认值为 60000。 仅在 REST 连接且 batchfetch 设置为 true 时生效。 -- TSDBDriver.PROPERTY_KEY_USE_SSL: 连接中是否使用 SSL。仅在 REST 连接时生效。 +- TSDBDriver.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT: 消息超时时间, 单位 ms， 默认值为 60000。 仅 WebSocket 连接下有效。 +- TSDBDriver.PROPERTY_KEY_USE_SSL: 连接中是否使用 SSL。仅在 WebSocket/REST 连接时生效。 - TSDBDriver.HTTP_POOL_SIZE: REST 并发请求大小，默认 20。 -- TSDBDriver.PROPERTY_KEY_ENABLE_COMPRESSION: 传输过程是否启用压缩。仅在使用 REST/Websocket 连接时生效。true: 启用，false: 不启用。默认为 false。 -- TSDBDriver.PROPERTY_KEY_ENABLE_AUTO_RECONNECT: 是否启用自动重连。仅在使用 Websocket 连接时生效。true: 启用，false: 不启用。默认为 false。 +- TSDBDriver.PROPERTY_KEY_ENABLE_COMPRESSION: 传输过程是否启用压缩。仅在使用 REST/WebSocket 连接时生效。true: 启用，false: 不启用。默认为 false。 +- TSDBDriver.PROPERTY_KEY_ENABLE_AUTO_RECONNECT: 是否启用自动重连。仅在使用 WebSocket 连接时生效。true: 启用，false: 不启用。默认为 false。 > **注意**：启用自动重连仅对简单执行 SQL 语句以及 无模式写入、数据订阅有效。对于参数绑定无效。自动重连仅对连接建立时通过参数指定数据库有效，对后面的 `use db` 语句切换数据库无效。 - TSDBDriver.PROPERTY_KEY_RECONNECT_INTERVAL_MS: 自动重连重试间隔，单位毫秒，默认值 2000。仅在 PROPERTY_KEY_ENABLE_AUTO_RECONNECT 为 true 时生效。 - TSDBDriver.PROPERTY_KEY_RECONNECT_RETRY_COUNT: 自动重连重试次数，默认值 3，仅在 PROPERTY_KEY_ENABLE_AUTO_RECONNECT 为 true 时生效。 -- TSDBDriver.PROPERTY_KEY_DISABLE_SSL_CERT_VALIDATION: 关闭 SSL 证书验证 。仅在使用 Websocket 连接时生效。true: 启用，false: 不启用。默认为 false。 +- TSDBDriver.PROPERTY_KEY_DISABLE_SSL_CERT_VALIDATION: 关闭 SSL 证书验证 。仅在使用 WebSocket 连接时生效。true: 启用，false: 不启用。默认为 false。 此外对 JDBC 原生连接，通过指定 URL 和 Properties 还可以指定其他参数，比如日志级别、SQL 长度等。 @@ -1154,7 +1167,7 @@ JDBC 驱动支持标准的 ResultSet 接口，提供了用于读取结果集中 PreparedStatement 允许使用预编译的 SQL 语句，这可以提高性能并提供参数化查询的能力，从而增加安全性。 JDBC 驱动提供了实现 PreparedStatement 接口的两个类： 1. 对应原生连接的 TSDBPreparedStatement -2. 对应 Websocket 连接的 TSWSPreparedStatement +2. 对应 WebSocket 连接的 TSWSPreparedStatement 因 JDBC 标准没有高性能绑定数据的接口，TSDBPreparedStatement 和 TSWSPreparedStatement 都新增了一些方法，用来扩展参数绑定能力。 > **注意**：由于 PreparedStatement 继承了 Statement 接口，因此对于这部分重复的接口不再赘述，请参考 Statement 接口中对应描述。 @@ -1347,8 +1360,8 @@ JDBC 标准不支持数据订阅，因此本章所有接口都是扩展接口。 - httpConnectTimeout: 创建连接超时参数，单位 ms，默认为 5000 ms。仅在 WebSocket 连接下有效。 - messageWaitTimeout: 数据传输超时参数，单位 ms，默认为 10000 ms。仅在 WebSocket 连接下有效。 - httpPoolSize: 同一个连接下最大并行请求数。仅在 WebSocket 连接下有效。 -- TSDBDriver.PROPERTY_KEY_ENABLE_COMPRESSION: 传输过程是否启用压缩。仅在使用 Websocket 连接时生效。true: 启用，false: 不启用。默认为 false。 -- TSDBDriver.PROPERTY_KEY_ENABLE_AUTO_RECONNECT: 是否启用自动重连。仅在使用 Websocket 连接时生效。true: 启用，false: 不启用。默认为 false。 +- TSDBDriver.PROPERTY_KEY_ENABLE_COMPRESSION: 传输过程是否启用压缩。仅在使用 WebSocket 连接时生效。true: 启用，false: 不启用。默认为 false。 +- TSDBDriver.PROPERTY_KEY_ENABLE_AUTO_RECONNECT: 是否启用自动重连。仅在使用 WebSocket 连接时生效。true: 启用，false: 不启用。默认为 false。 - TSDBDriver.PROPERTY_KEY_RECONNECT_INTERVAL_MS: 自动重连重试间隔，单位毫秒，默认值 2000。仅在 PROPERTY_KEY_ENABLE_AUTO_RECONNECT 为 true 时生效。 - TSDBDriver.PROPERTY_KEY_RECONNECT_RETRY_COUNT: 自动重连重试次数，默认值 3，仅在 PROPERTY_KEY_ENABLE_AUTO_RECONNECT 为 true 时生效。 diff --git a/docs/zh/14-reference/05-connector/26-rust.mdx b/docs/zh/14-reference/05-connector/26-rust.mdx index 1fcc1e3dcd..cfabed4c61 100644 --- a/docs/zh/14-reference/05-connector/26-rust.mdx +++ b/docs/zh/14-reference/05-connector/26-rust.mdx @@ -18,9 +18,9 @@ import RequestId from "./_request_id.mdx"; ## 连接方式 -`taos` 提供两种建立连接的方式。一般我们推荐使用 **Websocket 连接**。 +`taos` 提供两种建立连接的方式。一般我们推荐使用 **WebSocket 连接**。 - **原生连接**，它通过 TDengine 客户端驱动程序（taosc）连接 TDengine 运行实例。 -- **Websocket 连接**，它通过 taosAdapter 的 Websocket 接口连接 TDengine 运行实例。 +- **WebSocket 连接**，它通过 taosAdapter 的 WebSocket 接口连接 TDengine 运行实例。 你可以通过不同的 “特性（即 Cargo 关键字 `features`）” 来指定使用哪种连接器（默认同时支持）。 @@ -29,13 +29,13 @@ import RequestId from "./_request_id.mdx"; ## 支持的平台 原生连接支持的平台和 TDengine 客户端驱动支持的平台一致。 -Websocket 连接支持所有能运行 Rust 的平台。 +WebSocket 连接支持所有能运行 Rust 的平台。 ## 版本历史 | Rust 连接器版本 | TDengine 版本 | 主要功能 | | :----------------: | :--------------: | :--------------------------------------------------: | -| v0.12.3 | 3.3.0.0 or later | 优化了 Websocket 查询和插入性能，支持了 VARBINARY 和 GEOMETRY 类型 | +| v0.12.3 | 3.3.0.0 or later | 优化了 WebSocket 查询和插入性能，支持了 VARBINARY 和 GEOMETRY 类型 | | v0.12.0 | 3.2.3.0 or later | WS 支持压缩。 | | v0.11.0 | 3.2.0.0 | TMQ 功能优化。 | | v0.10.0 | 3.1.0.0 | WS endpoint 变更。 | @@ -115,15 +115,15 @@ DSN 描述字符串基本结构如下： - **driver**: 必须指定驱动名以便连接器选择何种方式创建连接，支持如下驱动名： - **taos**: 使用 TDengine 连接器驱动，默认是使用 taos 驱动。 - **tmq**: 使用 TMQ 订阅数据。 -- **protocol**: 显示指定以何种方式建立连接，例如：`taos+ws://localhost:6041` 指定以 Websocket 方式建立连接。 - - **http/ws**: 使用 Websocket 创建连接。 - - **https/wss**: 在 Websocket 连接方式下显示启用 SSL/TLS 连接。 +- **protocol**: 显示指定以何种方式建立连接，例如：`taos+ws://localhost:6041` 指定以 WebSocket 方式建立连接。 + - **http/ws**: 使用 WebSocket 创建连接。 + - **https/wss**: 在 WebSocket 连接方式下显示启用 SSL/TLS 连接。 - **username/password**: 用于创建连接的用户名及密码。 -- **host/port**: 指定创建连接的服务器及端口，当不指定服务器地址及端口时（`taos://`），原生连接默认为 `localhost:6030`，Websocket 连接默认为 `localhost:6041` 。 +- **host/port**: 指定创建连接的服务器及端口，当不指定服务器地址及端口时（`taos://`），原生连接默认为 `localhost:6030`，WebSocket 连接默认为 `localhost:6041` 。 - **database**: 指定默认连接的数据库名，可选参数。 - **params**：其他可选参数。 -一个完整的 DSN 描述字符串示例如下：`taos+ws://localhost:6041/test`， 表示使用 Websocket（`ws`）方式通过 `6041` 端口连接服务器 `localhost`，并指定默认数据库为 `test`。 +一个完整的 DSN 描述字符串示例如下：`taos+ws://localhost:6041/test`， 表示使用 WebSocket（`ws`）方式通过 `6041` 端口连接服务器 `localhost`，并指定默认数据库为 `test`。 #### TaosBuilder TaosBuilder 结构体主要提供了根据 DSN 构建 Taos 对象的方法，还提供了检查连接，以及获取客户端版本号等功能。 diff --git a/docs/zh/14-reference/05-connector/30-python.mdx b/docs/zh/14-reference/05-connector/30-python.mdx index 8e08bfc103..8436c30249 100644 --- a/docs/zh/14-reference/05-connector/30-python.mdx +++ b/docs/zh/14-reference/05-connector/30-python.mdx @@ -14,10 +14,10 @@ import RequestId from "./_request_id.mdx"; Python 连接器的源码托管在 [GitHub](https://github.com/taosdata/taos-connector-python)。 ## 连接方式 -`taospy`主要提供三种形式的连接器。一般我们推荐使用 **Websocket 连接**。 +`taospy`主要提供三种形式的连接器。一般我们推荐使用 **WebSocket 连接**。 - **原生连接**，对应 `taospy` 包的 `taos` 模块。通过 TDengine 客户端驱动程序（taosc）原生连接 TDengine 实例，支持数据写入、查询、数据订阅、schemaless 接口和参数绑定接口等功能。 - **REST 连接**，对应 `taospy` 包的 `taosrest` 模块。通过 taosAdapter 提供的 HTTP 接口连接 TDengine 实例，不支持 schemaless 和数据订阅等特性。 -- **Websocket 连接**，对应 `taos-ws-py` 包，可以选装。通过 taosAdapter 提供的 Websocket 接口连接 TDengine 实例，WebSocket 连接实现的功能集合和原生连接有少量不同。 +- **WebSocket 连接**，对应 `taos-ws-py` 包，可以选装。通过 taosAdapter 提供的 WebSocket 接口连接 TDengine 实例，WebSocket 连接实现的功能集合和原生连接有少量不同。 连接方式的详细介绍请参考：[连接方式](../../../develop/connect/#连接方式) @@ -48,9 +48,9 @@ Python 连接器的源码托管在 [GitHub](https://github.com/taosdata/taos-con |2.7.9|数据订阅支持获取消费进度和重置消费进度| |2.7.8|新增 `execute_many`| -|Python Websocket Connector 版本|主要变化| +|Python WebSocket Connector 版本|主要变化| |:----------------------------:|:-----:| -|0.3.2|优化 Websocket sql 查询和插入性能，修改 readme 和 文档，修复已知问题| +|0.3.2|优化 WebSocket sql 查询和插入性能，修改 readme 和 文档，修复已知问题| |0.2.9|已知问题修复| |0.2.5|1. 数据订阅支持获取消费进度和重置消费进度
2. 支持 schemaless
3. 支持 STMT| |0.2.4|数据订阅新增取消订阅方法| diff --git a/docs/zh/14-reference/05-connector/35-node.mdx b/docs/zh/14-reference/05-connector/35-node.mdx index bd2ca537e3..6ac34d2471 100644 --- a/docs/zh/14-reference/05-connector/35-node.mdx +++ b/docs/zh/14-reference/05-connector/35-node.mdx @@ -14,7 +14,7 @@ Node.js 连接器源码托管在 [GitHub](https://github.com/taosdata/taos-conne ## 连接方式 -Node.js 连接器目前仅支持 Websocket 连接器， 其通过 taosAdapter 提供的 Websocket 接口连接 TDengine 实例。 +Node.js 连接器目前仅支持 WebSocket 连接器， 其通过 taosAdapter 提供的 WebSocket 接口连接 TDengine 实例。 连接方式的详细介绍请参考：[连接方式](../../../develop/connect/#连接方式) @@ -48,7 +48,7 @@ Node.js 连接器目前仅支持 Websocket 连接器， 其通过 taosAdapter | 107 | unknown sql type in tdengine | 请检查 TDengine 支持的 Data Type 类型。 | | 108 | connection has been closed | 连接已经关闭，请检查 Connection 是否关闭后再次使用，或是连接是否正常。 | | 109 | fetch block data parse fail | 获取到的查询数据，解析失败 | -| 110 | websocket connection has reached its maximum limit | Websocket 连接达到上限 | +| 110 | websocket connection has reached its maximum limit | WebSocket 连接达到上限 | - [TDengine Node.js Connector Error Code](https://github.com/taosdata/taos-connector-node/blob/main/nodejs/src/common/wsError.ts) - TDengine 其他功能模块的报错，请参考 [错误码](../../../reference/error-code) @@ -104,7 +104,7 @@ Node.js 连接器目前仅支持 Websocket 连接器， 其通过 taosAdapter ## API 参考 -Node.js 连接器（`@tdengine/websocket`）, 其通过 taosAdapter 提供的 Websocket 接口连接 TDengine 实例。 +Node.js 连接器（`@tdengine/websocket`）, 其通过 taosAdapter 提供的 WebSocket 接口连接 TDengine 实例。 ### URL 规范 diff --git a/docs/zh/14-reference/05-connector/40-csharp.mdx b/docs/zh/14-reference/05-connector/40-csharp.mdx index 93f592fdd0..e4e778eeff 100644 --- a/docs/zh/14-reference/05-connector/40-csharp.mdx +++ b/docs/zh/14-reference/05-connector/40-csharp.mdx @@ -14,7 +14,7 @@ import RequestId from "./_request_id.mdx"; `TDengine.Connector` 提供两种形式的连接器 * **原生连接**，通过 TDengine 客户端驱动程序（taosc）原生连接 TDengine 实例，支持数据写入、查询、数据订阅、schemaless 接口和参数绑定接口等功能。 -* **Websocket 连接**，通过 taosAdapter 提供的 Websocket 接口连接 TDengine 实例，WebSocket 连接实现的功能集合和原生连接有少量不同。（自 v3.0.1 起） +* **WebSocket 连接**，通过 taosAdapter 提供的 WebSocket 接口连接 TDengine 实例，WebSocket 连接实现的功能集合和原生连接有少量不同。（自 v3.0.1 起） 连接方式的详细介绍请参考：[连接方式](../../../develop/connect/#连接方式) diff --git a/docs/zh/14-reference/05-connector/index.md b/docs/zh/14-reference/05-connector/index.md index 04a2ef6c1f..bd2cff6a3d 100644 --- a/docs/zh/14-reference/05-connector/index.md +++ b/docs/zh/14-reference/05-connector/index.md @@ -62,7 +62,7 @@ TDengine 版本更新往往会增加新的功能特性，列表中的连接器 | **连接管理** | 支持 | 支持 | 支持 | | **执行 SQL** | 支持 | 支持 | 支持 | -### 使用 Websocket 接口 +### 使用 WebSocket 接口 | **功能特性** | **Java** | **Python** | **Go** | **C#** | **Node.js** | **Rust** | **C/C++** | | ------------------- | -------- | ---------- | ------ | ------ | ----------- | -------- | --------- | diff --git a/docs/zh/28-releases/03-notes/3.3.3.0.md b/docs/zh/28-releases/03-notes/3.3.3.0.md index 405ca83d71..bb7bc0f831 100644 --- a/docs/zh/28-releases/03-notes/3.3.3.0.md +++ b/docs/zh/28-releases/03-notes/3.3.3.0.md @@ -10,7 +10,7 @@ description: 3.3.3.0 版本说明 4. TDengine支持macOS企业版客户端 [企业版] 5. taosX日志默认不写入syslog [企业版] 6. 服务端记录所有慢查询信息到log库 -7. show cluster machines 查询结果中添加服务端版本号 +7. show cluster machines 查询结果中添加服务端版本号 [企业版] 8. 删除保留关键字LEVEL/ENCODE/COMPRESS, 可以作为列名/表名/数据库名等使用 9. 禁止动态修改临时目录 10. round 函数：支持四舍五入的精度 diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index e6d750468e..58c1707e1f 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -754,7 +754,7 @@ int32_t streamMetaGetNumOfTasks(SStreamMeta* pMeta); int32_t streamMetaAcquireTaskNoLock(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, SStreamTask** pTask); int32_t streamMetaAcquireTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, SStreamTask** pTask); void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask); -void streamMetaAcquireOneTask(SStreamTask* pTask); +int32_t streamMetaAcquireOneTask(SStreamTask* pTask); void streamMetaClear(SStreamMeta* pMeta); void streamMetaInitBackend(SStreamMeta* pMeta); int32_t streamMetaCommit(SStreamMeta* pMeta); diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 74ab0bf484..f95b3f20ca 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -50,6 +50,7 @@ typedef struct { int32_t rollPeriod; // secs int64_t retentionSize; int64_t segSize; + int64_t committed; EWalType level; // wal level int32_t encryptAlgorithm; char encryptKey[ENCRYPT_KEY_LEN + 1]; diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index 209c376f30..a6aca2fddf 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -199,6 +199,7 @@ typedef struct { SArray *preLineTagKV; SArray *maxTagKVs; SArray *maxColKVs; + SArray *escapedStringList; SSmlLineInfo preLine; STableMeta *currSTableMeta; diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index d5cca55701..80f583bbee 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -479,6 +479,7 @@ int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs) } clearColValArraySml(info->currTableDataCtx->pValues); + taosArrayClearP(info->escapedStringList, taosMemoryFree); if (unlikely(ret != TSDB_CODE_SUCCESS)) { smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); return ret; @@ -1608,6 +1609,7 @@ void smlDestroyInfo(SSmlHandle *info) { taosArrayDestroy(info->valueJsonArray); taosArrayDestroyEx(info->preLineTagKV, freeSSmlKv); + taosArrayDestroyP(info->escapedStringList, taosMemoryFree); if (!info->dataFormat) { for (int i = 0; i < info->lineNum; i++) { @@ -1667,8 +1669,9 @@ int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle) { info->tagJsonArray = taosArrayInit(8, POINTER_BYTES); info->valueJsonArray = taosArrayInit(8, POINTER_BYTES); info->preLineTagKV = taosArrayInit(8, sizeof(SSmlKv)); - - if (info->tagJsonArray == NULL || info->valueJsonArray == NULL || info->preLineTagKV == NULL) { + info->escapedStringList = taosArrayInit(8, POINTER_BYTES); + if (info->tagJsonArray == NULL || info->valueJsonArray == NULL || + info->preLineTagKV == NULL || info->escapedStringList == NULL) { uError("SML:0x%" PRIx64 " failed to allocate memory", info->id); code = terrno; goto FAILED; @@ -1949,6 +1952,7 @@ int32_t smlClearForRerun(SSmlHandle *info) { } } + taosArrayClearP(info->escapedStringList, taosMemoryFree); (void)memset(&info->preLine, 0, sizeof(SSmlLineInfo)); info->currSTableMeta = NULL; info->currTableDataCtx = NULL; diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index e620ca9b0c..c1f3431698 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -451,6 +451,13 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL if (info->dataFormat) { bool isAligned = isSmlColAligned(info, cnt, &kv); + if (kv.type == TSDB_DATA_TYPE_BINARY && valueEscaped) { + if (taosArrayPush(info->escapedStringList, &kv.value) == NULL){ + freeSSmlKv(&kv); + return terrno; + } + kv.value = NULL; + } freeSSmlKv(&kv); if(!isAligned){ return TSDB_CODE_SUCCESS; @@ -459,10 +466,12 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL if (currElement->colArray == NULL) { currElement->colArray = taosArrayInit_s(sizeof(SSmlKv), 1); if (currElement->colArray == NULL) { + freeSSmlKv(&kv); return terrno; } } if (taosArrayPush(currElement->colArray, &kv) == NULL){ // reserve for timestamp + freeSSmlKv(&kv); return terrno; } } diff --git a/source/client/src/clientStmt2.c b/source/client/src/clientStmt2.c index b78e0d0f56..2f046b61d6 100644 --- a/source/client/src/clientStmt2.c +++ b/source/client/src/clientStmt2.c @@ -1200,6 +1200,22 @@ static int stmtAddBatch2(TAOS_STMT2* stmt) { STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_ADD_BATCH)); + if (pStmt->sql.stbInterlaceMode) { + int64_t startUs2 = taosGetTimestampUs(); + pStmt->stat.addBatchUs += startUs2 - startUs; + + pStmt->sql.siInfo.tableColsReady = false; + + SStmtQNode* param = NULL; + STMT_ERR_RET(stmtAllocQNodeFromBuf(&pStmt->sql.siInfo.tbBuf, (void**)¶m)); + param->restoreTbCols = true; + param->next = NULL; + + stmtEnqueue(pStmt, param); + + return TSDB_CODE_SUCCESS; + } + STMT_ERR_RET(stmtCacheBlock(pStmt)); return TSDB_CODE_SUCCESS; @@ -1403,10 +1419,10 @@ int stmtBindBatch2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* bind, int32_t colIdx) { if (pStmt->sql.stbInterlaceMode) { STMT_ERR_RET(stmtAppendTablePostHandle(pStmt, param)); + } else { + STMT_ERR_RET(stmtAddBatch2(pStmt)); } - STMT_ERR_RET(stmtAddBatch2(pStmt)); - pStmt->stat.bindDataUs4 += taosGetTimestampUs() - startUs4; return TSDB_CODE_SUCCESS; @@ -1609,24 +1625,12 @@ int stmtExec2(TAOS_STMT2* stmt, int* affected_rows) { return pStmt->errCode; } - STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_EXECUTE)); - if (pStmt->sql.stbInterlaceMode) { - int64_t startUs2 = taosGetTimestampUs(); - pStmt->stat.addBatchUs += startUs2 - startUs; - - pStmt->sql.siInfo.tableColsReady = false; - - SStmtQNode* param = NULL; - STMT_ERR_RET(stmtAllocQNodeFromBuf(&pStmt->sql.siInfo.tbBuf, (void**)¶m)); - param->restoreTbCols = true; - param->next = NULL; - - stmtEnqueue(pStmt, param); - - return TSDB_CODE_SUCCESS; + STMT_ERR_RET(stmtAddBatch2(pStmt)); } + STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_EXECUTE)); + if (STMT_TYPE_QUERY != pStmt->sql.type) { if (pStmt->sql.stbInterlaceMode) { int64_t startTs = taosGetTimestampUs(); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 5372533455..60d1f34cf7 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2625,6 +2625,8 @@ int32_t dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf uError("func %s failed to convert to ucs charset since %s", __func__, tstrerror(code)); lino = __LINE__; goto _exit; + } else { // reset the length value + code = TSDB_CODE_SUCCESS; } len += tsnprintf(dumpBuf + len, size - len, " %15s |", pBuf); if (len >= size - 1) goto _exit; diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index ddef0537f8..d15fafbda0 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -16,6 +16,7 @@ #define _DEFAULT_SOURCE #include "dmMgmt.h" #include "mnode.h" +#include "osFile.h" #include "tconfig.h" #include "tglobal.h" #include "version.h" @@ -181,6 +182,9 @@ static void dmSetSignalHandle() { } #endif } +extern bool generateNewMeta; + +extern bool generateNewMeta; static int32_t dmParseArgs(int32_t argc, char const *argv[]) { global.startTime = taosGetTimestampMs(); @@ -220,6 +224,8 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) { global.dumpSdb = true; } else if (strcmp(argv[i], "-dTxn") == 0) { global.deleteTrans = true; + } else if (strcmp(argv[i], "-r") == 0) { + generateNewMeta = true; } else if (strcmp(argv[i], "-E") == 0) { if (i < argc - 1) { if (strlen(argv[++i]) >= PATH_MAX) { @@ -415,6 +421,9 @@ int mainWindows(int argc, char **argv) { return code; } int ret = dmUpdateEncryptKey(global.encryptKey, toLogFile); + if (taosCloseFile(&pFile) != 0) { + encryptError("failed to close file:%p", pFile); + } taosCloseLog(); taosCleanupArgs(); return ret; diff --git a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h index 18b3f66a60..cbf1959e75 100644 --- a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h +++ b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h @@ -28,6 +28,7 @@ typedef struct SDnodeMgmt { const char *path; const char *name; TdThread statusThread; + TdThread statusInfoThread; TdThread notifyThread; TdThread monitorThread; TdThread auditThread; @@ -49,6 +50,7 @@ typedef struct SDnodeMgmt { // dmHandle.c SArray *dmGetMsgHandles(); void dmSendStatusReq(SDnodeMgmt *pMgmt); +void dmUpdateStatusInfo(SDnodeMgmt *pMgmt); void dmSendNotifyReq(SDnodeMgmt *pMgmt, SNotifyReq *pReq); int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t dmProcessAuthRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); @@ -62,7 +64,9 @@ int32_t dmProcessCreateEncryptKeyReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); // dmWorker.c int32_t dmPutNodeMsgToMgmtQueue(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t dmStartStatusThread(SDnodeMgmt *pMgmt); +int32_t dmStartStatusInfoThread(SDnodeMgmt *pMgmt); void dmStopStatusThread(SDnodeMgmt *pMgmt); +void dmStopStatusInfoThread(SDnodeMgmt *pMgmt); int32_t dmStartNotifyThread(SDnodeMgmt *pMgmt); void dmStopNotifyThread(SDnodeMgmt *pMgmt); int32_t dmStartMonitorThread(SDnodeMgmt *pMgmt); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index f1f3a3bee7..d0f8fbae87 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -22,6 +22,8 @@ extern SConfig *tsCfg; +SMonVloadInfo tsVinfo = {0}; + static void dmUpdateDnodeCfg(SDnodeMgmt *pMgmt, SDnodeCfg *pCfg) { int32_t code = 0; if (pMgmt->pData->dnodeId == 0 || pMgmt->pData->clusterId == 0) { @@ -123,6 +125,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { int32_t code = 0; SStatusReq req = {0}; + dDebug("send status req to mnode, statusSeq:%d, begin to mgnt lock", pMgmt->statusSeq); (void)taosThreadRwlockRdlock(&pMgmt->pData->lock); req.sver = tsVersion; req.dnodeVer = pMgmt->pData->dnodeVer; @@ -161,14 +164,24 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { memcpy(req.clusterCfg.charset, tsCharset, TD_LOCALE_LEN); (void)taosThreadRwlockUnlock(&pMgmt->pData->lock); - SMonVloadInfo vinfo = {0}; - (*pMgmt->getVnodeLoadsFp)(&vinfo); - req.pVloads = vinfo.pVloads; + dDebug("send status req to mnode, statusSeq:%d, begin to get vnode loads", pMgmt->statusSeq); + if (taosThreadMutexLock(&pMgmt->pData->statusInfolock) != 0) { + dError("failed to lock status info lock"); + return; + } + req.pVloads = tsVinfo.pVloads; + tsVinfo.pVloads = NULL; + if (taosThreadMutexUnlock(&pMgmt->pData->statusInfolock) != 0) { + dError("failed to unlock status info lock"); + return; + } + dDebug("send status req to mnode, statusSeq:%d, begin to get mnode loads", pMgmt->statusSeq); SMonMloadInfo minfo = {0}; (*pMgmt->getMnodeLoadsFp)(&minfo); req.mload = minfo.load; + dDebug("send status req to mnode, statusSeq:%d, begin to get qnode loads", pMgmt->statusSeq); (*pMgmt->getQnodeLoadsFp)(&req.qload); pMgmt->statusSeq++; @@ -206,6 +219,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { int8_t epUpdated = 0; (void)dmGetMnodeEpSet(pMgmt->pData, &epSet); + dDebug("send status req to mnode, statusSeq:%d, begin to send rpc msg", pMgmt->statusSeq); code = rpcSendRecvWithTimeout(pMgmt->msgCb.statusRpc, &epSet, &rpcMsg, &rpcRsp, &epUpdated, tsStatusInterval * 5 * 1000); if (code != 0) { @@ -226,6 +240,28 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { dmProcessStatusRsp(pMgmt, &rpcRsp); } +void dmUpdateStatusInfo(SDnodeMgmt *pMgmt) { + SMonVloadInfo vinfo = {0}; + dDebug("begin to get vnode loads"); + (*pMgmt->getVnodeLoadsFp)(&vinfo); + dDebug("begin to lock status info"); + if (taosThreadMutexLock(&pMgmt->pData->statusInfolock) != 0) { + dError("failed to lock status info lock"); + return; + } + if (tsVinfo.pVloads == NULL) { + tsVinfo.pVloads = vinfo.pVloads; + vinfo.pVloads = NULL; + } else { + taosArrayDestroy(vinfo.pVloads); + vinfo.pVloads = NULL; + } + if (taosThreadMutexUnlock(&pMgmt->pData->statusInfolock) != 0) { + dError("failed to unlock status info lock"); + return; + } +} + void dmSendNotifyReq(SDnodeMgmt *pMgmt, SNotifyReq *pReq) { int32_t contLen = tSerializeSNotifyReq(NULL, 0, pReq); if (contLen < 0) { diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c index 22c2b2f5b2..ed156ac1ec 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c @@ -22,6 +22,9 @@ static int32_t dmStartMgmt(SDnodeMgmt *pMgmt) { if ((code = dmStartStatusThread(pMgmt)) != 0) { return code; } + if ((code = dmStartStatusInfoThread(pMgmt)) != 0) { + return code; + } #if defined(TD_ENTERPRISE) if ((code = dmStartNotifyThread(pMgmt)) != 0) { return code; @@ -44,6 +47,7 @@ static void dmStopMgmt(SDnodeMgmt *pMgmt) { dmStopMonitorThread(pMgmt); dmStopAuditThread(pMgmt); dmStopStatusThread(pMgmt); + dmStopStatusInfoThread(pMgmt); #if defined(TD_ENTERPRISE) dmStopNotifyThread(pMgmt); #endif diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 58b86b20b1..7f802f3837 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -47,6 +47,35 @@ static void *dmStatusThreadFp(void *param) { return NULL; } +static void *dmStatusInfoThreadFp(void *param) { + SDnodeMgmt *pMgmt = param; + int64_t lastTime = taosGetTimestampMs(); + setThreadName("dnode-status-info"); + + int32_t upTimeCount = 0; + int64_t upTime = 0; + + while (1) { + taosMsleep(200); + if (pMgmt->pData->dropped || pMgmt->pData->stopped) break; + + int64_t curTime = taosGetTimestampMs(); + if (curTime < lastTime) lastTime = curTime; + float interval = (curTime - lastTime) / 1000.0f; + if (interval >= tsStatusInterval) { + dmUpdateStatusInfo(pMgmt); + lastTime = curTime; + + if ((upTimeCount = ((upTimeCount + 1) & 63)) == 0) { + upTime = taosGetOsUptime() - tsDndStartOsUptime; + tsDndUpTime = TMAX(tsDndUpTime, upTime); + } + } + } + + return NULL; +} + SDmNotifyHandle dmNotifyHdl = {.state = 0}; #define TIMESERIES_STASH_NUM 5 static void *dmNotifyThreadFp(void *param) { @@ -280,6 +309,22 @@ int32_t dmStartStatusThread(SDnodeMgmt *pMgmt) { return 0; } +int32_t dmStartStatusInfoThread(SDnodeMgmt *pMgmt) { + int32_t code = 0; + TdThreadAttr thAttr; + (void)taosThreadAttrInit(&thAttr); + (void)taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); + if (taosThreadCreate(&pMgmt->statusInfoThread, &thAttr, dmStatusInfoThreadFp, pMgmt) != 0) { + code = TAOS_SYSTEM_ERROR(errno); + dError("failed to create status Info thread since %s", tstrerror(code)); + return code; + } + + (void)taosThreadAttrDestroy(&thAttr); + tmsgReportStartup("dnode-status-info", "initialized"); + return 0; +} + void dmStopStatusThread(SDnodeMgmt *pMgmt) { if (taosCheckPthreadValid(pMgmt->statusThread)) { (void)taosThreadJoin(pMgmt->statusThread, NULL); @@ -287,6 +332,13 @@ void dmStopStatusThread(SDnodeMgmt *pMgmt) { } } +void dmStopStatusInfoThread(SDnodeMgmt *pMgmt) { + if (taosCheckPthreadValid(pMgmt->statusInfoThread)) { + (void)taosThreadJoin(pMgmt->statusInfoThread, NULL); + taosThreadClear(&pMgmt->statusInfoThread); + } +} + int32_t dmStartNotifyThread(SDnodeMgmt *pMgmt) { int32_t code = 0; TdThreadAttr thAttr; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index 3cf0382eba..20618dbdf3 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -213,13 +213,13 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal) taosQueueGetThreadId(pVnode->pApplyW.queue)); tMultiWorkerCleanup(&pVnode->pApplyW); - dInfo("vgId:%d, wait for vnode query queue:%p is empty", pVnode->vgId, pVnode->pQueryQ); - while (!taosQueueEmpty(pVnode->pQueryQ)) taosMsleep(10); - dInfo("vgId:%d, wait for vnode fetch queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pFetchQ, taosQueueGetThreadId(pVnode->pFetchQ)); while (!taosQueueEmpty(pVnode->pFetchQ)) taosMsleep(10); + dInfo("vgId:%d, wait for vnode query queue:%p is empty", pVnode->vgId, pVnode->pQueryQ); + while (!taosQueueEmpty(pVnode->pQueryQ)) taosMsleep(10); + tqNotifyClose(pVnode->pImpl->pTq); dInfo("vgId:%d, wait for vnode stream queue:%p is empty", pVnode->vgId, pVnode->pStreamQ); while (!taosQueueEmpty(pVnode->pStreamQ)) taosMsleep(10); diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index 277dd2e02a..9f1c292a90 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -214,10 +214,12 @@ int32_t dmInitVars(SDnode *pDnode) { } (void)taosThreadRwlockInit(&pData->lock, NULL); + (void)taosThreadMutexInit(&pData->statusInfolock, NULL); (void)taosThreadMutexInit(&pDnode->mutex, NULL); return 0; } +extern SMonVloadInfo tsVinfo; void dmClearVars(SDnode *pDnode) { for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) { SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype]; @@ -253,6 +255,25 @@ void dmClearVars(SDnode *pDnode) { (void)taosThreadRwlockUnlock(&pData->lock); (void)taosThreadRwlockDestroy(&pData->lock); + + dDebug("begin to lock status info when thread exit"); + if (taosThreadMutexLock(&pData->statusInfolock) != 0) { + dError("failed to lock status info lock"); + return; + } + if (tsVinfo.pVloads != NULL) { + taosArrayDestroy(tsVinfo.pVloads); + tsVinfo.pVloads = NULL; + } + if (taosThreadMutexUnlock(&pData->statusInfolock) != 0) { + dError("failed to unlock status info lock"); + return; + } + if (taosThreadMutexDestroy(&pData->statusInfolock) != 0) { + dError("failed to destroy status info lock"); + } + memset(&pData->statusInfolock, 0, sizeof(pData->statusInfolock)); + (void)taosThreadMutexDestroy(&pDnode->mutex); memset(&pDnode->mutex, 0, sizeof(pDnode->mutex)); } diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h index b5842acbad..de20f807e9 100644 --- a/source/dnode/mgmt/node_util/inc/dmUtil.h +++ b/source/dnode/mgmt/node_util/inc/dmUtil.h @@ -144,6 +144,7 @@ typedef struct { char machineId[TSDB_MACHINE_ID_LEN + 1]; EEncryptAlgor encryptAlgorigthm; EEncryptScope encryptScope; + TdThreadMutex statusInfolock; } SDnodeData; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index bee971b966..685ad2b7a5 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -515,6 +515,7 @@ static int32_t mndInitWal(SMnode *pMnode) { .fsyncPeriod = 0, .rollPeriod = -1, .segSize = -1, + .committed = -1, .retentionPeriod = 0, .retentionSize = 0, .level = TAOS_WAL_FSYNC, diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 69d3de25fc..a4327b777f 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -1783,7 +1783,7 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) { static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SStreamObj *pStream = NULL; - int32_t code = 0; + int32_t code = 0; if ((code = grantCheckExpire(TSDB_GRANT_STREAMS)) < 0) { return code; @@ -1811,6 +1811,7 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) { return 0; } + mInfo("stream:%s,%" PRId64 " start to resume stream from pause", resumeReq.name, pStream->uid); if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pStream->targetDb) != 0) { sdbRelease(pMnode->pSdb, pStream); return -1; diff --git a/source/dnode/mnode/impl/src/mndStreamTransAct.c b/source/dnode/mnode/impl/src/mndStreamTransAct.c index 4e0bf97587..139ea4f147 100644 --- a/source/dnode/mnode/impl/src/mndStreamTransAct.c +++ b/source/dnode/mnode/impl/src/mndStreamTransAct.c @@ -61,7 +61,6 @@ static int32_t doSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTa static int32_t doSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask) { SVDropStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVDropStreamTaskReq)); if (pReq == NULL) { - // terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno; } @@ -93,7 +92,6 @@ static int32_t doSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamTask *pT if (pReq == NULL) { mError("failed to malloc in resume stream, size:%" PRIzu ", code:%s", sizeof(SVResumeStreamTaskReq), tstrerror(TSDB_CODE_OUT_OF_MEMORY)); - // terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno; } @@ -106,19 +104,18 @@ static int32_t doSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamTask *pT bool hasEpset = false; int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId); if (code != TSDB_CODE_SUCCESS || (!hasEpset)) { - terrno = code; taosMemoryFree(pReq); - return terrno; + return code; } code = setTransAction(pTrans, pReq, sizeof(SVResumeStreamTaskReq), TDMT_STREAM_TASK_RESUME, &epset, 0, TSDB_CODE_VND_INVALID_VGROUP_ID); if (code != 0) { taosMemoryFree(pReq); - return terrno; + return code; } mDebug("set the resume action for trans:%d", pTrans->id); - return 0; + return code; } static int32_t doSetDropActionFromId(SMnode *pMnode, STrans *pTrans, SOrphanTask* pTask) { diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index fcf7905cd1..4268d73746 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -474,6 +474,21 @@ static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans) { // pTrans->startFunc = 0; } + if (pTrans->stage == TRN_STAGE_COMMIT) { + pTrans->stage = TRN_STAGE_COMMIT_ACTION; + mInfo("trans:%d, stage from commit to commitAction since perform update action", pTrans->id); + } + + if (pTrans->stage == TRN_STAGE_ROLLBACK) { + pTrans->stage = TRN_STAGE_UNDO_ACTION; + mInfo("trans:%d, stage from rollback to undoAction since perform update action", pTrans->id); + } + + if (pTrans->stage == TRN_STAGE_PRE_FINISH) { + pTrans->stage = TRN_STAGE_FINISH; + mInfo("trans:%d, stage from pre-finish to finished since perform update action", pTrans->id); + } + return 0; } @@ -563,17 +578,17 @@ static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pOld, STrans *pNew) { if (pOld->stage == TRN_STAGE_COMMIT) { pOld->stage = TRN_STAGE_COMMIT_ACTION; - mTrace("trans:%d, stage from commit to commitAction since perform update action", pNew->id); + mInfo("trans:%d, stage from commit to commitAction since perform update action", pNew->id); } if (pOld->stage == TRN_STAGE_ROLLBACK) { pOld->stage = TRN_STAGE_UNDO_ACTION; - mTrace("trans:%d, stage from rollback to undoAction since perform update action", pNew->id); + mInfo("trans:%d, stage from rollback to undoAction since perform update action", pNew->id); } if (pOld->stage == TRN_STAGE_PRE_FINISH) { pOld->stage = TRN_STAGE_FINISH; - mTrace("trans:%d, stage from pre-finish to finished since perform update action", pNew->id); + mInfo("trans:%d, stage from pre-finish to finished since perform update action", pNew->id); } return 0; @@ -1252,8 +1267,9 @@ int32_t mndTransProcessRsp(SRpcMsg *pRsp) { pAction->errCode = pRsp->code; pTrans->lastErrorNo = pRsp->code; - mInfo("trans:%d, %s:%d response is received, code:0x%x, accept:0x%x retry:0x%x", transId, - mndTransStr(pAction->stage), action, pRsp->code, pAction->acceptableCode, pAction->retryCode); + mInfo("trans:%d, %s:%d response is received, received code:0x%x(%s), accept:0x%x(%s) retry:0x%x(%s)", transId, + mndTransStr(pAction->stage), action, pRsp->code, tstrerror(pRsp->code), pAction->acceptableCode, + tstrerror(pAction->acceptableCode), pAction->retryCode, tstrerror(pAction->retryCode)); } else { mInfo("trans:%d, invalid action, index:%d, code:0x%x", transId, action, pRsp->code); } @@ -1294,7 +1310,7 @@ static void mndTransResetActions(SMnode *pMnode, STrans *pTrans, SArray *pArray) } } -// execute at bottom half +// execute in sync context static int32_t mndTransWriteSingleLog(SMnode *pMnode, STrans *pTrans, STransAction *pAction, bool topHalf) { if (pAction->rawWritten) return 0; if (topHalf) { @@ -1320,7 +1336,7 @@ static int32_t mndTransWriteSingleLog(SMnode *pMnode, STrans *pTrans, STransActi TAOS_RETURN(code); } -// execute at top half +// execute in trans context static int32_t mndTransSendSingleMsg(SMnode *pMnode, STrans *pTrans, STransAction *pAction, bool topHalf) { if (pAction->msgSent) return 0; if (mndCannotExecuteTransAction(pMnode, topHalf)) { @@ -1469,8 +1485,8 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans, bool topHalf) { int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->redoActions, topHalf); if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS && code != TSDB_CODE_MND_TRANS_CTX_SWITCH) { - mError("trans:%d, failed to execute redoActions since:%s, code:0x%x, topHalf:%d", pTrans->id, terrstr(), terrno, - topHalf); + mError("trans:%d, failed to execute redoActions since:%s, code:0x%x, topHalf(TransContext):%d", pTrans->id, + terrstr(), terrno, topHalf); } return code; } @@ -1478,7 +1494,8 @@ static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans, bool t static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans, bool topHalf) { int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->undoActions, topHalf); if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS && code != TSDB_CODE_MND_TRANS_CTX_SWITCH) { - mError("trans:%d, failed to execute undoActions since %s. topHalf:%d", pTrans->id, terrstr(), topHalf); + mError("trans:%d, failed to execute undoActions since %s. topHalf(TransContext):%d", pTrans->id, terrstr(), + topHalf); } return code; } @@ -1486,7 +1503,8 @@ static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans, bool t static int32_t mndTransExecuteCommitActions(SMnode *pMnode, STrans *pTrans, bool topHalf) { int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->commitActions, topHalf); if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS && code != TSDB_CODE_MND_TRANS_CTX_SWITCH) { - mError("trans:%d, failed to execute commitActions since %s. topHalf:%d", pTrans->id, terrstr(), topHalf); + mError("trans:%d, failed to execute commitActions since %s. topHalf(TransContext):%d", pTrans->id, terrstr(), + topHalf); } return code; } @@ -1500,11 +1518,15 @@ static int32_t mndTransExecuteActionsSerial(SMnode *pMnode, STrans *pTrans, SArr return code; } - mInfo("trans:%d, execute %d actions serial, current action:%d", pTrans->id, numOfActions, pTrans->actionPos); + mInfo("trans:%d, execute %d actions serial, begin at action:%d, stage:%s", pTrans->id, numOfActions, + pTrans->actionPos, mndTransStr(pTrans->stage)); for (int32_t action = pTrans->actionPos; action < numOfActions; ++action) { STransAction *pAction = taosArrayGet(pActions, action); + mInfo("trans:%d, current action:%d, stage:%s, actionType(0:log,1:msg):%d", pTrans->id, pTrans->actionPos, + mndTransStr(pAction->stage), pAction->actionType); + code = mndTransExecSingleAction(pMnode, pTrans, pAction, topHalf); if (code == 0) { if (pAction->msgSent) { @@ -1536,8 +1558,8 @@ static int32_t mndTransExecuteActionsSerial(SMnode *pMnode, STrans *pTrans, SArr if (mndCannotExecuteTransAction(pMnode, topHalf)) { pTrans->lastErrorNo = code; pTrans->code = code; - mInfo("trans:%d, %s:%d, topHalf:%d, not execute next action, code:%s", pTrans->id, mndTransStr(pAction->stage), - action, topHalf, tstrerror(code)); + mInfo("trans:%d, %s:%d, topHalf(TransContext):%d, not execute next action, code:%s", pTrans->id, + mndTransStr(pAction->stage), action, topHalf, tstrerror(code)); break; } @@ -1561,7 +1583,8 @@ static int32_t mndTransExecuteActionsSerial(SMnode *pMnode, STrans *pTrans, SArr break; } else if (code == pAction->retryCode || code == TSDB_CODE_SYN_PROPOSE_NOT_READY || code == TSDB_CODE_SYN_RESTORING || code == TSDB_CODE_SYN_NOT_LEADER) { - mInfo("trans:%d, %s:%d receive code:0x%x and retry", pTrans->id, mndTransStr(pAction->stage), pAction->id, code); + mInfo("trans:%d, %s:%d receive code:0x%x(%s) and retry", pTrans->id, mndTransStr(pAction->stage), pAction->id, + code, tstrerror(code)); pTrans->lastErrorNo = code; taosMsleep(300); action--; @@ -1570,8 +1593,8 @@ static int32_t mndTransExecuteActionsSerial(SMnode *pMnode, STrans *pTrans, SArr terrno = code; pTrans->lastErrorNo = code; pTrans->code = code; - mInfo("trans:%d, %s:%d receive code:0x%x and wait another schedule, failedTimes:%d", pTrans->id, - mndTransStr(pAction->stage), pAction->id, code, pTrans->failedTimes); + mInfo("trans:%d, %s:%d receive code:0x%x(%s) and wait another schedule, failedTimes:%d", pTrans->id, + mndTransStr(pAction->stage), pAction->id, code, tstrerror(code), pTrans->failedTimes); break; } } @@ -1647,8 +1670,8 @@ static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans, bool } else { continueExec = false; } - mInfo("trans:%d, cannot execute redo action stage, topHalf:%d, continueExec:%d, code:%s", pTrans->id, topHalf, - continueExec, tstrerror(code)); + mInfo("trans:%d, cannot execute redo action stage, topHalf(TransContext):%d, continueExec:%d, code:%s", pTrans->id, + topHalf, continueExec, tstrerror(code)); return continueExec; } @@ -1680,7 +1703,9 @@ static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans, bool } pTrans->stage = TRN_STAGE_ROLLBACK; - mError("trans:%d, stage from redoAction to rollback since %s", pTrans->id, terrstr()); + pTrans->actionPos = 0; + mError("trans:%d, stage from redoAction to rollback since %s, and set actionPos to %d", pTrans->id, terrstr(), + pTrans->actionPos); continueExec = true; } else { mError("trans:%d, stage keep on redoAction since %s, failedTimes:%d", pTrans->id, terrstr(), pTrans->failedTimes); @@ -1691,6 +1716,7 @@ static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans, bool return continueExec; } +// in trans context static bool mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans, bool topHalf) { if (mndCannotExecuteTransAction(pMnode, topHalf)) return false; @@ -1765,6 +1791,7 @@ static bool mndTransPerformUndoActionStage(SMnode *pMnode, STrans *pTrans, bool return continueExec; } +// in trans context static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans, bool topHalf) { if (mndCannotExecuteTransAction(pMnode, topHalf)) return false; @@ -1773,8 +1800,6 @@ static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans, bool to if (code == 0) { pTrans->stage = TRN_STAGE_UNDO_ACTION; - pTrans->actionPos = 0; - mInfo("trans:%d, stage from rollback to undoAction, actionPos:%d", pTrans->id, pTrans->actionPos); continueExec = true; } else { pTrans->failedTimes++; @@ -1829,7 +1854,7 @@ void mndTransExecuteImp(SMnode *pMnode, STrans *pTrans, bool topHalf) { bool continueExec = true; while (continueExec) { - mInfo("trans:%d, continue to execute, stage:%s createTime:%" PRId64 " topHalf:%d", pTrans->id, + mInfo("trans:%d, continue to execute, stage:%s createTime:%" PRId64 " topHalf(TransContext):%d", pTrans->id, mndTransStr(pTrans->stage), pTrans->createdTime, topHalf); pTrans->lastExecTime = taosGetTimestampMs(); switch (pTrans->stage) { diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 1bd4317234..fc98d6578b 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -81,6 +81,9 @@ typedef struct SCommitInfo SCommitInfo; typedef struct SCompactInfo SCompactInfo; typedef struct SQueryNode SQueryNode; +#define VNODE_META_TMP_DIR "meta.tmp" +#define VNODE_META_BACKUP_DIR "meta.backup" + #define VNODE_META_DIR "meta" #define VNODE_TSDB_DIR "tsdb" #define VNODE_TQ_DIR "tq" diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index f062505ac7..659ba3f777 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -133,7 +133,7 @@ static void doScan(SMeta *pMeta) { } } -int32_t metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) { +static int32_t metaOpenImpl(SVnode *pVnode, SMeta **ppMeta, const char *metaDir, int8_t rollback) { SMeta *pMeta = NULL; int32_t code = 0; int32_t lino; @@ -144,7 +144,11 @@ int32_t metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) { // create handle vnodeGetPrimaryDir(pVnode->path, pVnode->diskPrimary, pVnode->pTfs, path, TSDB_FILENAME_LEN); offset = strlen(path); - snprintf(path + offset, TSDB_FILENAME_LEN - offset - 1, "%s%s", TD_DIRSEP, VNODE_META_DIR); + snprintf(path + offset, TSDB_FILENAME_LEN - offset - 1, "%s%s", TD_DIRSEP, metaDir); + + if (strncmp(metaDir, VNODE_META_TMP_DIR, strlen(VNODE_META_TMP_DIR)) == 0) { + taosRemoveDir(path); + } if ((pMeta = taosMemoryCalloc(1, sizeof(*pMeta) + strlen(path) + 1)) == NULL) { TSDB_CHECK_CODE(code = terrno, lino, _exit); @@ -245,6 +249,188 @@ _exit: return code; } +bool generateNewMeta = false; + +static int32_t metaGenerateNewMeta(SMeta **ppMeta) { + SMeta *pNewMeta = NULL; + SMeta *pMeta = *ppMeta; + SVnode *pVnode = pMeta->pVnode; + + metaInfo("vgId:%d start to generate new meta", TD_VID(pMeta->pVnode)); + + // Open a new meta for orgainzation + int32_t code = metaOpenImpl(pMeta->pVnode, &pNewMeta, VNODE_META_TMP_DIR, false); + if (code) { + return code; + } + + code = metaBegin(pNewMeta, META_BEGIN_HEAP_NIL); + if (code) { + return code; + } + + // i == 0, scan super table + // i == 1, scan normal table and child table + for (int i = 0; i < 2; i++) { + TBC *uidCursor = NULL; + int32_t counter = 0; + + code = tdbTbcOpen(pMeta->pUidIdx, &uidCursor, NULL); + if (code) { + metaError("vgId:%d failed to open uid index cursor, reason:%s", TD_VID(pVnode), tstrerror(code)); + return code; + } + + code = tdbTbcMoveToFirst(uidCursor); + if (code) { + metaError("vgId:%d failed to move to first, reason:%s", TD_VID(pVnode), tstrerror(code)); + tdbTbcClose(uidCursor); + return code; + } + + for (;;) { + const void *pKey; + int kLen; + const void *pVal; + int vLen; + + if (tdbTbcGet(uidCursor, &pKey, &kLen, &pVal, &vLen) < 0) { + break; + } + + tb_uid_t uid = *(tb_uid_t *)pKey; + SUidIdxVal *pUidIdxVal = (SUidIdxVal *)pVal; + if ((i == 0 && (pUidIdxVal->suid && pUidIdxVal->suid == uid)) // super table + || (i == 1 && (pUidIdxVal->suid == 0 || pUidIdxVal->suid != uid)) // normal table and child table + ) { + counter++; + if (i == 0) { + metaInfo("vgId:%d counter:%d new meta handle %s table uid:%" PRId64, TD_VID(pVnode), counter, "super", uid); + } else { + metaInfo("vgId:%d counter:%d new meta handle %s table uid:%" PRId64, TD_VID(pVnode), counter, + pUidIdxVal->suid == 0 ? "normal" : "child", uid); + } + + // fetch table entry + void *value = NULL; + int valueSize = 0; + if (tdbTbGet(pMeta->pTbDb, + &(STbDbKey){ + .version = pUidIdxVal->version, + .uid = uid, + }, + sizeof(uid), &value, &valueSize) == 0) { + SDecoder dc = {0}; + SMetaEntry me = {0}; + tDecoderInit(&dc, value, valueSize); + if (metaDecodeEntry(&dc, &me) == 0) { + if (metaHandleEntry(pNewMeta, &me) != 0) { + metaError("vgId:%d failed to handle entry, uid:%" PRId64, TD_VID(pVnode), uid); + } + } + tDecoderClear(&dc); + } + tdbFree(value); + } + + code = tdbTbcMoveToNext(uidCursor); + if (code) { + metaError("vgId:%d failed to move to next, reason:%s", TD_VID(pVnode), tstrerror(code)); + return code; + } + } + + tdbTbcClose(uidCursor); + } + + code = metaCommit(pNewMeta, pNewMeta->txn); + if (code) { + metaError("vgId:%d failed to commit, reason:%s", TD_VID(pVnode), tstrerror(code)); + return code; + } + + code = metaFinishCommit(pNewMeta, pNewMeta->txn); + if (code) { + metaError("vgId:%d failed to finish commit, reason:%s", TD_VID(pVnode), tstrerror(code)); + return code; + } + + if ((code = metaBegin(pNewMeta, META_BEGIN_HEAP_NIL)) != 0) { + metaError("vgId:%d failed to begin new meta, reason:%s", TD_VID(pVnode), tstrerror(code)); + } + metaClose(&pNewMeta); + metaInfo("vgId:%d finish to generate new meta", TD_VID(pVnode)); + return 0; +} + +int32_t metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) { + if (generateNewMeta) { + char path[TSDB_FILENAME_LEN] = {0}; + char oldMetaPath[TSDB_FILENAME_LEN] = {0}; + char newMetaPath[TSDB_FILENAME_LEN] = {0}; + char backupMetaPath[TSDB_FILENAME_LEN] = {0}; + + vnodeGetPrimaryDir(pVnode->path, pVnode->diskPrimary, pVnode->pTfs, path, TSDB_FILENAME_LEN); + snprintf(oldMetaPath, sizeof(oldMetaPath) - 1, "%s%s%s", path, TD_DIRSEP, VNODE_META_DIR); + snprintf(newMetaPath, sizeof(newMetaPath) - 1, "%s%s%s", path, TD_DIRSEP, VNODE_META_TMP_DIR); + snprintf(backupMetaPath, sizeof(backupMetaPath) - 1, "%s%s%s", path, TD_DIRSEP, VNODE_META_BACKUP_DIR); + + bool oldMetaExist = taosCheckExistFile(oldMetaPath); + bool newMetaExist = taosCheckExistFile(newMetaPath); + bool backupMetaExist = taosCheckExistFile(backupMetaPath); + + if ((!backupMetaExist && !oldMetaExist && newMetaExist) // case 2 + || (backupMetaExist && !oldMetaExist && !newMetaExist) // case 4 + || (backupMetaExist && oldMetaExist && newMetaExist) // case 8 + ) { + metaError("vgId:%d invalid meta state, please check", TD_VID(pVnode)); + return TSDB_CODE_FAILED; + } else if ((backupMetaExist && oldMetaExist && !newMetaExist) // case 7 + || (!backupMetaExist && !oldMetaExist && !newMetaExist) // case 1 + ) { + return metaOpenImpl(pVnode, ppMeta, VNODE_META_DIR, rollback); + } else if (backupMetaExist && !oldMetaExist && newMetaExist) { + if (taosRenameFile(newMetaPath, oldMetaPath) != 0) { + metaError("vgId:%d failed to rename new meta to old meta, reason:%s", TD_VID(pVnode), tstrerror(terrno)); + return terrno; + } + return metaOpenImpl(pVnode, ppMeta, VNODE_META_DIR, rollback); + } else { + int32_t code = metaOpenImpl(pVnode, ppMeta, VNODE_META_DIR, rollback); + if (code) { + return code; + } + + code = metaGenerateNewMeta(ppMeta); + if (code) { + metaError("vgId:%d failed to generate new meta, reason:%s", TD_VID(pVnode), tstrerror(code)); + } + + metaClose(ppMeta); + if (taosRenameFile(oldMetaPath, backupMetaPath) != 0) { + metaError("vgId:%d failed to rename old meta to backup, reason:%s", TD_VID(pVnode), tstrerror(terrno)); + return terrno; + } + + // rename the new meta to old meta + if (taosRenameFile(newMetaPath, oldMetaPath) != 0) { + metaError("vgId:%d failed to rename new meta to old meta, reason:%s", TD_VID(pVnode), tstrerror(terrno)); + return terrno; + } + code = metaOpenImpl(pVnode, ppMeta, VNODE_META_DIR, false); + if (code) { + metaError("vgId:%d failed to open new meta, reason:%s", TD_VID(pVnode), tstrerror(code)); + return code; + } + } + + } else { + return metaOpenImpl(pVnode, ppMeta, VNODE_META_DIR, rollback); + } + + return TSDB_CODE_SUCCESS; +} + int32_t metaUpgrade(SVnode *pVnode, SMeta **ppMeta) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino; @@ -310,6 +496,7 @@ void metaULock(SMeta *pMeta) { static void metaCleanup(SMeta **ppMeta) { SMeta *pMeta = *ppMeta; if (pMeta) { + metaInfo("vgId:%d meta clean up, path:%s", TD_VID(pMeta->pVnode), pMeta->path); if (pMeta->pEnv) metaAbort(pMeta); if (pMeta->pCache) metaCacheClose(pMeta); #ifdef BUILD_NO_CALL diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 08ee422126..21d12ef77d 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -2985,9 +2985,6 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { } } end: - if (terrno != 0) { - ret = terrno; - } tDecoderClear(&dc); tdbFree(pData); return ret; diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 6b7e857120..a00e92997c 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -692,7 +692,7 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen STaskId id = {.streamId = pReq->streamId, .taskId = pReq->taskId}; SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); if ((ppTask != NULL) && ((*ppTask) != NULL)) { - streamMetaAcquireOneTask(*ppTask); + int32_t unusedRetRef = streamMetaAcquireOneTask(*ppTask); SStreamTask* pTask = *ppTask; if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { @@ -1119,10 +1119,6 @@ static int32_t tqProcessTaskResumeImpl(void* handle, SStreamTask* pTask, int64_t int32_t vgId = pMeta->vgId; int32_t code = 0; - if (pTask == NULL) { - return -1; - } - streamTaskResume(pTask); ETaskStatus status = streamTaskGetStatus(pTask).state; @@ -1150,7 +1146,6 @@ static int32_t tqProcessTaskResumeImpl(void* handle, SStreamTask* pTask, int64_t } } - streamMetaReleaseTask(pMeta, pTask); return code; } @@ -1173,6 +1168,7 @@ int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* m code = tqProcessTaskResumeImpl(handle, pTask, sversion, pReq->igUntreated, fromVnode); if (code != 0) { + streamMetaReleaseTask(pMeta, pTask); return code; } @@ -1186,6 +1182,7 @@ int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* m streamMutexUnlock(&pHTask->lock); code = tqProcessTaskResumeImpl(handle, pHTask, sversion, pReq->igUntreated, fromVnode); + streamMetaReleaseTask(pMeta, pHTask); } return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 89a51eb0f5..1ac65fb94d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -3064,9 +3064,8 @@ static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow, bool *pI iMax[nMax] = i; max[nMax++] = pIter->input[i].pRow; - } else { - pIter->input[i].next = false; } + pIter->input[i].next = false; } } diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.c b/source/dnode/vnode/src/tsdb/tsdbFSet2.c index fc681f9753..a0ae58ac96 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.c @@ -602,14 +602,14 @@ int32_t tsdbTFileSetInitRef(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fs SSttLvl *lvl; code = tsdbSttLvlInitRef(pTsdb, lvl1, &lvl); if (code) { - taosMemoryFree(lvl); + tsdbSttLvlClear(&lvl); tsdbTFileSetClear(fset); return code; } code = TARRAY2_APPEND(fset[0]->lvlArr, lvl); if (code) { - taosMemoryFree(lvl); + tsdbSttLvlClear(&lvl); tsdbTFileSetClear(fset); return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index d4b906fe2a..c4971e27cf 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -855,6 +855,7 @@ static int32_t loadFileBlockBrinInfo(STsdbReader* pReader, SArray* pIndexList, S STableBlockScanInfo** p = taosArrayGetLast(pTableScanInfoList); if (p == NULL) { clearBrinBlockIter(&iter); + tsdbError("invalid param, empty in tablescanInfoList, %s", pReader->idStr); return TSDB_CODE_INVALID_PARA; } @@ -5256,7 +5257,7 @@ int32_t tsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) { // NOTE: the following codes is used to perform test for suspend/resume for tsdbReader when it blocks the commit // the data should be ingested in round-robin and all the child tables should be createted before ingesting data // the version range of query will be used to identify the correctness of suspend/resume functions. - // this function will blocked before loading the SECOND block from vnode-buffer, and restart itself from sst-files + // this function will be blocked before loading the SECOND block from vnode-buffer, and restart itself from sst-files #if SUSPEND_RESUME_TEST if (!pReader->status.suspendInvoked && !pReader->status.loadFromFile) { tsem_wait(&pReader->resumeAfterSuspend); @@ -5909,6 +5910,7 @@ int32_t tsdbGetTableSchema(SMeta* pMeta, int64_t uid, STSchema** pSchema, int64_ } else if (mr.me.type == TSDB_NORMAL_TABLE) { // do nothing } else { code = TSDB_CODE_INVALID_PARA; + tsdbError("invalid mr.me.type:%d, code:%s", mr.me.type, tstrerror(code)); metaReaderClear(&mr); return code; } diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index d3acea4766..7c789e84ae 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -45,6 +45,7 @@ const SVnodeCfg vnodeCfgDefault = {.vgId = -1, .retentionPeriod = -1, .rollPeriod = 0, .segSize = 0, + .committed = 0, .retentionSize = -1, .level = TAOS_WAL_WRITE, .clearFiles = 0, diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 4a4d305f25..dae2b3a5ec 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -257,6 +257,7 @@ int vnodeLoadInfo(const char *dir, SVnodeInfo *pInfo) { code = vnodeDecodeInfo(pData, pInfo); TSDB_CHECK_CODE(code, lino, _exit); + pInfo->config.walCfg.committed = pInfo->state.committed; _exit: if (code) { if (pFile) { diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 371eaa0774..dd13c975cf 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -633,47 +633,44 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg } break; case TDMT_STREAM_TASK_DEPLOY: { - int32_t code = tqProcessTaskDeployReq(pVnode->pTq, ver, pReq, len); - if (code != TSDB_CODE_SUCCESS) { - terrno = code; + if ((code = tqProcessTaskDeployReq(pVnode->pTq, ver, pReq, len)) != TSDB_CODE_SUCCESS) { goto _err; } } break; case TDMT_STREAM_TASK_DROP: { - if (tqProcessTaskDropReq(pVnode->pTq, pMsg->pCont, pMsg->contLen) < 0) { + if ((code = tqProcessTaskDropReq(pVnode->pTq, pMsg->pCont, pMsg->contLen)) < 0) { goto _err; } } break; case TDMT_STREAM_TASK_UPDATE_CHKPT: { - if (tqProcessTaskUpdateCheckpointReq(pVnode->pTq, pMsg->pCont, pMsg->contLen) < 0) { + if ((code = tqProcessTaskUpdateCheckpointReq(pVnode->pTq, pMsg->pCont, pMsg->contLen)) < 0) { goto _err; } } break; case TDMT_STREAM_CONSEN_CHKPT: { - if (pVnode->restored) { - if (tqProcessTaskConsenChkptIdReq(pVnode->pTq, pMsg) < 0) { - goto _err; - } + if (pVnode->restored && (code = tqProcessTaskConsenChkptIdReq(pVnode->pTq, pMsg)) < 0) { + goto _err; } + } break; case TDMT_STREAM_TASK_PAUSE: { if (pVnode->restored && vnodeIsLeader(pVnode) && - tqProcessTaskPauseReq(pVnode->pTq, ver, pMsg->pCont, pMsg->contLen) < 0) { + (code = tqProcessTaskPauseReq(pVnode->pTq, ver, pMsg->pCont, pMsg->contLen)) < 0) { goto _err; } } break; case TDMT_STREAM_TASK_RESUME: { if (pVnode->restored && vnodeIsLeader(pVnode) && - tqProcessTaskResumeReq(pVnode->pTq, ver, pMsg->pCont, pMsg->contLen) < 0) { + (code = tqProcessTaskResumeReq(pVnode->pTq, ver, pMsg->pCont, pMsg->contLen)) < 0) { goto _err; } } break; case TDMT_VND_STREAM_TASK_RESET: { - if (pVnode->restored && vnodeIsLeader(pVnode)) { - if (tqProcessTaskResetReq(pVnode->pTq, pMsg) < 0) { + if (pVnode->restored && vnodeIsLeader(pVnode) && + (code = tqProcessTaskResetReq(pVnode->pTq, pMsg)) < 0) { goto _err; } - } + } break; case TDMT_VND_ALTER_CONFIRM: needCommit = pVnode->config.hashChange; @@ -693,10 +690,10 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg case TDMT_VND_DROP_INDEX: vnodeProcessDropIndexReq(pVnode, ver, pReq, len, pRsp); break; - case TDMT_VND_STREAM_CHECK_POINT_SOURCE: + case TDMT_VND_STREAM_CHECK_POINT_SOURCE: // always return true tqProcessTaskCheckPointSourceReq(pVnode->pTq, pMsg, pRsp); break; - case TDMT_VND_STREAM_TASK_UPDATE: + case TDMT_VND_STREAM_TASK_UPDATE: // always return true tqProcessTaskUpdateReq(pVnode->pTq, pMsg); break; case TDMT_VND_COMPACT: @@ -752,7 +749,7 @@ _exit: _err: vError("vgId:%d, process %s request failed since %s, ver:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), - tstrerror(terrno), ver); + tstrerror(code), ver); return code; } diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 27a43f7523..95c73763bf 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -551,7 +551,7 @@ void appendTagFields(char* buf, int32_t* len, STableCfg* pCfg) { (int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); } - *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, sizeof(type) - (VARSTR_HEADER_SIZE + *len), "%s`%s` %s", + *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), "%s`%s` %s", ((i > 0) ? ", " : ""), pSchema->name, type); } } diff --git a/source/libs/executor/src/timesliceoperator.c b/source/libs/executor/src/timesliceoperator.c index 70bf26405e..2ea300ace8 100644 --- a/source/libs/executor/src/timesliceoperator.c +++ b/source/libs/executor/src/timesliceoperator.c @@ -278,7 +278,7 @@ static bool checkNullRow(SExprSupp* pExprSup, SSDataBlock* pSrcBlock, int32_t in } static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock, - SSDataBlock* pSrcBlock, int32_t index, bool beforeTs, SExecTaskInfo* pTaskInfo, bool genAfterBlock) { + SSDataBlock* pSrcBlock, int32_t index, bool beforeTs, SExecTaskInfo* pTaskInfo) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; int32_t rows = pResBlock->info.rows; @@ -427,7 +427,7 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp break; } - if (start.key == INT64_MIN || end.key == INT64_MIN || genAfterBlock) { + if (start.key == INT64_MIN || end.key == INT64_MIN) { colDataSetNULL(pDst, rows); break; } @@ -463,13 +463,8 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp break; } - if (genAfterBlock && rows == 0) { - hasInterp = false; - break; - } - SGroupKeys* pkey = taosArrayGet(pSliceInfo->pNextRow, srcSlot); - if (pkey->isNull == false && !genAfterBlock) { + if (pkey->isNull == false) { code = colDataSetVal(pDst, rows, pkey->pData, false); QUERY_CHECK_CODE(code, lino, _end); } else { @@ -841,7 +836,7 @@ static void doTimesliceImpl(SOperatorInfo* pOperator, STimeSliceOperatorInfo* pS int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); if (nextTs > pSliceInfo->current) { while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) { - if (!genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, pBlock, i, false, pTaskInfo, false) && + if (!genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, pBlock, i, false, pTaskInfo) && pSliceInfo->fillType == TSDB_FILL_LINEAR) { break; } else { @@ -869,7 +864,7 @@ static void doTimesliceImpl(SOperatorInfo* pOperator, STimeSliceOperatorInfo* pS doKeepLinearInfo(pSliceInfo, pBlock, i); while (pSliceInfo->current < ts && pSliceInfo->current <= pSliceInfo->win.ekey) { - if (!genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, pBlock, i, true, pTaskInfo, false) && + if (!genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, pBlock, i, true, pTaskInfo) && pSliceInfo->fillType == TSDB_FILL_LINEAR) { break; } else { @@ -914,12 +909,13 @@ static void genInterpAfterDataBlock(STimeSliceOperatorInfo* pSliceInfo, SOperato SSDataBlock* pResBlock = pSliceInfo->pRes; SInterval* pInterval = &pSliceInfo->interval; - if (pSliceInfo->pPrevGroupKey == NULL) { + if (pSliceInfo->fillType == TSDB_FILL_NEXT || pSliceInfo->fillType == TSDB_FILL_LINEAR || + pSliceInfo->pPrevGroupKey == NULL) { return; } while (pSliceInfo->current <= pSliceInfo->win.ekey) { - (void)genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, NULL, index, false, pOperator->pTaskInfo, true); + (void)genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, NULL, index, false, pOperator->pTaskInfo); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); } diff --git a/source/libs/function/src/tpercentile.c b/source/libs/function/src/tpercentile.c index 29c48460c0..429ab52a8d 100644 --- a/source/libs/function/src/tpercentile.c +++ b/source/libs/function/src/tpercentile.c @@ -224,19 +224,18 @@ int32_t tBucketDoubleHash(tMemBucket *pBucket, const void *value, int32_t *index *index = -1; - if (v > pBucket->range.dMaxVal || v < pBucket->range.dMinVal) { + if (v > pBucket->range.dMaxVal || v < pBucket->range.dMinVal || isnan(v)) { return TSDB_CODE_SUCCESS; } // divide a range of [dMinVal, dMaxVal] into 1024 buckets double span = pBucket->range.dMaxVal - pBucket->range.dMinVal; - if (span < pBucket->numOfSlots) { - int32_t delta = (int32_t)(v - pBucket->range.dMinVal); - *index = (delta % pBucket->numOfSlots); + if (fabs(span) < DBL_EPSILON) { + *index = 0; } else { double slotSpan = span / pBucket->numOfSlots; *index = (int32_t)((v - pBucket->range.dMinVal) / slotSpan); - if (v == pBucket->range.dMaxVal) { + if (fabs(v - pBucket->range.dMaxVal) < DBL_EPSILON) { *index -= 1; } } @@ -583,48 +582,52 @@ int32_t getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction *result = getIdenticalDataVal(pMemBucket, i); return TSDB_CODE_SUCCESS; } - // try next round - pMemBucket->times += 1; - // qDebug("MemBucket:%p, start next round data bucketing, time:%d", pMemBucket, pMemBucket->times); - - pMemBucket->range = pSlot->range; - pMemBucket->total = 0; - - resetSlotInfo(pMemBucket); - - int32_t groupId = getGroupId(pMemBucket->numOfSlots, i, pMemBucket->times - 1); + tMemBucket *tmpBucket = NULL; + int32_t code = tMemBucketCreate(pMemBucket->bytes, pMemBucket->type, pSlot->range.dMinVal, pSlot->range.dMaxVal, + false, &tmpBucket); + if (TSDB_CODE_SUCCESS != code) { + tMemBucketDestroy(&tmpBucket); + return code; + } + int32_t groupId = getGroupId(pMemBucket->numOfSlots, i, pMemBucket->times); SArray* list; void *p = taosHashGet(pMemBucket->groupPagesMap, &groupId, sizeof(groupId)); if (p != NULL) { list = *(SArray **)p; if (list == NULL || list->size <= 0) { + tMemBucketDestroy(&tmpBucket); return -1; } } else { + tMemBucketDestroy(&tmpBucket); return -1; } for (int32_t f = 0; f < list->size; ++f) { int32_t *pageId = taosArrayGet(list, f); if (NULL == pageId) { + tMemBucketDestroy(&tmpBucket); return TSDB_CODE_OUT_OF_RANGE; } SFilePage *pg = getBufPage(pMemBucket->pBuffer, *pageId); if (pg == NULL) { + tMemBucketDestroy(&tmpBucket); return terrno; } - int32_t code = tMemBucketPut(pMemBucket, pg->data, (int32_t)pg->num); + code = tMemBucketPut(tmpBucket, pg->data, (int32_t)pg->num); if (code != TSDB_CODE_SUCCESS) { + tMemBucketDestroy(&tmpBucket); return code; } setBufPageDirty(pg, true); releaseBufPage(pMemBucket->pBuffer, pg); } - - return getPercentileImpl(pMemBucket, count - num, fraction, result); + code = getPercentileImpl(tmpBucket, count - num, fraction, result); + tMemBucketDestroy(&tmpBucket); + return code; } } else { num += pSlot->info.size; diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 98676160cb..9c2977d289 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -247,7 +247,10 @@ int32_t generateSyntaxErrMsgExt(SMsgBuf* pBuf, int32_t errCode, const char* pFor } int32_t buildInvalidOperationMsg(SMsgBuf* pBuf, const char* msg) { - strncpy(pBuf->buf, msg, pBuf->len); + if (pBuf->buf) { + strncpy(pBuf->buf, msg, pBuf->len); + } + return TSDB_CODE_TSC_INVALID_OPERATION; } @@ -259,7 +262,6 @@ int32_t buildInvalidOperationMsgExt(SMsgBuf* pBuf, const char* pFormat, ...) { return TSDB_CODE_TSC_INVALID_OPERATION; } - int32_t buildSyntaxErrMsg(SMsgBuf* pBuf, const char* additionalInfo, const char* sourceStr) { if (pBuf == NULL) return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; const char* msgFormat1 = "syntax error near \'%s\'"; @@ -328,7 +330,7 @@ STableMeta* tableMetaDup(const STableMeta* pTableMeta) { STableMeta* p = taosMemoryMalloc(size + schemaExtSize); if (NULL == p) return NULL; - memcpy(p, pTableMeta, schemaExtSize+size); + memcpy(p, pTableMeta, schemaExtSize + size); if (hasSchemaExt) { p->schemaExt = (SSchemaExt*)(((char*)p) + size); } else { @@ -453,7 +455,7 @@ int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, voi // strcpy(val.colName, colName); val.pKey = jsonKey; retCode = taosHashPut(keyHash, jsonKey, keyLen, &keyLen, - CHAR_BYTES); // add key to hash to remove dumplicate, value is useless + CHAR_BYTES); // add key to hash to remove dumplicate, value is useless if (TSDB_CODE_SUCCESS != retCode) { goto end; } @@ -649,7 +651,7 @@ static int32_t buildTableReq(SHashObj* pTablesHash, SArray** pTables) { char* pKey = taosHashGetKey(p, &len); char fullName[TSDB_TABLE_FNAME_LEN] = {0}; strncpy(fullName, pKey, len); - SName name = {0}; + SName name = {0}; int32_t code = tNameFromString(&name, fullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); if (TSDB_CODE_SUCCESS == code) { if (NULL == taosArrayPush(*pTables, &name)) { @@ -816,9 +818,10 @@ int32_t buildCatalogReq(const SParseMetaCache* pMetaCache, SCatalogReq* pCatalog return code; } -int32_t createSelectStmtImpl(bool isDistinct, SNodeList* pProjectionList, SNode* pTable, SNodeList* pHint, SNode** ppSelect) { +int32_t createSelectStmtImpl(bool isDistinct, SNodeList* pProjectionList, SNode* pTable, SNodeList* pHint, + SNode** ppSelect) { SSelectStmt* select = NULL; - int32_t code = nodesMakeNode(QUERY_NODE_SELECT_STMT, (SNode**)&select); + int32_t code = nodesMakeNode(QUERY_NODE_SELECT_STMT, (SNode**)&select); if (NULL == select) { return code; } @@ -861,7 +864,7 @@ int32_t getMetaDataFromHash(const char* pKey, int32_t len, SHashObj* pHash, void static int32_t putTableDataToCache(const SArray* pTableReq, const SArray* pTableData, SHashObj** pTable) { int32_t ntables = taosArrayGetSize(pTableReq); for (int32_t i = 0; i < ntables; ++i) { - char fullName[TSDB_TABLE_FNAME_LEN]; + char fullName[TSDB_TABLE_FNAME_LEN]; int32_t code = tNameExtractFullName(taosArrayGet(pTableReq, i), fullName); if (TSDB_CODE_SUCCESS != code) { return code; @@ -892,7 +895,7 @@ static int32_t putDbTableDataToCache(const SArray* pDbReq, const SArray* pTableD STablesReq* pReq = taosArrayGet(pDbReq, i); int32_t ntables = taosArrayGetSize(pReq->pTables); for (int32_t j = 0; j < ntables; ++j) { - char fullName[TSDB_TABLE_FNAME_LEN]; + char fullName[TSDB_TABLE_FNAME_LEN]; int32_t code = tNameExtractFullName(taosArrayGet(pReq->pTables, j), fullName); if (TSDB_CODE_SUCCESS != code) { return code; @@ -1088,7 +1091,7 @@ int32_t buildTableMetaFromViewMeta(STableMeta** pMeta, SViewMeta* pViewMeta) { } int32_t getViewMetaFromCache(SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta) { - char fullName[TSDB_TABLE_FNAME_LEN]; + char fullName[TSDB_TABLE_FNAME_LEN]; int32_t code = tNameExtractFullName(pName, fullName); if (TSDB_CODE_SUCCESS != code) { return code; @@ -1139,7 +1142,7 @@ int32_t reserveTableVgroupInCacheExt(const SName* pName, SParseMetaCache* pMetaC } int32_t getTableVgroupFromCache(SParseMetaCache* pMetaCache, const SName* pName, SVgroupInfo* pVgroup) { - char fullName[TSDB_TABLE_FNAME_LEN]; + char fullName[TSDB_TABLE_FNAME_LEN]; int32_t code = tNameExtractFullName(pName, fullName); if (TSDB_CODE_SUCCESS != code) { return code; @@ -1299,7 +1302,7 @@ int32_t reserveTableCfgInCache(int32_t acctId, const char* pDb, const char* pTab return reserveTableReqInCache(acctId, pDb, pTable, &pMetaCache->pTableCfg); } -int32_t reserveTableTSMAInfoInCache(int32_t acctId, const char *pDb, const char *pTable, SParseMetaCache *pMetaCache) { +int32_t reserveTableTSMAInfoInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache) { return reserveTableReqInCache(acctId, pDb, pTable, &pMetaCache->pTableTSMAs); } @@ -1308,9 +1311,10 @@ int32_t reserveTSMAInfoInCache(int32_t acctId, const char* pDb, const char* pTsm } int32_t getTableIndexFromCache(SParseMetaCache* pMetaCache, const SName* pName, SArray** pIndexes) { - char fullName[TSDB_TABLE_FNAME_LEN]; + char fullName[TSDB_TABLE_FNAME_LEN]; int32_t code = tNameExtractFullName(pName, fullName); - if (TSDB_CODE_SUCCESS != code) return code;; + if (TSDB_CODE_SUCCESS != code) return code; + ; SArray* pSmaIndexes = NULL; code = getMetaDataFromHash(fullName, strlen(fullName), pMetaCache->pTableIndex, (void**)&pSmaIndexes); if (TSDB_CODE_SUCCESS == code && NULL != pSmaIndexes) { @@ -1323,7 +1327,7 @@ int32_t getTableIndexFromCache(SParseMetaCache* pMetaCache, const SName* pName, } int32_t getTableTsmasFromCache(SParseMetaCache* pMetaCache, const SName* pTbName, SArray** pTsmas) { - char tbFName[TSDB_TABLE_FNAME_LEN]; + char tbFName[TSDB_TABLE_FNAME_LEN]; int32_t code = tNameExtractFullName(pTbName, tbFName); if (TSDB_CODE_SUCCESS != code) { return code; @@ -1337,7 +1341,7 @@ int32_t getTableTsmasFromCache(SParseMetaCache* pMetaCache, const SName* pTbName } int32_t getTsmaFromCache(SParseMetaCache* pMetaCache, const SName* pTsmaName, STableTSMAInfo** pTsma) { - char tsmaFName[TSDB_TABLE_FNAME_LEN]; + char tsmaFName[TSDB_TABLE_FNAME_LEN]; int32_t code = tNameExtractFullName(pTsmaName, tsmaFName); if (TSDB_CODE_SUCCESS != code) { return code; @@ -1349,7 +1353,7 @@ int32_t getTsmaFromCache(SParseMetaCache* pMetaCache, const SName* pTsmaName, ST return TSDB_CODE_PAR_INTERNAL_ERROR; } *pTsma = taosArrayGetP(pTsmaRsp->pTsmas, 0); - } else if (code == TSDB_CODE_PAR_INTERNAL_ERROR){ + } else if (code == TSDB_CODE_PAR_INTERNAL_ERROR) { code = TSDB_CODE_MND_SMA_NOT_EXIST; } return code; @@ -1410,7 +1414,7 @@ err: } int32_t getTableCfgFromCache(SParseMetaCache* pMetaCache, const SName* pName, STableCfg** pOutput) { - char fullName[TSDB_TABLE_FNAME_LEN]; + char fullName[TSDB_TABLE_FNAME_LEN]; int32_t code = tNameExtractFullName(pName, fullName); if (TSDB_CODE_SUCCESS != code) { return code; diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index 0011d1c70c..69014d5b1c 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -502,6 +502,10 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int } int32_t qWorkerProcessCQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_t ts) { + if (NULL == node || NULL == qWorkerMgmt || NULL == pMsg) { + QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + int32_t code = 0; int8_t status = 0; bool queryDone = false; diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 7180c58404..ddc4812b55 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -329,7 +329,7 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, if (len < 0) { QW_TASK_ELOG("invalid length from dsGetDataLength, length:%" PRId64 "", len); - QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + QW_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); } if (len == 0) { @@ -337,18 +337,18 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, code = dsGetDataBlock(ctx->sinkHandle, &output); if (code) { QW_TASK_ELOG("dsGetDataBlock failed, code:%x - %s", code, tstrerror(code)); - QW_ERR_RET(code); + QW_ERR_JRET(code); } QW_TASK_DLOG("no more data in sink and query end, fetched blocks %d rows %" PRId64, pOutput->numOfBlocks, pOutput->numOfRows); if (!ctx->dynamicTask) { - QW_ERR_RET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCC, ctx->dynamicTask)); + QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCC, ctx->dynamicTask)); } if (NULL == pRsp) { - QW_ERR_RET(qwMallocFetchRsp(!ctx->localExec, len, &pRsp)); + QW_ERR_JRET(qwMallocFetchRsp(!ctx->localExec, len, &pRsp)); *pOutput = output; } else { pOutput->queryEnd = output.queryEnd; @@ -368,7 +368,7 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, *dataLen += len + PAYLOAD_PREFIX_LEN; *pRawDataLen += rawLen + PAYLOAD_PREFIX_LEN; - QW_ERR_RET(qwMallocFetchRsp(!ctx->localExec, *dataLen, &pRsp)); + QW_ERR_JRET(qwMallocFetchRsp(!ctx->localExec, *dataLen, &pRsp)); // set the serialize start position output.pData = pRsp->data + *dataLen - (len + PAYLOAD_PREFIX_LEN); @@ -380,7 +380,7 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, code = dsGetDataBlock(ctx->sinkHandle, &output); if (code) { QW_TASK_ELOG("dsGetDataBlock failed, code:%x - %s", code, tstrerror(code)); - QW_ERR_RET(code); + QW_ERR_JRET(code); } pOutput->queryEnd = output.queryEnd; @@ -399,7 +399,7 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, if (DS_BUF_EMPTY == pOutput->bufStatus && pOutput->queryEnd) { QW_TASK_DLOG("task all data fetched and done, fetched blocks %d rows %" PRId64, pOutput->numOfBlocks, pOutput->numOfRows); - QW_ERR_RET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCC, ctx->dynamicTask)); + QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCC, ctx->dynamicTask)); break; } @@ -416,8 +416,11 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, } } +_return: + *rspMsg = pRsp; - return TSDB_CODE_SUCCESS; + + return code; } int32_t qwGetDeleteResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SDeleteRes *pRes) { @@ -472,6 +475,12 @@ int32_t qwQuickRspFetchReq(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SQWMsg *qwMsg, int32 code = qwGetQueryResFromSink(QW_FPARAMS(), ctx, &dataLen, &rawLen, &rsp, &sOutput); } + if (code) { + qwFreeFetchRsp(rsp); + rsp = NULL; + dataLen = 0; + } + if (NULL == rsp && TSDB_CODE_SUCCESS == code) { return TSDB_CODE_SUCCESS; } @@ -877,10 +886,11 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { break; } + qwFreeFetchRsp(rsp); + rsp = NULL; + if (code && QW_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); - qwFreeFetchRsp(rsp); - rsp = NULL; qwMsg->connInfo = ctx->dataConnInfo; code = qwBuildAndSendFetchRsp(ctx->fetchMsgType + 1, &qwMsg->connInfo, NULL, 0, code); @@ -1432,6 +1442,8 @@ void qWorkerDestroy(void **qWorkerMgmt) { while (0 == destroyed) { taosMsleep(2); } + + *qWorkerMgmt = NULL; } int32_t qWorkerGetStat(SReadHandle *handle, void *qWorkerMgmt, SQWorkerStat *pStat) { diff --git a/source/libs/stream/src/streamCheckStatus.c b/source/libs/stream/src/streamCheckStatus.c index 75bcc326b3..c1c54b3c0b 100644 --- a/source/libs/stream/src/streamCheckStatus.c +++ b/source/libs/stream/src/streamCheckStatus.c @@ -299,7 +299,7 @@ void streamTaskStartMonitorCheckRsp(SStreamTask* pTask) { return; } - /*SStreamTask* p = */ streamMetaAcquireOneTask(pTask); // add task ref here + int32_t unusedRetRef = streamMetaAcquireOneTask(pTask); // add task ref here streamTaskInitTaskCheckInfo(pInfo, &pTask->outputInfo, taosGetTimestampMs()); int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1); diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index e44bca123b..be914d9746 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -347,7 +347,8 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock if (old == 0) { int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1); stDebug("s-task:%s start checkpoint-trigger monitor in 10s, ref:%d ", pTask->id.idStr, ref); - streamMetaAcquireOneTask(pTask); + + int32_t unusedRetRef = streamMetaAcquireOneTask(pTask); streamTmrStart(checkpointTriggerMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, "trigger-recv-monitor"); pTmrInfo->launchChkptId = pActiveInfo->activeId; diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 133663ac28..62d60ff664 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -1162,7 +1162,7 @@ int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask) { if (old == 0) { int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1); stDebug("s-task:%s start checkpoint-ready monitor in 10s, ref:%d ", pTask->id.idStr, ref); - streamMetaAcquireOneTask(pTask); + int32_t unusedRetRef = streamMetaAcquireOneTask(pTask); streamTmrStart(chkptReadyMsgSendMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, "chkpt-ready-monitor"); diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 44c9e76906..7e9b60b61a 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -753,12 +753,17 @@ int32_t streamMetaAcquireTask(SStreamMeta* pMeta, int64_t streamId, int32_t task return code; } -void streamMetaAcquireOneTask(SStreamTask* pTask) { +int32_t streamMetaAcquireOneTask(SStreamTask* pTask) { int32_t ref = atomic_add_fetch_32(&pTask->refCnt, 1); stTrace("s-task:%s acquire task, ref:%d", pTask->id.idStr, ref); + return ref; } void streamMetaReleaseTask(SStreamMeta* UNUSED_PARAM(pMeta), SStreamTask* pTask) { + if (pTask == NULL) { + return; + } + int32_t taskId = pTask->id.taskId; int32_t ref = atomic_sub_fetch_32(&pTask->refCnt, 1); @@ -862,7 +867,7 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); if (ppTask) { pTask = *ppTask; - // it is an fill-history task, remove the related stream task's id that points to it + // it is a fill-history task, remove the related stream task's id that points to it if (pTask->info.fillHistory == 0) { int32_t ret = atomic_sub_fetch_32(&pMeta->numOfStreamTasks, 1); } diff --git a/source/libs/stream/src/streamSched.c b/source/libs/stream/src/streamSched.c index 98920e6f70..cdaa603e38 100644 --- a/source/libs/stream/src/streamSched.c +++ b/source/libs/stream/src/streamSched.c @@ -22,7 +22,7 @@ static void streamTaskSchedHelper(void* param, void* tmrId); void streamSetupScheduleTrigger(SStreamTask* pTask) { int64_t delaySchema = pTask->info.delaySchedParam; if (delaySchema != 0 && pTask->info.fillHistory == 0) { - int32_t ref = atomic_add_fetch_32(&pTask->refCnt, 1); + int32_t ref = streamMetaAcquireOneTask(pTask); stDebug("s-task:%s setup scheduler trigger, ref:%d delay:%" PRId64 " ms", pTask->id.idStr, ref, pTask->info.delaySchedParam); @@ -63,7 +63,11 @@ int32_t streamTaskSchedTask(SMsgCb* pMsgCb, int32_t vgId, int64_t streamId, int3 pRunReq->reqType = execType; SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)}; - return tmsgPutToQueue(pMsgCb, STREAM_QUEUE, &msg); + int32_t code = tmsgPutToQueue(pMsgCb, STREAM_QUEUE, &msg); + if (code) { + stError("vgId:%d failed to put msg into stream queue, code:%s, %x", vgId, tstrerror(code), taskId); + } + return code; } void streamTaskClearSchedIdleInfo(SStreamTask* pTask) { pTask->status.schedIdleTime = 0; } @@ -76,7 +80,7 @@ void streamTaskResumeInFuture(SStreamTask* pTask) { pTask->status.schedIdleTime, ref); // add one ref count for task - streamMetaAcquireOneTask(pTask); + int32_t unusedRetRef = streamMetaAcquireOneTask(pTask); streamTmrStart(streamTaskResumeHelper, pTask->status.schedIdleTime, pTask, streamTimer, &pTask->schedInfo.pIdleTimer, pTask->pMeta->vgId, "resume-task-tmr"); } diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 71a2ed3e4a..727701e03e 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -258,10 +258,12 @@ void tFreeStreamTask(SStreamTask* pTask) { if (pTask->inputq.queue) { streamQueueClose(pTask->inputq.queue, pTask->id.taskId); + pTask->inputq.queue = NULL; } if (pTask->outputq.queue) { streamQueueClose(pTask->outputq.queue, pTask->id.taskId); + pTask->outputq.queue = NULL; } if (pTask->exec.qmsg) { @@ -275,6 +277,7 @@ void tFreeStreamTask(SStreamTask* pTask) { if (pTask->exec.pWalReader != NULL) { walCloseReader(pTask->exec.pWalReader); + pTask->exec.pWalReader = NULL; } streamClearChkptReadyMsg(pTask->chkInfo.pActiveInfo); diff --git a/source/libs/stream/src/streamTaskSm.c b/source/libs/stream/src/streamTaskSm.c index 3501d30be4..c3a2742aa2 100644 --- a/source/libs/stream/src/streamTaskSm.c +++ b/source/libs/stream/src/streamTaskSm.c @@ -501,9 +501,10 @@ int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent even if (pTrans == NULL) { ETaskStatus s = pSM->current.state; - if (s != TASK_STATUS__DROPPING && s != TASK_STATUS__PAUSE && s != TASK_STATUS__STOP && - s != TASK_STATUS__UNINIT && s != TASK_STATUS__READY) { - stError("s-task:%s invalid task status:%s on handling event:%s success", id, pSM->current.name, GET_EVT_NAME(pSM->prev.evt)); + if (s != TASK_STATUS__DROPPING && s != TASK_STATUS__PAUSE && s != TASK_STATUS__STOP && s != TASK_STATUS__UNINIT && + s != TASK_STATUS__READY) { + stError("s-task:%s invalid task status:%s on handling event:%s success", id, pSM->current.name, + GET_EVT_NAME(pSM->prev.evt)); } // the pSM->prev.evt may be 0, so print string is not appropriate. @@ -521,11 +522,15 @@ int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent even return TSDB_CODE_STREAM_INVALID_STATETRANS; } - keepPrevInfo(pSM); + // repeat pause will not overwrite the previous pause state + if (pSM->current.state != TASK_STATUS__PAUSE || pTrans->next.state != TASK_STATUS__PAUSE) { + keepPrevInfo(pSM); + pSM->current = pTrans->next; + } else { + stDebug("s-task:%s repeat pause evt recv, not update prev status", id); + } - pSM->current = pTrans->next; pSM->pActiveTrans = NULL; - // todo remove it // todo: handle the error code // on success callback, add into lock if necessary, or maybe we should add an option for this? diff --git a/source/libs/stream/src/streamTimer.c b/source/libs/stream/src/streamTimer.c index 8b77fe7cb1..0da9acfd1d 100644 --- a/source/libs/stream/src/streamTimer.c +++ b/source/libs/stream/src/streamTimer.c @@ -56,7 +56,7 @@ void streamTmrStart(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* pParam, void* } } - stDebug("vgId:%d start %s tmr succ", vgId, pMsg); + stTrace("vgId:%d start %s tmr succ", vgId, pMsg); } void streamTmrStop(tmr_h tmrId) { diff --git a/source/libs/sync/inc/syncEnv.h b/source/libs/sync/inc/syncEnv.h index 0376920e8a..caf0e88457 100644 --- a/source/libs/sync/inc/syncEnv.h +++ b/source/libs/sync/inc/syncEnv.h @@ -24,7 +24,6 @@ extern "C" { #define TIMER_MAX_MS 0x7FFFFFFF #define PING_TIMER_MS 5000 -#define HEARTBEAT_TICK_NUM 20 typedef struct SSyncEnv { uint8_t isStart; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 451e82c7d4..3d37cdb560 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -977,9 +977,10 @@ static int32_t syncHbTimerStart(SSyncNode* pSyncNode, SSyncTimer* pSyncTimer) { pData->logicClock = pSyncTimer->logicClock; pData->execTime = tsNow + pSyncTimer->timerMS; - sTrace("vgId:%d, start hb timer, rid:%" PRId64 " addr:%" PRId64, pSyncNode->vgId, pData->rid, pData->destId.addr); + sTrace("vgId:%d, start hb timer, rid:%" PRId64 " addr:%" PRId64 " at %d", pSyncNode->vgId, pData->rid, + pData->destId.addr, pSyncTimer->timerMS); - TAOS_CHECK_RETURN(taosTmrReset(pSyncTimer->timerCb, pSyncTimer->timerMS / HEARTBEAT_TICK_NUM, (void*)(pData->rid), + TAOS_CHECK_RETURN(taosTmrReset(pSyncTimer->timerCb, pSyncTimer->timerMS, (void*)(pData->rid), syncEnv()->pTimerManager, &pSyncTimer->pTimer)); } else { code = TSDB_CODE_SYN_INTERNAL_ERROR; @@ -2711,7 +2712,8 @@ static void syncNodeEqPeerHeartbeatTimer(void* param, void* tmrId) { return; } - sTrace("vgId:%d, eq peer hb timer, rid:%" PRId64 " addr:%" PRId64, pSyncNode->vgId, hbDataRid, pData->destId.addr); + sTrace("vgId:%d, peer hb timer execution, rid:%" PRId64 " addr:%" PRId64, pSyncNode->vgId, hbDataRid, + pData->destId.addr); if (pSyncNode->totalReplicaNum > 1) { int64_t timerLogicClock = atomic_load_64(&pSyncTimer->logicClock); @@ -2753,13 +2755,12 @@ static void syncNodeEqPeerHeartbeatTimer(void* param, void* tmrId) { if (ret != 0) { sError("vgId:%d, failed to send heartbeat since %s", pSyncNode->vgId, tstrerror(ret)); } - } else { } if (syncIsInit()) { - // sTrace("vgId:%d, reset peer hb timer", pSyncNode->vgId); - if ((code = taosTmrReset(syncNodeEqPeerHeartbeatTimer, pSyncTimer->timerMS / HEARTBEAT_TICK_NUM, - (void*)hbDataRid, syncEnv()->pTimerManager, &pSyncTimer->pTimer)) != 0) { + sTrace("vgId:%d, reset peer hb timer at %d", pSyncNode->vgId, pSyncTimer->timerMS); + if ((code = taosTmrReset(syncNodeEqPeerHeartbeatTimer, pSyncTimer->timerMS, (void*)hbDataRid, + syncEnv()->pTimerManager, &pSyncTimer->pTimer)) != 0) { sError("vgId:%d, reset peer hb timer error, %s", pSyncNode->vgId, tstrerror(code)); syncNodeRelease(pSyncNode); syncHbTimerDataRelease(pData); diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index c688a6cc6a..2333a4a6a2 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -1446,6 +1446,9 @@ static int tdbBtreeDecodePayload(SPage *pPage, const SCell *pCell, int nHeader, return ret; } ofpCell = tdbPageGetCell(ofp, 0); + if (ofpCell == NULL) { + return TSDB_CODE_INVALID_DATA_FMT; + } if (nLeft <= ofp->maxLocal - sizeof(SPgno)) { bytes = nLeft; diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 9ade5e5638..d9666eb02f 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -282,6 +282,17 @@ static int32_t walRebuildFileInfoSet(SArray* metaLogList, SArray* actualLogList) } static void walAlignVersions(SWal* pWal) { + if (pWal->cfg.committed > 0 && pWal->cfg.committed != pWal->vers.snapshotVer) { + wWarn("vgId:%d, snapshotVer:%" PRId64 " in wal is different from commited:%" PRId64 + ". in vnode/mnode. align with it.", + pWal->cfg.vgId, pWal->vers.snapshotVer, pWal->cfg.committed); + pWal->vers.snapshotVer = pWal->cfg.committed; + } + if (pWal->vers.snapshotVer < 0 && pWal->vers.firstVer > 0) { + wWarn("vgId:%d, snapshotVer:%" PRId64 " in wal is an invalid value. align it with firstVer:%" PRId64 ".", + pWal->cfg.vgId, pWal->vers.snapshotVer, pWal->vers.firstVer); + pWal->vers.snapshotVer = pWal->vers.firstVer; + } if (pWal->vers.firstVer > pWal->vers.snapshotVer + 1) { wWarn("vgId:%d, firstVer:%" PRId64 " is larger than snapshotVer:%" PRId64 " + 1. align with it.", pWal->cfg.vgId, pWal->vers.firstVer, pWal->vers.snapshotVer); @@ -400,6 +411,17 @@ static int32_t walTrimIdxFile(SWal* pWal, int32_t fileIdx) { TAOS_RETURN(TSDB_CODE_SUCCESS); } +void printFileSet(SArray* fileSet) { + int32_t sz = taosArrayGetSize(fileSet); + for (int32_t i = 0; i < sz; i++) { + SWalFileInfo* pFileInfo = taosArrayGet(fileSet, i); + wInfo("firstVer:%" PRId64 ", lastVer:%" PRId64 ", fileSize:%" PRId64 ", syncedOffset:%" PRId64 ", createTs:%" PRId64 + ", closeTs:%" PRId64, + pFileInfo->firstVer, pFileInfo->lastVer, pFileInfo->fileSize, pFileInfo->syncedOffset, pFileInfo->createTs, + pFileInfo->closeTs); + } +} + int32_t walCheckAndRepairMeta(SWal* pWal) { // load log files, get first/snapshot/last version info int32_t code = 0; @@ -460,6 +482,10 @@ int32_t walCheckAndRepairMeta(SWal* pWal) { taosArraySort(actualLog, compareWalFileInfo); + wInfo("vgId:%d, wal path:%s, actual log file num:%d", pWal->cfg.vgId, pWal->path, + (int32_t)taosArrayGetSize(actualLog)); + printFileSet(actualLog); + int metaFileNum = taosArrayGetSize(pWal->fileInfoSet); int actualFileNum = taosArrayGetSize(actualLog); int64_t firstVerPrev = pWal->vers.firstVer; @@ -474,6 +500,10 @@ int32_t walCheckAndRepairMeta(SWal* pWal) { TAOS_RETURN(code); } + wInfo("vgId:%d, wal path:%s, meta log file num:%d", pWal->cfg.vgId, pWal->path, + (int32_t)taosArrayGetSize(pWal->fileInfoSet)); + printFileSet(pWal->fileInfoSet); + int32_t sz = taosArrayGetSize(pWal->fileInfoSet); // scan and determine the lastVer @@ -533,6 +563,7 @@ int32_t walCheckAndRepairMeta(SWal* pWal) { // repair ts of files TAOS_CHECK_RETURN(walRepairLogFileTs(pWal, &updateMeta)); + printFileSet(pWal->fileInfoSet); // update meta file if (updateMeta) { TAOS_CHECK_RETURN(walSaveMeta(pWal)); @@ -1124,6 +1155,10 @@ int32_t walLoadMeta(SWal* pWal) { (void)taosCloseFile(&pFile); taosMemoryFree(buf); + wInfo("vgId:%d, load meta file: %s, fileInfoSet size:%d", pWal->cfg.vgId, fnameStr, + (int32_t)taosArrayGetSize(pWal->fileInfoSet)); + printFileSet(pWal->fileInfoSet); + TAOS_RETURN(code); } diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index 3b23a2db80..d8a58efe4e 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -91,7 +91,8 @@ static int32_t walInitLock(SWal *pWal) { } SWal *walOpen(const char *path, SWalCfg *pCfg) { - SWal *pWal = taosMemoryCalloc(1, sizeof(SWal)); + int32_t code = 0; + SWal *pWal = taosMemoryCalloc(1, sizeof(SWal)); if (pWal == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return NULL; @@ -160,17 +161,20 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { pWal->writeHead.magic = WAL_MAGIC; // load meta - if (walLoadMeta(pWal) < 0) { - wInfo("vgId:%d, failed to load meta since %s", pWal->cfg.vgId, tstrerror(terrno)); + code = walLoadMeta(pWal); + if (code < 0) { + wWarn("vgId:%d, failed to load meta since %s", pWal->cfg.vgId, tstrerror(code)); } - if (walCheckAndRepairMeta(pWal) < 0) { - wError("vgId:%d, cannot open wal since repair meta file failed", pWal->cfg.vgId); + code = walCheckAndRepairMeta(pWal); + if (code < 0) { + wError("vgId:%d, cannot open wal since repair meta file failed since %s", pWal->cfg.vgId, tstrerror(code)); goto _err; } - if (walCheckAndRepairIdx(pWal) < 0) { - wError("vgId:%d, cannot open wal since repair idx file failed", pWal->cfg.vgId); + code = walCheckAndRepairIdx(pWal); + if (code < 0) { + wError("vgId:%d, cannot open wal since repair idx file failed since %s", pWal->cfg.vgId, tstrerror(code)); goto _err; } diff --git a/source/libs/wal/test/walMetaTest.cpp b/source/libs/wal/test/walMetaTest.cpp index 8bd4de0a89..a0285f1363 100644 --- a/source/libs/wal/test/walMetaTest.cpp +++ b/source/libs/wal/test/walMetaTest.cpp @@ -127,6 +127,7 @@ class WalRetentionEnv : public ::testing::Test { SWalCfg cfg; cfg.rollPeriod = -1; cfg.segSize = -1; + cfg.committed =-1; cfg.retentionPeriod = -1; cfg.retentionSize = 0; cfg.rollPeriod = 0; diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index 7989a2468b..b94bb512e2 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -200,6 +200,7 @@ void* taosArrayPop(SArray* pArray) { void* taosArrayGet(const SArray* pArray, size_t index) { if (NULL == pArray) { terrno = TSDB_CODE_INVALID_PARA; + uError("failed to return value from array of null ptr"); return NULL; } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 1c94a5f2e4..35f0bc73eb 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -314,7 +314,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CLOG_IS_NULL, "Transaction commitlog TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_NETWORK_UNAVAILL, "Unable to establish connection While execute transaction and will continue in the background") TAOS_DEFINE_ERROR(TSDB_CODE_MND_LAST_TRANS_NOT_FINISHED, "Last Transaction not finished") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_SYNC_TIMEOUT, "Sync timeout While execute transaction and will continue in the background") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CTX_SWITCH, "Transaction context switch") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CTX_SWITCH, "Wrong transaction execution context") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CONFLICT_COMPACT, "Transaction not completed due to conflict with compact") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_UNKNOW_ERROR, "Unknown transaction error") diff --git a/tests/army/db-encrypt/basic.py b/tests/army/db-encrypt/basic.py index 8d30bbcfe2..a93b85f0da 100644 --- a/tests/army/db-encrypt/basic.py +++ b/tests/army/db-encrypt/basic.py @@ -13,6 +13,7 @@ from frame.srvCtl import * from frame.caseBase import * from frame import * from frame.autogen import * +from frame import epath # from frame.server.dnodes import * # from frame.server.cluster import * @@ -20,7 +21,9 @@ from frame.autogen import * class TDTestCase(TBase): def init(self, conn, logSql, replicaVar=1): + updatecfgDict = {'dDebugFlag':131} super(TDTestCase, self).init(conn, logSql, replicaVar=1, checkColName="c1") + self.valgrind = 0 self.db = "test" self.stb = "meters" @@ -50,9 +53,36 @@ class TDTestCase(TBase): tdSql.error("create encrypt_key '12345678abcdefghi'") tdSql.error("create database test ENCRYPT_ALGORITHM 'sm4'") + def recreate_dndoe_encrypt_key(self): + """ + Description: From the jira TS-5507, the encrypt key can be recreated. + create: + 2024-09-23 created by Charles + update: + None + """ + # taosd path + taosd_path = epath.binPath() + tdLog.info(f"taosd_path: {taosd_path}") + # dnode2 path + dndoe2_path = tdDnodes.getDnodeDir(2) + dnode2_data_path = os.sep.join([dndoe2_path, "data"]) + dnode2_cfg_path = os.sep.join([dndoe2_path, "cfg"]) + tdLog.info(f"dnode2_path: {dnode2_data_path}") + # stop dnode2 + tdDnodes.stoptaosd(2) + tdLog.info("stop dndoe2") + # delete dndoe2 data + cmd = f"rm -rf {dnode2_data_path}" + os.system(cmd) + # recreate the encrypt key for dnode2 + os.system(f"{os.sep.join([taosd_path, 'taosd'])} -y '1234567890' -c {dnode2_cfg_path}") + tdLog.info("test case: recreate the encrypt key for dnode2 passed") + def run(self): self.create_encrypt_db_error() self.create_encrypt_db() + self.recreate_dndoe_encrypt_key() def stop(self): tdSql.close() diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 81ffaf0d4e..09216add82 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -10,7 +10,7 @@ # army-test # ,,y,army,./pytest.sh python3 ./test.py -f multi-level/mlevel_basic.py -N 3 -L 3 -D 2 -,,y,army,./pytest.sh python3 ./test.py -f db-encrypt/basic.py +,,y,army,./pytest.sh python3 ./test.py -f db-encrypt/basic.py -N 3 -M 3 ,,n,army,python3 ./test.py -f storage/s3/s3Basic.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f cluster/snapshot.py -N 3 -L 3 -D 2 ,,y,army,./pytest.sh python3 ./test.py -f query/function/test_func_elapsed.py @@ -223,6 +223,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/agg_group_NotReturnValue.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/agg_group_NotReturnValue.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/agg_group_NotReturnValue.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/td-32548.py ,,y,system-test,./pytest.sh python3 ./test.py -f 3-enterprise/restore/restoreDnode.py -N 5 -M 3 -i False ,,y,system-test,./pytest.sh python3 ./test.py -f 3-enterprise/restore/restoreVnode.py -N 5 -M 3 -i False @@ -1254,7 +1255,7 @@ ,,y,script,./test.sh -f tsim/query/unionall_as_table.sim ,,y,script,./test.sh -f tsim/query/multi_order_by.sim ,,y,script,./test.sh -f tsim/query/sys_tbname.sim -,,y,script,./test.sh -f tsim/query/sort-pre-cols.sim +,,y,script,./test.sh -f tsim/query/sort-pre-cols.sim ,,y,script,./test.sh -f tsim/query/groupby.sim ,,y,script,./test.sh -f tsim/query/groupby_distinct.sim ,,y,script,./test.sh -f tsim/query/event.sim @@ -1262,7 +1263,7 @@ ,,y,script,./test.sh -f tsim/query/emptyTsRange.sim ,,y,script,./test.sh -f tsim/query/emptyTsRange_scl.sim ,,y,script,./test.sh -f tsim/query/partitionby.sim -,,y,script,./test.sh -f tsim/query/tableCount.sim +,,y,script,./test.sh -f tsim/query/tableCount.sim ,,y,script,./test.sh -f tsim/query/show_db_table_kind.sim ,,y,script,./test.sh -f tsim/query/bi_star_table.sim ,,y,script,./test.sh -f tsim/query/bi_tag_scan.sim @@ -1532,8 +1533,8 @@ ,,n,script,./test.sh -f tsim/tagindex/sma_and_tag_index.sim ,,y,script,./test.sh -f tsim/tagindex/indexOverflow.sim ,,y,script,./test.sh -f tsim/view/view.sim -,,y,script,./test.sh -f tsim/query/cache_last.sim -,,y,script,./test.sh -f tsim/query/const.sim +,,y,script,./test.sh -f tsim/query/cache_last.sim +,,y,script,./test.sh -f tsim/query/const.sim ,,y,script,./test.sh -f tsim/query/nestedJoinView.sim @@ -1566,4 +1567,3 @@ ,,n,docs-examples-test,bash rust.sh ,,n,docs-examples-test,bash go.sh ,,n,docs-examples-test,bash test_R.sh - diff --git a/tests/system-test/2-query/interp.py b/tests/system-test/2-query/interp.py index 3cdf52725a..bcfc389d7b 100644 --- a/tests/system-test/2-query/interp.py +++ b/tests/system-test/2-query/interp.py @@ -907,7 +907,7 @@ class TDTestCase: ## {. . .} tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(next)") - tdSql.checkRows(13) + tdSql.checkRows(12) tdSql.checkData(0, 0, 5) tdSql.checkData(1, 0, 5) tdSql.checkData(2, 0, 10) @@ -920,7 +920,6 @@ class TDTestCase: tdSql.checkData(9, 0, 15) tdSql.checkData(10, 0, 15) tdSql.checkData(11, 0, 15) - tdSql.checkData(12, 0, None) ## {} ... tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:01', '2020-02-01 00:00:04') every(1s) fill(next)") @@ -958,12 +957,10 @@ class TDTestCase: ## ..{.} tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:13', '2020-02-01 00:00:17') every(1s) fill(next)") - tdSql.checkRows(5) + tdSql.checkRows(3) tdSql.checkData(0, 0, 15) tdSql.checkData(1, 0, 15) tdSql.checkData(2, 0, 15) - tdSql.checkData(3, 0, None) - tdSql.checkData(4, 0, None) ## ... {} tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:16', '2020-02-01 00:00:19') every(1s) fill(next)") @@ -1275,7 +1272,7 @@ class TDTestCase: tdSql.checkData(8, 1, True) tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(next)") - tdSql.checkRows(13) + tdSql.checkRows(12) tdSql.checkCols(3) tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') @@ -1290,7 +1287,6 @@ class TDTestCase: tdSql.checkData(9, 0, '2020-02-01 00:00:13.000') tdSql.checkData(10, 0, '2020-02-01 00:00:14.000') tdSql.checkData(11, 0, '2020-02-01 00:00:15.000') - tdSql.checkData(12, 0, '2020-02-01 00:00:16.000') tdSql.checkData(0, 1, True) tdSql.checkData(1, 1, False) @@ -1304,7 +1300,6 @@ class TDTestCase: tdSql.checkData(9, 1, True) tdSql.checkData(10, 1, True) tdSql.checkData(11, 1, False) - tdSql.checkData(12, 1, True) tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:05', '2020-02-01 00:00:15') every(2s) fill(next)") tdSql.checkRows(6) @@ -1682,13 +1677,9 @@ class TDTestCase: ## | . | { | .} | tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(next)") - tdSql.checkRows(6) + tdSql.checkRows(2) tdSql.checkData(0, 0, 15) tdSql.checkData(1, 0, 15) - tdSql.checkData(2, 0, None) - tdSql.checkData(3, 0, None) - tdSql.checkData(4, 0, None) - tdSql.checkData(5, 0, None) # test fill linear @@ -2741,7 +2732,7 @@ class TDTestCase: tdSql.checkData(4, i, 15) tdSql.query(f"select interp(c0),interp(c1),interp(c2),interp(c3) from {dbname}.{tbname} range('2020-02-09 00:00:05', '2020-02-13 00:00:05') every(1d) fill(next)") - tdSql.checkRows(5) + tdSql.checkRows(3) tdSql.checkCols(4) for i in range (tdSql.queryCols): @@ -2837,7 +2828,7 @@ class TDTestCase: # test fill next tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname2} range('2020-02-02 00:00:00', '2020-02-02 00:00:18') every(1s) fill(next)") - tdSql.checkRows(19) + tdSql.checkRows(18) tdSql.checkCols(3) tdSql.checkData(0, 0, '2020-02-02 00:00:00.000') @@ -2860,7 +2851,6 @@ class TDTestCase: tdSql.checkData(15, 2, None) tdSql.checkData(16, 2, None) tdSql.checkData(17, 2, None) - tdSql.checkData(18, 2, None) tdSql.checkData(17, 0, '2020-02-02 00:00:17.000') @@ -3091,7 +3081,7 @@ class TDTestCase: # test fill linear tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname2} range('2020-02-02 00:00:00', '2020-02-02 00:00:18') every(1s) fill(linear)") - tdSql.checkRows(18) + tdSql.checkRows(17) tdSql.checkCols(3) tdSql.checkData(0, 0, '2020-02-02 00:00:01.000') @@ -3113,9 +3103,8 @@ class TDTestCase: tdSql.checkData(14, 2, None) tdSql.checkData(15, 2, None) tdSql.checkData(16, 2, None) - tdSql.checkData(17, 2, None) - tdSql.checkData(17, 0, '2020-02-02 00:00:18.000') + tdSql.checkData(16, 0, '2020-02-02 00:00:17.000') tdLog.printNoPrefix("==========step13:test error cases") @@ -3231,7 +3220,7 @@ class TDTestCase: tdSql.checkData(17, 1, True) tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname} range('2020-02-01 00:00:00', '2020-02-01 00:00:18') every(1s) fill(next)") - tdSql.checkRows(19) + tdSql.checkRows(18) tdSql.checkData(0, 0, '2020-02-01 00:00:00.000') tdSql.checkData(0, 1, True) @@ -3254,12 +3243,9 @@ class TDTestCase: tdSql.checkData(15, 2, 15) tdSql.checkData(16, 2, 17) tdSql.checkData(17, 2, 17) - tdSql.checkData(18, 2, None) tdSql.checkData(17, 0, '2020-02-01 00:00:17.000') tdSql.checkData(17, 1, False) - tdSql.checkData(18, 0, '2020-02-01 00:00:18.000') - tdSql.checkData(18, 1, True) tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname} range('2020-02-01 00:00:00', '2020-02-01 00:00:18') every(1s) fill(linear)") tdSql.checkRows(17) @@ -3376,24 +3362,24 @@ class TDTestCase: tdSql.query(f"select tbname, _irowts, _isfilled, interp(c0) from {dbname}.{stbname} partition by tbname range('2020-02-01 00:00:00', '2020-02-01 00:00:18') every(1s) fill(next)") - tdSql.checkRows(57) - for i in range(0, 19): + tdSql.checkRows(48) + for i in range(0, 14): tdSql.checkData(i, 0, 'ctb1') - for i in range(19, 38): + for i in range(14, 30): tdSql.checkData(i, 0, 'ctb2') - for i in range(38, 57): + for i in range(30, 48): tdSql.checkData(i, 0, 'ctb3') tdSql.checkData(0, 1, '2020-02-01 00:00:00.000') - tdSql.checkData(18, 1, '2020-02-01 00:00:18.000') + tdSql.checkData(13, 1, '2020-02-01 00:00:13.000') - tdSql.checkData(19, 1, '2020-02-01 00:00:00.000') - tdSql.checkData(37, 1, '2020-02-01 00:00:18.000') + tdSql.checkData(14, 1, '2020-02-01 00:00:00.000') + tdSql.checkData(29, 1, '2020-02-01 00:00:15.000') - tdSql.checkData(38, 1, '2020-02-01 00:00:00.000') - tdSql.checkData(56, 1, '2020-02-01 00:00:18.000') + tdSql.checkData(30, 1, '2020-02-01 00:00:00.000') + tdSql.checkData(47, 1, '2020-02-01 00:00:17.000') for i in range(0, 2): tdSql.checkData(i, 3, 1) @@ -3404,33 +3390,24 @@ class TDTestCase: for i in range(8, 14): tdSql.checkData(i, 3, 13) - for i in range(14, 19): - tdSql.checkData(i, 3, None) - - for i in range(19, 23): + for i in range(14, 18): tdSql.checkData(i, 3, 3) - for i in range(23, 29): + for i in range(18, 24): tdSql.checkData(i, 3, 9) - for i in range(29, 35): + for i in range(24, 30): tdSql.checkData(i, 3, 15) - for i in range(35, 38): - tdSql.checkData(i, 3, None) - - for i in range(38, 44): + for i in range(30, 36): tdSql.checkData(i, 3, 5) - for i in range(44, 50): + for i in range(36, 42): tdSql.checkData(i, 3, 11) - for i in range(50, 56): + for i in range(42, 48): tdSql.checkData(i, 3, 17) - for i in range(56, 57): - tdSql.checkData(i, 3, None) - tdSql.query(f"select tbname, _irowts, _isfilled, interp(c0) from {dbname}.{stbname} partition by tbname range('2020-02-01 00:00:00', '2020-02-01 00:00:18') every(1s) fill(linear)") tdSql.checkRows(39) @@ -3473,7 +3450,7 @@ class TDTestCase: tdSql.checkRows(90) tdSql.query(f"select c0, _irowts, _isfilled, interp(c0) from {dbname}.{stbname} partition by c0 range('2020-02-01 00:00:00', '2020-02-01 00:00:18') every(1s) fill(next)") - tdSql.checkRows(171) + tdSql.checkRows(90) tdSql.query(f"select c0, _irowts, _isfilled, interp(c0) from {dbname}.{stbname} partition by c0 range('2020-02-01 00:00:00', '2020-02-01 00:00:18') every(1s) fill(linear)") tdSql.checkRows(9) @@ -3490,7 +3467,7 @@ class TDTestCase: tdSql.checkRows(48) tdSql.query(f"select t1, _irowts, _isfilled, interp(c0) from {dbname}.{stbname} partition by t1 range('2020-02-01 00:00:00', '2020-02-01 00:00:18') every(1s) fill(next)") - tdSql.checkRows(57) + tdSql.checkRows(48) tdSql.query(f"select t1, _irowts, _isfilled, interp(c0) from {dbname}.{stbname} partition by t1 range('2020-02-01 00:00:00', '2020-02-01 00:00:18') every(1s) fill(linear)") tdSql.checkRows(39) @@ -4386,7 +4363,7 @@ class TDTestCase: tdSql.query(f"select _irowts, _isfilled, interp(c0, 1) from {dbname}.{tbname_null} range('2020-02-02 00:00:01', '2020-02-02 00:00:11') every(1s) fill(next)") - tdSql.checkRows(11) + tdSql.checkRows(9) tdSql.checkData(0, 1, False) tdSql.checkData(1, 1, True) tdSql.checkData(2, 1, False) @@ -4396,8 +4373,6 @@ class TDTestCase: tdSql.checkData(6, 1, True) tdSql.checkData(7, 1, False) tdSql.checkData(8, 1, False) - tdSql.checkData(9, 1, True) - tdSql.checkData(10, 1, True) tdSql.checkData(0, 2, 1) tdSql.checkData(1, 2, 3) @@ -4408,13 +4383,11 @@ class TDTestCase: tdSql.checkData(6, 2, 8) tdSql.checkData(7, 2, 8) tdSql.checkData(8, 2, 9) - tdSql.checkData(9, 2, None) - tdSql.checkData(10, 2, None) tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_null} where c0 is not null range('2020-02-02 00:00:01', '2020-02-02 00:00:11') every(1s) fill(next)") - tdSql.checkRows(11) + tdSql.checkRows(9) tdSql.checkData(0, 1, False) tdSql.checkData(1, 1, True) tdSql.checkData(2, 1, False) @@ -4424,9 +4397,6 @@ class TDTestCase: tdSql.checkData(6, 1, True) tdSql.checkData(7, 1, False) tdSql.checkData(8, 1, False) - tdSql.checkData(9, 1, True) - tdSql.checkData(10, 1, True) - tdSql.checkData(0, 2, 1) tdSql.checkData(1, 2, 3) @@ -4437,8 +4407,6 @@ class TDTestCase: tdSql.checkData(6, 2, 8) tdSql.checkData(7, 2, 8) tdSql.checkData(8, 2, 9) - tdSql.checkData(9, 2, None) - tdSql.checkData(10, 2, None) # super table tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_null} range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(next)") @@ -4475,7 +4443,7 @@ class TDTestCase: tdSql.query(f"select _irowts, _isfilled, interp(c0, 1) from {dbname}.{stbname_null} range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(next)") - tdSql.checkRows(9) + tdSql.checkRows(8) tdSql.checkData(0, 1, False) tdSql.checkData(1, 1, True) tdSql.checkData(2, 1, True) @@ -4484,7 +4452,6 @@ class TDTestCase: tdSql.checkData(5, 1, True) tdSql.checkData(6, 1, False) tdSql.checkData(7, 1, False) - tdSql.checkData(8, 1, True) tdSql.checkData(0, 2, 1) tdSql.checkData(1, 2, 9) @@ -4494,12 +4461,11 @@ class TDTestCase: tdSql.checkData(5, 2, 13) tdSql.checkData(6, 2, 13) tdSql.checkData(7, 2, 15) - tdSql.checkData(8, 2, None) tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_null} where c0 is not null range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(next)") - tdSql.checkRows(9) + tdSql.checkRows(8) tdSql.checkData(0, 1, False) tdSql.checkData(1, 1, True) tdSql.checkData(2, 1, True) @@ -4508,7 +4474,6 @@ class TDTestCase: tdSql.checkData(5, 1, True) tdSql.checkData(6, 1, False) tdSql.checkData(7, 1, False) - tdSql.checkData(8, 1, True) tdSql.checkData(0, 2, 1) tdSql.checkData(1, 2, 9) @@ -4518,37 +4483,36 @@ class TDTestCase: tdSql.checkData(5, 2, 13) tdSql.checkData(6, 2, 13) tdSql.checkData(7, 2, 15) - tdSql.checkData(8, 2, None) tdSql.query(f"select tbname, _irowts, _isfilled, interp(c0, 1) from {dbname}.{stbname_null} partition by tbname range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(next)") - tdSql.checkRows(18) - for i in range(0, 9): + tdSql.checkRows(15) + for i in range(0, 7): tdSql.checkData(i, 0, 'ctb1_null') - for i in range(9, 18): + for i in range(7, 15): tdSql.checkData(i, 0, 'ctb2_null') tdSql.checkData(0, 1, '2020-02-01 00:00:01.000') - tdSql.checkData(8, 1, '2020-02-01 00:00:17.000') + tdSql.checkData(6, 1, '2020-02-01 00:00:13.000') - tdSql.checkData(9, 1, '2020-02-01 00:00:01.000') - tdSql.checkData(17, 1, '2020-02-01 00:00:17.000') + tdSql.checkData(7, 1, '2020-02-01 00:00:01.000') + tdSql.checkData(14, 1, '2020-02-01 00:00:15.000') tdSql.query(f"select tbname, _irowts, _isfilled, interp(c0) from {dbname}.{stbname_null} where c0 is not null partition by tbname range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(next)") - tdSql.checkRows(18) - for i in range(0, 9): + tdSql.checkRows(15) + for i in range(0, 7): tdSql.checkData(i, 0, 'ctb1_null') - for i in range(9, 18): + for i in range(7, 15): tdSql.checkData(i, 0, 'ctb2_null') tdSql.checkData(0, 1, '2020-02-01 00:00:01.000') - tdSql.checkData(8, 1, '2020-02-01 00:00:17.000') + tdSql.checkData(6, 1, '2020-02-01 00:00:13.000') - tdSql.checkData(9, 1, '2020-02-01 00:00:01.000') - tdSql.checkData(17, 1, '2020-02-01 00:00:17.000') + tdSql.checkData(7, 1, '2020-02-01 00:00:01.000') + tdSql.checkData(14, 1, '2020-02-01 00:00:15.000') # fill linear # normal table diff --git a/tests/system-test/2-query/slow_query_basic.py b/tests/system-test/2-query/slow_query_basic.py new file mode 100644 index 0000000000..10595028a7 --- /dev/null +++ b/tests/system-test/2-query/slow_query_basic.py @@ -0,0 +1,66 @@ +import random +import string +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * +import numpy as np + + +class TDTestCase: + updatecfgDict = {'slowLogThresholdTest': ''} + updatecfgDict["slowLogThresholdTest"] = 0 + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def getPath(self, tool="taosBenchmark"): + if (platform.system().lower() == 'windows'): + tool = tool + ".exe" + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + paths = [] + for root, dirs, files in os.walk(projPath): + if ((tool) in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + paths.append(os.path.join(root, tool)) + break + if (len(paths) == 0): + tdLog.exit("taosBenchmark not found!") + return + else: + tdLog.info("taosBenchmark found in %s" % paths[0]) + return paths[0] + + def taosBenchmark(self, param): + binPath = self.getPath() + cmd = f"{binPath} {param}" + tdLog.info(cmd) + os.system(cmd) + + def testSlowQuery(self): + self.taosBenchmark(" -d db -t 2 -v 2 -n 1000000 -y") + sql = "select count(*) from db.meters" + for i in range(10): + tdSql.query(sql) + tdSql.checkData(0, 0, 2 * 1000000) + + def run(self): + self.testSlowQuery() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/sml.py b/tests/system-test/2-query/sml.py index e28f3b1edd..4a8760cd67 100644 --- a/tests/system-test/2-query/sml.py +++ b/tests/system-test/2-query/sml.py @@ -105,6 +105,10 @@ class TDTestCase: tdSql.query(f"select * from ts3724.`stb2.`") tdSql.checkRows(1) + tdSql.query(f"select * from ts5528.device_log_yuelan_cs1") + tdSql.checkRows(2) + tdSql.checkData(0, 1, '{"deviceId":"星宇公司-861701069493741","headers":{"_uid":"4e3599eacd62834995c77b38ad95f88d","creatorId":"1199596756811550720","deviceNmae":"861701069493741","productId":"yuelan","productName":"悦蓝cat1穿戴设备"},"messageType":"REPORT_PROPERTY","properties":{"lat":35.265527067449185,"lng":118.49713144245987,"location":"118.49713144245987,35.265527067449185"},"timestamp":1728719963230}') + tdSql.checkData(1, 1, '{"deviceId":"星宇公司-861701069065507","headers":{"_uid":"9045d6b78b4ffaf1e2d244e912ebbff8","creatorId":"1199596756811550720","deviceNmae":"861701069065507","productId":"yuelan","productName":"悦蓝cat1穿戴设备"},"messageType":"REPORT_PROPERTY","properties":{"lat":36.788241914043425,"lng":119.15042325460891,"location":"119.15042325460891,36.788241914043425"},"timestamp":1728719964105}') # tdSql.query(f"select * from td24559.stb order by _ts") # tdSql.checkRows(4) # tdSql.checkData(0, 2, "POINT (4.343000 89.342000)") diff --git a/tests/system-test/2-query/td-32548.py b/tests/system-test/2-query/td-32548.py new file mode 100644 index 0000000000..45611b8372 --- /dev/null +++ b/tests/system-test/2-query/td-32548.py @@ -0,0 +1,32 @@ +from util.cases import * +from util.sql import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + + tdSql.execute("drop database if exists td_32548;") + tdSql.execute("create database td_32548 cachemodel 'last_row' keep 3650,3650,3650;") + + def run(self): + tdSql.execute("use td_32548;") + + tdSql.execute("create table ntb1 (ts timestamp, ival int);") + tdSql.execute("insert into ntb1 values ('2024-07-08 17:54:49.675', 54);") + + tdSql.execute("flush database td_32548;") + + tdSql.execute("insert into ntb1 values ('2024-07-08 17:53:49.675', 53);") + tdSql.execute("insert into ntb1 values ('2024-07-08 17:52:49.675', 52);") + tdSql.execute("delete from ntb1 where ts = '2024-07-08 17:54:49.675';") + + tdSql.query('select last_row(ts) from ntb1;') + tdSql.checkData(0, 0, '2024-07-08 17:53:49.675') + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/win-test-file b/tests/system-test/win-test-file index e86047bca8..c3047efdd7 100644 --- a/tests/system-test/win-test-file +++ b/tests/system-test/win-test-file @@ -925,3 +925,4 @@ python3 ./test.py -f 99-TDcase/TD-20582.py python3 ./test.py -f 5-taos-tools/taosbenchmark/insertMix.py -N 3 python3 ./test.py -f 5-taos-tools/taosbenchmark/stt.py -N 3 python3 ./test.py -f eco-system/meta/database/keep_time_offset.py +python3 ./test.py -f 2-query/slow_query_basic.py diff --git a/tools/auto/stmt2Performance/json/template.json b/tools/auto/stmt2Performance/json/template.json index 659c5966a4..8c54c5be22 100644 --- a/tools/auto/stmt2Performance/json/template.json +++ b/tools/auto/stmt2Performance/json/template.json @@ -16,14 +16,14 @@ { "dbinfo": { "name": "dbrate", - "drop": "yes", - "vgroups": 2 + "vgroups": 1, + "drop": "yes" }, "super_tables": [ { "name": "meters", "child_table_exists": "no", - "childtable_count": 10, + "childtable_count": 1, "childtable_prefix": "d", "insert_mode": "@STMT_MODE", "interlace_rows": @INTERLACE_MODE, diff --git a/tools/auto/stmt2Performance/stmt2Perf.py b/tools/auto/stmt2Performance/stmt2Perf.py index e7a4d5ecbe..4d99f2483d 100644 --- a/tools/auto/stmt2Performance/stmt2Perf.py +++ b/tools/auto/stmt2Performance/stmt2Perf.py @@ -198,16 +198,20 @@ def findContextValue(context, label): def writeTemplateInfo(resultFile): # create info - context = readFileContext(templateFile) + context = readFileContext(templateFile) vgroups = findContextValue(context, "vgroups") childCount = findContextValue(context, "childtable_count") insertRows = findContextValue(context, "insert_rows") - line = f"vgroups = {vgroups}\nchildtable_count = {childCount}\ninsert_rows = {insertRows}\n\n" + bindVGroup = findContextValue(context, "thread_bind_vgroup") + nThread = findContextValue(context, "thread_count") + if bindVGroup.lower().find("yes") != -1: + nThread = vgroups + line = f"thread_bind_vgroup = {bindVGroup}\nvgroups = {vgroups}\nchildtable_count = {childCount}\ninsert_rows = {insertRows}\ninsertThreads = {nThread} \n\n" print(line) appendFileContext(resultFile, line) -def totalCompressRate(stmt, interlace, resultFile, writeSpeed, querySpeed): +def totalCompressRate(stmt, interlace, resultFile, spent, spentReal, writeSpeed, writeReal, min, avg, p90, p99, max, querySpeed): global Number # flush command = 'taos -s "flush database dbrate;"' @@ -220,7 +224,7 @@ def totalCompressRate(stmt, interlace, resultFile, writeSpeed, querySpeed): # read compress rate command = 'taos -s "show table distributed dbrate.meters\G;"' rets = runRetList(command) - print(rets) + #print(rets) str1 = rets[5] arr = str1.split(" ") @@ -234,46 +238,88 @@ def totalCompressRate(stmt, interlace, resultFile, writeSpeed, querySpeed): str2 = arr[6] pos = str2.find("=[") rate = str2[pos+2:] - print("rate =" + rate) # total data file size #dataSize = getFolderSize(f"{dataDir}/vnode/") #dataSizeMB = int(dataSize/1024/1024) # appand to file - + + # %("No", "stmtMode", "interlaceRows", "spent", "spent-real", "writeSpeed", "write-real", "query-QPS", "dataSize", "rate") Number += 1 - context = "%10s %10s %15s %10s %10s %30s %15s\n"%( Number, stmt, interlace, str(totalSize)+" MB", rate+"%", writeSpeed + " Records/second", querySpeed) + ''' + context = "%2s %6s %10s %10s %10s %15s %15s %16s %16s %16s %16s %16s %8s %8s %8s\n"%( + Number, stmt, interlace, spent + "s", spentReal + "s", writeSpeed + " rows/s", writeReal + " rows/s", + min, avg, p90, p99, max, + querySpeed, str(totalSize) + " MB", rate + "%") + ''' + context = "%2s %8s %10s %10s %16s %16s %12s %12s %12s %12s %12s %12s %10s %10s %10s\n"%( + Number, stmt, interlace, spent + "s", spentReal + "s", writeSpeed + "r/s", writeReal + "r/s", + min, avg, p90, p99, max + "ms", + querySpeed, str(totalSize) + " MB", rate + "%") + showLog(context) appendFileContext(resultFile, context) +def cutEnd(line, start, endChar): + pos = line.find(endChar, start) + if pos == -1: + return line[start:] + return line[start : pos] + +def findValue(context, pos, key, endChar,command): + pos = context.find(key, pos) + if pos == -1: + print(f"error, run command={command} output not found \"{key}\" keyword. context={context}") + exit(1) + pos += len(key) + value = cutEnd(context, pos, endChar) + return (value, pos) + def testWrite(jsonFile): command = f"taosBenchmark -f {jsonFile}" output, context = run(command, 60000) + print(context) + # SUCC: Spent 0.960248 (real 0.947154) seconds to insert rows: 100000 with 1 thread(s) into dbrate 104139.76 (real 105579.45) records/second - # find second real - pos = context.find("(real ") + # spent + key = "Spent " + pos = -1 + pos1 = 0 + while pos1 != -1: # find last "Spent " + pos1 = context.find(key, pos1) + if pos1 != -1: + pos = pos1 # update last found + pos1 += len(key) if pos == -1: - print(f"error, run command={command} output not found first \"(real\" keyword. error={context}") + print(f"error, run command={command} output not found \"{key}\" keyword. context={context}") exit(1) - pos = context.find("(real ", pos + 5) + pos += len(key) + spent = cutEnd(context, pos, ".") + + # spent-real + spentReal, pos = findValue(context, pos, "(real ", ".", command) + + # writeSpeed + key = "into " + pos = context.find(key, pos) if pos == -1: - print(f"error, run command={command} output not found second \"(real\" keyword. error={context}") - exit(1) - - pos += 5 - length = len(context) - while pos < length and context[pos] == ' ': - pos += 1 - end = context.find(".", pos) - if end == -1: - print(f"error, run command={command} output not found second \".\" keyword. error={context}") + print(f"error, run command={command} output not found \"{key}\" keyword. context={context}") exit(1) + pos += len(key) + writeSpeed, pos = findValue(context, pos, " ", ".", command) + # writeReal + writeReal, pos = findValue(context, pos, "(real ", ".", command) - speed = context[pos: end] - #print(f"write pos ={pos} end={end} speed={speed}\n output={context} \n") - return speed + # delay + min, pos = findValue(context, pos, "min: ", ",", command) + avg, pos = findValue(context, pos, "avg: ", ",", command) + p90, pos = findValue(context, pos, "p90: ", ",", command) + p99, pos = findValue(context, pos, "p99: ", ",", command) + max, pos = findValue(context, pos, "max: ", "ms", command) + + return (spent, spentReal, writeSpeed, writeReal, min, avg, p90, p99, max) def testQuery(): command = f"taosBenchmark -f json/query.json" @@ -308,13 +354,13 @@ def doTest(stmt, interlace, resultFile): # run taosBenchmark t1 = time.time() - writeSpeed = testWrite(jsonFile) + spent, spentReal, writeSpeed, writeReal, min, avg, p90, p99, max = testWrite(jsonFile) t2 = time.time() # total write speed querySpeed = testQuery() # total compress rate - totalCompressRate(stmt, interlace, resultFile, writeSpeed, querySpeed) + totalCompressRate(stmt, interlace, resultFile, spent, spentReal, writeSpeed, writeReal, min, avg, p90, p99, max, querySpeed) def main(): @@ -333,7 +379,17 @@ def main(): # json info writeTemplateInfo(resultFile) # head - context = "\n%10s %10s %15s %10s %10s %30s %15s\n"%("No", "stmtMode", "interlaceRows", "dataSize", "rate", "writeSpeed", "query-QPS") + ''' + context = "%3s %8s %10s %10s %10s %15s %15s %10s %10s %10s %10s %10s %8s %8s %8s\n"%( + "No", "stmtMode", "interlace", "spent", "spent-real", "writeSpeed", "write-real", + "min", "avg", "p90", "p99", "max", + "query-QPS", "dataSize", "rate") + ''' + context = "%2s %8s %10s %10s %16s %16s %12s %12s %12s %12s %12s %12s %10s %10s %10s\n"%( + "No", "stmtMode", "interlace", "spent", "spent-real", "writeSpeed", "write-real", + "min", "avg", "p90", "p99", "max", + "query-QPS", "dataSize", "rate") + appendFileContext(resultFile, context) diff --git a/tools/auto/testCompression/testCompression.py b/tools/auto/testCompression/testCompression.py index 281a097f8a..ee922a1a23 100644 --- a/tools/auto/testCompression/testCompression.py +++ b/tools/auto/testCompression/testCompression.py @@ -134,8 +134,6 @@ def getMatch(datatype, algo): def generateJsonFile(algo): - print(f"doTest algo: {algo} \n") - # replace datatype context = readFileContext(templateFile) # replace compress @@ -192,8 +190,6 @@ def findContextValue(context, label): ends = [',','}',']', 0] while context[end] not in ends: end += 1 - - print(f"start = {start} end={end}\n") return context[start:end] @@ -281,10 +277,10 @@ def testQuery(): # INFO: Spend 6.7350 second completed total queries: 10, the QPS of all threads: 1.485 speed = None - for i in range(20, len(lines)): + for i in range(0, len(lines)): # find second real + context = lines[i] pos = context.find("the QPS of all threads:") - context = lines[26] if pos == -1 : continue pos += 24 @@ -302,7 +298,6 @@ def doTest(algo, resultFile): print(f"doTest algo: {algo} \n") #cleanAndStartTaosd() - # json jsonFile = generateJsonFile(algo) diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index a3830d2518..55fa336221 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -2098,12 +2098,47 @@ int sml_td29373_Test() { return code; } +int sml_ts5528_test(){ + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "drop database if exists ts5528"); + taos_free_result(pRes); + + pRes = taos_query(taos, "create database if not exists ts5528"); + taos_free_result(pRes); + + // check column name duplication + char *sql[] = { + "device_log_yuelan_cs1,deviceId=861701069493741 content=\"{\\\"deviceId\\\":\\\"星宇公司-861701069493741\\\",\\\"headers\\\":{\\\"_uid\\\":\\\"4e3599eacd62834995c77b38ad95f88d\\\",\\\"creatorId\\\":\\\"1199596756811550720\\\",\\\"deviceNmae\\\":\\\"861701069493741\\\",\\\"productId\\\":\\\"yuelan\\\",\\\"productName\\\":\\\"悦蓝cat1穿戴设备\\\"},\\\"messageType\\\":\\\"REPORT_PROPERTY\\\",\\\"properties\\\":{\\\"lat\\\":35.265527067449185,\\\"lng\\\":118.49713144245987,\\\"location\\\":\\\"118.49713144245987,35.265527067449185\\\"},\\\"timestamp\\\":1728719963230}\",createTime=1728719963230i64,id=\"4e3599eacd62834995c77b38ad95f88d\",messageId=\"\",timestamp=1728719963230i64,type=\"reportProperty\" 1728719963230", + "device_log_yuelan_cs1,deviceId=861701069065507 content=\"{\\\"deviceId\\\":\\\"星宇公司-861701069065507\\\",\\\"headers\\\":{\\\"_uid\\\":\\\"9045d6b78b4ffaf1e2d244e912ebbff8\\\",\\\"creatorId\\\":\\\"1199596756811550720\\\",\\\"deviceNmae\\\":\\\"861701069065507\\\",\\\"productId\\\":\\\"yuelan\\\",\\\"productName\\\":\\\"悦蓝cat1穿戴设备\\\"},\\\"messageType\\\":\\\"REPORT_PROPERTY\\\",\\\"properties\\\":{\\\"lat\\\":36.788241914043425,\\\"lng\\\":119.15042325460891,\\\"location\\\":\\\"119.15042325460891,36.788241914043425\\\"},\\\"timestamp\\\":1728719964105}\",createTime=1728719964105i64,id=\"9045d6b78b4ffaf1e2d244e912ebbff8\",messageId=\"\",timestamp=1728719964105i64,type=\"reportProperty\" 1728719964105", + }; + pRes = taos_query(taos, "use ts5528"); + taos_free_result(pRes); + + for( int i = 0; i < 2; i++){ + int32_t totalRows = 0; + pRes = taos_schemaless_insert_raw(taos, sql[i], strlen(sql[i]), &totalRows, TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + int code = taos_errno(pRes); + taos_free_result(pRes); + if (code != 0) { + taos_close(taos); + return code; + } + } + taos_close(taos); + printf("%s result success\n", __FUNCTION__); + return 0; +} + int main(int argc, char *argv[]) { if (argc == 2) { taos_options(TSDB_OPTION_CONFIGDIR, argv[1]); } int ret = 0; + ret = sml_ts5528_test(); + ASSERT(!ret); ret = sml_td29691_Test(); ASSERT(ret); ret = sml_td29373_Test();