xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

change

This commit is contained in:
zyyang 2020-12-14 13:05:43 +08:00
parent cf453e39f3
commit c8cceb68d3
6 changed files with 112 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
tests/examples/JDBC/taosdemo/pom.xml
View File

@ -24,7 +24,7 @@
<dependency>
<groupId>com.taosdata.jdbc</groupId>
<artifactId>taos-jdbcdriver</artifactId>
<version>2.0.14</version>
<version>2.0.15</version>
</dependency>
<!-- mysql -->
<dependency>

96
tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/components/TaosDemoCommandLineRunner.java
View File

@ -14,7 +14,6 @@ import org.springframework.boot.CommandLineRunner;
import org.springframework.stereotype.Component;
import java.util.*;
import java.util.concurrent.TimeUnit;
@Component
@ -30,9 +29,8 @@ public class TaosDemoCommandLineRunner implements CommandLineRunner {
private SuperTableMeta superTableMeta;
private List<SubTableMeta> subTableMetaList;
private List<SubTableValue> subTableValueList;
private List<List<SubTableValue>> dataList;
// private List<SubTableValue> subTableValueList;
// private List<List<SubTableValue>> dataList;
@Override
public void run(String... args) throws Exception {
@ -44,7 +42,7 @@ public class TaosDemoCommandLineRunner implements CommandLineRunner {
System.exit(0);
}
// 准备数据
prepareData(config);
prepareMetaData(config);
// 创建数据库
createDatabaseTask(config);
// 建表
@ -52,6 +50,7 @@ public class TaosDemoCommandLineRunner implements CommandLineRunner {
// 插入
insertTask(config);
// 查询: 1. 生成查询语句, 2. 执行查询
// 删除表
if (config.dropTable) {
superTableService.drop(config.database, config.superTable);
@ -62,7 +61,6 @@ public class TaosDemoCommandLineRunner implements CommandLineRunner {
private void createDatabaseTask(JdbcTaosdemoConfig config) {
long start = System.currentTimeMillis();
Map<String, String> databaseParam = new HashMap<>();
databaseParam.put("database", config.database);
databaseParam.put("keep", Integer.toString(config.keep));
@ -72,9 +70,8 @@ public class TaosDemoCommandLineRunner implements CommandLineRunner {
databaseService.dropDatabase(config.database);
databaseService.createDatabase(databaseParam);
databaseService.useDatabase(config.database);
long end = System.currentTimeMillis();
logger.info(">>> insert time cost : " + (end - start) + " ms.");
logger.info(">>> create database time cost : " + (end - start) + " ms.");
}
// 建超级表，三种方式：1. 指定SQL，2. 指定field和tags的个数，3. 默认
@ -92,30 +89,71 @@ public class TaosDemoCommandLineRunner implements CommandLineRunner {
private void insertTask(JdbcTaosdemoConfig config) {
long start = System.currentTimeMillis();
int numOfThreadsForInsert = config.numOfThreadsForInsert;
int sleep = config.sleep;
if (config.autoCreateTable) {
// 批量插入，自动建表
dataList.stream().forEach(subTableValues -> {
subTableService.insertAutoCreateTable(subTableValues, numOfThreadsForInsert);
sleep(sleep);
});
} else {
dataList.stream().forEach(subTableValues -> {
subTableService.insert(subTableValues, numOfThreadsForInsert);
sleep(sleep);
});
int numOfTables = config.numOfTables;
int numOfTablesPerSQL = config.numOfTablesPerSQL;
int numOfRowsPerTable = config.numOfRowsPerTable;
int numOfValuesPerSQL = config.numOfValuesPerSQL;
if (numOfRowsPerTable < numOfValuesPerSQL)
numOfValuesPerSQL = numOfRowsPerTable;
if (numOfTables < numOfTablesPerSQL)
numOfTablesPerSQL = numOfTables;
//table
for (int tableCnt = 0; tableCnt < numOfTables; ) {
int tableSize = numOfTablesPerSQL;
if (tableCnt + tableSize > numOfTables) {
tableSize = numOfTables - tableCnt;
}
// row
for (int rowCnt = 0; rowCnt < numOfRowsPerTable; ) {
int rowSize = numOfValuesPerSQL;
if (rowCnt + rowSize > numOfRowsPerTable) {
rowSize = numOfRowsPerTable - rowCnt;
}
/***********************************************/
// 生成数据
List<SubTableValue> data = SubTableValueGenerator.generate(subTableMetaList, tableCnt, tableSize, rowSize, config.startTime, config.timeGap);
// 乱序
if (config.order != 0) {
SubTableValueGenerator.disrupt(data, config.rate, config.range);
}
// insert
if (config.autoCreateTable) {
subTableService.insertAutoCreateTable(data, config.numOfThreadsForInsert, config.frequency);
} else {
subTableService.insert(data, config.numOfThreadsForInsert, config.frequency);
}
/***********************************************/
rowCnt += rowSize;
}
tableCnt += tableSize;
}
// 批量插入，自动建表
// dataList.stream().forEach(subTableValues -> {
// subTableService.insertAutoCreateTable(subTableValues, config.numOfThreadsForInsert, config.frequency);
// });
// subTableService.insertAutoCreateTable(subTableMetaList, config.numOfTables, config.tablePrefix, config.numOfThreadsForInsert, config.frequency);
// } else {
// dataList.stream().forEach(subTableValues -> {
// subTableService.insert(subTableValues, config.numOfThreadsForInsert, config.frequency);
// });
// subTableService.insert(subTableMetaList, config.numOfTables, config.tablePrefix, config.numOfThreadsForInsert, config.frequency);
// }
long end = System.currentTimeMillis();
logger.info(">>> insert time cost : " + (end - start) + " ms.");
}
private void prepareData(JdbcTaosdemoConfig config) {
private void prepareMetaData(JdbcTaosdemoConfig config) {
long start = System.currentTimeMillis();
// 超级表的meta
superTableMeta = createSupertable(config);
// 子表的meta
subTableMetaList = SubTableMetaGenerator.generate(superTableMeta, config.numOfTables, config.tablePrefix);
/*
// 子表的data
subTableValueList = SubTableValueGenerator.generate(subTableMetaList, config.numOfRowsPerTable, config.startTime, config.timeGap);
// 如果有乱序，给数据搞乱
@ -128,8 +166,9 @@ public class TaosDemoCommandLineRunner implements CommandLineRunner {
int numOfRowsPerTable = config.numOfRowsPerTable;
int numOfValuesPerSQL = config.numOfValuesPerSQL;
dataList = SubTableValueGenerator.split(subTableValueList, numOfTables, numOfTablesPerSQL, numOfRowsPerTable, numOfValuesPerSQL);
*/
long end = System.currentTimeMillis();
logger.info(">>> prepare data time cost : " + (end - start) + " ms.");
logger.info(">>> prepare meta data time cost : " + (end - start) + " ms.");
}
private SuperTableMeta createSupertable(JdbcTaosdemoConfig config) {
@ -139,6 +178,8 @@ public class TaosDemoCommandLineRunner implements CommandLineRunner {
if (config.superTableSQL != null) {
// use a sql to create super table
tableMeta = SuperTableMetaGenerator.generate(config.superTableSQL);
if (config.database != null && !config.database.isEmpty())
tableMeta.setDatabase(config.database);
} else if (config.numOfFields == 0) {
// default sql = "create table test.weather (ts timestamp, temperature float, humidity int) tags(location nchar(64), groupId int)";
SuperTableMeta superTableMeta = new SuperTableMeta();
@ -161,14 +202,5 @@ public class TaosDemoCommandLineRunner implements CommandLineRunner {
return tableMeta;
}
private static void sleep(int sleep) {
if (sleep <= 0)
return;
try {
TimeUnit.MILLISECONDS.sleep(sleep);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}

46
tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SubTableService.java
View File

@ -11,6 +11,7 @@ import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
@Service
public class SubTableService extends AbstractService {
@ -51,18 +52,22 @@ public class SubTableService extends AbstractService {
/*************************************************************************************************************************/
// 插入：多线程，多表
public int insert(List<SubTableValue> subTableValues, int threadSize) {
public int insert(List<SubTableValue> subTableValues, int threadSize, int frequency) {
ExecutorService executor = Executors.newFixedThreadPool(threadSize);
Future<Integer> future = executor.submit(() -> insert(subTableValues));
executor.shutdown();
//TODO：
sleep(1000);
return getAffectRows(future);
}
// 插入：多线程，多表, 自动建表
public int insertAutoCreateTable(List<SubTableValue> subTableValues, int threadSize) {
public int insertAutoCreateTable(List<SubTableValue> subTableValues, int threadSize, int frequency) {
ExecutorService executor = Executors.newFixedThreadPool(threadSize);
Future<Integer> future = executor.submit(() -> insertAutoCreateTable(subTableValues));
executor.shutdown();
return getAffectRows(future);
}
@ -86,33 +91,14 @@ public class SubTableService extends AbstractService {
return mapper.insertMultiTableMultiValuesUsingSuperTable(subTableValues);
}
// ExecutorService executors = Executors.newFixedThreadPool(threadSize);
// int count = 0;
//
// //
// List<SubTableValue> subTableValues = new ArrayList<>();
// for (int tableIndex = 1; tableIndex <= numOfTablesPerSQL; tableIndex++) {
// // each table
// SubTableValue subTableValue = new SubTableValue();
// subTableValue.setDatabase();
// subTableValue.setName();
// subTableValue.setSupertable();
//
// List<RowValue> values = new ArrayList<>();
// for (int valueCnt = 0; valueCnt < numOfValuesPerSQL; valueCnt++) {
// List<FieldValue> fields = new ArrayList<>();
// for (int fieldInd = 0; fieldInd <; fieldInd++) {
// FieldValue<Object> field = new FieldValue<>("", "");
// fields.add(field);
// }
// RowValue row = new RowValue();
// row.setFields(fields);
// values.add(row);
// }
// subTableValue.setValues(values);
// subTableValues.add(subTableValue);
// }
private static void sleep(int sleep) {
if (sleep <= 0)
return;
try {
TimeUnit.MILLISECONDS.sleep(sleep);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}

33
tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/SubTableValueGenerator.java
View File

@ -12,21 +12,7 @@ import java.util.List;
public class SubTableValueGenerator {
public static List<SubTableValue> generate(List<SubTableMeta> subTableMetaList, int numOfRowsPerTable, long start, long timeGap) {
List<SubTableValue> subTableValueList = new ArrayList<>();
subTableMetaList.stream().forEach((subTableMeta) -> {
// insert into xxx.xxx using xxxx tags(...) values(),()...
SubTableValue subTableValue = new SubTableValue();
subTableValue.setDatabase(subTableMeta.getDatabase());
subTableValue.setName(subTableMeta.getName());
subTableValue.setSupertable(subTableMeta.getSupertable());
subTableValue.setTags(subTableMeta.getTags());
TimeStampUtil.TimeTuple tuple = TimeStampUtil.range(start, timeGap, numOfRowsPerTable);
List<RowValue> values = FieldValueGenerator.generate(tuple.start, tuple.end, tuple.timeGap, subTableMeta.getFields());
subTableValue.setValues(values);
subTableValueList.add(subTableValue);
});
return subTableValueList;
return generate(subTableMetaList, 0, subTableMetaList.size(), numOfRowsPerTable, start, timeGap);
}
public static void disrupt(List<SubTableValue> subTableValueList, int rate, long range) {
@ -38,12 +24,10 @@ public class SubTableValueGenerator {
public static List<List<SubTableValue>> split(List<SubTableValue> subTableValueList, int numOfTables, int numOfTablesPerSQL, int numOfRowsPerTable, int numOfValuesPerSQL) {
List<List<SubTableValue>> dataList = new ArrayList<>();
if (numOfRowsPerTable < numOfValuesPerSQL)
numOfValuesPerSQL = numOfRowsPerTable;
if (numOfTables < numOfTablesPerSQL)
numOfTablesPerSQL = numOfTables;
//table
for (int tableCnt = 0; tableCnt < numOfTables; ) {
int tableSize = numOfTablesPerSQL;
@ -81,4 +65,19 @@ public class SubTableValueGenerator {
split(null, 99, 10, 99, 10);
}
public static List<SubTableValue> generate(List<SubTableMeta> subTableMetaList, int tableCnt, int tableSize, int rowSize, long startTime, long timeGap) {
List<SubTableValue> subTableValueList = new ArrayList<>();
for (int i = 0; i < tableSize; i++) {
SubTableMeta subTableMeta = subTableMetaList.get(tableCnt + i);
SubTableValue subTableValue = new SubTableValue();
subTableValue.setDatabase(subTableMeta.getDatabase());
subTableValue.setName(subTableMeta.getName());
subTableValue.setSupertable(subTableMeta.getSupertable());
subTableValue.setTags(subTableMeta.getTags());
TimeStampUtil.TimeTuple tuple = TimeStampUtil.range(startTime, timeGap, rowSize);
List<RowValue> values = FieldValueGenerator.generate(tuple.start, tuple.end, tuple.timeGap, subTableMeta.getFields());
subTableValue.setValues(values);
}
return subTableValueList;
}
}

19
tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/utils/JdbcTaosdemoConfig.java
View File

@ -21,18 +21,18 @@ public final class JdbcTaosdemoConfig {
public String superTableSQL;
//sub table
public String tablePrefix = "t";
public int numOfTables = 100;
public int numOfTables = 1;
public int numOfThreadsForCreate = 1;
// insert task
public boolean autoCreateTable;
public int numOfRowsPerTable = 100;
public int numOfRowsPerTable = 1;
public int numOfThreadsForInsert = 1;
public int numOfTablesPerSQL = 10;
public int numOfValuesPerSQL = 10;
public int numOfTablesPerSQL = 1;
public int numOfValuesPerSQL = 1;
public long startTime;
public long timeGap;
public int sleep = 0;
public int order = 0;
public int frequency;
public int order;
public int rate = 10;
public long range = 1000l;
// select task
@ -74,11 +74,10 @@ public final class JdbcTaosdemoConfig {
System.out.println("-numOfValuesPerSQL The number of value per SQL. Default is 1");
System.out.println("-startTime start time for insert task, The format is \"yyyy-MM-dd HH:mm:ss.SSS\".");
System.out.println("-timeGap the number of time gap. Default is 1000 ms");
System.out.println("-sleep The number of milliseconds for sleep after each insert. default is 0");
System.out.println("-frequency the number of records per second inserted into one table. default is 0, do not control frequency");
System.out.println("-order Insert mode--0: In order, 1: Out of order. Default is in order");
System.out.println("-rate The proportion of data out of order. effective only if order is 1. min 0, max 100, default is 10");
System.out.println("-range The range of data out of order. effective only if order is 1. default is 1000 ms");
// query task
// System.out.println("-sqlFile The select sql file");
// drop task
@ -174,8 +173,8 @@ public final class JdbcTaosdemoConfig {
if ("-timeGap".equals(args[i]) && i < args.length - 1) {
timeGap = Long.parseLong(args[++i]);
}
if ("-sleep".equals(args[i]) && i < args.length - 1) {
sleep = Integer.parseInt(args[++i]);
if ("-frequency".equals(args[i]) && i < args.length - 1) {
frequency = Integer.parseInt(args[++i]);
}
if ("-order".equals(args[i]) && i < args.length - 1) {
order = Integer.parseInt(args[++i]);

7
tests/examples/JDBC/taosdemo/src/main/resources/application.properties
View File

@ -3,11 +3,16 @@
#spring.datasource.username=root
#spring.datasource.password=123456
spring.datasource.url=jdbc:TAOS://master:6030/?charset=UTF-8&locale=en_US.UTF-8&timezone=UTC-8
spring.datasource.url=jdbc:TAOS://:/?charset=UTF-8&locale=en_US.UTF-8&timezone=UTC-8
spring.datasource.driver-class-name=com.taosdata.jdbc.TSDBDriver
spring.datasource.username=root
spring.datasource.password=taosdata
#spring.datasource.url=jdbc:TAOS-RS://:/?charset=UTF-8&locale=en_US.UTF-8&timezone=UTC-8
#spring.datasource.driver-class-name=com.taosdata.jdbc.rs.RestfulDriver
#spring.datasource.username=root
#spring.datasource.password=taosdata
spring.datasource.hikari.maximum-pool-size=10
spring.datasource.hikari.minimum-idle=10
spring.datasource.hikari.max-lifetime=600000