xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #346 from fangpanpan/master 

upload code for cassandra/tdengine test
This commit is contained in:
hzcheng 2019-08-14 14:13:10 +08:00 committed by GitHub
parent 747e93260c c9ebc4e33e
commit 62f9a90f9c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 516 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
tests/comparisonTest/cassandra/application.conf Normal file
View File

@ -0,0 +1,5 @@
datastax-java-driver {
basic.request {
timeout = 200000 seconds
}
}

131
tests/comparisonTest/cassandra/cassandratest/pom.xml Normal file
View File

@ -0,0 +1,131 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.cassandra.test</groupId>
<artifactId>cassandratest</artifactId>
<version>1.0-SNAPSHOT</version>
<packaging>jar</packaging>
<build>
<pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-plugins</artifactId>
<version>30</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>3.0.0</version>
</plugin>
</plugins>
</pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>3.1.0</version>
<configuration>
<archive>
<manifest>
<mainClass>CassandraTest</mainClass>
</manifest>
</archive>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>2.3.2</version>
<configuration>
<source>8</source>
<target>8</target>
</configuration>
</plugin>
</plugins>
</build>
<name>cassandratest</name>
<!-- FIXME change it to the project's website -->
<url>http://www.example.com</url>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
</properties>
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.datastax.oss</groupId>
<artifactId>java-driver-core</artifactId>
<version>4.1.0</version>
</dependency>
<dependency>
<groupId>com.datastax.oss</groupId>
<artifactId>java-driver-query-builder</artifactId>
<version>4.1.0</version>
</dependency>
<dependency>
<groupId>com.datastax.oss</groupId>
<artifactId>java-driver-mapper-runtime</artifactId>
<version>4.1.0</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.4</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.7</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.5</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.5</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-1.2-api</artifactId>
<version>2.8.2</version>
</dependency>
</dependencies>
</project>

200
tests/comparisonTest/cassandra/cassandratest/src/main/java/com/cassandra/test/CassandraTest.java Normal file
View File

@ -0,0 +1,200 @@
import com.datastax.oss.driver.api.core.CqlSession;
import com.datastax.oss.driver.api.core.cql.*;
import com.datastax.oss.driver.api.core.session.*;
import com.datastax.oss.driver.api.core.config.*;
import com.datastax.oss.driver.api.core.cql.ResultSet;
import com.datastax.oss.driver.api.core.cql.Row;
//import com.datastax.driver.core.Cluster;
//import com.datastax.driver.core.Cluster;
import java.io.BufferedWriter;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.FileReader;
import java.io.IOException;
import java.text.DecimalFormat;
import java.util.Random;
import java.math.*;
import java.lang.reflect.Method;
public class CassandraTest{
public static void main(String args[]) {
// begin to parse argument
String datadir = "/home/ubuntu/testdata";
String sqlfile = "/home/ubuntu/fang/cassandra/q1.txt";
String cfgfile = "/home/ubuntu/fang/cassandra/application.conf";
boolean q4flag = false;
int numOfRows = 1000000;
int numOfFiles =0;
int numOfClients =0;
int rowsPerRequest =0;
for (int i = 0; i < args.length; ++i) {
if (args[i].equalsIgnoreCase("-dataDir")) {
if (i < args.length - 1) {
datadir = args[++i];
}
} else if (args[i].equalsIgnoreCase("-numofFiles")) {
if (i < args.length - 1) {
numOfFiles = Integer.parseInt(args[++i]);
}
} else if (args[i].equalsIgnoreCase("-rowsPerRequest")) {
if (i < args.length - 1) {
rowsPerRequest = Integer.parseInt(args[++i]);
}
} else if (args[i].equalsIgnoreCase("-writeClients")) {
if (i < args.length - 1) {
numOfClients = Integer.parseInt(args[++i]);
}
} else if (args[i].equalsIgnoreCase("-sql")) {
sqlfile = args[++i];
} else if (args[i].equalsIgnoreCase("-timetest")) {
q4flag = true;
} else if (args[i].equalsIgnoreCase("-conf")) {
cfgfile = args[++i];
}
}
// file below to make sure no timeout error
File confile = new File(cfgfile);
System.out.println("parameters\n");
if (numOfFiles >0) {
// write data
System.out.printf("----dataDir:%s\n", datadir);
System.out.printf("----numOfFiles:%d\n", numOfFiles);
System.out.printf("----numOfClients:%d\n", numOfClients);
System.out.printf("----rowsPerRequest:%d\n", rowsPerRequest);
// connect to cassandra server
System.out.printf("----connecting to cassandra server\n");
try {
CqlSession session = CqlSession.builder()
.withConfigLoader(DriverConfigLoader.fromFile(confile))
.build();
session.execute("drop keyspace if exists cassandra");
session.execute("CREATE KEYSPACE if not exists cassandra WITH replication = {'class':'SimpleStrategy', 'replication_factor':1}");
if (q4flag) {
session.execute("create table if not exists cassandra.test (devid int, devname text, devgroup int, ts bigint, minute bigint, temperature int, humidity float ,primary key (minute,ts,devgroup,devid,devname))");
} else {
session.execute("create table if not exists cassandra.test (devid int, devname text, devgroup int, ts bigint, temperature int, humidity float ,primary key (devgroup,devid,devname,ts))");
}
session.close();
System.out.printf("----created keyspace cassandra and table test\n");
// begin to insert data
System.out.printf("----begin to insert data\n");
long startTime = System.currentTimeMillis();
int a = numOfFiles/numOfClients;
int b = numOfFiles%numOfClients;
int last = 0;
WriteThread[] writethreads = new WriteThread[numOfClients];
int[] wargs = new int[2]; // data file start, end
wargs[0] = numOfRows; //rows to be read from each file
wargs[1] = rowsPerRequest;
int fstart =0;
int fend =0;
for (int i = 0; i<numOfClients; ++i) {
if (i<b) {
fstart = last;
fend = last+a;
last = last+a+1;
writethreads[i] = new WriteThread(fstart,fend,wargs,datadir,q4flag);
System.out.printf("----Thread %d begin to write\n",i);
writethreads[i].start();
} else {
fstart = last;
fend = last+a-1;
last = last+a;
writethreads[i] = new WriteThread(fstart,fend,wargs,datadir,q4flag);
System.out.printf("----Thread %d begin to write\n",i);
writethreads[i].start();
}
}
for (int i =0; i<numOfClients; ++i) {
try {
writethreads[i].join();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
long stopTime = System.currentTimeMillis();
float elapseTime = stopTime - startTime;
elapseTime = elapseTime/1000;
float speeds = numOfRows*numOfFiles/elapseTime;
System.out.printf("---- insertation speed: %f Rows/Second\n",speeds);
} catch (Exception ex) {
ex.printStackTrace();
System.exit(1);
} finally {
System.out.printf("---- insertion end\n");
}
// above:write part; below: read part;
} else {
// query data begin
System.out.printf("----sql command file:%s\n", sqlfile);
// connect to cassandra server
try {
CqlSession session = CqlSession.builder()
.withConfigLoader(DriverConfigLoader.fromFile(confile))
.build();
//session.execute("use cassandra;");
BufferedReader br = null;
String line = "";
try {
br = new BufferedReader(new FileReader(sqlfile));
while ((line = br.readLine()) != null && line.length()>10) {
long startTime = System.currentTimeMillis();
// begin to query one line command //
// end querying one line command
try {
ResultSet results = session.execute(line);
long icounter = 0;
for (Row row : results) {
icounter++;
}
long stopTime = System.currentTimeMillis();
float elapseTime = stopTime - startTime;
elapseTime = elapseTime/1000;
System.out.printf("----spend %f seconds to query: %s\n", elapseTime, line);
} catch (Exception ex) {
ex.printStackTrace();
System.out.printf("---- query failed!\n");
System.exit(1);
}
}
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
if (br != null) {
try {
br.close();
} catch (IOException e) {
e.printStackTrace();
}
}
session.close();
}
} catch (Exception ex) {
ex.printStackTrace();
} finally {
System.out.println("query end:----\n");
}
} // end write or query
System.exit(0);
}// end main
}// end class

99
tests/comparisonTest/cassandra/cassandratest/src/main/java/com/cassandra/test/WriteThread.java Normal file
View File

@ -0,0 +1,99 @@
import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.math.*;
import com.datastax.oss.driver.api.core.CqlSession;
import com.datastax.oss.driver.api.core.cql.*;
import com.datastax.oss.driver.api.core.session.*;
import com.datastax.oss.driver.api.core.config.*;
public class WriteThread extends Thread {
private int[] wargs; // fstart, fend, rows to be read, rows perrequest
private String fdir;
private int fstart;
private int fend;
private boolean q4flag;
public WriteThread (int fstart, int fend,int[] wargs, String fdir, boolean q4flag) {
this.fstart = fstart;
this.fend = fend;
this.fdir = fdir;
this.wargs = wargs;
this.q4flag = q4flag;
}
// begin to insert in this thread
public void run() {
/*
// this configuration file makes sure no timeout error
File confile = new File("/home/ubuntu/fang/cassandra/application.conf");
*/
// connect to server
try {
CqlSession session = CqlSession.builder()
//.withConfigLoader(DriverConfigLoader.fromFile(confile))
.build();
//session.execute("use cassandra");
int tominute = 6000;
for (int i=fstart; i<=fend; i++) {
String csvfile;
csvfile = fdir + "/testdata"+ Integer.toString(i)+".csv";
BufferedReader br = null;
String line = "";
String cvsSplitBy = " ";
try {
br = new BufferedReader(new FileReader(csvfile));
System.out.println("---- begin to read file " +csvfile+"\n");
for (int itotalrow =0; itotalrow<wargs[0]; itotalrow=itotalrow+wargs[1]) {
String cqlstr = "BEGIN BATCH ";
for (int irow =0; irow<wargs[1]; ++irow) {
line = br.readLine();
if (line !=null) {
String[] meter = line.split(cvsSplitBy);
BigInteger tminute = new BigInteger(meter[3]);
tminute = tminute.divide(BigInteger.valueOf(tominute));
if (q4flag) {
cqlstr = cqlstr + "insert into cassandra.test (devid,devname,devgroup,ts, minute,temperature,humidity) values ";
cqlstr = cqlstr +"("+meter[0] +"," +"'" +meter[1] +"'" +"," +meter[2] +"," + meter[3] +",";
cqlstr = cqlstr +tminute.toString() +"," +meter[4] +"," +meter[5] +");";
} else {
cqlstr = cqlstr + "insert into cassandra.test (devid,devname,devgroup,ts,temperature,humidity) values ";
cqlstr = cqlstr +"("+meter[0] +"," +"'" +meter[1] +"'" +"," +meter[2] +"," + meter[3] +",";
cqlstr = cqlstr +meter[4] +"," +meter[5] +");";
}
} // if this line is not null
}//end row iteration in one batch
cqlstr = cqlstr+" APPLY BATCH;";
try {
//System.out.println(cqlstr+"----\n");
session.execute(cqlstr);
} catch (Exception ex) {
ex.printStackTrace();
}
}// end one file reading
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
if (br != null) {
try {
br.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}//end file iteration
session.close();
} catch (Exception ex) {
ex.printStackTrace();
}
}//end run
}//end class

11
tests/comparisonTest/cassandra/q1.txt Normal file
View File

@ -0,0 +1,11 @@
select * from cassandra.test where devgroup=0 allow filtering;
select * from cassandra.test where devgroup=10 allow filtering;
select * from cassandra.test where devgroup=20 allow filtering;
select * from cassandra.test where devgroup=30 allow filtering;
select * from cassandra.test where devgroup=40 allow filtering;
select * from cassandra.test where devgroup=50 allow filtering;
select * from cassandra.test where devgroup=60 allow filtering;
select * from cassandra.test where devgroup=70 allow filtering;
select * from cassandra.test where devgroup=80 allow filtering;
select * from cassandra.test where devgroup=90 allow filtering;

50
tests/comparisonTest/cassandra/q2.txt Normal file
View File

@ -0,0 +1,50 @@
select count(*) from cassandra.test where devgroup<10 allow filtering;
select count(*) from cassandra.test where devgroup<20 allow filtering;
select count(*) from cassandra.test where devgroup<30 allow filtering;
select count(*) from cassandra.test where devgroup<40 allow filtering;
select count(*) from cassandra.test where devgroup<50 allow filtering;
select count(*) from cassandra.test where devgroup<60 allow filtering;
select count(*) from cassandra.test where devgroup<70 allow filtering;
select count(*) from cassandra.test where devgroup<80 allow filtering;
select count(*) from cassandra.test where devgroup<90 allow filtering;
select count(*) from cassandra.test allow filtering;
select avg(temperature) from cassandra.test where devgroup<10 allow filtering;
select avg(temperature) from cassandra.test where devgroup<20 allow filtering;
select avg(temperature) from cassandra.test where devgroup<30 allow filtering;
select avg(temperature) from cassandra.test where devgroup<40 allow filtering;
select avg(temperature) from cassandra.test where devgroup<50 allow filtering;
select avg(temperature) from cassandra.test where devgroup<60 allow filtering;
select avg(temperature) from cassandra.test where devgroup<70 allow filtering;
select avg(temperature) from cassandra.test where devgroup<80 allow filtering;
select avg(temperature) from cassandra.test where devgroup<90 allow filtering;
select avg(temperature) from cassandra.test allow filtering;
select sum(temperature) from cassandra.test where devgroup<10 allow filtering;
select sum(temperature) from cassandra.test where devgroup<20 allow filtering;
select sum(temperature) from cassandra.test where devgroup<30 allow filtering;
select sum(temperature) from cassandra.test where devgroup<40 allow filtering;
select sum(temperature) from cassandra.test where devgroup<50 allow filtering;
select sum(temperature) from cassandra.test where devgroup<60 allow filtering;
select sum(temperature) from cassandra.test where devgroup<70 allow filtering;
select sum(temperature) from cassandra.test where devgroup<80 allow filtering;
select sum(temperature) from cassandra.test where devgroup<90 allow filtering;
select sum(temperature) from cassandra.test allow filtering;
select max(temperature) from cassandra.test where devgroup<10 allow filtering;
select max(temperature) from cassandra.test where devgroup<20 allow filtering;
select max(temperature) from cassandra.test where devgroup<30 allow filtering;
select max(temperature) from cassandra.test where devgroup<40 allow filtering;
select max(temperature) from cassandra.test where devgroup<50 allow filtering;
select max(temperature) from cassandra.test where devgroup<60 allow filtering;
select max(temperature) from cassandra.test where devgroup<70 allow filtering;
select max(temperature) from cassandra.test where devgroup<80 allow filtering;
select max(temperature) from cassandra.test where devgroup<90 allow filtering;
select max(temperature) from cassandra.test allow filtering;
select min(temperature) from cassandra.test where devgroup<10 allow filtering;
select min(temperature) from cassandra.test where devgroup<20 allow filtering;
select min(temperature) from cassandra.test where devgroup<30 allow filtering;
select min(temperature) from cassandra.test where devgroup<40 allow filtering;
select min(temperature) from cassandra.test where devgroup<50 allow filtering;
select min(temperature) from cassandra.test where devgroup<60 allow filtering;
select min(temperature) from cassandra.test where devgroup<70 allow filtering;
select min(temperature) from cassandra.test where devgroup<80 allow filtering;
select min(temperature) from cassandra.test where devgroup<90 allow filtering;
select min(temperature) from cassandra.test allow filtering;

10
tests/comparisonTest/cassandra/q3.txt Normal file
View File

@ -0,0 +1,10 @@
select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<10 group by devgroup allow filtering;
select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<20 group by devgroup allow filtering;
select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<30 group by devgroup allow filtering;
select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<40 group by devgroup allow filtering;
select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<50 group by devgroup allow filtering;
select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<60 group by devgroup allow filtering;
select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<70 group by devgroup allow filtering;
select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<80 group by devgroup allow filtering;
select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<90 group by devgroup allow filtering;
select count(temperature), sum(temperature), avg(temperature) from cassandra.test group by devgroup allow filtering;

10
tests/comparisonTest/cassandra/q4.txt Normal file
View File

@ -0,0 +1,10 @@
select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<10 group by minute allow filtering;
select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<20 group by minute allow filtering;
select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<30 group by minute allow filtering;
select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<40 group by minute allow filtering;
select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<50 group by minute allow filtering;
select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<60 group by minute allow filtering;
select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<70 group by minute allow filtering;
select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<80 group by minute allow filtering;
select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<90 group by minute allow filtering;
select count(temperature), sum(temperature), avg(temperature) from cassandra.test group by minute;