xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #2706 from taosdata/feature/sangshuduo/perftest-script-improve 

Feature/sangshuduo/perftest script improve
This commit is contained in:
Shengliang Guan 2020-07-18 17:34:15 +08:00 committed by GitHub
parent e633174a84 49f8c9d822
commit dd905b33b7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 177 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
tests/comparisonTest/tdengine/q5.txt Normal file
View File

@ -0,0 +1 @@
select * from db.devices;

75
tests/comparisonTest/tdengine/tdengineTest.c
View File

@ -13,8 +13,9 @@ typedef struct {
char sql[256];
char dataDir[256];
int filesNum;
int writeClients;
int clients;
int rowsPerRequest;
int write;
} ProArgs;
typedef struct {
@ -41,7 +42,7 @@ int main(int argc, char *argv[]) {
statis.totalRows = 0;
parseArg(argc, argv);
if (arguments.writeClients > 0) {
if (arguments.write) {
writeData();
} else {
readData();
@ -52,7 +53,7 @@ void parseArg(int argc, char *argv[]) {
strcpy(arguments.sql, "./sqlCmd.txt");
strcpy(arguments.dataDir, "./testdata");
arguments.filesNum = 2;
arguments.writeClients = 0;
arguments.clients = 1;
arguments.rowsPerRequest = 100;
for (int i = 1; i < argc; ++i) {
@ -83,12 +84,12 @@ void parseArg(int argc, char *argv[]) {
exit(EXIT_FAILURE);
}
}
else if (strcmp(argv[i], "-writeClients") == 0) {
else if (strcmp(argv[i], "-clients") == 0) {
if (i < argc - 1) {
arguments.writeClients = atoi(argv[++i]);
arguments.clients = atoi(argv[++i]);
}
else {
fprintf(stderr, "'-writeClients' requires a parameter, default:%d\n", arguments.writeClients);
fprintf(stderr, "'-clients' requires a parameter, default:%d\n", arguments.clients);
exit(EXIT_FAILURE);
}
}
@ -101,6 +102,9 @@ void parseArg(int argc, char *argv[]) {
exit(EXIT_FAILURE);
}
}
else if (strcmp(argv[i], "-w") == 0) {
arguments.write = 1;
}
}
}
@ -215,7 +219,7 @@ void writeDataImp(void *param) {
void writeData() {
printf("write data\n");
printf("---- writeClients: %d\n", arguments.writeClients);
printf("---- clients: %d\n", arguments.clients);
printf("---- dataDir: %s\n", arguments.dataDir);
printf("---- numOfFiles: %d\n", arguments.filesNum);
printf("---- rowsPerRequest: %d\n", arguments.rowsPerRequest);
@ -243,12 +247,12 @@ void writeData() {
int64_t st = getTimeStampMs();
int a = arguments.filesNum / arguments.writeClients;
int b = arguments.filesNum % arguments.writeClients;
int a = arguments.filesNum / arguments.clients;
int b = arguments.filesNum % arguments.clients;
int last = 0;
ThreadObj *threads = calloc((size_t)arguments.writeClients, sizeof(ThreadObj));
for (int i = 0; i < arguments.writeClients; ++i) {
ThreadObj *threads = calloc((size_t)arguments.clients, sizeof(ThreadObj));
for (int i = 0; i < arguments.clients; ++i) {
ThreadObj *pthread = threads + i;
pthread_attr_t thattr;
pthread->threadId = i + 1;
@ -264,7 +268,7 @@ void writeData() {
pthread_create(&pthread->pid, &thattr, (void *(*)(void *))writeDataImp, pthread);
}
for (int i = 0; i < arguments.writeClients; i++) {
for (int i = 0; i < arguments.clients; i++) {
pthread_join(threads[i].pid, NULL);
}
@ -272,17 +276,15 @@ void writeData() {
float seconds = (float)elapsed / 1000;
float rs = (float)statis.totalRows / seconds;
free(threads);
printf("---- Spent %f seconds to insert %ld records, speed: %f Rows/Second\n", seconds, statis.totalRows, rs);
}
void readData() {
printf("read data\n");
printf("---- sql: %s\n", arguments.sql);
void *taos = taos_connect("127.0.0.1", "root", "taosdata", NULL, 0);
if (taos == NULL)
taos_error(taos);
void readDataImp(void *param)
{
ThreadObj *pThread = (ThreadObj *)param;
printf("Thread %d\n", pThread->threadId);
FILE *fp = fopen(arguments.sql, "r");
if (fp == NULL) {
printf("failed to open file %s\n", arguments.sql);
@ -290,6 +292,10 @@ void readData() {
}
printf("open file %s success\n", arguments.sql);
void *taos = taos_connect("127.0.0.1", "root", "taosdata", NULL, 0);
if (taos == NULL)
taos_error(taos);
char *line = NULL;
size_t len = 0;
while (!feof(fp)) {
@ -325,9 +331,36 @@ void readData() {
int64_t elapsed = getTimeStampMs() - st;
float seconds = (float)elapsed / 1000;
printf("---- Spent %f seconds to query: %s", seconds, line);
printf("---- Spent %f seconds to retrieve %d records, Thread:%d query: %s\n", seconds, rows, pThread->threadId, line);
}
fclose(fp);
}
void readData() {
printf("read data\n");
printf("---- sql: %s\n", arguments.sql);
printf("---- clients: %d\n", arguments.clients);
void *taos = taos_connect("127.0.0.1", "root", "taosdata", NULL, 0);
if (taos == NULL)
taos_error(taos);
ThreadObj *threads = calloc((size_t)arguments.clients, sizeof(ThreadObj));
for (int i = 0; i < arguments.clients; ++i) {
ThreadObj *pthread = threads + i;
pthread_attr_t thattr;
pthread->threadId = i + 1;
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
pthread_create(&pthread->pid, &thattr, (void *(*)(void *))readDataImp, pthread);
}
for (int i = 0; i < arguments.clients; i++) {
pthread_join(threads[i].pid, NULL);
}
free(threads);
}

107
tests/perftest-scripts/tdengineTestQ5Loop.sh Executable file
View File

@ -0,0 +1,107 @@
#!/bin/bash
DATA_DIR=/mnt/root/testdata
NUM_LOOP=5
function printTo {
if $verbose ; then
echo $1
fi
}
TDTESTQ5OUT=tdengineTestQ5.out
function runTest {
totalThroughput=0
for i in `seq 1 $NUM_LOOP`; do
for c in `seq 1 $clients`; do
records[$c]=0
spentTime[$c]=0
throughput[$c]=0
done
printTo "loop i:$i, $TDTEST_DIR/tdengineTest \
-clients $clients -sql q5.txt"
restartTaosd
beginMS=`date +%s%3N`
$TDTEST_DIR/tdengineTest \
-clients $clients -sql $TDTEST_DIR/q5.txt > $TDTESTQ5OUT
endMS=`date +%s%3N`
totalRecords=0
for c in `seq 1 $clients`; do
records[$c]=`grep Thread:$c $TDTESTQ5OUT | awk '{print $7}'`
totalRecords=`echo "$totalRecords + ${records[$c]}"|bc`
done
spending=`echo "scale=4; x = ($endMS - $beginMS)/1000; if (x<1) print 0; x"|bc`
throughput=`echo "scale=4; x= $totalRecords / $spending; if (x<1) print 0; x" | bc`
printTo "spending: $spending sec, throughput: $throughput"
totalThroughput=`echo "scale=4; x = $totalThroughput + $throughput; if(x<1) print 0; x"|bc`
done
avgThrougput=`echo "scale=4; x = $totalThroughput / $NUM_LOOP; if (x<1) print 0; x"|bc`
echo "avg Throughput: $avgThrougput"
}
function restartTaosd {
printTo "Stop taosd"
systemctl stop taosd
PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
while [ -n "$PID" ]
do
pkill -TERM -x taosd
sleep 1
PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
done
printTo "Start taosd"
$TAOSD_DIR/taosd > /dev/null 2>&1 &
sleep 10
}
################ Main ################
master=false
develop=true
verbose=false
clients=1
while : ; do
case $1 in
-v)
verbose=true
shift ;;
master)
master=true
develop=false
shift ;;
develop)
master=false
develop=true
shift ;;
-c)
clients=$2
shift 2;;
*)
break ;;
esac
done
if $master ; then
printTo "Test master branch.."
cp /mnt/root/cfg/master/taos.cfg /etc/taos/taos.cfg
WORK_DIR=/mnt/root/TDengine.master
else
printTo "Test develop branch.."
cp /mnt/root/cfg/10billion/taos.cfg /etc/taos/taos.cfg
WORK_DIR=/mnt/root/TDengine
fi
TAOSD_DIR=$WORK_DIR/debug/build/bin
TDTEST_DIR=$WORK_DIR/tests/comparisonTest/tdengine
runTest
printTo "Test done!"

27
tests/perftest-scripts/tdengineTestWriteLoop.sh
View File

@ -5,7 +5,6 @@ NUM_LOOP=5
NUM_OF_FILES=100
rowsPerRequest=(1 100 500 1000 2000)
numOfClients=(1 2 3 4 5 6 7)
function printTo {
if $verbose ; then
@ -15,7 +14,7 @@ function printTo {
function runTest {
printf "R/R, "
for c in ${numOfClients[@]}; do
for c in `seq 1 $clients`; do
if [ "$c" == "1" ]; then
printf "$c client, "
else
@ -26,7 +25,7 @@ function runTest {
for r in ${rowsPerRequest[@]}; do
printf "$r, "
for c in ${numOfClients[@]}; do
for c in `seq 1 $clients`; do
totalRPR=0
for i in `seq 1 $NUM_LOOP`; do
restartTaosd
@ -34,12 +33,12 @@ function runTest {
printTo "loop i:$i, $TDTEST_DIR/tdengineTest \
-dataDir $DATA_DIR \
-numOfFiles $NUM_OF_FILES \
-writeClients $c \
-w -clients $c \
-rowsPerRequest $r"
RPR=`$TDTEST_DIR/tdengineTest \
-dataDir $DATA_DIR \
-numOfFiles 1 \
-writeClients $c \
-w -clients $c \
-rowsPerRequest $r \
| grep speed | awk '{print $(NF-1)}'`
totalRPR=`echo "scale=4; $totalRPR + $RPR" | bc`
@ -73,25 +72,29 @@ function restartTaosd {
master=false
develop=true
verbose=false
clients=1
for arg in "$@"
do
case $arg in
while : ; do
case $1 in
-v)
verbose=true
;;
shift ;;
master)
master=true
develop=false
;;
shift ;;
develop)
master=false
develop=true
;;
shift ;;
-c)
clients=$2
shift 2;;
*)
;;
break ;;
esac
done