[TD-2846]<test>: update docker cluster scripts
This commit is contained in:
Ping Xiao
parent
42d5896ba1
commit
e53e414f51
|
|
@ -12,141 +12,89 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
import os
|
||||
import taos
|
||||
import random
|
||||
import argparse
|
||||
|
||||
class ClusterTestcase:
|
||||
class BuildDockerCluser:
|
||||
|
||||
def __init__(self, hostName, user, password, configDir, numOfNodes, clusterVersion):
|
||||
self.hostName = hostName
|
||||
self.user = user
|
||||
self.password = password
|
||||
self.configDir = configDir
|
||||
self.numOfNodes = numOfNodes
|
||||
self.clusterVersion = clusterVersion
|
||||
|
||||
def getConnection(self):
|
||||
self.conn = taos.connect(
|
||||
host = self.hostName,
|
||||
user = self.user,
|
||||
password = self.password,
|
||||
config = self.configDir)
|
||||
|
||||
def createDondes(self):
|
||||
self.cursor = self.conn.cursor()
|
||||
for i in range(2, self.numOfNodes + 1):
|
||||
self.cursor.execute("create dnode tdnode%d" % i)
|
||||
|
||||
def startArbitrator(self):
|
||||
print("start arbitrator")
|
||||
os.system("docker exec -d $(docker ps|grep tdnode1|awk '{print $1}') tarbitrator")
|
||||
|
||||
|
||||
def run(self):
|
||||
os.system("./buildClusterEnv.sh -n 3 -v 2.0.14.1")
|
||||
os.system("yes|taosdemo -h 172.27.0.7 -n 100 -t 100 -x")
|
||||
os.system("python3 ../../concurrent_inquiry.py -H 172.27.0.7 -T 4 -t 4 -l 10")
|
||||
if self.numOfNodes < 2 or self.numOfNodes > 5:
|
||||
print("the number of nodes must be between 2 and 5")
|
||||
exit(0)
|
||||
os.system("./buildClusterEnv.sh -n %d -v %s" % (self.numOfNodes, self.clusterVersion))
|
||||
self.getConnection()
|
||||
self.createDondes()
|
||||
self.startArbitrator()
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
'-H',
|
||||
'--host-name',
|
||||
'--host',
|
||||
action='store',
|
||||
default='127.0.0.1',
|
||||
default='tdnode1',
|
||||
type=str,
|
||||
help='host name to be connected (default: 127.0.0.1)')
|
||||
parser.add_argument(
|
||||
'-S',
|
||||
'--ts',
|
||||
action='store',
|
||||
default=1500000000000,
|
||||
type=int,
|
||||
help='insert data from timestamp (default: 1500000000000)')
|
||||
parser.add_argument(
|
||||
'-d',
|
||||
'--db-name',
|
||||
action='store',
|
||||
default='test',
|
||||
type=str,
|
||||
help='Database name to be created (default: test)')
|
||||
parser.add_argument(
|
||||
'-t',
|
||||
'--number-of-native-threads',
|
||||
action='store',
|
||||
default=10,
|
||||
type=int,
|
||||
help='Number of native threads (default: 10)')
|
||||
parser.add_argument(
|
||||
'-T',
|
||||
'--number-of-rest-threads',
|
||||
action='store',
|
||||
default=10,
|
||||
type=int,
|
||||
help='Number of rest threads (default: 10)')
|
||||
parser.add_argument(
|
||||
'-r',
|
||||
'--number-of-records',
|
||||
action='store',
|
||||
default=100,
|
||||
type=int,
|
||||
help='Number of record to be created for each table (default: 100)')
|
||||
parser.add_argument(
|
||||
'-c',
|
||||
'--create-table',
|
||||
action='store',
|
||||
default='0',
|
||||
type=int,
|
||||
help='whether gen data (default: 0)')
|
||||
parser.add_argument(
|
||||
'-p',
|
||||
'--subtb-name-prefix',
|
||||
action='store',
|
||||
default='t',
|
||||
type=str,
|
||||
help='subtable-name-prefix (default: t)')
|
||||
parser.add_argument(
|
||||
'-P',
|
||||
'--stb-name-prefix',
|
||||
action='store',
|
||||
default='st',
|
||||
type=str,
|
||||
help='stable-name-prefix (default: st)')
|
||||
parser.add_argument(
|
||||
'-b',
|
||||
'--probabilities',
|
||||
action='store',
|
||||
default='0.05',
|
||||
type=float,
|
||||
help='probabilities of join (default: 0.05)')
|
||||
parser.add_argument(
|
||||
'-l',
|
||||
'--loop-per-thread',
|
||||
action='store',
|
||||
default='100',
|
||||
type=int,
|
||||
help='loop per thread (default: 100)')
|
||||
help='host name to be connected (default: tdnode1)')
|
||||
parser.add_argument(
|
||||
'-u',
|
||||
'--user',
|
||||
action='store',
|
||||
action='store',
|
||||
default='root',
|
||||
type=str,
|
||||
help='user name')
|
||||
help='user (default: root)')
|
||||
parser.add_argument(
|
||||
'-w',
|
||||
'-p',
|
||||
'--password',
|
||||
action='store',
|
||||
default='root',
|
||||
action='store',
|
||||
default='taosdata',
|
||||
type=str,
|
||||
help='user name')
|
||||
help='password (default: taosdata)')
|
||||
parser.add_argument(
|
||||
'-c',
|
||||
'--config-dir',
|
||||
action='store',
|
||||
default='/etc/taos',
|
||||
type=str,
|
||||
help='configuration directory (default: /etc/taos)')
|
||||
parser.add_argument(
|
||||
'-n',
|
||||
'--number-of-tables',
|
||||
action='store',
|
||||
default=1000,
|
||||
type=int,
|
||||
help='Number of subtales per stable (default: 1000)')
|
||||
parser.add_argument(
|
||||
'-N',
|
||||
'--number-of-stables',
|
||||
'--num-of-nodes',
|
||||
action='store',
|
||||
default=2,
|
||||
type=int,
|
||||
help='Number of stables (default: 2)')
|
||||
help='number of nodes in the cluster (default: 2, min: 2, max: 5)')
|
||||
parser.add_argument(
|
||||
'-m',
|
||||
'--mix-stable-subtable',
|
||||
'-v',
|
||||
'--version',
|
||||
action='store',
|
||||
default=0,
|
||||
type=int,
|
||||
help='0:stable & substable ,1:subtable ,2:stable (default: 0)')
|
||||
default='2.0.14.1',
|
||||
type=str,
|
||||
help='the version of the cluster to be build, Default is 2.0.14.1')
|
||||
|
||||
args = parser.parse_args()
|
||||
q = ConcurrentInquiry(
|
||||
args.ts,args.host_name,args.user,args.password,args.db_name,
|
||||
args.stb_name_prefix,args.subtb_name_prefix,args.number_of_native_threads,args.number_of_rest_threads,
|
||||
args.probabilities,args.loop_per_thread,args.number_of_stables,args.number_of_tables ,args.number_of_records,
|
||||
args.mix_stable_subtable )
|
||||
|
||||
if args.create_table:
|
||||
q.gen_data()
|
||||
q.get_full()
|
||||
|
||||
#q.gen_query_sql()
|
||||
q.run()
|
||||
cluster = BuildDockerCluser(args.host, args.user, args.password, args.config_dir, args.num_of_nodes, args.version)
|
||||
cluster.run()
|
||||
|
|
@ -32,7 +32,7 @@ function addTaoscfg {
|
|||
do
|
||||
touch /data/node$i/cfg/taos.cfg
|
||||
echo 'firstEp tdnode1:6030' > /data/node$i/cfg/taos.cfg
|
||||
echo 'fqdn tdnode$i' >> /data/node$i/cfg/taos.cfg
|
||||
echo 'fqdn tdnode'$i >> /data/node$i/cfg/taos.cfg
|
||||
echo 'arbitrator tdnode1:6042' >> /data/node$i/cfg/taos.cfg
|
||||
done
|
||||
}
|
||||
|
|
@ -64,9 +64,9 @@ function prepareBuild {
|
|||
rm -rf $CURR_DIR/../../../../release/*
|
||||
fi
|
||||
|
||||
|
||||
if [ ! -e $DOCKER_DIR/TDengine-server-$VERSION-Linux-x64.tar.gz && ! -e TDengine-arbitrator-$VERSION-Linux-x64.tar.gz ]; then
|
||||
if [ ! -e $DOCKER_DIR/TDengine-server-$VERSION-Linux-x64.tar.gz ] || [ ! -e $DOCKER_DIR/TDengine-arbitrator-$VERSION-Linux-x64.tar.gz ]; then
|
||||
cd $CURR_DIR/../../../../packaging
|
||||
echo "generating TDeninger packages"
|
||||
./release.sh -v edge -n $VERSION >> /dev/null
|
||||
|
||||
if [ ! -e $CURR_DIR/../../../../release/TDengine-server-$VERSION-Linux-x64.tar.gz ]; then
|
||||
|
|
@ -92,26 +92,28 @@ function prepareBuild {
|
|||
}
|
||||
|
||||
function clusterUp {
|
||||
echo "docker compose start"
|
||||
|
||||
cd $DOCKER_DIR
|
||||
|
||||
PARAMETERS=PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION DIR2=TDengine-arbitrator-$VERSION VERSION=$VERSION
|
||||
cd $DOCKER_DIR
|
||||
|
||||
if [ $NUM_OF_NODES -eq 2 ]; then
|
||||
$PARAMETERS docker-compose up -d
|
||||
echo "create 2 dnodes"
|
||||
PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION DIR2=TDengine-arbitrator-$VERSION VERSION=$VERSION docker-compose up -d
|
||||
fi
|
||||
|
||||
if [ $NUM_OF_NODES -eq 3 ]; then
|
||||
$PARAMETERS docker-compose -f docker-compose.yml -f node3.yml up -d
|
||||
PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION DIR2=TDengine-arbitrator-$VERSION VERSION=$VERSION docker-compose -f docker-compose.yml -f node3.yml up -d
|
||||
fi
|
||||
|
||||
if [ $NUM_OF_NODES -eq 4 ]; then
|
||||
$PARAMETERS docker-compose -f docker-compose.yml -f node3.yml -f node4.yml up -d
|
||||
PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION DIR2=TDengine-arbitrator-$VERSION VERSION=$VERSION docker-compose -f docker-compose.yml -f node3.yml -f node4.yml up -d
|
||||
fi
|
||||
|
||||
if [ $NUM_OF_NODES -eq 5 ]; then
|
||||
$PARAMETERS docker-compose -f docker-compose.yml -f node3.yml -f node4.yml -f node5.yml up -d
|
||||
PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION DIR2=TDengine-arbitrator-$VERSION VERSION=$VERSION docker-compose -f docker-compose.yml -f node3.yml -f node4.yml -f node5.yml up -d
|
||||
fi
|
||||
|
||||
echo "docker compose finish"
|
||||
}
|
||||
|
||||
createDIR
|
||||
|
|
|
|||
|
|
@ -0,0 +1,55 @@
|
|||
{
|
||||
"filetype": "insert",
|
||||
"cfgdir": "/etc/taos",
|
||||
"host": "127.0.0.1",
|
||||
"port": 6030,
|
||||
"user": "root",
|
||||
"password": "taosdata",
|
||||
"thread_count": 4,
|
||||
"thread_count_create_tbl": 1,
|
||||
"result_file": "./insert_res.txt",
|
||||
"confirm_parameter_prompt": "no",
|
||||
"databases": [{
|
||||
"dbinfo": {
|
||||
"name": "db",
|
||||
"drop": "no",
|
||||
"replica": 1,
|
||||
"days": 2,
|
||||
"cache": 16,
|
||||
"blocks": 8,
|
||||
"precision": "ms",
|
||||
"keep": 365,
|
||||
"minRows": 100,
|
||||
"maxRows": 4096,
|
||||
"comp":2,
|
||||
"walLevel":1,
|
||||
"quorum":1,
|
||||
"fsync":3000,
|
||||
"update": 0
|
||||
},
|
||||
"super_tables": [{
|
||||
"name": "stb",
|
||||
"child_table_exists":"no",
|
||||
"childtable_count": 1,
|
||||
"childtable_prefix": "stb_",
|
||||
"auto_create_table": "no",
|
||||
"data_source": "rand",
|
||||
"insert_mode": "taosc",
|
||||
"insert_rate": 0,
|
||||
"insert_rows": 100000,
|
||||
"multi_thread_write_one_tbl": "no",
|
||||
"number_of_tbl_in_one_sql": 1,
|
||||
"rows_per_tbl": 100,
|
||||
"max_sql_len": 1024000,
|
||||
"disorder_ratio": 0,
|
||||
"disorder_range": 1000,
|
||||
"timestamp_step": 10,
|
||||
"start_timestamp": "2020-10-01 00:00:00.000",
|
||||
"sample_format": "csv",
|
||||
"sample_file": "./sample.csv",
|
||||
"tags_file": "",
|
||||
"columns": [{"type": "INT"}, {"type": "DOUBLE", "count":10}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}],
|
||||
"tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}]
|
||||
}]
|
||||
}]
|
||||
}
|
||||
|
|
@ -0,0 +1,142 @@
|
|||
###################################################################
|
||||
# Copyright (c) 2016 by TAOS Technologies, Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This file is proprietary and confidential to TAOS Technologies.
|
||||
# No part of this file may be reproduced, stored, transmitted,
|
||||
# disclosed or used in any form or by any means other than as
|
||||
# expressly provided by the written permission from Jianhui Tao
|
||||
#
|
||||
###################################################################
|
||||
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import os
|
||||
import random
|
||||
import argparse
|
||||
|
||||
class taosdemoxWrapper:
|
||||
|
||||
def __init__(self, host, metadata, database, tables, threads, configDir, replica,
|
||||
columnType, columnsPerTable, rowsPerTable, disorderRatio, disorderRange, charTypeLen):
|
||||
self.host = host
|
||||
self.metadata = metadata
|
||||
self.database = database
|
||||
self.tables = tables
|
||||
self.threads = threads
|
||||
self.configDir = configDir
|
||||
self.replica = replica
|
||||
self.columnType = columnType
|
||||
self.columnsPerTable = columnsPerTable
|
||||
self.rowsPerTable = rowsPerTable
|
||||
self.disorderRatio = disorderRatio
|
||||
self.disorderRange = disorderRange
|
||||
self.charTypeLen = charTypeLen
|
||||
|
||||
def run(self):
|
||||
if self.metadata is None:
|
||||
os.system("taosdemox -h %s -d %s -t %d -T %d -c %s -a %d -b %s -n %d -t %d -O %d -R %d -w %d -x -y"
|
||||
% (self.host, self.database, self.tables, self.threads, self.configDir, self.replica, self.columnType,
|
||||
self.rowsPerTable, self.disorderRatio, self.disorderRange, self.charTypeLen))
|
||||
else:
|
||||
os.system("taosdemox -f %s" % self.metadata)
|
||||
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
'-H',
|
||||
'--host-name',
|
||||
action='store',
|
||||
default='tdnode1',
|
||||
type=str,
|
||||
help='host name to be connected (default: tdnode1)')
|
||||
parser.add_argument(
|
||||
'-f',
|
||||
'--metadata',
|
||||
action='store',
|
||||
default=None,
|
||||
type=str,
|
||||
help='The meta data to execution procedure, if use -f, all other options invalid, Default is NULL')
|
||||
parser.add_argument(
|
||||
'-d',
|
||||
'--db-name',
|
||||
action='store',
|
||||
default='test',
|
||||
type=str,
|
||||
help='Database name to be created (default: test)')
|
||||
parser.add_argument(
|
||||
'-t',
|
||||
'--num-of-tables',
|
||||
action='store',
|
||||
default=10,
|
||||
type=int,
|
||||
help='Number of tables (default: 10000)')
|
||||
parser.add_argument(
|
||||
'-T',
|
||||
'--num-of-threads',
|
||||
action='store',
|
||||
default=10,
|
||||
type=int,
|
||||
help='Number of rest threads (default: 10)')
|
||||
parser.add_argument(
|
||||
'-c',
|
||||
'--config-dir',
|
||||
action='store',
|
||||
default='/etc/taos/',
|
||||
type=str,
|
||||
help='Configuration directory. (Default is /etc/taos/)')
|
||||
parser.add_argument(
|
||||
'-a',
|
||||
'--replica',
|
||||
action='store',
|
||||
default=100,
|
||||
type=int,
|
||||
help='Set the replica parameters of the database (default: 1, min: 1, max: 3)')
|
||||
parser.add_argument(
|
||||
'-b',
|
||||
'--column-type',
|
||||
action='store',
|
||||
default='int',
|
||||
type=str,
|
||||
help='the data_type of columns (default: TINYINT,SMALLINT,INT,BIGINT,FLOAT,DOUBLE,BINARY,NCHAR,BOOL,TIMESTAMP)')
|
||||
parser.add_argument(
|
||||
'-l',
|
||||
'--num-of-cols',
|
||||
action='store',
|
||||
default=10,
|
||||
type=int,
|
||||
help='The number of columns per record (default: 10)')
|
||||
parser.add_argument(
|
||||
'-n',
|
||||
'--num-of-rows',
|
||||
action='store',
|
||||
default=1000,
|
||||
type=int,
|
||||
help='Number of subtales per stable (default: 1000)')
|
||||
parser.add_argument(
|
||||
'-O',
|
||||
'--disorder-ratio',
|
||||
action='store',
|
||||
default=0,
|
||||
type=int,
|
||||
help=' (0: in order, > 0: disorder ratio, default: 0)')
|
||||
parser.add_argument(
|
||||
'-R',
|
||||
'--disorder-range',
|
||||
action='store',
|
||||
default=0,
|
||||
type=int,
|
||||
help='Out of order datas range, ms (default: 1000)')
|
||||
parser.add_argument(
|
||||
'-w',
|
||||
'--char-type-length',
|
||||
action='store',
|
||||
default=16,
|
||||
type=int,
|
||||
help='Out of order datas range, ms (default: 16)')
|
||||
|
||||
args = parser.parse_args()
|
||||
taosdemox = taosdemoxWrapper(args.host_name, args.metadata, args.db_name, args.num_of_tables,
|
||||
args.num_of_threads, args.config_dir, args.replica, args.column_type, args.num_of_cols,
|
||||
args.num_of_rows, args.disorder_ratio, args.disorder_range, args.char_type_length)
|
||||
taosdemox.run()
|
||||
Loading…
Reference in New Issue