xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

[TD-2846]<test>: update docker cluster scripts

This commit is contained in:
Ping Xiao 2021-02-06 17:50:21 +08:00
parent 58d4e1fa00
commit 508993a73b
7 changed files with 243 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
tests/pytest/cluster/clusterEnvSetup/Dockerfile
View File

@ -1,12 +1,13 @@
FROM ubuntu:latest AS builder
ARG PACKAGE=TDengine-server-1.6.5.10-Linux-x64.tar.gz
ARG TARBITRATORPKG=TDengine-tarbitrator-1.6.5.10-Linux-x64.tar.gz
ARG EXTRACTDIR=TDengine-enterprise-server
ARG TARBITRATORPKG=TDengine-tarbitrator-1.6.5.10-Linux-x64.tar.gz
ARG EXTRACTDIR2=TDengine-enterprise-arbitrator
ARG CONTENT=taos.tar.gz
WORKDIR /root
COPY ${PACKAGE} .
COPY ${TARBITRATORPKG} .
@ -14,6 +15,7 @@ RUN tar -zxf ${PACKAGE}
RUN tar -zxf ${TARBITRATORPKG}
RUN mv ${EXTRACTDIR}/driver ./lib
RUN tar -zxf ${EXTRACTDIR}/${CONTENT}
RUN mv ${EXTRACTDIR2}/bin/* /root/bin
FROM ubuntu:latest
@ -22,6 +24,7 @@ WORKDIR /root
RUN apt-get update
RUN apt-get install -y vim tmux net-tools
RUN echo 'alias ll="ls -l --color=auto"' >> /root/.bashrc
RUN ulimit -c unlimited
COPY --from=builder /root/bin/taosd /usr/bin
COPY --from=builder /root/bin/tarbitrator /usr/bin

130
tests/pytest/cluster/clusterEnvSetup/basic.py
View File

@ -13,6 +13,7 @@
import os
import random
import argparse
class ClusterTestcase:
@ -22,5 +23,130 @@ class ClusterTestcase:
os.system("yes|taosdemo -h 172.27.0.7 -n 100 -t 100 -x")
os.system("python3 ../../concurrent_inquiry.py -H 172.27.0.7 -T 4 -t 4 -l 10")
clusterTest = ClusterTestcase()
clusterTest.run()
parser = argparse.ArgumentParser()
parser.add_argument(
'-H',
'--host-name',
action='store',
default='127.0.0.1',
type=str,
help='host name to be connected (default: 127.0.0.1)')
parser.add_argument(
'-S',
'--ts',
action='store',
default=1500000000000,
type=int,
help='insert data from timestamp (default: 1500000000000)')
parser.add_argument(
'-d',
'--db-name',
action='store',
default='test',
type=str,
help='Database name to be created (default: test)')
parser.add_argument(
'-t',
'--number-of-native-threads',
action='store',
default=10,
type=int,
help='Number of native threads (default: 10)')
parser.add_argument(
'-T',
'--number-of-rest-threads',
action='store',
default=10,
type=int,
help='Number of rest threads (default: 10)')
parser.add_argument(
'-r',
'--number-of-records',
action='store',
default=100,
type=int,
help='Number of record to be created for each table (default: 100)')
parser.add_argument(
'-c',
'--create-table',
action='store',
default='0',
type=int,
help='whether gen data (default: 0)')
parser.add_argument(
'-p',
'--subtb-name-prefix',
action='store',
default='t',
type=str,
help='subtable-name-prefix (default: t)')
parser.add_argument(
'-P',
'--stb-name-prefix',
action='store',
default='st',
type=str,
help='stable-name-prefix (default: st)')
parser.add_argument(
'-b',
'--probabilities',
action='store',
default='0.05',
type=float,
help='probabilities of join (default: 0.05)')
parser.add_argument(
'-l',
'--loop-per-thread',
action='store',
default='100',
type=int,
help='loop per thread (default: 100)')
parser.add_argument(
'-u',
'--user',
action='store',
default='root',
type=str,
help='user name')
parser.add_argument(
'-w',
'--password',
action='store',
default='root',
type=str,
help='user name')
parser.add_argument(
'-n',
'--number-of-tables',
action='store',
default=1000,
type=int,
help='Number of subtales per stable (default: 1000)')
parser.add_argument(
'-N',
'--number-of-stables',
action='store',
default=2,
type=int,
help='Number of stables (default: 2)')
parser.add_argument(
'-m',
'--mix-stable-subtable',
action='store',
default=0,
type=int,
help='0:stable & substable ,1:subtable ,2:stable (default: 0)')
args = parser.parse_args()
q = ConcurrentInquiry(
args.ts,args.host_name,args.user,args.password,args.db_name,
args.stb_name_prefix,args.subtb_name_prefix,args.number_of_native_threads,args.number_of_rest_threads,
args.probabilities,args.loop_per_thread,args.number_of_stables,args.number_of_tables ,args.number_of_records,
args.mix_stable_subtable )
if args.create_table:
q.gen_data()
q.get_full()
#q.gen_query_sql()
q.run()

39
tests/pytest/cluster/clusterEnvSetup/buildClusterEnv.sh
View File

@ -27,9 +27,18 @@ do
esac
done
function addTaoscfg {
for i in {1..5}
do
touch /data/node$i/cfg/taos.cfg
echo 'firstEp tdnode1:6030' > /data/node$i/cfg/taos.cfg
echo 'fqdn tdnode$i' >> /data/node$i/cfg/taos.cfg
echo 'arbitrator tdnode1:6042' >> /data/node$i/cfg/taos.cfg
done
}
function createDIR {
for i in {1.. $2}
for i in {1..5}
do
mkdir -p /data/node$i/data
mkdir -p /data/node$i/log
@ -39,7 +48,7 @@ function createDIR {
}
function cleanEnv {
for i in {1..3}
for i in {1..5}
do
echo /data/node$i/data/*
rm -rf /data/node$i/data/*
@ -78,35 +87,35 @@ function prepareBuild {
rm -rf $DOCKER_DIR/*.yml
cd $CURR_DIR
cp docker-compose.yml $DOCKER_DIR
cp *.yml $DOCKER_DIR
cp Dockerfile $DOCKER_DIR
if [ $NUM_OF_NODES -eq 4 ]; then
cp ../node4.yml $DOCKER_DIR
fi
if [ $NUM_OF_NODES -eq 5 ]; then
cp ../node5.yml $DOCKER_DIR
fi
}
function clusterUp {
cd $DOCKER_DIR
PARAMETERS=PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION DIR2=TDengine-arbitrator-$VERSION VERSION=$VERSION
if [ $NUM_OF_NODES -eq 2 ]; then
$PARAMETERS docker-compose up -d
fi
if [ $NUM_OF_NODES -eq 3 ]; then
PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION VERSION=$VERSION docker-compose up -d
$PARAMETERS docker-compose -f docker-compose.yml -f node3.yml up -d
fi
if [ $NUM_OF_NODES -eq 4 ]; then
PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION VERSION=$VERSION docker-compose -f docker-compose.yml -f node4.yml up -d
$PARAMETERS docker-compose -f docker-compose.yml -f node3.yml -f node4.yml up -d
fi
if [ $NUM_OF_NODES -eq 5 ]; then
PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION VERSION=$VERSION docker-compose -f docker-compose.yml -f node4.yml -f node5.yml up -d
$PARAMETERS docker-compose -f docker-compose.yml -f node3.yml -f node4.yml -f node5.yml up -d
fi
}
cleanEnv
createDIR
cleanEnv
addTaoscfg
prepareBuild
clusterUp

64
tests/pytest/cluster/clusterEnvSetup/docker-compose.yml
View File

@ -8,6 +8,7 @@ services:
- PACKAGE=${PACKAGE}
- TARBITRATORPKG=${TARBITRATORPKG}
- EXTRACTDIR=${DIR}
- EXTRACTDIR2=${DIR2}
image: 'tdengine:${VERSION}'
container_name: 'tdnode1'
cap_add:
@ -19,6 +20,9 @@ services:
command: >
sh -c "ln -snf /usr/share/zoneinfo/$TZ /etc/localtime &&
echo $TZ > /etc/timezone &&
mkdir /coredump &&
echo 'kernel.core_pattern=/coredump/core_%e_%p' >> /etc/sysctl.conf &&
sysctl -p &&
exec my-main-application"
extra_hosts:
- "tdnode2:172.27.0.8"
@ -40,19 +44,16 @@ services:
target: /etc/taos
# bind core dump path
- type: bind
source: /data/node2/core
source: /data/node1/core
target: /coredump
- type: bind
source: /data
target: /root
hostname: tdnode1
networks:
taos_update_net:
ipv4_address: 172.27.0.7
command: >
sh -c "tarbitrator
&& taosd"
command: taosd
td2.0-node2:
build:
@ -71,6 +72,9 @@ services:
command: >
sh -c "ln -snf /usr/share/zoneinfo/$TZ /etc/localtime &&
echo $TZ > /etc/timezone &&
mkdir /coredump &&
echo 'kernel.core_pattern=/coredump/core_%e_%p' >> /etc/sysctl.conf &&
sysctl -p &&
exec my-main-application"
extra_hosts:
- "tdnode1:172.27.0.7"
@ -90,11 +94,10 @@ services:
- type: bind
source: /data/node2/cfg
target: /etc/taos
# bind configuration
# bind core dump path
- type: bind
source: /data/node2/core
target: /coredump
# bind core dump path
target: /coredump
- type: bind
source: /data
target: /root
@ -104,51 +107,6 @@ services:
ipv4_address: 172.27.0.8
command: taosd
td2.0-node3:
build:
context: .
args:
- PACKAGE=${PACKAGE}
- EXTRACTDIR=${DIR}
image: 'tdengine:${VERSION}'
container_name: 'tdnode3'
cap_add:
- ALL
stdin_open: true
tty: true
environment:
TZ: "Asia/Shanghai"
command: >
sh -c "ln -snf /usr/share/zoneinfo/$TZ /etc/localtime &&
echo $TZ > /etc/timezone &&
exec my-main-application"
extra_hosts:
- "tdnode1:172.27.0.7"
- "tdnode2:172.27.0.8"
- "tdnode4:172.27.0.10"
- "tdnode5:172.27.0.11"
volumes:
# bind data directory
- type: bind
source: /data/node3/data
target: /var/lib/taos
# bind log directory
- type: bind
source: /data/node3/log
target: /var/log/taos
# bind configuration
- type: bind
source: /data/node3/cfg
target: /etc/taos
# bind core dump path
- type: bind
source: /data
target: /root
hostname: tdnode3
networks:
taos_update_net:
ipv4_address: 172.27.0.9
command: taosd
networks:
taos_update_net:

54
tests/pytest/cluster/clusterEnvSetup/node3.yml Normal file
View File

@ -0,0 +1,54 @@
version: '3.7'
services:
td2.0-node3:
build:
context: .
args:
- PACKAGE=${PACKAGE}
- EXTRACTDIR=${DIR}
image: 'tdengine:${VERSION}'
container_name: 'tdnode3'
cap_add:
- ALL
stdin_open: true
tty: true
environment:
TZ: "Asia/Shanghai"
command: >
sh -c "ln -snf /usr/share/zoneinfo/$TZ /etc/localtime &&
echo $TZ > /etc/timezone &&
mkdir /coredump &&
echo 'kernel.core_pattern=/coredump/core_%e_%p' >> /etc/sysctl.conf &&
sysctl -p &&
exec my-main-application"
extra_hosts:
- "tdnode1:172.27.0.7"
- "tdnode2:172.27.0.8"
- "tdnode4:172.27.0.10"
- "tdnode5:172.27.0.11"
volumes:
# bind data directory
- type: bind
source: /data/node3/data
target: /var/lib/taos
# bind log directory
- type: bind
source: /data/node3/log
target: /var/log/taos
# bind configuration
- type: bind
source: /data/node3/cfg
target: /etc/taos
# bind core dump path
- type: bind
source: /data/node3/core
target: /coredump
- type: bind
source: /data
target: /root
hostname: tdnode3
networks:
taos_update_net:
ipv4_address: 172.27.0.9
command: taosd

14
tests/pytest/cluster/clusterEnvSetup/node4.yml
View File

@ -7,7 +7,7 @@ services:
args:
- PACKAGE=${PACKAGE}
- EXTRACTDIR=${DIR}
image: 'tdengine:2.0.13.1'
image: 'tdengine:${VERSION}'
container_name: 'tdnode4'
cap_add:
- ALL
@ -17,8 +17,16 @@ services:
TZ: "Asia/Shanghai"
command: >
sh -c "ln -snf /usr/share/zoneinfo/$TZ /etc/localtime &&
echo $TZ > /etc/timezone &&
echo $TZ > /etc/timezone &&
mkdir /coredump &&
echo 'kernel.core_pattern=/coredump/core_%e_%p' >> /etc/sysctl.conf &&
sysctl -p &&
exec my-main-application"
extra_hosts:
- "tdnode2:172.27.0.8"
- "tdnode3:172.27.0.9"
- "tdnode4:172.27.0.10"
- "tdnode5:172.27.0.11"
volumes:
# bind data directory
- type: bind
@ -34,7 +42,7 @@ services:
target: /etc/taos
# bind core dump path
- type: bind
source: /data/node2/core
source: /data/node4/core
target: /coredump
- type: bind
source: /data

16
tests/pytest/cluster/clusterEnvSetup/node5.yml
View File

@ -7,7 +7,7 @@ services:
args:
- PACKAGE=${PACKAGE}
- EXTRACTDIR=${DIR}
image: 'tdengine:2.0.13.1'
image: 'tdengine:${VERSION}'
container_name: 'tdnode5'
cap_add:
- ALL
@ -18,7 +18,15 @@ services:
command: >
sh -c "ln -snf /usr/share/zoneinfo/$TZ /etc/localtime &&
echo $TZ > /etc/timezone &&
mkdir /coredump &&
echo 'kernel.core_pattern=/coredump/core_%e_%p' >> /etc/sysctl.conf &&
sysctl -p &&
exec my-main-application"
extra_hosts:
- "tdnode2:172.27.0.8"
- "tdnode3:172.27.0.9"
- "tdnode4:172.27.0.10"
- "tdnode5:172.27.0.11"
volumes:
# bind data directory
- type: bind
@ -34,11 +42,7 @@ services:
target: /etc/taos
# bind core dump path
- type: bind
source: /data/node2/core
target: /coredump
# bind core dump path
- type: bind
source: /data/node2/core
source: /data/node5/core
target: /coredump
- type: bind
source: /data