[TD-2846]<test>: update docker cluster scripts

tests/pytest/cluster/clusterEnvSetup/basic.py

@@ -12,141 +12,89 @@
 # -*- coding: utf-8 -*-
 
 import os
+import taos
 import random
 import argparse
 
-class ClusterTestcase:
+class BuildDockerCluser:
+
+    def __init__(self, hostName, user, password, configDir, numOfNodes, clusterVersion):
+        self.hostName = hostName
+        self.user = user
+        self.password = password
+        self.configDir = configDir
+        self.numOfNodes = numOfNodes
+        self.clusterVersion = clusterVersion        
+
+    def getConnection(self):
+        self.conn = taos.connect(
+            host = self.hostName,
+            user = self.user,
+            password = self.password,
+            config = self.configDir)
+
+    def createDondes(self):
+        self.cursor = self.conn.cursor()        
+        for i in range(2, self.numOfNodes + 1):            
+            self.cursor.execute("create dnode tdnode%d" % i)
+    
+    def startArbitrator(self):
+        print("start arbitrator")
+        os.system("docker exec -d $(docker ps|grep tdnode1|awk '{print $1}') tarbitrator")
 
-        
     def run(self):
-        os.system("./buildClusterEnv.sh -n 3 -v 2.0.14.1")
+        if self.numOfNodes < 2 or self.numOfNodes > 5:
-        os.system("yes|taosdemo -h 172.27.0.7 -n 100 -t 100 -x")
+            print("the number of nodes must be between 2 and 5")
-        os.system("python3 ../../concurrent_inquiry.py -H 172.27.0.7 -T 4 -t 4 -l 10")
+            exit(0)        
+        os.system("./buildClusterEnv.sh -n %d -v %s" % (self.numOfNodes, self.clusterVersion))
+        self.getConnection()
+        self.createDondes()
+        self.startArbitrator() 
 
 parser = argparse.ArgumentParser()
 parser.add_argument(
     '-H',
-    '--host-name',
+    '--host',
     action='store',
-    default='127.0.0.1',
+    default='tdnode1',
     type=str,
-    help='host name to be connected (default: 127.0.0.1)')
+    help='host name to be connected (default: tdnode1)')
-parser.add_argument(
-    '-S',
-    '--ts',
-    action='store',
-    default=1500000000000,
-    type=int,
-    help='insert data from timestamp (default: 1500000000000)')
-parser.add_argument(
-    '-d',
-    '--db-name',
-    action='store',
-    default='test',
-    type=str,
-    help='Database name to be created (default: test)')
-parser.add_argument(
-    '-t',
-    '--number-of-native-threads',
-    action='store',
-    default=10,
-    type=int,
-    help='Number of native threads (default: 10)')
-parser.add_argument(
-    '-T',
-    '--number-of-rest-threads',
-    action='store',
-    default=10,
-    type=int,
-    help='Number of rest threads (default: 10)')
-parser.add_argument(
-    '-r',
-    '--number-of-records',
-    action='store',
-    default=100,
-    type=int,
-    help='Number of record to be created for each table  (default: 100)')
-parser.add_argument(
-    '-c',
-    '--create-table',
-    action='store',
-    default='0',
-    type=int,
-    help='whether gen data (default: 0)')
-parser.add_argument(
-    '-p',
-    '--subtb-name-prefix',
-    action='store',
-    default='t',
-    type=str,
-    help='subtable-name-prefix (default: t)')
-parser.add_argument(
-    '-P',
-    '--stb-name-prefix',
-    action='store',
-    default='st',
-    type=str,
-    help='stable-name-prefix (default: st)')
-parser.add_argument(
-    '-b',
-    '--probabilities',
-    action='store',
-    default='0.05',
-    type=float,
-    help='probabilities of join (default: 0.05)')
-parser.add_argument(
-    '-l',
-    '--loop-per-thread',
-    action='store',
-    default='100',
-    type=int,
-    help='loop per thread (default: 100)')
 parser.add_argument(
     '-u',
     '--user',
-    action='store', 
+    action='store',
     default='root',
     type=str,
-    help='user name')
+    help='user (default: root)')
 parser.add_argument(
-    '-w',
+    '-p',
     '--password',
-    action='store', 
+    action='store',
-    default='root',
+    default='taosdata',
     type=str,
-    help='user name')
+    help='password (default: taosdata)')
+parser.add_argument(
+    '-c',
+    '--config-dir',
+    action='store',
+    default='/etc/taos',
+    type=str,
+    help='configuration directory (default: /etc/taos)')
 parser.add_argument(
     '-n',
-    '--number-of-tables',
+    '--num-of-nodes',
-    action='store',
-    default=1000,
-    type=int,
-    help='Number of subtales per stable (default: 1000)')
-parser.add_argument(
-    '-N',
-    '--number-of-stables',
     action='store',
     default=2,
     type=int,
-    help='Number of stables  (default: 2)')
+    help='number of nodes in the cluster (default: 2, min: 2, max: 5)')
 parser.add_argument(
-    '-m',
+    '-v',
-    '--mix-stable-subtable',
+    '--version',
     action='store',
-    default=0,
+    default='2.0.14.1',
-    type=int,
+    type=str,
-    help='0:stable & substable ,1:subtable ,2:stable (default: 0)')
+    help='the version of the cluster to be build, Default is 2.0.14.1')
 
 args = parser.parse_args()
-q = ConcurrentInquiry(
+cluster = BuildDockerCluser(args.host, args.user, args.password, args.config_dir, args.num_of_nodes, args.version)
-    args.ts,args.host_name,args.user,args.password,args.db_name,
+cluster.run()
-                args.stb_name_prefix,args.subtb_name_prefix,args.number_of_native_threads,args.number_of_rest_threads,
-                args.probabilities,args.loop_per_thread,args.number_of_stables,args.number_of_tables ,args.number_of_records,
-                args.mix_stable_subtable )
-
-if args.create_table: 
-    q.gen_data()
-q.get_full()
-
-#q.gen_query_sql()
-q.run()

tests/pytest/cluster/clusterEnvSetup/buildClusterEnv.sh

@@ -32,7 +32,7 @@ function addTaoscfg {
   do 
     touch /data/node$i/cfg/taos.cfg
     echo 'firstEp          tdnode1:6030' > /data/node$i/cfg/taos.cfg
-    echo 'fqdn             tdnode$i' >> /data/node$i/cfg/taos.cfg
+    echo 'fqdn             tdnode'$i >> /data/node$i/cfg/taos.cfg
     echo 'arbitrator       tdnode1:6042' >> /data/node$i/cfg/taos.cfg
   done
 }
@@ -64,9 +64,9 @@ function prepareBuild {
     rm -rf $CURR_DIR/../../../../release/*
   fi
 
-
+  if [ ! -e $DOCKER_DIR/TDengine-server-$VERSION-Linux-x64.tar.gz ] || [ ! -e $DOCKER_DIR/TDengine-arbitrator-$VERSION-Linux-x64.tar.gz ]; then
-  if [ ! -e $DOCKER_DIR/TDengine-server-$VERSION-Linux-x64.tar.gz && ! -e TDengine-arbitrator-$VERSION-Linux-x64.tar.gz ]; then
     cd $CURR_DIR/../../../../packaging
+    echo "generating TDeninger packages"
     ./release.sh -v edge -n $VERSION >> /dev/null
 
     if [ ! -e $CURR_DIR/../../../../release/TDengine-server-$VERSION-Linux-x64.tar.gz ]; then
@@ -92,26 +92,28 @@ function prepareBuild {
 }
 
 function clusterUp {
+  echo "docker compose start"
   
-  cd $DOCKER_DIR
+  cd $DOCKER_DIR  
-
-  PARAMETERS=PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION DIR2=TDengine-arbitrator-$VERSION VERSION=$VERSION
 
   if [ $NUM_OF_NODES -eq 2 ]; then
-    $PARAMETERS docker-compose up -d
+    echo "create 2 dnodes"
+    PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION DIR2=TDengine-arbitrator-$VERSION VERSION=$VERSION docker-compose up -d
   fi
 
   if [ $NUM_OF_NODES -eq 3 ]; then
-    $PARAMETERS docker-compose -f docker-compose.yml -f node3.yml up -d
+    PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION DIR2=TDengine-arbitrator-$VERSION VERSION=$VERSION  docker-compose -f docker-compose.yml -f node3.yml up -d
   fi
 
   if [ $NUM_OF_NODES -eq 4 ]; then
-    $PARAMETERS docker-compose -f docker-compose.yml -f node3.yml -f node4.yml up -d
+    PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION DIR2=TDengine-arbitrator-$VERSION VERSION=$VERSION  docker-compose -f docker-compose.yml -f node3.yml -f node4.yml up -d
   fi
 
   if [ $NUM_OF_NODES -eq 5 ]; then
-    $PARAMETERS docker-compose -f docker-compose.yml -f node3.yml -f node4.yml -f node5.yml up -d
+    PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION DIR2=TDengine-arbitrator-$VERSION VERSION=$VERSION  docker-compose -f docker-compose.yml -f node3.yml -f node4.yml -f node5.yml up -d
   fi
+
+  echo "docker compose finish"
 }
 
 createDIR

tests/pytest/cluster/clusterEnvSetup/insert.json

@@ -0,0 +1,55 @@
+{
+	"filetype": "insert",
+	"cfgdir": "/etc/taos",
+	"host": "127.0.0.1",
+	"port": 6030,
+	"user": "root",
+	"password": "taosdata",
+	"thread_count": 4,
+	"thread_count_create_tbl": 1,
+	"result_file": "./insert_res.txt",
+        "confirm_parameter_prompt": "no",
+	"databases": [{
+		"dbinfo": {
+			"name": "db",
+			"drop": "no",
+			"replica": 1,
+			"days": 2,
+			"cache": 16,
+			"blocks": 8,
+			"precision": "ms",
+			"keep": 365,
+			"minRows": 100,
+			"maxRows": 4096,
+			"comp":2,
+			"walLevel":1,
+			"quorum":1,
+			"fsync":3000,
+			"update": 0
+		},
+		"super_tables": [{
+			"name": "stb",
+                        "child_table_exists":"no",
+			"childtable_count": 1,
+			"childtable_prefix": "stb_",
+			"auto_create_table": "no",
+			"data_source": "rand",
+			"insert_mode": "taosc",
+			"insert_rate": 0,
+			"insert_rows": 100000,
+			"multi_thread_write_one_tbl": "no",
+			"number_of_tbl_in_one_sql": 1,
+			"rows_per_tbl": 100,
+			"max_sql_len": 1024000,
+			"disorder_ratio": 0,
+			"disorder_range": 1000,
+			"timestamp_step": 10,
+			"start_timestamp": "2020-10-01 00:00:00.000",
+			"sample_format": "csv",
+			"sample_file": "./sample.csv",
+			"tags_file": "",
+			"columns": [{"type": "INT"}, {"type": "DOUBLE", "count":10}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}],
+			"tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}]
+		}]
+	}]
+}

tests/pytest/cluster/clusterEnvSetup/taosdemoxWrapper.py

@@ -0,0 +1,142 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import os
+import random
+import argparse
+
+class taosdemoxWrapper:
+
+    def __init__(self, host, metadata, database, tables, threads, configDir, replica, 
+        columnType, columnsPerTable, rowsPerTable, disorderRatio, disorderRange, charTypeLen):
+        self.host = host
+        self.metadata = metadata
+        self.database = database
+        self.tables = tables
+        self.threads = threads
+        self.configDir = configDir
+        self.replica = replica
+        self.columnType = columnType
+        self.columnsPerTable = columnsPerTable
+        self.rowsPerTable = rowsPerTable
+        self.disorderRatio = disorderRatio
+        self.disorderRange = disorderRange
+        self.charTypeLen = charTypeLen
+        
+    def run(self):
+        if self.metadata is None:
+            os.system("taosdemox -h %s -d %s -t %d -T %d -c %s -a %d -b %s -n %d -t %d -O %d -R %d -w %d -x -y" 
+            % (self.host, self.database, self.tables, self.threads, self.configDir, self.replica, self.columnType,
+            self.rowsPerTable, self.disorderRatio, self.disorderRange, self.charTypeLen))
+        else:
+            os.system("taosdemox -f %s" % self.metadata)
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    '-H',
+    '--host-name',
+    action='store',
+    default='tdnode1',
+    type=str,
+    help='host name to be connected (default: tdnode1)')
+parser.add_argument(
+    '-f',
+    '--metadata',
+    action='store',
+    default=None,
+    type=str,
+    help='The meta data to execution procedure, if use -f, all other options invalid, Default is NULL')
+parser.add_argument(
+    '-d',
+    '--db-name',
+    action='store',
+    default='test',
+    type=str,
+    help='Database name to be created (default: test)')
+parser.add_argument(
+    '-t',
+    '--num-of-tables',
+    action='store',
+    default=10,
+    type=int,
+    help='Number of tables (default: 10000)')
+parser.add_argument(
+    '-T',
+    '--num-of-threads',
+    action='store',
+    default=10,
+    type=int,
+    help='Number of rest threads (default: 10)')
+parser.add_argument(
+    '-c',
+    '--config-dir',
+    action='store',
+    default='/etc/taos/',
+    type=str,
+    help='Configuration directory. (Default is /etc/taos/)')
+parser.add_argument(
+    '-a',
+    '--replica',
+    action='store',
+    default=100,
+    type=int,
+    help='Set the replica parameters of the database  (default: 1, min: 1, max: 3)')
+parser.add_argument(
+    '-b',
+    '--column-type',
+    action='store',
+    default='int',
+    type=str,
+    help='the data_type of columns (default: TINYINT,SMALLINT,INT,BIGINT,FLOAT,DOUBLE,BINARY,NCHAR,BOOL,TIMESTAMP)')
+parser.add_argument(
+    '-l',
+    '--num-of-cols',
+    action='store',
+    default=10,
+    type=int,
+    help='The number of columns per record (default: 10)')
+parser.add_argument(
+    '-n',
+    '--num-of-rows',
+    action='store',
+    default=1000,
+    type=int,
+    help='Number of subtales per stable (default: 1000)')
+parser.add_argument(
+    '-O',
+    '--disorder-ratio',
+    action='store',
+    default=0,
+    type=int,
+    help=' (0: in order, > 0: disorder ratio, default: 0)')
+parser.add_argument(
+    '-R',
+    '--disorder-range',
+    action='store',
+    default=0,
+    type=int,
+    help='Out of order datas range, ms (default: 1000)')
+parser.add_argument(
+    '-w',
+    '--char-type-length',
+    action='store',
+    default=16,
+    type=int,
+    help='Out of order datas range, ms (default: 16)')
+    
+args = parser.parse_args()
+taosdemox = taosdemoxWrapper(args.host_name, args.metadata, args.db_name, args.num_of_tables, 
+        args.num_of_threads, args.config_dir, args.replica, args.column_type, args.num_of_cols, 
+        args.num_of_rows, args.disorder_ratio, args.disorder_range, args.char_type_length)
+taosdemox.run()