xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

[TD-1837] <test> add test cases

This commit is contained in:
Ping Xiao 2020-11-13 19:09:59 +08:00
parent a2114b22f8
commit 7ca81e4271
1 changed files with 104 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

119
tests/pytest/insert/restfulInsert.py
View File

@ -15,17 +15,20 @@ import requests
import threading import threading
import random import random
import time import time
import argparse
class RestfulInsert: class RestfulInsert:
def init(self): def __init__(self, host, dbname, threads, tables, records, batchSize, tbNamePerfix, outOfOrder):
self.header = {'Authorization': 'Basic cm9vdDp0YW9zZGF0YQ=='} self.header = {'Authorization': 'Basic cm9vdDp0YW9zZGF0YQ=='}
self.url = "http://127.0.0.1:6041/rest/sql" self.url = "http://%s:6041/rest/sql" % host
self.ts = 1500000000000 self.ts = 1500000000000
self.numOfThreads = 20 self.dbname = dbname
self.numOfTables = 10000 self.numOfThreads = threads
self.recordsPerTable = 10000 self.numOfTables = tables
self.batchSize = 1000 self.recordsPerTable = records
self.tableNamePerfix = 't' self.batchSize = batchSize
self.tableNamePerfix = tbNamePerfix
self.outOfOrder = outOfOrder
def createTable(self, threadID): def createTable(self, threadID):
tablesPerThread = int (self.numOfTables / self.numOfThreads) tablesPerThread = int (self.numOfTables / self.numOfThreads)
@ -33,7 +36,7 @@ class RestfulInsert:
for i in range(tablesPerThread): for i in range(tablesPerThread):
tableID = threadID * tablesPerThread tableID = threadID * tablesPerThread
name = 'beijing' if tableID % 2 == 0 else 'shanghai' name = 'beijing' if tableID % 2 == 0 else 'shanghai'
data = "create table test.%s%d using test.meters tags(%d, '%s')" % (self.tableNamePerfix, tableID + i, tableID + i, name) data = "create table %s.%s%d using %s.meters tags(%d, '%s')" % (self.dbname, self.tableNamePerfix, tableID + i, self.dbname, tableID + i, name)
requests.post(self.url, data, headers = self.header) requests.post(self.url, data, headers = self.header)
def insertData(self, threadID): def insertData(self, threadID):
@ -43,17 +46,42 @@ class RestfulInsert:
tableID = i + threadID * tablesPerThread tableID = i + threadID * tablesPerThread
start = self.ts start = self.ts
for j in range(int(self.recordsPerTable / self.batchSize)): for j in range(int(self.recordsPerTable / self.batchSize)):
data = "insert into test.%s%d values" % (self.tableNamePerfix, tableID) data = "insert into %s.%s%d values" % (self.dbname, self.tableNamePerfix, tableID)
values = []
for k in range(self.batchSize): for k in range(self.batchSize):
data += "(%d, %d, %d, %d)" % (start + j * self.batchSize + k, random.randint(1, 100), random.randint(1, 100), random.randint(1, 100)) data += "(%d, %d, %d, %d)" % (start + j * self.batchSize + k, random.randint(1, 100), random.randint(1, 100), random.randint(1, 100))
requests.post(self.url, data, headers = self.header)
def insertUnlimitedData(self, threadID):
print("thread %d started" % threadID)
tablesPerThread = int (self.numOfTables / self.numOfThreads)
while True:
i = 0
start = self.ts
for i in range(tablesPerThread):
tableID = i + threadID * tablesPerThread
data = "insert into %s.%s%d values" % (self.dbname, self.tableNamePerfix, tableID)
values = []
for k in range(self.batchSize):
values.append("(%d, %d, %d, %d)" % (start + j * self.batchSize + k, random.randint(1, 100), random.randint(1, 100), random.randint(1, 100)))
if(self.outOfOrder == False):
for k in range(len(values)):
data += values[k]
else:
random.shuffle(values)
for k in range(len(values)):
data += values[k]
requests.post(self.url, data, headers = self.header) requests.post(self.url, data, headers = self.header)
def run(self): def run(self):
data = "drop database if exists test" data = "drop database if exists %s" % self.dbname
requests.post(self.url, data, headers = self.header) requests.post(self.url, data, headers = self.header)
data = "create database test" data = "create database %s" % self.dbname
requests.post(self.url, data, headers = self.header) requests.post(self.url, data, headers = self.header)
data = "create table test.meters(ts timestamp, f1 int, f2 int, f3 int) tags(id int, loc nchar(20))" data = "create table %s.meters(ts timestamp, f1 int, f2 int, f3 int) tags(id int, loc nchar(20))" % self.dbname
requests.post(self.url, data, headers = self.header) requests.post(self.url, data, headers = self.header)
threads = [] threads = []
@ -70,7 +98,10 @@ class RestfulInsert:
threads = [] threads = []
startTime = time.time() startTime = time.time()
for i in range(self.numOfThreads): for i in range(self.numOfThreads):
thread = threading.Thread(target=self.insertData, args=(i,)) if(self.recordsPerTable != -1):
thread = threading.Thread(target=self.insertData, args=(i,))
else:
thread = threading.Thread(target=self.insertUnlimitedData, args=(i,))
thread.start() thread.start()
threads.append(thread) threads.append(thread)
@ -78,6 +109,62 @@ class RestfulInsert:
threads[i].join() threads[i].join()
print("inserting %d records takes %d seconds" % (self.numOfTables * self.recordsPerTable, (time.time() - startTime))) print("inserting %d records takes %d seconds" % (self.numOfTables * self.recordsPerTable, (time.time() - startTime)))
ri = RestfulInsert() parser = argparse.ArgumentParser()
ri.init() parser.add_argument(
'-H',
'--host-name',
action='store',
default='127.0.0.1',
type=str,
help='host name to be connected (default: 127.0.0.1)')
parser.add_argument(
'-d',
'--db-name',
action='store',
default='test',
type=str,
help='Database name to be created (default: test)')
parser.add_argument(
'-t',
'--number-of-threads',
action='store',
default=10,
type=int,
help='Number of threads to create tables and insert datas (default: 10)')
parser.add_argument(
'-T',
'--number-of-tables',
action='store',
default=1000,
type=int,
help='Number of tables to be created (default: 1000)')
parser.add_argument(
'-r',
'--number-of-records',
action='store',
default=1000,
type=int,
help='Number of record to be created for each table (default: 1000, -1 for unlimited records)')
parser.add_argument(
'-s',
'--batch-size',
action='store',
default='1000',
type=int,
help='Number of tables to be created (default: 1000)')
parser.add_argument(
'-p',
'--table-name-prefix',
action='store',
default='t',
type=str,
help='Number of tables to be created (default: 1000)')
parser.add_argument(
'-o',
'--out-of-order',
action='store_true',
help='The order of test data (default: False)')
args = parser.parse_args()
ri = RestfulInsert(args.host_name, args.db_name, args.number_of_threads, args.number_of_tables, args.number_of_records, args.batch_size, args.table_name_prefix, args.out_of_order)
ri.run() ri.run()