xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

update query script

This commit is contained in:
Ping Xiao 2022-11-04 15:39:11 +08:00
parent 4be6e0f55e
commit d2a675dbcb
1 changed files with 395 additions and 339 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

180
tests/pytest/concurrent_inquiry.py
View File

@ -21,19 +21,28 @@ import argparse
import datetime import datetime
import string import string
from requests.auth import HTTPBasicAuth from requests.auth import HTTPBasicAuth
func_list=['avg','count','twa','sum','stddev','leastsquares','min',
'max','first','last','top','bottom','percentile','apercentile', func_list = ['abs', 'acos', 'asin', 'atan', 'ceil', 'cos', 'floor', 'log', 'pow', 'round', 'sin', 'sqrt', 'tan',
'last_row','diff','spread','distinct'] 'char_length', 'concat', 'concat_ws', 'length', 'lower', 'ltrim', 'rtrim', 'substr', 'upper',
'cast', 'to_iso8601', 'to_json', 'to_unixtimestamp', 'now', 'timediff', 'timetruncate', 'timezone', 'today',
'apercentile', 'avg', 'count', 'elapsed', 'leastsquares', 'spread', 'stddev', 'sum', 'hyperloglog', 'histogram', 'percentile',
'bottom', 'first', 'interp', 'last', 'last_row', 'max', 'min', 'mode', 'sample', 'tail', 'top', 'unique',
'csum', 'derivative', 'diff', 'irate', 'mavg', 'statecount', 'stateduration', 'twa',
'database', 'client_version', 'server_version', 'server_status']
condition_list = [ condition_list = [
"where _c0 > now -10d ", "where _c0 > now -10d ",
'interval(10s)', 'interval(10s)',
'limit 10', 'limit 10',
'group by', 'group by',
'partition by',
'order by', 'order by',
'fill(null)' 'fill(null)'
] ]
where_list = ['_c0>now-10d', ' <50', 'like', ' is null', 'in'] where_list = ['_c0>now-10d', ' <50', 'like', ' is null', 'in']
class ConcurrentInquiry: class ConcurrentInquiry:
# def __init__(self,ts=1500000001000,host='127.0.0.1',user='root',password='taosdata',dbname='test', # def __init__(self,ts=1500000001000,host='127.0.0.1',user='root',password='taosdata',dbname='test',
# stb_prefix='st',subtb_prefix='t',n_Therads=10,r_Therads=10,probabilities=0.05,loop=5, # stb_prefix='st',subtb_prefix='t',n_Therads=10,r_Therads=10,probabilities=0.05,loop=5,
@ -66,6 +75,7 @@ class ConcurrentInquiry:
self.max_ts = datetime.datetime.now() self.max_ts = datetime.datetime.now()
self.min_ts = datetime.datetime.now() - datetime.timedelta(days=5) self.min_ts = datetime.datetime.now() - datetime.timedelta(days=5)
self.replay = replay self.replay = replay
def SetThreadsNum(self, num): def SetThreadsNum(self, num):
self.numOfTherads = num self.numOfTherads = num
@ -109,7 +119,8 @@ class ConcurrentInquiry:
self.subtb_tag_list.append(tag) self.subtb_tag_list.append(tag)
def get_timespan(self, cl): # 获取时间跨度(仅第一个超级表) def get_timespan(self, cl): # 获取时间跨度(仅第一个超级表)
sql = 'select first(_c0),last(_c0) from ' + self.dbname + '.' + self.stb_list[0] + ';' sql = 'select first(_c0),last(_c0) from ' + \
self.dbname + '.' + self.stb_list[0] + ';'
print(sql) print(sql)
cl.execute(sql) cl.execute(sql)
for data in cl: for data in cl:
@ -121,9 +132,9 @@ class ConcurrentInquiry:
user = self.user user = self.user
password = self.password password = self.password
conn = taos.connect( conn = taos.connect(
host, host='%s' % host,
user, user='%s' % user,
password, password='%s' % password,
) )
cl = conn.cursor() cl = conn.cursor()
self.r_stb_list(cl) self.r_stb_list(cl)
@ -141,16 +152,23 @@ class ConcurrentInquiry:
for i in range(random.randint(0, len(tlist))): for i in range(random.randint(0, len(tlist))):
c = random.choice(where_list) c = random.choice(where_list)
if c == '_c0>now-10d': if c == '_c0>now-10d':
rdate = self.min_ts + (self.max_ts - self.min_ts)/10 * random.randint(-11,11) rdate = self.min_ts + \
conlist = ' _c0 ' + random.choice(['<','>','>=','<=','<>']) + "'" + str(rdate) + "'" (self.max_ts - self.min_ts)/10 * random.randint(-11, 11)
conlist = ' _c0 ' + \
random.choice(['<', '>', '>=', '<=', '<>']
) + "'" + str(rdate) + "'"
if self.random_pick(): if self.random_pick():
l.append(conlist) l.append(conlist)
else: l.append(c) else:
l.append(c)
elif '<50' in c: elif '<50' in c:
conlist = ' ' + random.choice(tlist) + random.choice(['<','>','>=','<=','<>']) + str(random.randrange(-100,100)) conlist = ' ' + random.choice(tlist) + random.choice(
['<', '>', '>=', '<=', '<>']) + str(random.randrange(-100, 100))
l.append(conlist) l.append(conlist)
elif 'is null' in c: elif 'is null' in c:
conlist = ' ' + random.choice(tlist) + random.choice([' is null',' is not null']) conlist = ' ' + \
random.choice(tlist) + \
random.choice([' is null', ' is not null'])
l.append(conlist) l.append(conlist)
elif 'in' in c: elif 'in' in c:
in_list = [] in_list = []
@ -161,26 +179,33 @@ class ConcurrentInquiry:
in_list.append(temp) in_list.append(temp)
temp1 = [] temp1 = []
for i in range(random.randint(0, 100)): for i in range(random.randint(0, 100)):
temp1.append("'" + ''.join(random.sample(string.ascii_letters, random.randint(0,10))) + "'") temp1.append(
"'" + ''.join(random.sample(string.ascii_letters, random.randint(0, 10))) + "'")
in_list.append(temp1) in_list.append(temp1)
in_list.append(['NULL', 'NULL']) in_list.append(['NULL', 'NULL'])
conlist = ' ' + random.choice(tlist) + ' in (' + ','.join(random.choice(in_list)) + ')' conlist = ' ' + \
random.choice(tlist) + ' in (' + \
','.join(random.choice(in_list)) + ')'
l.append(conlist) l.append(conlist)
else: else:
s_all = string.ascii_letters s_all = string.ascii_letters
conlist = ' ' + random.choice(tlist) + " like \'%" + random.choice(s_all) + "%\' " conlist = ' ' + \
random.choice(tlist) + " like \'%" + \
random.choice(s_all) + "%\' "
l.append(conlist) l.append(conlist)
return 'where '+random.choice([' and ', ' or ']).join(l) return 'where '+random.choice([' and ', ' or ']).join(l)
def con_interval(self, tlist, col_list, tag_list): def con_interval(self, tlist, col_list, tag_list):
interval = 'interval(' + str(random.randint(0,20)) + random.choice(['a','s','d','w','n','y']) + ')' interval = 'interval(' + str(random.randint(0, 20)) + \
random.choice(['a', 's', 'd', 'w', 'n', 'y']) + ')'
return interval return interval
def con_limit(self, tlist, col_list, tag_list): def con_limit(self, tlist, col_list, tag_list):
rand1 = str(random.randint(0, 1000)) rand1 = str(random.randint(0, 1000))
rand2 = str(random.randint(0, 1000)) rand2 = str(random.randint(0, 1000))
return random.choice(['limit ' + rand1, 'limit ' + rand1 + ' offset '+rand2, return random.choice(['limit ' + rand1, 'limit ' + rand1 + ' offset '+rand2,
' slimit ' + rand1,' slimit ' + rand1 + ' offset ' + rand2,'limit '+rand1 + ' slimit '+ rand2, ' slimit ' + rand1, ' slimit ' + rand1 + ' offset ' +
rand2, 'limit '+rand1 + ' slimit ' + rand2,
'limit ' + rand1 + ' offset' + rand2 + ' slimit ' + rand1 + ' soffset ' + rand2]) 'limit ' + rand1 + ' offset' + rand2 + ' slimit ' + rand1 + ' soffset ' + rand2])
def con_fill(self, tlist, col_list, tag_list): def con_fill(self, tlist, col_list, tag_list):
@ -201,7 +226,8 @@ class ConcurrentInquiry:
return 'state_window(' + random.choice(tlist + tag_list) + ')' return 'state_window(' + random.choice(tlist + tag_list) + ')'
def con_session_window(self, tlist, col_list, tag_list): def con_session_window(self, tlist, col_list, tag_list):
session_window = 'session_window(' + random.choice(tlist + tag_list) + ',' + str(random.randint(0,20)) + random.choice(['a','s','d','w','n','y']) + ')' session_window = 'session_window(' + random.choice(tlist + tag_list) + ',' + str(
random.randint(0, 20)) + random.choice(['a', 's', 'd', 'w', 'n', 'y']) + ')'
return session_window return session_window
def gen_subquery_sql(self): def gen_subquery_sql(self):
@ -242,17 +268,20 @@ class ConcurrentInquiry:
else: else:
sql = sql+','.join(sel_col_list) # select col & func sql = sql+','.join(sel_col_list) # select col & func
sql = sql + ' from (' + subsql + ') ' sql = sql + ' from (' + subsql + ') '
con_func=[self.con_where,self.con_interval,self.con_limit,self.con_group,self.con_order,self.con_fill,self.con_state_window,self.con_session_window] con_func = [self.con_where, self.con_interval, self.con_limit, self.con_group,
self.con_order, self.con_fill, self.con_state_window, self.con_session_window]
sel_con = random.sample(con_func, random.randint(0, len(con_func))) sel_con = random.sample(con_func, random.randint(0, len(con_func)))
sel_con_list = [] sel_con_list = []
for i in sel_con: for i in sel_con:
sel_con_list.append(i(tlist, col_list, tag_list)) # 获取对应的条件函数 sel_con_list.append(i(tlist, col_list, tag_list)) # 获取对应的条件函数
sql+=' '.join(sel_con_list) # condition # condition
sql += ' '.join(sel_con_list)
# print(sql) # print(sql)
return sql return sql
def gen_query_sql(self, subquery=0): # 生成查询语句 def gen_query_sql(self, subquery=0): # 生成查询语句
tbi=random.randint(0,len(self.subtb_list)+len(self.stb_list)) #随机决定查询哪张表 tbi = random.randint(0, len(self.subtb_list) +
len(self.stb_list)) # 随机决定查询哪张表
tbname = '' tbname = ''
col_list = [] col_list = []
tag_list = [] tag_list = []
@ -297,17 +326,20 @@ class ConcurrentInquiry:
else: else:
sql = sql+','.join(sel_col_list) # select col & func sql = sql+','.join(sel_col_list) # select col & func
if self.mix_table == 0: if self.mix_table == 0:
sql = sql + ' from '+random.choice(self.stb_list+self.subtb_list)+' ' sql = sql + ' from ' + \
random.choice(self.stb_list+self.subtb_list)+' '
elif self.mix_table == 1: elif self.mix_table == 1:
sql = sql + ' from '+random.choice(self.subtb_list)+' ' sql = sql + ' from '+random.choice(self.subtb_list)+' '
else: else:
sql = sql + ' from '+random.choice(self.stb_list)+' ' sql = sql + ' from '+random.choice(self.stb_list)+' '
con_func=[self.con_where,self.con_interval,self.con_limit,self.con_group,self.con_order,self.con_fill,self.con_state_window,self.con_session_window] con_func = [self.con_where, self.con_interval, self.con_limit, self.con_group,
self.con_order, self.con_fill, self.con_state_window, self.con_session_window]
sel_con = random.sample(con_func, random.randint(0, len(con_func))) sel_con = random.sample(con_func, random.randint(0, len(con_func)))
sel_con_list = [] sel_con_list = []
for i in sel_con: for i in sel_con:
sel_con_list.append(i(tlist, col_list, tag_list)) # 获取对应的条件函数 sel_con_list.append(i(tlist, col_list, tag_list)) # 获取对应的条件函数
sql+=' '.join(sel_con_list) # condition # condition
sql += ' '.join(sel_con_list)
# print(sql) # print(sql)
return (sql, loop) return (sql, loop)
@ -323,32 +355,42 @@ class ConcurrentInquiry:
subtable = True subtable = True
tbname = random.sample(self.subtb_list, 2) tbname = random.sample(self.subtb_list, 2)
for i in tbname: for i in tbname:
col_list.append(self.subtb_stru_list[self.subtb_list.index(i)]) col_list.append(
tag_list.append(self.subtb_stru_list[self.subtb_list.index(i)]) self.subtb_stru_list[self.subtb_list.index(i)])
col_intersection = list(set(col_list[0]).intersection(set(col_list[1]))) tag_list.append(
tag_intersection = list(set(tag_list[0]).intersection(set(tag_list[1]))) self.subtb_stru_list[self.subtb_list.index(i)])
col_intersection = list(
set(col_list[0]).intersection(set(col_list[1])))
tag_intersection = list(
set(tag_list[0]).intersection(set(tag_list[1])))
else: else:
tbname = random.sample(self.stb_list, 2) tbname = random.sample(self.stb_list, 2)
for i in tbname: for i in tbname:
col_list.append(self.stb_stru_list[self.stb_list.index(i)]) col_list.append(self.stb_stru_list[self.stb_list.index(i)])
tag_list.append(self.stb_stru_list[self.stb_list.index(i)]) tag_list.append(self.stb_stru_list[self.stb_list.index(i)])
col_intersection = list(set(col_list[0]).intersection(set(col_list[1]))) col_intersection = list(
tag_intersection = list(set(tag_list[0]).intersection(set(tag_list[1]))) set(col_list[0]).intersection(set(col_list[1])))
tag_intersection = list(
set(tag_list[0]).intersection(set(tag_list[1])))
elif self.mix_table == 1: elif self.mix_table == 1:
subtable = True subtable = True
tbname = random.sample(self.subtb_list, 2) tbname = random.sample(self.subtb_list, 2)
for i in tbname: for i in tbname:
col_list.append(self.subtb_stru_list[self.subtb_list.index(i)]) col_list.append(self.subtb_stru_list[self.subtb_list.index(i)])
tag_list.append(self.subtb_stru_list[self.subtb_list.index(i)]) tag_list.append(self.subtb_stru_list[self.subtb_list.index(i)])
col_intersection = list(set(col_list[0]).intersection(set(col_list[1]))) col_intersection = list(
tag_intersection = list(set(tag_list[0]).intersection(set(tag_list[1]))) set(col_list[0]).intersection(set(col_list[1])))
tag_intersection = list(
set(tag_list[0]).intersection(set(tag_list[1])))
else: else:
tbname = random.sample(self.stb_list, 2) tbname = random.sample(self.stb_list, 2)
for i in tbname: for i in tbname:
col_list.append(self.stb_stru_list[self.stb_list.index(i)]) col_list.append(self.stb_stru_list[self.stb_list.index(i)])
tag_list.append(self.stb_stru_list[self.stb_list.index(i)]) tag_list.append(self.stb_stru_list[self.stb_list.index(i)])
col_intersection = list(set(col_list[0]).intersection(set(col_list[1]))) col_intersection = list(
tag_intersection = list(set(tag_list[0]).intersection(set(tag_list[1]))) set(col_list[0]).intersection(set(col_list[1])))
tag_intersection = list(
set(tag_list[0]).intersection(set(tag_list[1])))
con_rand = random.randint(0, len(condition_list)) con_rand = random.randint(0, len(condition_list))
col_rand = random.randint(0, len(col_list)) col_rand = random.randint(0, len(col_list))
tag_rand = random.randint(0, len(tag_list)) tag_rand = random.randint(0, len(tag_list))
@ -359,8 +401,10 @@ class ConcurrentInquiry:
if bool(random.getrandbits(1)): if bool(random.getrandbits(1)):
sql += '*' sql += '*'
else: else:
sel_col_tag.append('t1.' + str(random.choice(col_list[0] + tag_list[0]))) sel_col_tag.append(
sel_col_tag.append('t2.' + str(random.choice(col_list[1] + tag_list[1]))) 't1.' + str(random.choice(col_list[0] + tag_list[0])))
sel_col_tag.append(
't2.' + str(random.choice(col_list[1] + tag_list[1])))
sel_col_list = [] sel_col_list = []
random.shuffle(func_list) random.shuffle(func_list)
if self.random_pick(): if self.random_pick():
@ -382,17 +426,21 @@ class ConcurrentInquiry:
else: else:
sql += ','.join(sel_col_tag) sql += ','.join(sel_col_tag)
sql = sql + ' from '+ str(tbname[0]) +' t1,' + str(tbname[1]) + ' t2 ' #select col & func sql = sql + ' from ' + \
str(tbname[0]) + ' t1,' + str(tbname[1]) + \
' t2 ' # select col & func
join_section = None join_section = None
temp = None temp = None
if subtable: if subtable:
temp = random.choices(col_intersection) temp = random.choices(col_intersection)
join_section = temp.pop() join_section = temp.pop()
sql += 'where t1._c0 = t2._c0 and ' + 't1.' + str(join_section) + '=t2.' + str(join_section) sql += 'where t1._c0 = t2._c0 and ' + 't1.' + \
str(join_section) + '=t2.' + str(join_section)
else: else:
temp = random.choices(col_intersection+tag_intersection) temp = random.choices(col_intersection+tag_intersection)
join_section = temp.pop() join_section = temp.pop()
sql += 'where t1._c0 = t2._c0 and ' + 't1.' + str(join_section) + '=t2.' + str(join_section) sql += 'where t1._c0 = t2._c0 and ' + 't1.' + \
str(join_section) + '=t2.' + str(join_section)
return sql return sql
def random_pick(self): def random_pick(self):
@ -400,7 +448,8 @@ class ConcurrentInquiry:
cumulative_probability = 0.0 cumulative_probability = 0.0
for item, item_probability in zip(self.ifjoin, self.probabilities): for item, item_probability in zip(self.ifjoin, self.probabilities):
cumulative_probability += item_probability cumulative_probability += item_probability
if x < cumulative_probability:break if x < cumulative_probability:
break
return item return item
def gen_data(self): def gen_data(self):
@ -412,9 +461,9 @@ class ConcurrentInquiry:
user = self.user user = self.user
password = self.password password = self.password
conn = taos.connect( conn = taos.connect(
host, host='%s' % host,
user, user='%s' % user,
password, password='%s' % password,
) )
cl = conn.cursor() cl = conn.cursor()
cl.execute("drop database if exists %s;" % self.dbname) cl.execute("drop database if exists %s;" % self.dbname)
@ -427,10 +476,12 @@ class ConcurrentInquiry:
for j in range(subtableNum): for j in range(subtableNum):
if j % 100 == 0: if j % 100 == 0:
sql = "create table %s using %s tags(NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL)" % \ sql = "create table %s using %s tags(NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL)" % \
(self.subtb_prefix+str(k)+'_'+str(j),self.stb_prefix+str(k)) (self.subtb_prefix+str(k)+'_' +
str(j), self.stb_prefix+str(k))
else: else:
sql = "create table %s using %s tags(%d,%d,%d,%d,%d,%d,%d,'%s','%s',%d,%d,%d,%d)" % \ sql = "create table %s using %s tags(%d,%d,%d,%d,%d,%d,%d,'%s','%s',%d,%d,%d,%d)" % \
(self.subtb_prefix+str(k)+'_'+str(j),self.stb_prefix+str(k),j,j/2.0,j%41,j%51,j%53,j*1.0,j%2,'taos'+str(j),'涛思'+str(j), j%43, j%23 , j%17 , j%3167) (self.subtb_prefix+str(k)+'_'+str(j), self.stb_prefix+str(k), j, j/2.0, j % 41, j %
51, j % 53, j*1.0, j % 2, 'taos'+str(j), '涛思'+str(j), j % 43, j % 23, j % 17, j % 3167)
print(sql) print(sql)
cl.execute(sql) cl.execute(sql)
for i in range(insertRows): for i in range(insertRows):
@ -481,15 +532,14 @@ class ConcurrentInquiry:
nRows = rj['rows'] if ('rows' in rj) else 0 nRows = rj['rows'] if ('rows' in rj) else 0
return nRows return nRows
def query_thread_n(self, threadID): # 使用原生python接口查询 def query_thread_n(self, threadID): # 使用原生python接口查询
host = self.host host = self.host
user = self.user user = self.user
password = self.password password = self.password
conn = taos.connect( conn = taos.connect(
host, host='%s' % host,
user, user='%s' % user,
password, password='%s' % password,
) )
cl = conn.cursor() cl = conn.cursor()
cl.execute("use %s;" % self.dbname) cl.execute("use %s;" % self.dbname)
@ -522,7 +572,8 @@ class ConcurrentInquiry:
if err_uec in str(e) and loop > 0: if err_uec in str(e) and loop > 0:
exit(-1) exit(-1)
loop -= 1 loop -= 1
if loop == 0: break if loop == 0:
break
fo.close() fo.close()
cl.close() cl.close()
conn.close() conn.close()
@ -533,9 +584,9 @@ class ConcurrentInquiry:
user = self.user user = self.user
password = self.password password = self.password
conn = taos.connect( conn = taos.connect(
host, host='%s' % host,
user, user='%s' % user,
password, password='%s' % password,
) )
cl = conn.cursor() cl = conn.cursor()
cl.execute("use %s;" % self.dbname) cl.execute("use %s;" % self.dbname)
@ -591,7 +642,8 @@ class ConcurrentInquiry:
if err_uec in str(e) and loop > 0: if err_uec in str(e) and loop > 0:
exit(-1) exit(-1)
loop -= 1 loop -= 1
if loop == 0: break if loop == 0:
break
fo.close() fo.close()
print("Thread %d: finishing" % threadID) print("Thread %d: finishing" % threadID)
@ -623,23 +675,28 @@ class ConcurrentInquiry:
threads = [] threads = []
if self.replay: # whether replay if self.replay: # whether replay
for i in range(self.n_numOfTherads): for i in range(self.n_numOfTherads):
thread = threading.Thread(target=self.query_thread_nr, args=(i,)) thread = threading.Thread(
target=self.query_thread_nr, args=(i,))
threads.append(thread) threads.append(thread)
thread.start() thread.start()
for i in range(self.r_numOfTherads): for i in range(self.r_numOfTherads):
thread = threading.Thread(target=self.query_thread_rr, args=(i,)) thread = threading.Thread(
target=self.query_thread_rr, args=(i,))
threads.append(thread) threads.append(thread)
thread.start() thread.start()
else: else:
for i in range(self.n_numOfTherads): for i in range(self.n_numOfTherads):
thread = threading.Thread(target=self.query_thread_n, args=(i,)) thread = threading.Thread(
target=self.query_thread_n, args=(i,))
threads.append(thread) threads.append(thread)
thread.start() thread.start()
for i in range(self.r_numOfTherads): for i in range(self.r_numOfTherads):
thread = threading.Thread(target=self.query_thread_r, args=(i,)) thread = threading.Thread(
target=self.query_thread_r, args=(i,))
threads.append(thread) threads.append(thread)
thread.start() thread.start()
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument( parser.add_argument(
'-H', '-H',
@ -729,9 +786,9 @@ parser.add_argument(
'-w', '-w',
'--password', '--password',
action='store', action='store',
default='root', default='taosdata',
type=str, type=str,
help='user name') help='password')
parser.add_argument( parser.add_argument(
'-n', '-n',
'--number-of-tables', '--number-of-tables',
@ -774,4 +831,3 @@ q.get_full()
# q.gen_query_sql() # q.gen_query_sql()
q.run() q.run()