update query script
This commit is contained in:
Ping Xiao
parent
4be6e0f55e
commit
d2a675dbcb
|
|
@ -21,19 +21,28 @@ import argparse
|
|||
import datetime
|
||||
import string
|
||||
from requests.auth import HTTPBasicAuth
|
||||
func_list=['avg','count','twa','sum','stddev','leastsquares','min',
|
||||
'max','first','last','top','bottom','percentile','apercentile',
|
||||
'last_row','diff','spread','distinct']
|
||||
|
||||
func_list = ['abs', 'acos', 'asin', 'atan', 'ceil', 'cos', 'floor', 'log', 'pow', 'round', 'sin', 'sqrt', 'tan',
|
||||
'char_length', 'concat', 'concat_ws', 'length', 'lower', 'ltrim', 'rtrim', 'substr', 'upper',
|
||||
'cast', 'to_iso8601', 'to_json', 'to_unixtimestamp', 'now', 'timediff', 'timetruncate', 'timezone', 'today',
|
||||
'apercentile', 'avg', 'count', 'elapsed', 'leastsquares', 'spread', 'stddev', 'sum', 'hyperloglog', 'histogram', 'percentile',
|
||||
'bottom', 'first', 'interp', 'last', 'last_row', 'max', 'min', 'mode', 'sample', 'tail', 'top', 'unique',
|
||||
'csum', 'derivative', 'diff', 'irate', 'mavg', 'statecount', 'stateduration', 'twa',
|
||||
'database', 'client_version', 'server_version', 'server_status']
|
||||
|
||||
condition_list = [
|
||||
"where _c0 > now -10d ",
|
||||
'interval(10s)',
|
||||
'limit 10',
|
||||
'group by',
|
||||
'partition by',
|
||||
'order by',
|
||||
'fill(null)'
|
||||
|
||||
]
|
||||
|
||||
where_list = ['_c0>now-10d', ' <50', 'like', ' is null', 'in']
|
||||
|
||||
|
||||
class ConcurrentInquiry:
|
||||
# def __init__(self,ts=1500000001000,host='127.0.0.1',user='root',password='taosdata',dbname='test',
|
||||
# stb_prefix='st',subtb_prefix='t',n_Therads=10,r_Therads=10,probabilities=0.05,loop=5,
|
||||
|
|
@ -66,6 +75,7 @@ class ConcurrentInquiry:
|
|||
self.max_ts = datetime.datetime.now()
|
||||
self.min_ts = datetime.datetime.now() - datetime.timedelta(days=5)
|
||||
self.replay = replay
|
||||
|
||||
def SetThreadsNum(self, num):
|
||||
self.numOfTherads = num
|
||||
|
||||
|
|
@ -109,7 +119,8 @@ class ConcurrentInquiry:
|
|||
self.subtb_tag_list.append(tag)
|
||||
|
||||
def get_timespan(self, cl): # 获取时间跨度(仅第一个超级表)
|
||||
sql = 'select first(_c0),last(_c0) from ' + self.dbname + '.' + self.stb_list[0] + ';'
|
||||
sql = 'select first(_c0),last(_c0) from ' + \
|
||||
self.dbname + '.' + self.stb_list[0] + ';'
|
||||
print(sql)
|
||||
cl.execute(sql)
|
||||
for data in cl:
|
||||
|
|
@ -121,9 +132,9 @@ class ConcurrentInquiry:
|
|||
user = self.user
|
||||
password = self.password
|
||||
conn = taos.connect(
|
||||
host,
|
||||
user,
|
||||
password,
|
||||
host='%s' % host,
|
||||
user='%s' % user,
|
||||
password='%s' % password,
|
||||
)
|
||||
cl = conn.cursor()
|
||||
self.r_stb_list(cl)
|
||||
|
|
@ -141,16 +152,23 @@ class ConcurrentInquiry:
|
|||
for i in range(random.randint(0, len(tlist))):
|
||||
c = random.choice(where_list)
|
||||
if c == '_c0>now-10d':
|
||||
rdate = self.min_ts + (self.max_ts - self.min_ts)/10 * random.randint(-11,11)
|
||||
conlist = ' _c0 ' + random.choice(['<','>','>=','<=','<>']) + "'" + str(rdate) + "'"
|
||||
rdate = self.min_ts + \
|
||||
(self.max_ts - self.min_ts)/10 * random.randint(-11, 11)
|
||||
conlist = ' _c0 ' + \
|
||||
random.choice(['<', '>', '>=', '<=', '<>']
|
||||
) + "'" + str(rdate) + "'"
|
||||
if self.random_pick():
|
||||
l.append(conlist)
|
||||
else: l.append(c)
|
||||
else:
|
||||
l.append(c)
|
||||
elif '<50' in c:
|
||||
conlist = ' ' + random.choice(tlist) + random.choice(['<','>','>=','<=','<>']) + str(random.randrange(-100,100))
|
||||
conlist = ' ' + random.choice(tlist) + random.choice(
|
||||
['<', '>', '>=', '<=', '<>']) + str(random.randrange(-100, 100))
|
||||
l.append(conlist)
|
||||
elif 'is null' in c:
|
||||
conlist = ' ' + random.choice(tlist) + random.choice([' is null',' is not null'])
|
||||
conlist = ' ' + \
|
||||
random.choice(tlist) + \
|
||||
random.choice([' is null', ' is not null'])
|
||||
l.append(conlist)
|
||||
elif 'in' in c:
|
||||
in_list = []
|
||||
|
|
@ -161,26 +179,33 @@ class ConcurrentInquiry:
|
|||
in_list.append(temp)
|
||||
temp1 = []
|
||||
for i in range(random.randint(0, 100)):
|
||||
temp1.append("'" + ''.join(random.sample(string.ascii_letters, random.randint(0,10))) + "'")
|
||||
temp1.append(
|
||||
"'" + ''.join(random.sample(string.ascii_letters, random.randint(0, 10))) + "'")
|
||||
in_list.append(temp1)
|
||||
in_list.append(['NULL', 'NULL'])
|
||||
conlist = ' ' + random.choice(tlist) + ' in (' + ','.join(random.choice(in_list)) + ')'
|
||||
conlist = ' ' + \
|
||||
random.choice(tlist) + ' in (' + \
|
||||
','.join(random.choice(in_list)) + ')'
|
||||
l.append(conlist)
|
||||
else:
|
||||
s_all = string.ascii_letters
|
||||
conlist = ' ' + random.choice(tlist) + " like \'%" + random.choice(s_all) + "%\' "
|
||||
conlist = ' ' + \
|
||||
random.choice(tlist) + " like \'%" + \
|
||||
random.choice(s_all) + "%\' "
|
||||
l.append(conlist)
|
||||
return 'where '+random.choice([' and ', ' or ']).join(l)
|
||||
|
||||
def con_interval(self, tlist, col_list, tag_list):
|
||||
interval = 'interval(' + str(random.randint(0,20)) + random.choice(['a','s','d','w','n','y']) + ')'
|
||||
interval = 'interval(' + str(random.randint(0, 20)) + \
|
||||
random.choice(['a', 's', 'd', 'w', 'n', 'y']) + ')'
|
||||
return interval
|
||||
|
||||
def con_limit(self, tlist, col_list, tag_list):
|
||||
rand1 = str(random.randint(0, 1000))
|
||||
rand2 = str(random.randint(0, 1000))
|
||||
return random.choice(['limit ' + rand1, 'limit ' + rand1 + ' offset '+rand2,
|
||||
' slimit ' + rand1,' slimit ' + rand1 + ' offset ' + rand2,'limit '+rand1 + ' slimit '+ rand2,
|
||||
' slimit ' + rand1, ' slimit ' + rand1 + ' offset ' +
|
||||
rand2, 'limit '+rand1 + ' slimit ' + rand2,
|
||||
'limit ' + rand1 + ' offset' + rand2 + ' slimit ' + rand1 + ' soffset ' + rand2])
|
||||
|
||||
def con_fill(self, tlist, col_list, tag_list):
|
||||
|
|
@ -201,7 +226,8 @@ class ConcurrentInquiry:
|
|||
return 'state_window(' + random.choice(tlist + tag_list) + ')'
|
||||
|
||||
def con_session_window(self, tlist, col_list, tag_list):
|
||||
session_window = 'session_window(' + random.choice(tlist + tag_list) + ',' + str(random.randint(0,20)) + random.choice(['a','s','d','w','n','y']) + ')'
|
||||
session_window = 'session_window(' + random.choice(tlist + tag_list) + ',' + str(
|
||||
random.randint(0, 20)) + random.choice(['a', 's', 'd', 'w', 'n', 'y']) + ')'
|
||||
return session_window
|
||||
|
||||
def gen_subquery_sql(self):
|
||||
|
|
@ -242,17 +268,20 @@ class ConcurrentInquiry:
|
|||
else:
|
||||
sql = sql+','.join(sel_col_list) # select col & func
|
||||
sql = sql + ' from (' + subsql + ') '
|
||||
con_func=[self.con_where,self.con_interval,self.con_limit,self.con_group,self.con_order,self.con_fill,self.con_state_window,self.con_session_window]
|
||||
con_func = [self.con_where, self.con_interval, self.con_limit, self.con_group,
|
||||
self.con_order, self.con_fill, self.con_state_window, self.con_session_window]
|
||||
sel_con = random.sample(con_func, random.randint(0, len(con_func)))
|
||||
sel_con_list = []
|
||||
for i in sel_con:
|
||||
sel_con_list.append(i(tlist, col_list, tag_list)) # 获取对应的条件函数
|
||||
sql+=' '.join(sel_con_list) # condition
|
||||
# condition
|
||||
sql += ' '.join(sel_con_list)
|
||||
# print(sql)
|
||||
return sql
|
||||
|
||||
def gen_query_sql(self, subquery=0): # 生成查询语句
|
||||
tbi=random.randint(0,len(self.subtb_list)+len(self.stb_list)) #随机决定查询哪张表
|
||||
tbi = random.randint(0, len(self.subtb_list) +
|
||||
len(self.stb_list)) # 随机决定查询哪张表
|
||||
tbname = ''
|
||||
col_list = []
|
||||
tag_list = []
|
||||
|
|
@ -297,17 +326,20 @@ class ConcurrentInquiry:
|
|||
else:
|
||||
sql = sql+','.join(sel_col_list) # select col & func
|
||||
if self.mix_table == 0:
|
||||
sql = sql + ' from '+random.choice(self.stb_list+self.subtb_list)+' '
|
||||
sql = sql + ' from ' + \
|
||||
random.choice(self.stb_list+self.subtb_list)+' '
|
||||
elif self.mix_table == 1:
|
||||
sql = sql + ' from '+random.choice(self.subtb_list)+' '
|
||||
else:
|
||||
sql = sql + ' from '+random.choice(self.stb_list)+' '
|
||||
con_func=[self.con_where,self.con_interval,self.con_limit,self.con_group,self.con_order,self.con_fill,self.con_state_window,self.con_session_window]
|
||||
con_func = [self.con_where, self.con_interval, self.con_limit, self.con_group,
|
||||
self.con_order, self.con_fill, self.con_state_window, self.con_session_window]
|
||||
sel_con = random.sample(con_func, random.randint(0, len(con_func)))
|
||||
sel_con_list = []
|
||||
for i in sel_con:
|
||||
sel_con_list.append(i(tlist, col_list, tag_list)) # 获取对应的条件函数
|
||||
sql+=' '.join(sel_con_list) # condition
|
||||
# condition
|
||||
sql += ' '.join(sel_con_list)
|
||||
# print(sql)
|
||||
return (sql, loop)
|
||||
|
||||
|
|
@ -323,32 +355,42 @@ class ConcurrentInquiry:
|
|||
subtable = True
|
||||
tbname = random.sample(self.subtb_list, 2)
|
||||
for i in tbname:
|
||||
col_list.append(self.subtb_stru_list[self.subtb_list.index(i)])
|
||||
tag_list.append(self.subtb_stru_list[self.subtb_list.index(i)])
|
||||
col_intersection = list(set(col_list[0]).intersection(set(col_list[1])))
|
||||
tag_intersection = list(set(tag_list[0]).intersection(set(tag_list[1])))
|
||||
col_list.append(
|
||||
self.subtb_stru_list[self.subtb_list.index(i)])
|
||||
tag_list.append(
|
||||
self.subtb_stru_list[self.subtb_list.index(i)])
|
||||
col_intersection = list(
|
||||
set(col_list[0]).intersection(set(col_list[1])))
|
||||
tag_intersection = list(
|
||||
set(tag_list[0]).intersection(set(tag_list[1])))
|
||||
else:
|
||||
tbname = random.sample(self.stb_list, 2)
|
||||
for i in tbname:
|
||||
col_list.append(self.stb_stru_list[self.stb_list.index(i)])
|
||||
tag_list.append(self.stb_stru_list[self.stb_list.index(i)])
|
||||
col_intersection = list(set(col_list[0]).intersection(set(col_list[1])))
|
||||
tag_intersection = list(set(tag_list[0]).intersection(set(tag_list[1])))
|
||||
col_intersection = list(
|
||||
set(col_list[0]).intersection(set(col_list[1])))
|
||||
tag_intersection = list(
|
||||
set(tag_list[0]).intersection(set(tag_list[1])))
|
||||
elif self.mix_table == 1:
|
||||
subtable = True
|
||||
tbname = random.sample(self.subtb_list, 2)
|
||||
for i in tbname:
|
||||
col_list.append(self.subtb_stru_list[self.subtb_list.index(i)])
|
||||
tag_list.append(self.subtb_stru_list[self.subtb_list.index(i)])
|
||||
col_intersection = list(set(col_list[0]).intersection(set(col_list[1])))
|
||||
tag_intersection = list(set(tag_list[0]).intersection(set(tag_list[1])))
|
||||
col_intersection = list(
|
||||
set(col_list[0]).intersection(set(col_list[1])))
|
||||
tag_intersection = list(
|
||||
set(tag_list[0]).intersection(set(tag_list[1])))
|
||||
else:
|
||||
tbname = random.sample(self.stb_list, 2)
|
||||
for i in tbname:
|
||||
col_list.append(self.stb_stru_list[self.stb_list.index(i)])
|
||||
tag_list.append(self.stb_stru_list[self.stb_list.index(i)])
|
||||
col_intersection = list(set(col_list[0]).intersection(set(col_list[1])))
|
||||
tag_intersection = list(set(tag_list[0]).intersection(set(tag_list[1])))
|
||||
col_intersection = list(
|
||||
set(col_list[0]).intersection(set(col_list[1])))
|
||||
tag_intersection = list(
|
||||
set(tag_list[0]).intersection(set(tag_list[1])))
|
||||
con_rand = random.randint(0, len(condition_list))
|
||||
col_rand = random.randint(0, len(col_list))
|
||||
tag_rand = random.randint(0, len(tag_list))
|
||||
|
|
@ -359,8 +401,10 @@ class ConcurrentInquiry:
|
|||
if bool(random.getrandbits(1)):
|
||||
sql += '*'
|
||||
else:
|
||||
sel_col_tag.append('t1.' + str(random.choice(col_list[0] + tag_list[0])))
|
||||
sel_col_tag.append('t2.' + str(random.choice(col_list[1] + tag_list[1])))
|
||||
sel_col_tag.append(
|
||||
't1.' + str(random.choice(col_list[0] + tag_list[0])))
|
||||
sel_col_tag.append(
|
||||
't2.' + str(random.choice(col_list[1] + tag_list[1])))
|
||||
sel_col_list = []
|
||||
random.shuffle(func_list)
|
||||
if self.random_pick():
|
||||
|
|
@ -382,17 +426,21 @@ class ConcurrentInquiry:
|
|||
else:
|
||||
sql += ','.join(sel_col_tag)
|
||||
|
||||
sql = sql + ' from '+ str(tbname[0]) +' t1,' + str(tbname[1]) + ' t2 ' #select col & func
|
||||
sql = sql + ' from ' + \
|
||||
str(tbname[0]) + ' t1,' + str(tbname[1]) + \
|
||||
' t2 ' # select col & func
|
||||
join_section = None
|
||||
temp = None
|
||||
if subtable:
|
||||
temp = random.choices(col_intersection)
|
||||
join_section = temp.pop()
|
||||
sql += 'where t1._c0 = t2._c0 and ' + 't1.' + str(join_section) + '=t2.' + str(join_section)
|
||||
sql += 'where t1._c0 = t2._c0 and ' + 't1.' + \
|
||||
str(join_section) + '=t2.' + str(join_section)
|
||||
else:
|
||||
temp = random.choices(col_intersection+tag_intersection)
|
||||
join_section = temp.pop()
|
||||
sql += 'where t1._c0 = t2._c0 and ' + 't1.' + str(join_section) + '=t2.' + str(join_section)
|
||||
sql += 'where t1._c0 = t2._c0 and ' + 't1.' + \
|
||||
str(join_section) + '=t2.' + str(join_section)
|
||||
return sql
|
||||
|
||||
def random_pick(self):
|
||||
|
|
@ -400,7 +448,8 @@ class ConcurrentInquiry:
|
|||
cumulative_probability = 0.0
|
||||
for item, item_probability in zip(self.ifjoin, self.probabilities):
|
||||
cumulative_probability += item_probability
|
||||
if x < cumulative_probability:break
|
||||
if x < cumulative_probability:
|
||||
break
|
||||
return item
|
||||
|
||||
def gen_data(self):
|
||||
|
|
@ -412,9 +461,9 @@ class ConcurrentInquiry:
|
|||
user = self.user
|
||||
password = self.password
|
||||
conn = taos.connect(
|
||||
host,
|
||||
user,
|
||||
password,
|
||||
host='%s' % host,
|
||||
user='%s' % user,
|
||||
password='%s' % password,
|
||||
)
|
||||
cl = conn.cursor()
|
||||
cl.execute("drop database if exists %s;" % self.dbname)
|
||||
|
|
@ -427,10 +476,12 @@ class ConcurrentInquiry:
|
|||
for j in range(subtableNum):
|
||||
if j % 100 == 0:
|
||||
sql = "create table %s using %s tags(NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL)" % \
|
||||
(self.subtb_prefix+str(k)+'_'+str(j),self.stb_prefix+str(k))
|
||||
(self.subtb_prefix+str(k)+'_' +
|
||||
str(j), self.stb_prefix+str(k))
|
||||
else:
|
||||
sql = "create table %s using %s tags(%d,%d,%d,%d,%d,%d,%d,'%s','%s',%d,%d,%d,%d)" % \
|
||||
(self.subtb_prefix+str(k)+'_'+str(j),self.stb_prefix+str(k),j,j/2.0,j%41,j%51,j%53,j*1.0,j%2,'taos'+str(j),'涛思'+str(j), j%43, j%23 , j%17 , j%3167)
|
||||
(self.subtb_prefix+str(k)+'_'+str(j), self.stb_prefix+str(k), j, j/2.0, j % 41, j %
|
||||
51, j % 53, j*1.0, j % 2, 'taos'+str(j), '涛思'+str(j), j % 43, j % 23, j % 17, j % 3167)
|
||||
print(sql)
|
||||
cl.execute(sql)
|
||||
for i in range(insertRows):
|
||||
|
|
@ -481,15 +532,14 @@ class ConcurrentInquiry:
|
|||
nRows = rj['rows'] if ('rows' in rj) else 0
|
||||
return nRows
|
||||
|
||||
|
||||
def query_thread_n(self, threadID): # 使用原生python接口查询
|
||||
host = self.host
|
||||
user = self.user
|
||||
password = self.password
|
||||
conn = taos.connect(
|
||||
host,
|
||||
user,
|
||||
password,
|
||||
host='%s' % host,
|
||||
user='%s' % user,
|
||||
password='%s' % password,
|
||||
)
|
||||
cl = conn.cursor()
|
||||
cl.execute("use %s;" % self.dbname)
|
||||
|
|
@ -522,7 +572,8 @@ class ConcurrentInquiry:
|
|||
if err_uec in str(e) and loop > 0:
|
||||
exit(-1)
|
||||
loop -= 1
|
||||
if loop == 0: break
|
||||
if loop == 0:
|
||||
break
|
||||
fo.close()
|
||||
cl.close()
|
||||
conn.close()
|
||||
|
|
@ -533,9 +584,9 @@ class ConcurrentInquiry:
|
|||
user = self.user
|
||||
password = self.password
|
||||
conn = taos.connect(
|
||||
host,
|
||||
user,
|
||||
password,
|
||||
host='%s' % host,
|
||||
user='%s' % user,
|
||||
password='%s' % password,
|
||||
)
|
||||
cl = conn.cursor()
|
||||
cl.execute("use %s;" % self.dbname)
|
||||
|
|
@ -591,7 +642,8 @@ class ConcurrentInquiry:
|
|||
if err_uec in str(e) and loop > 0:
|
||||
exit(-1)
|
||||
loop -= 1
|
||||
if loop == 0: break
|
||||
if loop == 0:
|
||||
break
|
||||
fo.close()
|
||||
print("Thread %d: finishing" % threadID)
|
||||
|
||||
|
|
@ -623,23 +675,28 @@ class ConcurrentInquiry:
|
|||
threads = []
|
||||
if self.replay: # whether replay
|
||||
for i in range(self.n_numOfTherads):
|
||||
thread = threading.Thread(target=self.query_thread_nr, args=(i,))
|
||||
thread = threading.Thread(
|
||||
target=self.query_thread_nr, args=(i,))
|
||||
threads.append(thread)
|
||||
thread.start()
|
||||
for i in range(self.r_numOfTherads):
|
||||
thread = threading.Thread(target=self.query_thread_rr, args=(i,))
|
||||
thread = threading.Thread(
|
||||
target=self.query_thread_rr, args=(i,))
|
||||
threads.append(thread)
|
||||
thread.start()
|
||||
else:
|
||||
for i in range(self.n_numOfTherads):
|
||||
thread = threading.Thread(target=self.query_thread_n, args=(i,))
|
||||
thread = threading.Thread(
|
||||
target=self.query_thread_n, args=(i,))
|
||||
threads.append(thread)
|
||||
thread.start()
|
||||
for i in range(self.r_numOfTherads):
|
||||
thread = threading.Thread(target=self.query_thread_r, args=(i,))
|
||||
thread = threading.Thread(
|
||||
target=self.query_thread_r, args=(i,))
|
||||
threads.append(thread)
|
||||
thread.start()
|
||||
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
'-H',
|
||||
|
|
@ -729,9 +786,9 @@ parser.add_argument(
|
|||
'-w',
|
||||
'--password',
|
||||
action='store',
|
||||
default='root',
|
||||
default='taosdata',
|
||||
type=str,
|
||||
help='user name')
|
||||
help='password')
|
||||
parser.add_argument(
|
||||
'-n',
|
||||
'--number-of-tables',
|
||||
|
|
@ -774,4 +831,3 @@ q.get_full()
|
|||
|
||||
# q.gen_query_sql()
|
||||
q.run()
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue