1602 lines
76 KiB
Python
1602 lines
76 KiB
Python
from random import randrange
|
|
import time
|
|
import threading
|
|
import secrets
|
|
from util.log import *
|
|
from util.sql import *
|
|
from util.cases import *
|
|
from util.dnodes import *
|
|
from util.common import *
|
|
# from tmqCommon import *
|
|
|
|
ROUND = 1000
|
|
|
|
class TSMA:
|
|
def __init__(self):
|
|
self.tsma_name = ''
|
|
self.db_name = ''
|
|
self.original_table_name = ''
|
|
self.funcs = []
|
|
self.cols = []
|
|
self.interval: str = ''
|
|
|
|
|
|
class UsedTsma:
|
|
TS_MIN = '-9223372036854775808'
|
|
TS_MAX = '9223372036854775806'
|
|
TSMA_RES_STB_POSTFIX = '_tsma_res_stb_'
|
|
|
|
def __init__(self) -> None:
|
|
self.name = '' # tsma name or table name
|
|
self.time_range_start: float = float(UsedTsma.TS_MIN)
|
|
self.time_range_end: float = float(UsedTsma.TS_MAX)
|
|
self.is_tsma_ = False
|
|
|
|
def __eq__(self, __value: object) -> bool:
|
|
if isinstance(__value, self.__class__):
|
|
return self.name == __value.name \
|
|
and self.time_range_start == __value.time_range_start \
|
|
and self.time_range_end == __value.time_range_end \
|
|
and self.is_tsma_ == __value.is_tsma_
|
|
else:
|
|
return False
|
|
|
|
def __ne__(self, __value: object) -> bool:
|
|
return not self.__eq__(__value)
|
|
|
|
def __str__(self) -> str:
|
|
return "%s: from %s to %s is_tsma: %d" % (self.name, self.time_range_start, self.time_range_end, self.is_tsma_)
|
|
|
|
def __repr__(self) -> str:
|
|
return self.__str__()
|
|
|
|
def setIsTsma(self):
|
|
self.is_tsma_ = self.name.endswith(self.TSMA_RES_STB_POSTFIX)
|
|
if not self.is_tsma_:
|
|
self.is_tsma_ = len(self.name) == 32 # for tsma output child table
|
|
|
|
class TSMAQueryContext:
|
|
def __init__(self) -> None:
|
|
self.sql = ''
|
|
self.used_tsmas: List[UsedTsma] = []
|
|
self.ignore_tsma_check_ = False
|
|
self.ignore_res_order_ = False
|
|
|
|
def __eq__(self, __value) -> bool:
|
|
if isinstance(__value, self.__class__):
|
|
if self.ignore_tsma_check_ or __value.ignore_tsma_check_:
|
|
return True
|
|
if len(self.used_tsmas) != len(__value.used_tsmas):
|
|
return False
|
|
for used_tsma1, used_tsma2 in zip(self.used_tsmas, __value.used_tsmas):
|
|
if not used_tsma1 == used_tsma2:
|
|
return False
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
def __ne__(self, __value: object) -> bool:
|
|
return self.__eq__(__value)
|
|
|
|
def __str__(self) -> str:
|
|
return str(self.used_tsmas)
|
|
|
|
def has_tsma(self) -> bool:
|
|
for tsma in self.used_tsmas:
|
|
if tsma.is_tsma_:
|
|
return True
|
|
return False
|
|
|
|
|
|
class TSMAQCBuilder:
|
|
def __init__(self) -> None:
|
|
self.qc_: TSMAQueryContext = TSMAQueryContext()
|
|
|
|
def get_qc(self) -> TSMAQueryContext:
|
|
return self.qc_
|
|
|
|
def with_sql(self, sql: str):
|
|
self.qc_.sql = sql
|
|
return self
|
|
|
|
def to_timestamp(self, ts: str) -> float:
|
|
if ts == UsedTsma.TS_MAX or ts == UsedTsma.TS_MIN:
|
|
return float(ts)
|
|
tdSql.query(
|
|
"select to_timestamp('%s', 'yyyy-mm-dd hh24-mi-ss.ms')" % (ts))
|
|
res = tdSql.queryResult[0][0]
|
|
return res.timestamp() * 1000
|
|
|
|
def md5(self, buf: str) -> str:
|
|
tdSql.query(f'select md5("{buf}")')
|
|
res = tdSql.queryResult[0][0]
|
|
return res
|
|
|
|
def should_query_with_table(self, tb_name: str, ts_begin: str = UsedTsma.TS_MIN, ts_end: str = UsedTsma.TS_MAX) -> 'TSMAQCBuilder':
|
|
used_tsma: UsedTsma = UsedTsma()
|
|
used_tsma.name = tb_name
|
|
used_tsma.time_range_start = self.to_timestamp(ts_begin)
|
|
used_tsma.time_range_end = self.to_timestamp(ts_end)
|
|
used_tsma.is_tsma_ = False
|
|
self.qc_.used_tsmas.append(used_tsma)
|
|
return self
|
|
|
|
def should_query_with_tsma_ctb(self, db_name: str, tsma_name: str, ctb_name: str, ts_begin: str = UsedTsma.TS_MIN, ts_end: str = UsedTsma.TS_MAX) -> 'TSMAQCBuilder':
|
|
used_tsma: UsedTsma = UsedTsma()
|
|
name = f'1.{db_name}.{tsma_name}_{ctb_name}'
|
|
used_tsma.name = self.md5(name)
|
|
used_tsma.time_range_start = self.to_timestamp(ts_begin)
|
|
used_tsma.time_range_end = self.to_timestamp(ts_end)
|
|
used_tsma.is_tsma_ = True
|
|
self.qc_.used_tsmas.append(used_tsma)
|
|
return self
|
|
|
|
def ignore_query_table(self):
|
|
self.qc_.ignore_tsma_check_ = True
|
|
return self
|
|
|
|
def ignore_res_order(self, ignore: bool):
|
|
self.qc_.ignore_res_order_ = ignore
|
|
return self
|
|
|
|
def should_query_with_tsma(self, tsma_name: str, ts_begin: str = UsedTsma.TS_MIN, ts_end: str = UsedTsma.TS_MAX, child_tb: bool = False) -> 'TSMAQCBuilder':
|
|
used_tsma: UsedTsma = UsedTsma()
|
|
if child_tb:
|
|
used_tsma.name = tsma_name
|
|
else:
|
|
used_tsma.name = tsma_name + UsedTsma.TSMA_RES_STB_POSTFIX
|
|
used_tsma.time_range_start = self.to_timestamp(ts_begin)
|
|
used_tsma.time_range_end = self.to_timestamp(ts_end)
|
|
used_tsma.is_tsma_ = True
|
|
self.qc_.used_tsmas.append(used_tsma)
|
|
return self
|
|
|
|
|
|
class TSMATester:
|
|
def __init__(self, tdSql: TDSql) -> None:
|
|
self.tsmas = []
|
|
self.tdSql: TDSql = tdSql
|
|
|
|
def explain_sql(self, sql: str):
|
|
tdSql.execute("alter local 'querySmaOptimize' '1'")
|
|
sql = "explain verbose true " + sql
|
|
tdSql.query(sql, queryTimes=1)
|
|
res = self.tdSql.queryResult
|
|
if self.tdSql.queryResult is None:
|
|
raise
|
|
return res
|
|
|
|
def get_tsma_query_ctx(self, sql: str):
|
|
explain_res = self.explain_sql(sql)
|
|
query_ctx: TSMAQueryContext = TSMAQueryContext()
|
|
query_ctx.sql = sql
|
|
query_ctx.used_tsmas = []
|
|
used_tsma: UsedTsma = UsedTsma()
|
|
for row in explain_res:
|
|
row = str(row)
|
|
if len(used_tsma.name) == 0:
|
|
idx = row.find("Table Scan on ")
|
|
if idx >= 0:
|
|
words = row[idx:].split(' ')
|
|
used_tsma.name = words[3]
|
|
used_tsma.setIsTsma()
|
|
else:
|
|
idx = row.find('Time Range:')
|
|
if idx >= 0:
|
|
row = row[idx:].split('[')[1]
|
|
row = row.split(']')[0]
|
|
words = row.split(',')
|
|
used_tsma.time_range_start = float(words[0].strip())
|
|
used_tsma.time_range_end = float(words[1].strip())
|
|
query_ctx.used_tsmas.append(used_tsma)
|
|
used_tsma = UsedTsma()
|
|
|
|
deduplicated_tsmas: list[UsedTsma] = []
|
|
if len(query_ctx.used_tsmas) > 0:
|
|
deduplicated_tsmas.append(query_ctx.used_tsmas[0])
|
|
for tsma in query_ctx.used_tsmas:
|
|
if tsma == deduplicated_tsmas[-1]:
|
|
continue
|
|
else:
|
|
deduplicated_tsmas.append(tsma)
|
|
query_ctx.used_tsmas = deduplicated_tsmas
|
|
|
|
return query_ctx
|
|
|
|
def check_explain(self, sql: str, expect: TSMAQueryContext) -> TSMAQueryContext:
|
|
query_ctx = self.get_tsma_query_ctx(sql)
|
|
if not query_ctx == expect:
|
|
tdLog.exit('check explain failed for sql: %s \nexpect: %s \nactual: %s' % (
|
|
sql, str(expect), str(query_ctx)))
|
|
elif expect.has_tsma():
|
|
tdLog.debug('check explain succeed for sql: %s \ntsma: %s' %
|
|
(sql, str(expect.used_tsmas)))
|
|
has_tsma = False
|
|
for tsma in query_ctx.used_tsmas:
|
|
has_tsma = has_tsma or tsma.is_tsma_
|
|
if not has_tsma and len(query_ctx.used_tsmas) > 1:
|
|
tdLog.exit(
|
|
f'explain err for sql: {sql}, has multi non tsmas, {query_ctx.used_tsmas}')
|
|
return query_ctx
|
|
|
|
def check_result(self, sql: str, skip_order: bool = False):
|
|
tdSql.execute("alter local 'querySmaOptimize' '1'")
|
|
tsma_res = tdSql.getResult(sql)
|
|
|
|
tdSql.execute("alter local 'querySmaOptimize' '0'")
|
|
no_tsma_res = tdSql.getResult(sql)
|
|
|
|
if no_tsma_res is None or tsma_res is None:
|
|
if no_tsma_res != tsma_res:
|
|
tdLog.exit("comparing tsma res for: %s got different rows of result: without tsma: %s, with tsma: %s" % (
|
|
sql, str(no_tsma_res), str(tsma_res)))
|
|
else:
|
|
return
|
|
|
|
if len(no_tsma_res) != len(tsma_res):
|
|
tdLog.exit("comparing tsma res for: %s got different rows of result: \nwithout tsma: %s\nwith tsma: %s" % (
|
|
sql, str(no_tsma_res), str(tsma_res)))
|
|
if skip_order:
|
|
try:
|
|
no_tsma_res.sort(
|
|
key=lambda x: [v is None for v in x] + list(x))
|
|
tsma_res.sort(key=lambda x: [v is None for v in x] + list(x))
|
|
except Exception as e:
|
|
tdLog.exit("comparing tsma res for: %s got different data: \nno tsma res: %s \n tsma res: %s err: %s" % (
|
|
sql, str(no_tsma_res), str(tsma_res), str(e)))
|
|
|
|
for row_no_tsma, row_tsma in zip(no_tsma_res, tsma_res):
|
|
if row_no_tsma != row_tsma:
|
|
tdLog.exit("comparing tsma res for: %s got different row data: no tsma row: %s, tsma row: %s \nno tsma res: %s \n tsma res: %s" % (
|
|
sql, str(row_no_tsma), str(row_tsma), str(no_tsma_res), str(tsma_res)))
|
|
tdLog.info('result check succeed for sql: %s. \n tsma-res: %s. \nno_tsma-res: %s' %
|
|
(sql, str(tsma_res), str(no_tsma_res)))
|
|
|
|
def check_sql(self, sql: str, expect: TSMAQueryContext):
|
|
tdLog.debug(f"start to check sql: {sql}")
|
|
actual_ctx = self.check_explain(sql, expect=expect)
|
|
tdLog.debug(f"ctx: {actual_ctx}")
|
|
if actual_ctx.has_tsma():
|
|
self.check_result(sql, expect.ignore_res_order_)
|
|
|
|
def check_sqls(self, sqls, expects):
|
|
for sql, query_ctx in zip(sqls, expects):
|
|
self.check_sql(sql, query_ctx)
|
|
|
|
|
|
class TSMATesterSQLGeneratorOptions:
|
|
def __init__(self) -> None:
|
|
self.ts_min: int = 1537146000000 - 1000 * 60 * 60
|
|
self.ts_max: int = 1537150999000 + 1000 * 60 * 60
|
|
self.times: int = 100
|
|
self.pk_col: str = 'ts'
|
|
self.column_prefix: str = 'c'
|
|
self.column_num: int = 9 # c1 - c10
|
|
self.tags_prefix: str = 't'
|
|
self.tag_num: int = 6 # t1 - t6
|
|
self.str_tag_idx: List = [2, 3]
|
|
self.child_table_name_prefix: str = 't'
|
|
self.child_table_num: int = 10 # t0 - t9
|
|
self.interval: bool = False
|
|
# 70% generating a partition by, 30% no partition by, same as group by
|
|
self.partition_by: bool = False
|
|
self.group_by: bool = False
|
|
# generating no ts range condition is also possible
|
|
self.where_ts_range: bool = False
|
|
self.where_tbname_func: bool = False
|
|
self.where_tag_func: bool = False
|
|
self.where_col_func: bool = False
|
|
self.slimit_max = 10
|
|
self.limit_max = 10
|
|
self.norm_tb = False
|
|
|
|
|
|
class TSMATesterSQLGeneratorRes:
|
|
def __init__(self):
|
|
self.has_where_ts_range: bool = False
|
|
self.has_interval: bool = False
|
|
self.partition_by: bool = False
|
|
self.group_by: bool = False
|
|
self.has_slimit: bool = False
|
|
self.has_limit: bool = False
|
|
self.has_user_order_by: bool = False
|
|
|
|
def can_ignore_res_order(self):
|
|
return not (self.has_limit and self.has_slimit)
|
|
|
|
|
|
class TSMATestSQLGenerator:
|
|
def __init__(self, opts: TSMATesterSQLGeneratorOptions = TSMATesterSQLGeneratorOptions()):
|
|
self.db_name_: str = ''
|
|
self.tb_name_: str = ''
|
|
self.ts_scan_range_: List[float] = [
|
|
float(UsedTsma.TS_MIN), float(UsedTsma.TS_MAX)]
|
|
self.agg_funcs_: List[str] = []
|
|
self.tsmas_: List[TSMA] = [] # currently created tsmas
|
|
self.opts_: TSMATesterSQLGeneratorOptions = opts
|
|
self.res_: TSMATesterSQLGeneratorRes = TSMATesterSQLGeneratorRes()
|
|
|
|
self.select_list_: List[str] = []
|
|
self.where_list_: List[str] = []
|
|
self.group_or_partition_by_list: List[str] = []
|
|
self.interval: str = ''
|
|
|
|
def get_depth_one_str_funcs(self, name: str) -> List[str]:
|
|
concat1 = f'CONCAT({name}, "_concat")'
|
|
concat2 = f'CONCAT({name}, {name})'
|
|
concat3 = f'CONCAT({name}, {name}, {name})'
|
|
start = random.randint(1, 3)
|
|
len = random.randint(0, 3)
|
|
substr = f'SUBSTR({name}, {start}, {len})'
|
|
lower = f'LOWER({name})'
|
|
ltrim = f'LTRIM({name})'
|
|
return [concat1, concat2, concat3, substr, substr, lower, lower, ltrim, name]
|
|
|
|
def generate_depthed_str_func(self, name: str, depth: int) -> str:
|
|
if depth == 1:
|
|
return random.choice(self.get_depth_one_str_funcs(name))
|
|
name = self.generate_depthed_str_func(name, depth - 1)
|
|
return random.choice(self.get_depth_one_str_funcs(name))
|
|
|
|
def generate_str_func(self, column_name: str, depth: int = 0) -> str:
|
|
if depth == 0:
|
|
depth = random.randint(1, 3)
|
|
|
|
ret = self.generate_depthed_str_func(column_name, depth)
|
|
tdLog.debug(f'generating str func: {ret}')
|
|
return ret
|
|
|
|
def get_random_type(self, funcs):
|
|
rand: int = randrange(1, len(funcs))
|
|
return funcs[rand-1]()
|
|
|
|
def generate_select_list(self, user_select_list: str, partition_by_list: str):
|
|
res = user_select_list
|
|
if self.res_.has_interval and random.random() < 0.8:
|
|
res = res + ',_wstart, _wend'
|
|
if self.res_.partition_by or self.res_.group_by and random.random() < 0.8:
|
|
res = res + f',{partition_by_list}'
|
|
return res
|
|
|
|
def generate_order_by(self, user_order_by: str, partition_by_list: str):
|
|
auto_order_by = 'ORDER BY'
|
|
has_limit = self.res_.has_limit or self.res_.has_slimit
|
|
if has_limit and (self.res_.group_by or self.res_.partition_by):
|
|
auto_order_by = f'{auto_order_by} {partition_by_list},'
|
|
if has_limit and self.res_.has_interval:
|
|
auto_order_by = f'{auto_order_by} _wstart, _wend,'
|
|
if len(user_order_by) > 0:
|
|
self.res_.has_user_order_by = True
|
|
auto_order_by = f'{auto_order_by} {user_order_by},'
|
|
if auto_order_by == 'ORDER BY':
|
|
return ''
|
|
else:
|
|
return auto_order_by[:-1]
|
|
|
|
def generate_one(self, select_list: str, possible_tbs: List, order_by_list: str, interval_list: List[str] = []) -> str:
|
|
tb = random.choice(possible_tbs)
|
|
where = self.generate_where()
|
|
interval = self.generate_interval(interval_list)
|
|
(partition_by, partition_by_list) = self.generate_partition_by()
|
|
limit = self.generate_limit()
|
|
auto_select_list = self.generate_select_list(
|
|
select_list, partition_by_list)
|
|
order_by = self.generate_order_by(order_by_list, partition_by_list)
|
|
sql = f"SELECT {auto_select_list} FROM {tb} {where} {partition_by} {partition_by_list} {interval} {order_by} {limit}"
|
|
tdLog.debug(sql)
|
|
return sql
|
|
|
|
def can_ignore_res_order(self):
|
|
return self.res_.can_ignore_res_order()
|
|
|
|
def generate_where(self) -> str:
|
|
v = random.random()
|
|
where = ''
|
|
if not self.opts_.norm_tb:
|
|
if v < 0.2:
|
|
where = f'{self.generate_tbname_where()}'
|
|
elif v < 0.5:
|
|
where = f'{self.generate_tag_where()}'
|
|
elif v < 0.7:
|
|
op = random.choice(['AND', 'OR'])
|
|
where = f'{self.generate_tbname_where()} {op} {self.generate_tag_where()}'
|
|
ts_where = self.generate_ts_where_range()
|
|
if len(ts_where) > 0 or len(where) > 0:
|
|
op = ''
|
|
if len(where) > 0 and len(ts_where) > 0:
|
|
op = random.choice(['AND', 'AND', 'AND', 'AND', 'OR'])
|
|
return f'WHERE {ts_where} {op} {where}'
|
|
return ''
|
|
|
|
def generate_str_equal_operator(self, column_name: str, opts: List) -> str:
|
|
opt = random.choice(opts)
|
|
return f'{column_name} = "{opt}"'
|
|
|
|
# TODO support it
|
|
def generate_str_in_operator(self, column_name: str, opts: List) -> str:
|
|
opt = random.choice(opts)
|
|
IN = f'"{",".join(opts)}"'
|
|
return f'{column_name} in ({IN})'
|
|
|
|
def generate_str_like_operator(self, column_name: str, opts: List) -> str:
|
|
opt = random.choice(opts)
|
|
return f'{column_name} like "{opt}"'
|
|
|
|
def generate_tbname_where(self) -> str:
|
|
tbs = []
|
|
for idx in range(1, self.opts_.tag_num + 1):
|
|
tbs.append(f'{self.opts_.child_table_name_prefix}{idx}')
|
|
|
|
if random.random() < 0.5:
|
|
return self.generate_str_equal_operator('tbname', tbs)
|
|
else:
|
|
return self.generate_str_like_operator('tbname', ['t%', '%2'])
|
|
|
|
def generate_tag_where(self) -> str:
|
|
idx = random.randrange(1, self.opts_.tag_num + 1)
|
|
if random.random() < 0.5 and idx in self.opts_.str_tag_idx:
|
|
if random.random() < 0.5:
|
|
return self.generate_str_equal_operator(f'{self.opts_.tags_prefix}{idx}', [f'tb{random.randint(1,100)}'])
|
|
else:
|
|
return self.generate_str_like_operator(f'{self.opts_.tags_prefix}{idx}', ['%1', 'tb%', 'tb1%', '%1%'])
|
|
else:
|
|
operator = random.choice(['>', '>=', '<', '<=', '=', '!='])
|
|
val = random.randint(1, 100)
|
|
return f'{self.opts_.tags_prefix}{idx} {operator} {val}'
|
|
|
|
def generate_timestamp(self, min: float = -1, max: float = 0) -> int:
|
|
milliseconds_aligned: float = random.randint(int(min), int(max))
|
|
seconds_aligned = int(milliseconds_aligned / 1000) * 1000
|
|
if seconds_aligned < min:
|
|
seconds_aligned = int(min)
|
|
minutes_aligned = int(milliseconds_aligned / 1000 / 60) * 1000 * 60
|
|
if minutes_aligned < min:
|
|
minutes_aligned = int(min)
|
|
hour_aligned = int(milliseconds_aligned / 1000 /
|
|
60 / 60) * 1000 * 60 * 60
|
|
if hour_aligned < min:
|
|
hour_aligned = int(min)
|
|
|
|
return random.choice([milliseconds_aligned, seconds_aligned, seconds_aligned, minutes_aligned, minutes_aligned, hour_aligned, hour_aligned])
|
|
|
|
def generate_ts_where_range(self):
|
|
if not self.opts_.where_ts_range:
|
|
return ''
|
|
left_operators = ['>', '>=', '']
|
|
right_operators = ['<', '<=', '']
|
|
left_operator = left_operators[random.randrange(0, 3)]
|
|
right_operator = right_operators[random.randrange(0, 3)]
|
|
a = ''
|
|
left_value = None
|
|
if left_operator:
|
|
left_value = self.generate_timestamp(
|
|
self.opts_.ts_min, self.opts_.ts_max)
|
|
a += f'{self.opts_.pk_col} {left_operator} {left_value}'
|
|
if right_operator:
|
|
if left_value:
|
|
start = left_value
|
|
else:
|
|
start = self.opts_.ts_min
|
|
right_value = self.generate_timestamp(start, self.opts_.ts_max)
|
|
if left_operator:
|
|
a += ' AND '
|
|
a += f'{self.opts_.pk_col} {right_operator} {right_value}'
|
|
# tdLog.debug(f'{self.opts_.pk_col} range with: {a}')
|
|
if len(a) > 0:
|
|
self.res_.has_where_ts_range = True
|
|
return a
|
|
|
|
def generate_limit(self) -> str:
|
|
ret = ''
|
|
can_have_slimit = self.res_.partition_by or self.res_.group_by
|
|
if can_have_slimit:
|
|
if random.random() < 0.4:
|
|
ret = f'SLIMIT {random.randint(0, self.opts_.slimit_max)}'
|
|
self.res_.has_slimit = True
|
|
if random.random() < 0.4:
|
|
self.res_.has_limit = True
|
|
ret = ret + f' LIMIT {random.randint(0, self.opts_.limit_max)}'
|
|
return ret
|
|
|
|
## if offset is True, offset cannot be the same as interval
|
|
def generate_random_offset_sliding(self, interval: str, offset: bool = False) -> str:
|
|
unit = interval[-1]
|
|
hasUnit = unit.isalpha()
|
|
if not hasUnit:
|
|
start = 1
|
|
if offset:
|
|
start = 2
|
|
ret: int = int(int(interval) / random.randint(start, 5))
|
|
return str(ret)
|
|
return ''
|
|
|
|
# add sliding offset
|
|
def generate_interval(self, intervals: List[str]) -> str:
|
|
if not self.opts_.interval:
|
|
return ''
|
|
if random.random() < 0.4: # no interval
|
|
return ''
|
|
value = random.choice(intervals)
|
|
self.res_.has_interval = True
|
|
has_offset = False
|
|
offset = ''
|
|
has_sliding = False
|
|
sliding = ''
|
|
num: int = int(value[:-1])
|
|
unit = value[-1]
|
|
if has_offset and num > 1:
|
|
offset = f', {self.generate_random_offset_sliding(value, True)}'
|
|
if has_sliding:
|
|
sliding = f'sliding({self.generate_random_offset_sliding(value)})'
|
|
return f'INTERVAL({value} {offset}) {sliding}'
|
|
|
|
def generate_tag_list(self):
|
|
used_tag_num = random.randrange(1, self.opts_.tag_num + 1)
|
|
ret = ''
|
|
for _ in range(used_tag_num):
|
|
tag_idx = random.randint(1, self.opts_.tag_num)
|
|
tag_name = self.opts_.tags_prefix + f'{tag_idx}'
|
|
if random.random() < 0.5 and tag_idx in self.opts_.str_tag_idx:
|
|
tag_func = self.generate_str_func(tag_name, 2)
|
|
else:
|
|
tag_func = tag_name
|
|
ret = ret + f'{tag_func},'
|
|
return ret[:-1]
|
|
|
|
def generate_tbname_tag_list(self):
|
|
tag_num = random.randrange(1, self.opts_.tag_num)
|
|
ret = ''
|
|
tbname_idx = random.randint(0, tag_num + 1)
|
|
for i in range(tag_num + 1):
|
|
if i == tbname_idx:
|
|
ret = ret + 'tbname,'
|
|
else:
|
|
tag_idx = random.randint(1, self.opts_.tag_num)
|
|
ret = ret + self.opts_.tags_prefix + f'{tag_idx},'
|
|
return ret[:-1]
|
|
|
|
def generate_partition_by(self):
|
|
if not self.opts_.partition_by and not self.opts_.group_by:
|
|
return ('', '')
|
|
# no partition or group
|
|
if random.random() < 0.3:
|
|
return ('', '')
|
|
ret = ''
|
|
rand = random.random()
|
|
if rand < 0.4:
|
|
if random.random() < 0.5:
|
|
ret = self.generate_str_func('tbname', 3)
|
|
else:
|
|
ret = 'tbname'
|
|
elif rand < 0.8:
|
|
ret = self.generate_tag_list()
|
|
else:
|
|
# tbname and tag
|
|
ret = self.generate_tbname_tag_list()
|
|
# tdLog.debug(f'partition by: {ret}')
|
|
if self.res_.has_interval or random.random() < 0.5:
|
|
self.res_.partition_by = True
|
|
return (str('PARTITION BY'), f'{ret}')
|
|
else:
|
|
self.res_.group_by = True
|
|
return (str('GROUP BY'), f'{ret}')
|
|
|
|
def generate_where_tbname(self) -> str:
|
|
return self.generate_str_func('tbname')
|
|
|
|
def generate_where_tag(self) -> str:
|
|
# tag_idx = random.randint(1, self.opts_.tag_num)
|
|
# tag = self.opts_.tags_prefix + str(tag_idx)
|
|
return self.generate_str_func('t3')
|
|
|
|
def generate_where_conditions(self) -> str:
|
|
|
|
pass
|
|
|
|
# generate func in tsmas(select list)
|
|
def _generate_agg_func_for_select(self) -> str:
|
|
pass
|
|
|
|
# order by, limit, having, subquery...
|
|
|
|
|
|
class TDTestCase:
|
|
updatecfgDict = {'asynclog': 0, 'ttlUnit': 1, 'ttlPushInterval': 5, 'ratioOfVnodeStreamThrea': 4, 'maxTsmaNum': 8}
|
|
|
|
def __init__(self):
|
|
self.vgroups = 4
|
|
self.ctbNum = 10
|
|
self.rowsPerTbl = 10000
|
|
self.duraion = '1h'
|
|
|
|
def init(self, conn, logSql, replicaVar=1):
|
|
self.replicaVar = int(replicaVar)
|
|
tdLog.debug(f"start to excute {__file__}")
|
|
tdSql.init(conn.cursor(), False)
|
|
self.tsma_tester: TSMATester = TSMATester(tdSql)
|
|
self.tsma_sql_generator: TSMATestSQLGenerator = TSMATestSQLGenerator()
|
|
|
|
def create_database(self, tsql, dbName, dropFlag=1, vgroups=2, replica=1, duration: str = '1d'):
|
|
if dropFlag == 1:
|
|
tsql.execute("drop database if exists %s" % (dbName))
|
|
|
|
tsql.execute("create database if not exists %s vgroups %d replica %d duration %s" % (
|
|
dbName, vgroups, replica, duration))
|
|
tdLog.debug("complete to create database %s" % (dbName))
|
|
return
|
|
|
|
def create_stable(self, tsql, paraDict):
|
|
colString = tdCom.gen_column_type_str(
|
|
colname_prefix=paraDict["colPrefix"], column_elm_list=paraDict["colSchema"])
|
|
tagString = tdCom.gen_tag_type_str(
|
|
tagname_prefix=paraDict["tagPrefix"], tag_elm_list=paraDict["tagSchema"])
|
|
sqlString = f"create table if not exists %s.%s (%s) tags (%s)" % (
|
|
paraDict["dbName"], paraDict["stbName"], colString, tagString)
|
|
tdLog.debug("%s" % (sqlString))
|
|
tsql.execute(sqlString)
|
|
return
|
|
|
|
def create_ctable(self, tsql=None, dbName='dbx', stbName='stb', ctbPrefix='ctb', ctbNum=1, ctbStartIdx=0):
|
|
for i in range(ctbNum):
|
|
sqlString = "create table %s.%s%d using %s.%s tags(%d, 'tb%d', 'tb%d', %d, %d, %d)" % (dbName, ctbPrefix, i+ctbStartIdx, dbName, stbName, (i+ctbStartIdx) % 5, i+ctbStartIdx + random.randint(
|
|
1, 100), i+ctbStartIdx + random.randint(1, 100), i+ctbStartIdx + random.randint(1, 100), i+ctbStartIdx + random.randint(1, 100), i+ctbStartIdx + random.randint(1, 100))
|
|
tsql.execute(sqlString)
|
|
|
|
tdLog.debug("complete to create %d child tables by %s.%s" %
|
|
(ctbNum, dbName, stbName))
|
|
return
|
|
|
|
def init_normal_tb(self, tsql, db_name: str, tb_name: str, rows: int, start_ts: int, ts_step: int):
|
|
sql = 'CREATE TABLE %s.%s (ts timestamp, c1 INT, c2 INT, c3 INT, c4 double, c5 VARCHAR(255))' % (
|
|
db_name, tb_name)
|
|
tsql.execute(sql)
|
|
sql = 'INSERT INTO %s.%s values' % (db_name, tb_name)
|
|
for j in range(rows):
|
|
sql += f'(%d, %d,%d,%d,{random.random()},"varchar_%d"),' % (start_ts + j * ts_step + randrange(500), j %
|
|
10 + randrange(200), j % 10, j % 10, j % 10 + randrange(100))
|
|
tsql.execute(sql)
|
|
|
|
def insert_data(self, tsql, dbName, ctbPrefix, ctbNum, rowsPerTbl, batchNum, startTs, tsStep):
|
|
tdLog.debug("start to insert data ............")
|
|
tsql.execute("use %s" % dbName)
|
|
pre_insert = "insert into "
|
|
sql = pre_insert
|
|
|
|
for i in range(ctbNum):
|
|
rowsBatched = 0
|
|
sql += " %s.%s%d values " % (dbName, ctbPrefix, i)
|
|
for j in range(rowsPerTbl):
|
|
if (i < ctbNum/2):
|
|
sql += "(%d, %d, %d, %d,%d,%d,%d,true,'binary%d', 'nchar%d') " % (startTs + j*tsStep + randrange(
|
|
500), j % 10 + randrange(100), j % 10 + randrange(200), j % 10, j % 10, j % 10, j % 10, j % 10, j % 10)
|
|
else:
|
|
sql += "(%d, %d, NULL, %d,NULL,%d,%d,true,'binary%d', 'nchar%d') " % (
|
|
startTs + j*tsStep + randrange(500), j % 10, j % 10, j % 10, j % 10, j % 10, j % 10)
|
|
rowsBatched += 1
|
|
if ((rowsBatched == batchNum) or (j == rowsPerTbl - 1)):
|
|
tsql.execute(sql)
|
|
rowsBatched = 0
|
|
if j < rowsPerTbl - 1:
|
|
sql = "insert into %s.%s%d values " % (dbName, ctbPrefix, i)
|
|
else:
|
|
sql = "insert into "
|
|
if sql != pre_insert:
|
|
tsql.execute(sql)
|
|
tdLog.debug("insert data ............ [OK]")
|
|
return
|
|
|
|
def init_data(self, ctb_num: int = 10, rows_per_ctb: int = 10000, start_ts: int = 1537146000000, ts_step: int = 500):
|
|
tdLog.printNoPrefix(
|
|
"======== prepare test env include database, stable, ctables, and insert data: ")
|
|
paraDict = {'dbName': 'test',
|
|
'dropFlag': 1,
|
|
'vgroups': 2,
|
|
'stbName': 'meters',
|
|
'colPrefix': 'c',
|
|
'tagPrefix': 't',
|
|
'colSchema': [{'type': 'INT', 'count': 1}, {'type': 'BIGINT', 'count': 1}, {'type': 'FLOAT', 'count': 1}, {'type': 'DOUBLE', 'count': 1}, {'type': 'smallint', 'count': 1}, {'type': 'tinyint', 'count': 1}, {'type': 'bool', 'count': 1}, {'type': 'binary', 'len': 10, 'count': 1}, {'type': 'nchar', 'len': 10, 'count': 1}],
|
|
'tagSchema': [{'type': 'INT', 'count': 1}, {'type': 'nchar', 'len': 20, 'count': 1}, {'type': 'binary', 'len': 20, 'count': 1}, {'type': 'BIGINT', 'count': 1}, {'type': 'smallint', 'count': 1}, {'type': 'DOUBLE', 'count': 1}],
|
|
'ctbPrefix': 't',
|
|
'ctbStartIdx': 0,
|
|
'ctbNum': ctb_num,
|
|
'rowsPerTbl': rows_per_ctb,
|
|
'batchNum': 3000,
|
|
'startTs': start_ts,
|
|
'tsStep': ts_step}
|
|
|
|
paraDict['vgroups'] = self.vgroups
|
|
paraDict['ctbNum'] = self.ctbNum
|
|
paraDict['rowsPerTbl'] = self.rowsPerTbl
|
|
|
|
tdLog.info("create database")
|
|
self.create_database(tsql=tdSql, dbName=paraDict["dbName"], dropFlag=paraDict["dropFlag"],
|
|
vgroups=paraDict["vgroups"], replica=self.replicaVar, duration=self.duraion)
|
|
|
|
tdLog.info("create stb")
|
|
self.create_stable(tsql=tdSql, paraDict=paraDict)
|
|
|
|
tdLog.info("create child tables")
|
|
self.create_ctable(tsql=tdSql, dbName=paraDict["dbName"],
|
|
stbName=paraDict["stbName"], ctbPrefix=paraDict["ctbPrefix"],
|
|
ctbNum=paraDict["ctbNum"], ctbStartIdx=paraDict["ctbStartIdx"])
|
|
self.insert_data(tsql=tdSql, dbName=paraDict["dbName"],
|
|
ctbPrefix=paraDict["ctbPrefix"], ctbNum=paraDict["ctbNum"],
|
|
rowsPerTbl=paraDict["rowsPerTbl"], batchNum=paraDict["batchNum"],
|
|
startTs=paraDict["startTs"], tsStep=paraDict["tsStep"])
|
|
self.init_normal_tb(tdSql, paraDict['dbName'], 'norm_tb',
|
|
paraDict['rowsPerTbl'], paraDict['startTs'], paraDict['tsStep'])
|
|
|
|
def wait_for_tsma_calculation(self, func_list: list, db: str, tb: str, interval: str, tsma_name: str, timeout_seconds: int =600):
|
|
start_time = time.time()
|
|
while True:
|
|
current_time = time.time()
|
|
if current_time - start_time > timeout_seconds:
|
|
error_message = f"Timeout occurred while waiting for TSMA calculation to complete."
|
|
tdLog.exit(error_message)
|
|
sql = 'select %s from %s.%s interval(%s)' % (
|
|
', '.join(func_list), db, tb, interval)
|
|
tdLog.debug(
|
|
f'waiting for tsma {db}.{tsma_name} to be useful with sql {sql}')
|
|
ctx: TSMAQueryContext = self.tsma_tester.get_tsma_query_ctx(sql)
|
|
if ctx.has_tsma():
|
|
if ctx.used_tsmas[0].name == tsma_name + UsedTsma.TSMA_RES_STB_POSTFIX:
|
|
break
|
|
elif len(ctx.used_tsmas[0].name) == 32:
|
|
name = f'1.{db}.{tsma_name}_{tb}'
|
|
if ctx.used_tsmas[0].name == TSMAQCBuilder().md5(name):
|
|
break
|
|
else:
|
|
time.sleep(1)
|
|
else:
|
|
time.sleep(1)
|
|
else:
|
|
time.sleep(1)
|
|
time.sleep(1)
|
|
|
|
def create_tsma(self, tsma_name: str, db: str, tb: str, func_list: list, interval: str, check_tsma_calculation : str=True):
|
|
tdSql.execute('use %s' % db)
|
|
sql = "CREATE TSMA %s ON %s.%s FUNCTION(%s) INTERVAL(%s)" % (
|
|
tsma_name, db, tb, ','.join(func_list), interval)
|
|
tdSql.execute(sql, queryTimes=1)
|
|
if check_tsma_calculation == True:
|
|
self.wait_for_tsma_calculation(func_list, db, tb, interval, tsma_name)
|
|
|
|
def create_error_tsma(self, tsma_name: str, db: str, tb: str, func_list: list, interval: str, expectedErrno: int):
|
|
tdSql.execute('use %s' % db)
|
|
sql = "CREATE TSMA %s ON %s.%s FUNCTION(%s) INTERVAL(%s)" % (
|
|
tsma_name, db, tb, ','.join(func_list), interval)
|
|
tdSql.error(sql, expectedErrno)
|
|
|
|
def create_recursive_tsma(self, base_tsma_name: str, new_tsma_name: str, db: str, interval: str, tb_name: str, func_list: List[str] = ['avg(c1)']):
|
|
tdSql.execute('use %s' % db, queryTimes=1)
|
|
sql = 'CREATE RECURSIVE TSMA %s ON %s.%s INTERVAL(%s)' % (
|
|
new_tsma_name, db, base_tsma_name, interval)
|
|
tdSql.execute(sql, queryTimes=1)
|
|
self.wait_for_tsma_calculation(
|
|
func_list, db, tb_name, interval, new_tsma_name)
|
|
|
|
def drop_tsma(self, tsma_name: str, db: str):
|
|
sql = 'DROP TSMA %s.%s' % (db, tsma_name)
|
|
tdSql.execute(sql, queryTimes=1)
|
|
|
|
def check_explain_res_has_row(self, plan_str_expect: str, explain_output):
|
|
plan_found = False
|
|
for row in explain_output:
|
|
if str(row).find(plan_str_expect) >= 0:
|
|
tdLog.debug("plan: [%s] found in: [%s]" %
|
|
(plan_str_expect, str(row)))
|
|
plan_found = True
|
|
break
|
|
if not plan_found:
|
|
tdLog.exit("plan: %s not found in res: [%s]" % (
|
|
plan_str_expect, str(explain_output)))
|
|
|
|
def check(self, ctxs: List):
|
|
for ctx in ctxs:
|
|
self.tsma_tester.check_sql(ctx.sql, ctx)
|
|
|
|
def test_query_with_tsma(self):
|
|
self.create_tsma('tsma1', 'test', 'meters', ['avg(c1)', 'avg(c2)'], '5m')
|
|
self.create_tsma('tsma2', 'test', 'meters', ['avg(c1)', 'avg(c2)'], '30m')
|
|
self.create_tsma('tsma5', 'test', 'norm_tb', ['avg(c1)', 'avg(c2)'], '10m')
|
|
|
|
self.test_query_with_tsma_interval()
|
|
self.test_query_with_tsma_agg()
|
|
self.test_recursive_tsma()
|
|
self.test_query_interval_sliding()
|
|
self.test_union()
|
|
self.test_query_child_table()
|
|
self.test_skip_tsma_hint()
|
|
self.test_long_tsma_name()
|
|
self.test_long_ctb_name()
|
|
self.test_add_tag_col()
|
|
self.test_modify_col_name_value()
|
|
self.test_alter_tag_val()
|
|
self.test_ins_tsma()
|
|
|
|
def test_ins_tsma(self):
|
|
tdSql.execute('use performance_schema')
|
|
tdSql.query('show performance_schema.tsmas')
|
|
tdSql.checkRows(0)
|
|
tdSql.execute('use test')
|
|
tdSql.query('show test.tsmas')
|
|
tdSql.checkRows(3)
|
|
tdSql.query('select * from information_schema.ins_tsmas')
|
|
tdSql.checkRows(3)
|
|
tdSql.execute('create database dd')
|
|
tdSql.execute('use dd')
|
|
tdSql.execute('create table dd.norm_tb (ts timestamp, c1 int)')
|
|
tdSql.execute('insert into dd.norm_tb values(now, 1)')
|
|
self.create_tsma('tsma_norm_tb_dd', 'dd', 'norm_tb', ['avg(c1)', 'sum(c1)', 'min(c1)'], '10m')
|
|
tdSql.query('show dd.tsmas')
|
|
tdSql.checkRows(1)
|
|
tdSql.query('select * from information_schema.ins_tsmas')
|
|
tdSql.checkRows(4)
|
|
tdSql.query('show test.tsmas')
|
|
tdSql.checkRows(3)
|
|
tdSql.execute('use test')
|
|
tdSql.query('show dd.tsmas')
|
|
tdSql.checkRows(1)
|
|
tdSql.execute('drop database dd')
|
|
tdSql.query('select * from information_schema.ins_tsmas')
|
|
tdSql.checkRows(3)
|
|
tdSql.execute('use test')
|
|
|
|
def test_alter_tag_val(self):
|
|
sql = 'alter table test.t1 set tag t1 = 999'
|
|
tdSql.error(sql, -2147471088)
|
|
|
|
def test_query_interval_sliding(self):
|
|
pass
|
|
|
|
def test_union(self):
|
|
ctxs = []
|
|
sql = 'select avg(c1) from test.meters union select avg(c1) from test.norm_tb'
|
|
ctx = TSMAQCBuilder().with_sql(sql).should_query_with_tsma('tsma2').should_query_with_tsma_ctb('test', 'tsma5', 'norm_tb').get_qc()
|
|
ctxs.append(ctx)
|
|
sql = 'select avg(c1), avg(c2) from test.meters where ts between "2018-09-17 09:00:00.000" and "2018-09-17 10:00:00.000" union select avg(c1), avg(c2) from test.meters where ts between "2018-09-17 09:00:00.200" and "2018-09-17 10:23:19.800"'
|
|
ctxs.append(TSMAQCBuilder().with_sql(sql)
|
|
.should_query_with_tsma('tsma2', '2018-09-17 09:00:00', '2018-09-17 09:59:59:999')
|
|
.should_query_with_table("meters", '2018-09-17 10:00:00', '2018-09-17 10:00:00')
|
|
.should_query_with_table('meters', '2018-09-17 09:00:00.200', '2018-09-17 09:29:59:999')
|
|
.should_query_with_tsma('tsma2', '2018-09-17 09:30:00', '2018-09-17 09:59:59.999')
|
|
.should_query_with_table('meters', '2018-09-17 10:00:00.000', '2018-09-17 10:23:19.800').get_qc())
|
|
self.check(ctxs)
|
|
|
|
tdSql.execute('create database db2')
|
|
tdSql.execute('use db2')
|
|
tdSql.execute('create table db2.norm_tb(ts timestamp, c2 int)')
|
|
tdSql.execute('insert into db2.norm_tb values(now, 1)')
|
|
tdSql.execute('insert into db2.norm_tb values(now, 2)')
|
|
self.create_tsma('tsma_db2_norm_t', 'db2', 'norm_tb', ['avg(c2)', 'last(ts)'], '10m')
|
|
sql = 'select avg(c1) as avg_c1 from test.meters union select avg(c2) from db2.norm_tb order by avg_c1'
|
|
self.check([TSMAQCBuilder().with_sql(sql).should_query_with_tsma('tsma2').should_query_with_tsma_ctb('db2', 'tsma_db2_norm_t', 'norm_tb').get_qc()])
|
|
tdSql.execute('drop database db2')
|
|
tdSql.execute('use test')
|
|
|
|
def test_modify_col_name_value(self):
|
|
tdSql.error('alter table test.norm_tb rename column c1 c1_new', -2147471088) ## tsma must be dropped
|
|
|
|
## modify tag name
|
|
tdSql.error('alter stable test.meters rename tag t1 t1_new;', -2147482637) ## stream must be dropped
|
|
|
|
def test_add_tag_col(self):
|
|
## query with newly add tag will skip all tsmas not have this tag
|
|
tdSql.execute('alter table test.meters add tag tag_new int', queryTimes=1)
|
|
sql = 'select avg(c1) from test.meters partition by tag_new'
|
|
self.check([TSMAQCBuilder().with_sql(sql).should_query_with_table('meters').get_qc()])
|
|
sql = 'select avg(c1) from test.meters partition by abs(tag_new)'
|
|
self.check([TSMAQCBuilder().with_sql(sql).should_query_with_table('meters').get_qc()])
|
|
sql = 'select avg(c1) from test.meters where abs(tag_new) > 100'
|
|
self.check([TSMAQCBuilder().with_sql(sql).should_query_with_table('meters').get_qc()])
|
|
|
|
tdSql.execute('alter table test.meters drop tag tag_new', queryTimes=1)
|
|
|
|
def generate_random_string(self, length):
|
|
letters_and_digits = string.ascii_lowercase
|
|
result_str = ''.join(random.choice(letters_and_digits) for i in range(length))
|
|
return result_str
|
|
|
|
def test_long_tsma_name(self):
|
|
name = self.generate_random_string(178)
|
|
tsma_func_list = ['avg(c2)', 'avg(c3)', 'min(c4)', 'max(c3)', 'sum(c2)', 'count(ts)', 'count(c2)', 'first(c5)', 'last(c5)', 'spread(c2)', 'stddev(c3)', 'hyperloglog(c5)', 'last(ts)']
|
|
self.create_tsma(name, 'test', 'meters', tsma_func_list, '55m')
|
|
sql = 'select last(c5), spread(c2) from test.meters interval(55m)'
|
|
ctx = TSMAQCBuilder().with_sql(sql).should_query_with_tsma(name).get_qc()
|
|
self.check([ctx])
|
|
tdSql.execute(f'drop tsma test.{name}')
|
|
|
|
name = self.generate_random_string(180)
|
|
tdSql.error(f'create tsma {name} on test.meters function({",".join(tsma_func_list)}) interval(1h)', -2147471087)
|
|
|
|
name = self.generate_random_string(179)
|
|
tdSql.error(f'create tsma {name} on test.meters function({",".join(tsma_func_list)}) interval(1h)', -2147471087)
|
|
|
|
name = self.generate_random_string(178)
|
|
self.create_recursive_tsma('tsma1', name, 'test', '60m', 'meters', ['avg(c1)','avg(c2)'])
|
|
sql = 'select avg(c1) from test.meters interval(60m)'
|
|
self.check([TSMAQCBuilder().with_sql(sql).should_query_with_tsma(name).get_qc()])
|
|
|
|
tdSql.execute(f'drop tsma test.{name}')
|
|
|
|
def test_long_ctb_name(self):
|
|
tb_name = self.generate_random_string(192)
|
|
tsma_name = self.generate_random_string(178)
|
|
tdSql.execute('create database db2')
|
|
tdSql.execute('use db2')
|
|
db_name = 'db2'
|
|
tdSql.execute(f'create table {db_name}.{tb_name}(ts timestamp, c2 int)')
|
|
tdSql.execute(f'insert into {db_name}.{tb_name} values(now, 1)')
|
|
tdSql.execute(f'insert into {db_name}.{tb_name} values(now, 2)')
|
|
self.create_tsma(tsma_name, 'db2', tb_name, ['avg(c2)', 'last(ts)'], '10m')
|
|
sql = f'select avg(c2), last(ts) from {db_name}.{tb_name}'
|
|
self.check([TSMAQCBuilder().with_sql(sql).should_query_with_tsma_ctb('db2', tsma_name, tb_name).get_qc()])
|
|
tdSql.execute('drop database db2')
|
|
tdSql.execute('use test')
|
|
|
|
def test_skip_tsma_hint(self):
|
|
ctxs = []
|
|
sql = 'select /*+ skip_tsma()*/avg(c1), avg(c2) from test.meters interval(5m)'
|
|
ctxs.append(TSMAQCBuilder().with_sql(sql).should_query_with_table('meters').get_qc())
|
|
|
|
sql = 'select avg(c1), avg(c2) from test.meters interval(5m)'
|
|
ctxs.append(TSMAQCBuilder().with_sql(sql).should_query_with_tsma('tsma1').get_qc())
|
|
self.check(ctxs)
|
|
|
|
def test_query_child_table(self):
|
|
sql = 'select avg(c1) from test.t1'
|
|
ctx = TSMAQCBuilder().with_sql(sql).should_query_with_tsma_ctb('test', 'tsma2', 't1', UsedTsma.TS_MIN, UsedTsma.TS_MAX).get_qc()
|
|
self.tsma_tester.check_sql(sql, ctx)
|
|
sql = 'select avg(c1) from test.t3'
|
|
ctx = TSMAQCBuilder().with_sql(sql).should_query_with_tsma_ctb('test', 'tsma2', 't3').get_qc()
|
|
self.tsma_tester.check_sql(sql, ctx)
|
|
|
|
def test_recursive_tsma(self):
|
|
tdSql.execute('drop tsma test.tsma2')
|
|
tsma_func_list = ['last(ts)', 'avg(c2)', 'avg(c3)', 'min(c4)', 'max(c3)', 'sum(c2)', 'count(ts)', 'count(c2)', 'first(c5)', 'last(c5)', 'spread(c2)', 'stddev(c3)', 'hyperloglog(c5)']
|
|
select_func_list: List[str] = tsma_func_list.copy()
|
|
select_func_list.append('count(*)')
|
|
self.create_tsma('tsma3', 'test', 'meters', tsma_func_list, '5m')
|
|
self.create_recursive_tsma(
|
|
'tsma3', 'tsma4', 'test', '20m', 'meters', tsma_func_list)
|
|
# now we have 5m, 10m, 30m, 1h 4 tsmas
|
|
sql = 'select avg(c2), "recursive test.tsma4" from test.meters'
|
|
ctx = TSMAQCBuilder().with_sql(sql).should_query_with_tsma(
|
|
'tsma4', UsedTsma.TS_MIN, UsedTsma.TS_MAX).get_qc()
|
|
#time.sleep(999999)
|
|
self.tsma_tester.check_sql(sql, ctx)
|
|
self.check(self.test_query_tsma_all(select_func_list))
|
|
self.create_recursive_tsma(
|
|
'tsma4', 'tsma6', 'test', '1h', 'meters', tsma_func_list)
|
|
ctx = TSMAQCBuilder().with_sql(sql).should_query_with_tsma(
|
|
'tsma6', UsedTsma.TS_MIN, UsedTsma.TS_MAX).get_qc()
|
|
self.tsma_tester.check_sql(sql, ctx)
|
|
|
|
self.check(self.test_query_tsma_all(select_func_list))
|
|
|
|
tdSql.error('drop tsma test.tsma3', -2147482491)
|
|
tdSql.error('drop tsma test.tsma4', -2147482491)
|
|
tdSql.execute('drop tsma test.tsma6')
|
|
tdSql.execute('drop tsma test.tsma4')
|
|
tdSql.execute('drop tsma test.tsma3')
|
|
self.create_tsma('tsma2', 'test', 'meters', ['avg(c1)', 'avg(c2)'], '30m')
|
|
|
|
# test query with dropped tsma tsma4 and tsma6
|
|
sql = 'select avg(c2), "test.tsma2" from test.meters'
|
|
ctx = TSMAQCBuilder().with_sql(sql).should_query_with_tsma(
|
|
'tsma2', UsedTsma.TS_MIN, UsedTsma.TS_MAX).get_qc()
|
|
self.check([ctx])
|
|
|
|
# test recrusive tsma on norm_tb
|
|
tsma_name = 'tsma_recursive_on_norm_tb'
|
|
self.create_recursive_tsma('tsma5', tsma_name, 'test', '20m', 'norm_tb', ['avg(c1)', 'avg(c2)'])
|
|
sql = 'select avg(c1), avg(c2), tbname from test.norm_tb partition by tbname interval(20m)'
|
|
self.check([TSMAQCBuilder().with_sql(sql).should_query_with_tsma_ctb('test', tsma_name, 'norm_tb').get_qc()])
|
|
tdSql.execute(f'drop tsma test.{tsma_name}')
|
|
|
|
def test_query_with_tsma_interval(self):
|
|
self.check(self.test_query_with_tsma_interval_possibly_partition())
|
|
self.check(self.test_query_with_tsma_interval_partition_by_col())
|
|
|
|
def test_query_tsma_all(self, func_list: List = ['avg(c1)', 'avg(c2)']) -> List:
|
|
ctxs = []
|
|
interval_list = ['1s', '5s', '59s', '60s', '1m', '120s', '10m', '20m',
|
|
'30m', '1h', '90m', '2h', '8h', '1d']
|
|
opts: TSMATesterSQLGeneratorOptions = TSMATesterSQLGeneratorOptions()
|
|
opts.interval = True
|
|
opts.where_ts_range = True
|
|
for _ in range(1, ROUND):
|
|
opts.partition_by = True
|
|
opts.group_by = True
|
|
opts.norm_tb = False
|
|
sql_generator = TSMATestSQLGenerator(opts)
|
|
sql = sql_generator.generate_one(
|
|
','.join(func_list), ['test.meters', 'test.meters', 'test.t1', 'test.t9'], '', interval_list)
|
|
ctxs.append(TSMAQCBuilder().with_sql(sql).ignore_query_table(
|
|
).ignore_res_order(sql_generator.can_ignore_res_order()).get_qc())
|
|
|
|
if random.random() > 0.7:
|
|
continue
|
|
opts.partition_by = False
|
|
opts.group_by = False
|
|
opts.norm_tb = True
|
|
sql_generator = TSMATestSQLGenerator(opts)
|
|
sql = sql_generator.generate_one(
|
|
','.join(func_list), ['test.norm_tb', 'test.t5'], '', interval_list)
|
|
ctxs.append(TSMAQCBuilder().with_sql(sql).ignore_query_table(
|
|
).ignore_res_order(sql_generator.can_ignore_res_order()).get_qc())
|
|
return ctxs
|
|
|
|
def test_query_with_tsma_interval_possibly_partition(self,db_name: str = 'test'):
|
|
ctxs: List[TSMAQueryContext] = []
|
|
sql = f'select avg(c1), avg(c2) from {db_name}.meters interval(5m)'
|
|
ctxs.append(TSMAQCBuilder().with_sql(sql)
|
|
.should_query_with_tsma('tsma1', UsedTsma.TS_MIN, UsedTsma.TS_MAX).get_qc())
|
|
|
|
sql = f'select avg(c1), avg(c2) from {db_name}.meters interval(10m)'
|
|
ctxs.append(TSMAQCBuilder().with_sql(sql)
|
|
.should_query_with_tsma('tsma1', UsedTsma.TS_MIN, UsedTsma.TS_MAX).get_qc())
|
|
sql = f'select avg(c1), avg(c2) from {db_name}.meters interval(30m)'
|
|
ctxs.append(TSMAQCBuilder().with_sql(sql)
|
|
.should_query_with_tsma('tsma2', UsedTsma.TS_MIN, UsedTsma.TS_MAX).get_qc())
|
|
sql = f'select avg(c1), avg(c2) from {db_name}.meters interval(60m)'
|
|
ctxs.append(TSMAQCBuilder().with_sql(sql)
|
|
.should_query_with_tsma('tsma2', UsedTsma.TS_MIN, UsedTsma.TS_MAX).get_qc())
|
|
sql = f'select avg(c1), avg(c2) from {db_name}.meters interval(60m, 30m) SLIDING(30m)'
|
|
ctxs.append(TSMAQCBuilder().with_sql(sql)
|
|
.should_query_with_tsma('tsma2', UsedTsma.TS_MIN, UsedTsma.TS_MAX).get_qc())
|
|
sql = f'select avg(c1), avg(c2) from {db_name}.meters interval(60m, 25m) SLIDING(25m)'
|
|
ctxs.append(TSMAQCBuilder().with_sql(sql)
|
|
.should_query_with_tsma('tsma1', UsedTsma.TS_MIN, UsedTsma.TS_MAX).get_qc())
|
|
|
|
sql = f"select avg(c1), avg(c2) from {db_name}.meters where ts >= '2018-09-17 09:00:00.009' and ts < '2018-09-17 10:23:19.665' interval(30m)"
|
|
ctxs.append(TSMAQCBuilder().with_sql(sql)
|
|
.should_query_with_table('meters', '2018-09-17 09:00:00.009', '2018-09-17 09:29:59.999')
|
|
.should_query_with_tsma('tsma2', '2018-09-17 09:30:00', '2018-09-17 09:59:59.999')
|
|
.should_query_with_table('meters', '2018-09-17 10:00:00.000', '2018-09-17 10:23:19.664').get_qc())
|
|
|
|
sql = f"select avg(c1), avg(c2) from {db_name}.meters where ts >= '2018-09-17 09:00:00.009' and ts < '2018-09-17 10:23:19.665' interval(30m, 25m) SLIDING(10m)"
|
|
ctxs.append(TSMAQCBuilder().with_sql(sql)
|
|
.should_query_with_table('meters', '2018-09-17 09:00:00.009', '2018-09-17 09:04:59.999')
|
|
.should_query_with_tsma('tsma1', '2018-09-17 09:05:00', '2018-09-17 09:54:59.999')
|
|
.should_query_with_table('meters', '2018-09-17 09:55:00.000', '2018-09-17 10:23:19.664').get_qc())
|
|
|
|
sql = f"SELECT avg(c1), avg(c2),_wstart, _wend,t3,t4,t5,t2 FROM {db_name}.meters WHERE ts >= '2018-09-17 8:00:00' AND ts < '2018-09-17 09:03:18.334' PARTITION BY t3,t4,t5,t2 INTERVAL(1d);"
|
|
ctxs.append(TSMAQCBuilder().with_sql(sql)
|
|
.should_query_with_table('meters', '2018-09-17 8:00:00', '2018-09-17 09:03:18.333').get_qc())
|
|
ctxs.extend(self.test_query_tsma_all())
|
|
return ctxs
|
|
|
|
def test_query_with_tsma_interval_partition_by_col(self):
|
|
return []
|
|
|
|
def test_query_with_tsma_agg(self):
|
|
self.check(self.test_query_with_tsma_agg_no_group_by())
|
|
self.check(self.test_query_with_tsma_agg_group_by_tbname())
|
|
self.check(self.test_query_with_tsma_with_having())
|
|
|
|
def test_query_with_tsma_agg_no_group_by(self, db_name: str = 'test'):
|
|
ctxs: List[TSMAQueryContext] = []
|
|
sql = f'select avg(c1), avg(c2) from {db_name}.meters'
|
|
ctxs.append(TSMAQCBuilder().with_sql(
|
|
sql).should_query_with_tsma('tsma2').get_qc())
|
|
|
|
sql = f'select avg(c1), avg(c2) from {db_name}.meters where ts between "2018-09-17 09:00:00.000" and "2018-09-17 10:00:00.000"'
|
|
ctxs.append(TSMAQCBuilder().with_sql(sql)
|
|
.should_query_with_tsma('tsma2', '2018-09-17 09:00:00', '2018-09-17 09:59:59:999')
|
|
.should_query_with_table("meters", '2018-09-17 10:00:00', '2018-09-17 10:00:00').get_qc())
|
|
|
|
sql = f'select avg(c1), avg(c2) from {db_name}.meters where ts between "2018-09-17 09:00:00.200" and "2018-09-17 10:23:19.800"'
|
|
ctxs.append(TSMAQCBuilder().with_sql(sql)
|
|
.should_query_with_table('meters', '2018-09-17 09:00:00.200', '2018-09-17 09:29:59:999')
|
|
.should_query_with_tsma('tsma2', '2018-09-17 09:30:00', '2018-09-17 09:59:59.999')
|
|
.should_query_with_table('meters', '2018-09-17 10:00:00.000', '2018-09-17 10:23:19.800').get_qc())
|
|
|
|
sql = f'select avg(c1) + avg(c2), avg(c2) from {db_name}.meters where ts between "2018-09-17 09:00:00.200" and "2018-09-17 10:23:19.800"'
|
|
ctxs.append(TSMAQCBuilder().with_sql(sql)
|
|
.should_query_with_table('meters', '2018-09-17 09:00:00.200', '2018-09-17 09:29:59:999')
|
|
.should_query_with_tsma('tsma2', '2018-09-17 09:30:00', '2018-09-17 09:59:59.999')
|
|
.should_query_with_table('meters', '2018-09-17 10:00:00.000', '2018-09-17 10:23:19.800').get_qc())
|
|
|
|
sql = f'select avg(c1) + avg(c2), avg(c2) + 1 from {db_name}.meters where ts between "2018-09-17 09:00:00.200" and "2018-09-17 10:23:19.800"'
|
|
ctxs.append(TSMAQCBuilder().with_sql(sql)
|
|
.should_query_with_table('meters', '2018-09-17 09:00:00.200', '2018-09-17 09:29:59:999')
|
|
.should_query_with_tsma('tsma2', '2018-09-17 09:30:00', '2018-09-17 09:59:59.999')
|
|
.should_query_with_table('meters', '2018-09-17 10:00:00.000', '2018-09-17 10:23:19.800').get_qc())
|
|
|
|
sql = f"select avg(c1) + 1, avg(c2) from {db_name}.meters where ts >= '2018-09-17 9:30:00.118' and ts < '2018-09-17 10:50:00'"
|
|
ctxs.append(TSMAQCBuilder().with_sql(sql)
|
|
.should_query_with_table('meters', '2018-09-17 9:30:00.118', '2018-09-17 9:59:59.999')
|
|
.should_query_with_tsma('tsma2', '2018-09-17 10:00:00', '2018-09-17 10:29:59.999')
|
|
.should_query_with_tsma('tsma1', '2018-09-17 10:30:00.000', '2018-09-17 10:49:59.999').get_qc())
|
|
|
|
sql = f"select avg(c1), avg(c2) from {db_name}.meters where ts >= '2018-09-17 9:00:00' and ts < '2018-09-17 9:45:00' limit 2"
|
|
ctxs.append(TSMAQCBuilder().with_sql(sql)
|
|
.should_query_with_tsma('tsma2', '2018-09-17 9:00:00', '2018-09-17 9:29:59.999')
|
|
.should_query_with_tsma('tsma1', '2018-09-17 9:30:00', '2018-09-17 9:44:59.999').get_qc())
|
|
|
|
sql = f'select avg(c1) + avg(c2) from {db_name}.meters where tbname like "%t1%"'
|
|
ctxs.append(TSMAQCBuilder().with_sql(
|
|
sql).should_query_with_tsma('tsma2').get_qc())
|
|
|
|
sql = f'select avg(c1), avg(c2) from {db_name}.meters where c1 is not NULL'
|
|
ctxs.append(TSMAQCBuilder().with_sql(
|
|
sql).should_query_with_table('meters').get_qc())
|
|
|
|
sql = f'select avg(c1), avg(c2), spread(c4) from {db_name}.meters'
|
|
ctxs.append(TSMAQCBuilder().with_sql(
|
|
sql).should_query_with_table('meters').get_qc())
|
|
|
|
sql = f'select avg(c1), avg(c2) from {db_name}.meters where tbname = \'t1\''
|
|
ctxs.append(TSMAQCBuilder().with_sql(
|
|
sql).should_query_with_tsma('tsma2').get_qc())
|
|
|
|
sql = f'select avg(c1), avg(c2) from {db_name}.meters where tbname = \'t1\' or tbname = \'t2\''
|
|
ctxs.append(TSMAQCBuilder().with_sql(
|
|
sql).should_query_with_tsma('tsma2').get_qc())
|
|
|
|
sql = f'''select avg(c1), avg(c2) from {db_name}.meters where tbname = 't1' and c1 is not NULL'''
|
|
ctxs.append(TSMAQCBuilder().with_sql(
|
|
sql).should_query_with_table('meters').get_qc())
|
|
|
|
sql = f'select avg(c1+c2) from {db_name}.meters'
|
|
ctxs.append(TSMAQCBuilder().with_sql(
|
|
sql).should_query_with_table('meters').get_qc())
|
|
|
|
sql = f'select avg(c1), avg(c2) from {db_name}.meters where ts >= "2018-09-17 9:25:00" and ts < "2018-09-17 10:00:00" limit 6'
|
|
ctxs.append(TSMAQCBuilder().with_sql(sql).should_query_with_tsma('tsma1', '2018-09-17 9:25:00', '2018-09-17 9:29:59.999')
|
|
.should_query_with_tsma('tsma2', '2018-09-17 9:30:00', '2018-09-17 9:59:59.999').get_qc())
|
|
|
|
return ctxs
|
|
|
|
def test_query_with_tsma_agg_group_by_tbname(self, db_name: str = 'test'):
|
|
ctxs: List[TSMAQueryContext] = []
|
|
sql = f'select avg(c1) as a, avg(c2) as b, tbname from {db_name}.meters group by tbname order by tbname, a, b'
|
|
ctxs.append(TSMAQCBuilder().with_sql(
|
|
sql).should_query_with_tsma('tsma2').get_qc())
|
|
|
|
sql = f'select avg(c1) as a, avg(c2) + 1 as b, tbname from {db_name}.meters where c1 > 10 group by tbname order by tbname, a, b'
|
|
ctxs.append(TSMAQCBuilder().with_sql(
|
|
sql).should_query_with_table('meters').get_qc())
|
|
|
|
sql = f'select avg(c1) + avg(c2) as a, avg(c2) + 1 as b, tbname from {db_name}.meters where ts between "2018-09-17 09:00:00.200" and "2018-09-17 10:23:19.800" group by tbname order by tbname, a, b'
|
|
ctxs.append(TSMAQCBuilder().with_sql(sql)
|
|
.should_query_with_table('meters', '2018-09-17 09:00:00.200', '2018-09-17 09:29:59:999')
|
|
.should_query_with_tsma('tsma2', '2018-09-17 09:30:00', '2018-09-17 09:59:59.999')
|
|
.should_query_with_table('meters', '2018-09-17 10:00:00.000', '2018-09-17 10:23:19.800').get_qc())
|
|
|
|
sql = f'select avg(c1) + avg(c2) + 3 as a, substr(tbname, 1) as c from {db_name}.meters group by substr(tbname, 1) order by c, a'
|
|
ctxs.append(TSMAQCBuilder().with_sql(
|
|
sql).should_query_with_tsma('tsma2').get_qc())
|
|
|
|
sql = f'select avg(c1) + avg(c2) as a, avg(c2) + 1 as b, substr(tbname, 1, 1) as c from {db_name}.meters where ts between "2018-09-17 09:00:00.200" and "2018-09-17 10:23:19.800" group by substr(tbname, 1, 1) order by c, a, b'
|
|
ctxs.append(TSMAQCBuilder().with_sql(sql)
|
|
.should_query_with_table('meters', '2018-09-17 09:00:00.200', '2018-09-17 09:29:59:999')
|
|
.should_query_with_tsma('tsma2', '2018-09-17 09:30:00', '2018-09-17 09:59:59.999')
|
|
.should_query_with_table('meters', '2018-09-17 10:00:00.000', '2018-09-17 10:23:19.800').get_qc())
|
|
|
|
sql = f'select avg(c1), tbname from {db_name}.meters group by tbname having avg(c1) > 0 order by tbname'
|
|
ctxs.append(TSMAQCBuilder().with_sql(
|
|
sql).should_query_with_tsma('tsma2').get_qc())
|
|
sql = f'select avg(c1), tbname from {db_name}.meters group by tbname having avg(c1) > 0 and tbname = "t1"'
|
|
ctxs.append(TSMAQCBuilder().with_sql(
|
|
sql).should_query_with_tsma('tsma2').get_qc())
|
|
|
|
sql = f'select avg(c1), tbname from {db_name}.meters group by tbname having avg(c1) > 0 and tbname = "t1" order by tbname'
|
|
ctxs.append(TSMAQCBuilder().with_sql(
|
|
sql).should_query_with_tsma('tsma2').get_qc())
|
|
|
|
sql = f'select avg(c1) + 1, tbname from {db_name}.meters group by tbname having avg(c1) > 0 and tbname = "t1" order by tbname'
|
|
ctxs.append(TSMAQCBuilder().with_sql(
|
|
sql).should_query_with_tsma('tsma2').get_qc())
|
|
sql = f'select avg(c1) + 1, tbname from {db_name}.meters group by tbname having avg(c1) > 0 and tbname like "t%" order by tbname'
|
|
ctxs.append(TSMAQCBuilder().with_sql(
|
|
sql).should_query_with_tsma('tsma2').get_qc())
|
|
|
|
return ctxs
|
|
|
|
def test_query_with_tsma_with_having(self):
|
|
return []
|
|
|
|
def test_ddl(self):
|
|
self.test_create_tsma()
|
|
self.test_drop_tsma()
|
|
self.test_tb_ddl_with_created_tsma()
|
|
|
|
|
|
def run(self):
|
|
self.init_data()
|
|
self.test_ddl()
|
|
self.test_query_with_tsma()
|
|
# bug to fix
|
|
# self.test_flush_query()
|
|
|
|
#cluster test
|
|
cluster_dnode_list = tdSql.get_cluseter_dnodes()
|
|
clust_dnode_nums = len(cluster_dnode_list)
|
|
if clust_dnode_nums > 1:
|
|
self.test_redistribute_vgroups()
|
|
|
|
def test_create_tsma(self):
|
|
function_name = sys._getframe().f_code.co_name
|
|
tdLog.debug(f'-----{function_name}------')
|
|
self.test_create_tsma_on_stable()
|
|
self.test_create_tsma_on_norm_table()
|
|
self.test_create_tsma_on_child_table()
|
|
self.test_create_recursive_tsma()
|
|
self.test_create_tsma_maxlist_function()
|
|
self.test_create_diffrent_tsma_name()
|
|
self.test_create_illegal_tsma_sql()
|
|
# self.test_drop_stable() ## drop stable and recreate a stable
|
|
# self.test_drop_ctable()
|
|
self.test_drop_db()
|
|
|
|
def wait_query(self, sql: str, expected_row_num: int, timeout_in_seconds: float):
|
|
timeout = timeout_in_seconds
|
|
tdSql.query(sql)
|
|
while timeout > 0 and tdSql.getRows() != expected_row_num:
|
|
tdLog.debug(f'start to wait query: {sql} to return {expected_row_num}, got: {tdSql.getRows()}, remain: {timeout_in_seconds - timeout}')
|
|
time.sleep(1)
|
|
timeout = timeout - 1
|
|
tdSql.query(sql)
|
|
if timeout <= 0:
|
|
tdLog.exit(f'failed to wait query: {sql} to return {expected_row_num} rows timeout: {timeout_in_seconds}s')
|
|
else:
|
|
tdLog.debug(f'wait query succeed: {sql} to return {expected_row_num}, got: {tdSql.getRows()}')
|
|
|
|
def test_drop_tsma(self):
|
|
function_name = sys._getframe().f_code.co_name
|
|
tdLog.debug(f'-----{function_name}------')
|
|
self.create_tsma('tsma1', 'test', 'meters', [
|
|
'avg(c1)', 'avg(c2)'], '5m')
|
|
self.create_recursive_tsma('tsma1', 'tsma2', 'test', '15m', 'meters')
|
|
|
|
# drop recursive tsma first
|
|
tdSql.error('drop tsma test.tsma1', -2147482491)
|
|
tdSql.execute('drop tsma test.tsma2', queryTimes=1)
|
|
tdSql.execute('drop tsma test.tsma1', queryTimes=1)
|
|
self.wait_query('show transactions', 0, 10)
|
|
tdSql.execute('drop database test', queryTimes=1)
|
|
|
|
self.init_data()
|
|
|
|
def test_drop_db(self):
|
|
function_name = sys._getframe().f_code.co_name
|
|
tdLog.debug(f'-----{function_name}------')
|
|
tdSql.execute('create database nsdb precision "ns"', queryTimes=1)
|
|
tdSql.execute('use nsdb', queryTimes=1)
|
|
tdSql.execute(
|
|
'create table nsdb.meters(ts timestamp, c1 int, c2 int) tags(t1 int, t2 int)', queryTimes=1)
|
|
# TODO insert data
|
|
self.create_tsma('tsma1', 'nsdb', 'meters', [
|
|
'avg(c1)', 'avg(c2)'], '5m')
|
|
self.create_recursive_tsma('tsma1', 'tsma2', 'nsdb', '10m', 'meters')
|
|
tdSql.query('select avg(c1) from nsdb.meters', queryTimes=1)
|
|
tdSql.execute('drop database nsdb')
|
|
|
|
def test_tb_ddl_with_created_tsma(self):
|
|
function_name = sys._getframe().f_code.co_name
|
|
tdLog.debug(f'-----{function_name}------')
|
|
tdSql.execute('create database nsdb precision "ns"', queryTimes=1)
|
|
tdSql.execute('use nsdb', queryTimes=1)
|
|
tdSql.execute(
|
|
'create table nsdb.meters(ts timestamp, c1 int, c2 int) tags(t1 int, t2 int)', queryTimes=1)
|
|
self.create_tsma('tsma1', 'nsdb', 'meters', ['avg(c1)', 'avg(c2)'], '5m')
|
|
# drop column, drop tag
|
|
tdSql.error('alter table nsdb.meters drop column c1', -2147482637)
|
|
tdSql.error('alter table nsdb.meters drop tag t1', -2147482637)
|
|
tdSql.error('alter table nsdb.meters drop tag t2', -
|
|
2147482637) # Stream must be dropped first
|
|
tdSql.execute('drop tsma nsdb.tsma1', queryTimes=1)
|
|
|
|
# add tag
|
|
tdSql.execute('alter table nsdb.meters add tag t3 int', queryTimes=1)
|
|
# Invalid tsma func param, only one non-tag column allowed
|
|
tdSql.error(
|
|
'create tsma tsma1 on nsdb.meters function(avg(c1), avg(c2), avg(t3)) interval(5m)', -2147471096)
|
|
|
|
tdSql.execute('alter table nsdb.meters drop tag t3', queryTimes=1)
|
|
self.wait_query('show transactions', 0, 10)
|
|
tdSql.execute('drop database nsdb')
|
|
|
|
# drop norm table
|
|
self.create_tsma('tsma_norm_tb', 'test', 'norm_tb', ['avg(c1)', 'avg(c2)'], '5m')
|
|
tdSql.error('drop table test.norm_tb', -2147471088)
|
|
|
|
# drop no tsma table
|
|
tdSql.execute('drop table test.t2, test.t1')
|
|
|
|
# test ttl drop table
|
|
self.create_tsma('tsma1', 'test', 'meters', ['avg(c1)', 'avg(c2)'], '5m')
|
|
tdSql.execute('alter table test.t0 ttl 2', queryTimes=1)
|
|
tdSql.execute('flush database test')
|
|
self.wait_query('show test.tables like "%t0"', 0, 10)
|
|
|
|
# test drop multi tables
|
|
tdSql.execute('drop table test.t3, test.t4')
|
|
self.wait_query('show test.tables like "%t3"', 0, 1)
|
|
self.wait_query('show test.tables like "%t4"', 0, 1)
|
|
|
|
tdSql.query('show test.tables like "%tsma%"')
|
|
tdSql.checkRows(0)
|
|
|
|
# test drop stream
|
|
tdSql.error('drop stream tsma1', -2147471088) ## TSMA must be dropped first
|
|
|
|
self.wait_query('show transactions', 0, 10)
|
|
tdSql.execute('drop database test', queryTimes=1)
|
|
self.init_data()
|
|
|
|
def test_create_tsma_on_stable(self):
|
|
function_name = sys._getframe().f_code.co_name
|
|
tdLog.debug(f'-----{function_name}------')
|
|
tdSql.execute('create database nsdb precision "ns"', queryTimes=1)
|
|
tdSql.execute('use nsdb', queryTimes=1)
|
|
tdSql.execute(
|
|
'create table nsdb.meters(ts timestamp, c1 int, c2 int, c3 varchar(255)) tags(t1 int, t2 int)', queryTimes=1)
|
|
self.create_tsma('tsma1', 'nsdb', 'meters', ['avg(c1)', 'avg(c2)'], '5m')
|
|
# Invalid tsma interval, 1ms ~ 1h is allowed
|
|
tdSql.error(
|
|
'create tsma tsma2 on nsdb.meters function(avg(c1), avg(c2)) interval(2h)', -2147471097)
|
|
tdSql.error(
|
|
'create tsma tsma2 on nsdb.meters function(avg(c1), avg(c2)) interval(3601s)', -2147471097)
|
|
tdSql.error(
|
|
'create tsma tsma2 on nsdb.meters function(avg(c1), avg(c2)) interval(3600001a)', -2147471097)
|
|
tdSql.error(
|
|
'create tsma tsma2 on nsdb.meters function(avg(c1), avg(c2)) interval(3600001000u)', -2147471097)
|
|
tdSql.error(
|
|
'create tsma tsma2 on nsdb.meters function(avg(c1), avg(c2)) interval(999999b)', -2147471097)
|
|
tdSql.error(
|
|
'create tsma tsma2 on nsdb.meters function(avg(c1), avg(c2)) interval(999u)', -2147471097)
|
|
# invalid tsma func param
|
|
tdSql.error(
|
|
'create tsma tsma2 on nsdb.meters function(avg(c1, c2), avg(c2)) interval(10m)', -2147471096)
|
|
# invalid param data type
|
|
tdSql.error(
|
|
'create tsma tsma2 on nsdb.meters function(avg(ts), avg(c2)) interval(10m)', -2147473406)
|
|
tdSql.error(
|
|
'create tsma tsma2 on nsdb.meters function(avg(c3), avg(c2)) interval(10m)', -2147473406)
|
|
# invalid tsma func param
|
|
tdSql.error(
|
|
'create tsma tsma2 on nsdb.meters function(avg(c1+1), avg(c2)) interval(10m)', -2147471096)
|
|
# invalid tsma func param
|
|
tdSql.error(
|
|
'create tsma tsma2 on nsdb.meters function(avg(c1*c2), avg(c2)) interval(10m)', -2147471096)
|
|
|
|
# sma already exists in different db
|
|
# SMA already exists in db # Stream already exists
|
|
tdSql.error(
|
|
'create tsma tsma1 on test.meters function(avg(c1), avg(c2)) interval(10m)', -2147482496)
|
|
|
|
|
|
# Invalid tsma interval, error format,including sliding and interval_offset
|
|
tdSql.error(
|
|
'create tsma tsma_error_interval on nsdb.meters function(count(c2)) interval(10)') #syntax error
|
|
tdSql.error(
|
|
'create tsma tsma_error_interval on nsdb.meters function(count(c2)) interval("10m")')
|
|
tdSql.error(
|
|
'create tsma tsma_error_interval on nsdb.meters function(count(c2)) interval(10,10m)')
|
|
tdSql.error(
|
|
'create tsma tsma_error_interval on nsdb.meters function(count(c2)) interval(10s,10m)')
|
|
tdSql.error(
|
|
'create tsma tsma_error_interval on nsdb.meters function(count(c2)) interval(10s) sliding(1m)')
|
|
|
|
# max tsma num 8
|
|
self.create_tsma('tsma2', 'nsdb', 'meters', ['avg(c1)', 'avg(c2)'], '10s')
|
|
self.create_tsma('tsma_test3', 'test', 'meters', ['avg(c1)', 'avg(c2)'], '100s')
|
|
self.create_tsma('tsma4', 'nsdb', 'meters', ['avg(c1)', 'avg(c2)'], '101s')
|
|
self.create_tsma('tsma5', 'nsdb', 'meters', ['avg(c1)', 'count(ts)'], '102s')
|
|
self.create_tsma('tsma6', 'nsdb', 'meters', ['avg(c1)', 'avg(c2)'], '103s')
|
|
self.create_tsma('tsma7', 'nsdb', 'meters', ['avg(c1)', 'count(c2)'], '104s')
|
|
self.create_tsma('tsma8', 'test', 'meters', ['avg(c1)', 'sum(c2)'], '105s')
|
|
tdSql.error('create tsma tsma9 on nsdb.meters function(count(ts), count(c1), sum(c2)) interval(99s)', -2147482490)
|
|
tdSql.error('create recursive tsma tsma9 on test.tsma8 interval(210s)', -2147482490)
|
|
|
|
# modify maxTsmaNum para
|
|
tdSql.error('alter dnode 1 "maxTsmaNum" "13";')
|
|
tdSql.error('alter dnode 1 "maxTsmaNum" "-1";')
|
|
|
|
# tdSql.error('alter dnode 1 "maxTsmaNum" "abc";')
|
|
# tdSql.error('alter dnode 1 "maxTsmaNum" "1.2";')
|
|
|
|
tdSql.execute("alter dnode 1 'maxTsmaNum' '0';", queryTimes=1)
|
|
tdSql.error('create tsma tsma9 on nsdb.meters function(count(ts), count(c1), sum(c2)) interval(99s)', -2147482490)
|
|
tdSql.execute("alter dnode 1 'maxTsmaNum' '12';", queryTimes=1)
|
|
tdSql.execute('create tsma tsma9 on nsdb.meters function(count(ts), count(c1), sum(c2)) interval(109s)')
|
|
tdSql.execute('create tsma tsma10 on nsdb.meters function(count(ts), count(c1), sum(c2)) interval(110s)')
|
|
tdSql.execute('create tsma tsma11 on nsdb.meters function(count(ts), count(c1), sum(c2)) interval(111s)')
|
|
tdSql.execute('create tsma tsma12 on nsdb.meters function(count(ts), count(c1), sum(c2)) interval(112s)')
|
|
tdSql.query("show nsdb.tsmas", queryTimes=1)
|
|
print(tdSql.queryResult)
|
|
tdSql.query("show test.tsmas", queryTimes=1)
|
|
print(tdSql.queryResult)
|
|
tdSql.error('create tsma tsma13 on nsdb.meters function(count(ts), count(c1), sum(c2)) interval(113s)', -2147482490)
|
|
|
|
|
|
# drop tsma
|
|
tdSql.execute('drop tsma nsdb.tsma1', queryTimes=1)
|
|
tdSql.execute('use test', queryTimes=1)
|
|
tdSql.execute(
|
|
'create tsma tsma1 on nsdb.meters function(avg(c1), avg(c2)) interval(10m)', queryTimes=1)
|
|
self.wait_for_tsma_calculation(
|
|
['avg(c1)', 'avg(c2)'], 'nsdb', 'meters', '10m', 'tsma1')
|
|
tdSql.execute('drop tsma nsdb.tsma1', queryTimes=1)
|
|
|
|
tdSql.error(
|
|
'create tsma tsma1 on test.meters function(avg(c1), avg(c2)) interval(2h)', -2147471097)
|
|
self.wait_query('show transactions', 0, 10)
|
|
tdSql.execute('drop database nsdb')
|
|
|
|
def test_create_tsma_on_norm_table(self):
|
|
function_name = sys._getframe().f_code.co_name
|
|
tdLog.debug(f'-----{function_name}------')
|
|
|
|
def test_create_tsma_on_child_table(self):
|
|
function_name = sys._getframe().f_code.co_name
|
|
tdLog.debug(f'-----{function_name}------')
|
|
# Invalid table to create tsma, only stable or normal table allowed
|
|
tdSql.error(
|
|
'create tsma tsma1 on test.t1 function(avg(c1), avg(c2)) interval(1m)', -2147471098)
|
|
|
|
def test_create_recursive_tsma(self):
|
|
function_name = sys._getframe().f_code.co_name
|
|
tdLog.debug(f'-----{function_name}------')
|
|
tdSql.execute('use test')
|
|
self.create_tsma('tsma1', 'test', 'meters', [
|
|
'avg(c1)', 'avg(c2)'], '5m')
|
|
sql = 'create recursive tsma tsma2 on test.tsma1 function(avg(c1)) interval(1m)'
|
|
tdSql.error(sql, -2147473920) # syntax error
|
|
|
|
sql = 'create recursive tsma tsma2 on test.tsma1 interval(1m)'
|
|
tdSql.error(sql, -2147471099) # invalid tsma parameter
|
|
|
|
sql = 'create recursive tsma tsma2 on test.tsma1 interval(7m)'
|
|
tdSql.error(sql, -2147471099) # invalid tsma parameter
|
|
|
|
sql = 'create recursive tsma tsma2 on test.tsma1 interval(11m)'
|
|
tdSql.error(sql, -2147471099) # invalid tsma parameter
|
|
|
|
self.create_recursive_tsma('tsma1', 'tsma2', 'test', '20m', 'meters')
|
|
|
|
sql = 'create recursive tsma tsma3 on test.tsma2 interval(30m)'
|
|
tdSql.error(sql, -2147471099) # invalid tsma parameter
|
|
|
|
self.create_recursive_tsma('tsma2', 'tsma3', 'test', '40m', 'meters')
|
|
|
|
tdSql.execute('drop tsma test.tsma3', queryTimes=1)
|
|
tdSql.execute('drop tsma test.tsma2', queryTimes=1)
|
|
tdSql.execute('drop tsma test.tsma1', queryTimes=1)
|
|
|
|
def test_create_tsma_maxlist_function(self):
|
|
function_name = sys._getframe().f_code.co_name
|
|
tdLog.debug(f'-----{function_name}------')
|
|
json_file = "2-query/compa4096_tsma.json"
|
|
tdCom.update_json_file_replica(json_file, self.replicaVar)
|
|
os.system(f"taosBenchmark -f {json_file} -y ")
|
|
# max number of list is 4093: 4096 - 3 - 2(原始表tag个数) - 1(tbname)
|
|
tdSql.execute('use db4096')
|
|
|
|
self.create_tsma('tsma_4050', 'db4096', 'stb0', self.generate_tsma_function_list_columns(4050), '5m',check_tsma_calculation=False)
|
|
|
|
self.create_tsma('tsma_4090', 'db4096', 'stb0', self.generate_tsma_function_list_columns(4090), '6m',check_tsma_calculation=False)
|
|
|
|
self.create_error_tsma('tsma_4091', 'db4096', 'stb0', self.generate_tsma_function_list_columns(4091), '5m', -2147473856) #Too many columns
|
|
|
|
self.drop_tsma('tsma_4050', 'db4096')
|
|
self.drop_tsma('tsma_4090', 'db4096')
|
|
|
|
def generate_tsma_function_list_columns(self,max_column: int =4093):
|
|
columns = []
|
|
self.tsma_support_func = ["max", "min", "count", "sum"]
|
|
num_items = len(self.tsma_support_func)
|
|
for i in range(max_column):
|
|
random_index = secrets.randbelow(num_items)
|
|
tsma_column_element = self.tsma_support_func[random_index] + '(c' + str(i) + ')'
|
|
columns.append(tsma_column_element)
|
|
return columns
|
|
|
|
def test_create_diffrent_tsma_name(self):
|
|
function_name = sys._getframe().f_code.co_name
|
|
tdLog.debug(f'-----{function_name}------')
|
|
tdSql.execute('use test')
|
|
|
|
self.create_tsma('tsma_repeat', 'test', 'meters', ['avg(c1)', 'avg(c2)'], '5m')
|
|
tdSql.error('create tsma tsma_repeat on test.meters function(avg(c1), avg(c2)) interval(10m)', -2147482496) # DB error: SMA already exists in db
|
|
|
|
# same name with dbname, stable and child table
|
|
self.test_create_and_drop_tsma('meters', 'test', 'meters', ['count(c1)', 'avg(c2)'], '5m')
|
|
self.test_create_and_drop_tsma('t1', 'test', 'meters', ['count(c1)', 'avg(c2)'], '5m')
|
|
self.test_create_and_drop_tsma('test', 'test', 'meters', ['count(c1)', 'avg(c2)'], '5m')
|
|
|
|
# tsma name is key word
|
|
tdSql.error("CREATE TSMA tsma ON test.meters FUNCTION(avg(c1)) INTERVAL(5m); ", -2147473920) # syntax error near
|
|
tdSql.error("CREATE TSMA bool ON test.meters FUNCTION(avg(c1)) INTERVAL(5m); ", -2147473920)
|
|
|
|
# tsma name is illegal
|
|
tdSql.error("CREATE TSMA 129_tsma ON test.meters FUNCTION(count(c1)) INTERVAL(5m); ", -2147473920)
|
|
tdSql.error("CREATE TSMA T*\-sma129_ ON test.meters FUNCTION(count(c1)) INTERVAL(5m); ", -2147473920)
|
|
tdSql.error("CREATE TSMA Tsma_repeat ON test.meters FUNCTION(count(c1)) INTERVAL(5m); ", -2147482496)
|
|
|
|
# tsma name include escape character
|
|
|
|
tdSql.execute("CREATE TSMA `129_tsma` ON test.meters FUNCTION(count(c3)) INTERVAL(5m); ")
|
|
tdSql.execute("CREATE TSMA `129_Tsma` ON test.meters FUNCTION(count(c3)) INTERVAL(5m); ")
|
|
tdSql.execute("CREATE TSMA `129_T*\-sma` ON test.meters FUNCTION(count(c3)) INTERVAL(5m); ")
|
|
tdSql.execute("drop tsma test.`129_tsma`")
|
|
tdSql.execute("drop tsma test.`129_Tsma`")
|
|
tdSql.execute("drop tsma test.`129_T*\-sma`")
|
|
|
|
self.drop_tsma('tsma_repeat', 'test')
|
|
|
|
|
|
def test_create_and_drop_tsma(self, tsma_name: str, db_name: str = 'test', table_name: str = 'meters', func_list: List = ['avg(c1)', 'avg(c2)'], interval: str = '5m'):
|
|
function_name = sys._getframe().f_code.co_name
|
|
tdLog.debug(f'-----{function_name}------')
|
|
tdSql.execute('use test')
|
|
self.create_tsma(tsma_name, db_name, table_name , func_list, interval)
|
|
self.drop_tsma(tsma_name, db_name)
|
|
|
|
def test_create_illegal_tsma_sql(self):
|
|
function_name = sys._getframe().f_code.co_name
|
|
tdLog.debug(f'-----{function_name}------')
|
|
tdSql.execute('use test')
|
|
|
|
# DB error: Table does not exist
|
|
tdSql.error('create tsma tsma_illegal on test.meterss function(count(c5), avg(c2)) interval(5m)',-2147473917)
|
|
|
|
# syntax error near
|
|
tdSql.error('create tsma tsma_illegal on test.meters function() interval(5m)',-2147473920)
|
|
tdSql.error('create tsma tsma_illegal on test.meters function("count(c5)") interval(5m)',-2147473920)
|
|
tdSql.error('create tsma tsma_illegal on test.meters function(count(c5)+1) interval(5m)',-2147473920)
|
|
tdSql.error('create tsma tsma_illegal on test.meters function(avg(c1)+avg(c2)) interval(5m)',-2147473920)
|
|
|
|
# Invalid tsma func param, only one column allowed
|
|
tdSql.error('create tsma tsma_illegal on test.meters function(count(1)) interval(5m)',-2147471096)
|
|
tdSql.error('create tsma tsma_illegal on test.meters function(count(c1,c2)) interval(5m)',-2147471096)
|
|
tdSql.error('create tsma tsma_illegal on test.meters function(count(_wstart)) interval(5m)',-2147471096)
|
|
tdSql.error('create tsma tsma_illegal on test.meters function(count(_wend)) interval(5m)',-2147471096)
|
|
tdSql.error('create tsma tsma_illegal on test.meters function(count(_wduration)) interval(5m)',-2147471096)
|
|
|
|
# Tsma func not supported
|
|
# TODO add all of funcs not supported
|
|
tdSql.error('create tsma tsma_illegal on test.meters function(abs(c1)) interval(5m)',-2147471095)
|
|
tdSql.error('create tsma tsma_illegal on test.meters function(last_row(c1)) interval(5m)',-2147471095)
|
|
|
|
# mixed tsma func not supported
|
|
tdSql.error('create tsma tsma_illegal on test.meters function(last(c1),last_row(c1)) interval(5m)',-2147471095)
|
|
|
|
# Invalid function para type
|
|
tdSql.error('create tsma tsma_illegal on test.meters function(avg(c8)) interval(5m)',-2147473406)
|
|
|
|
def test_flush_query(self):
|
|
tdSql.execute('insert into test.norm_tb (ts,c1_new,c2) values (now,1,2)(now+1s,2,3)(now+2s,2,3)(now+3s,2,3) (now+4s,1,2)(now+5s,2,3)(now+6s,2,3)(now+7s,2,3); select /*+ skip_tsma()*/ avg(c1_new),avg(c2) from test.norm_tb interval(10m);select avg(c1_new),avg(c2) from test.norm_tb interval(10m);select * from information_schema.ins_stream_tasks;', queryTimes=1)
|
|
tdSql.execute('flush database test', queryTimes=1)
|
|
tdSql.query('select count(*) from test.meters', queryTimes=1)
|
|
tdSql.checkData(0,0,100000)
|
|
tdSql.query('select count(*) from test.norm_tb', queryTimes=1)
|
|
tdSql.checkData(0,0,10008)
|
|
tdSql.execute('flush database test', queryTimes=1)
|
|
tdSql.query('select count(*) from test.meters', queryTimes=1)
|
|
tdSql.checkData(0,0,100000)
|
|
tdSql.query('select count(*) from test.norm_tb', queryTimes=1)
|
|
tdSql.checkData(0,0,10008)
|
|
|
|
def test_redistribute_vgroups(self):
|
|
tdSql.redistribute_db_all_vgroups('test', self.replicaVar)
|
|
tdSql.redistribute_db_all_vgroups('db4096', self.replicaVar)
|
|
|
|
# def test_replica_dnode(self):
|
|
|
|
# def test_split_dnode(self):
|
|
|
|
|
|
def stop(self):
|
|
tdSql.close()
|
|
tdLog.success(f"{__file__} successfully executed")
|
|
|
|
|
|
event = threading.Event()
|
|
|
|
tdCases.addLinux(__file__, TDTestCase())
|
|
tdCases.addWindows(__file__, TDTestCase())
|