474 lines
20 KiB
Python
474 lines
20 KiB
Python
###################################################################
|
||
# Copyright (c) 2016 by TAOS Technologies, Inc.
|
||
# All rights reserved.
|
||
#
|
||
# This file is proprietary and confidential to TAOS Technologies.
|
||
# No part of this file may be reproduced, stored, transmitted,
|
||
# disclosed or used in any form or by any means other than as
|
||
# expressly provided by the written permission from Jianhui Tao
|
||
#
|
||
###################################################################
|
||
|
||
# -*- coding: utf-8 -*-
|
||
|
||
import sys
|
||
import subprocess
|
||
import random
|
||
import math
|
||
import numpy as np
|
||
import inspect
|
||
import re
|
||
|
||
from util.log import *
|
||
from util.cases import *
|
||
from util.sql import *
|
||
from util.dnodes import *
|
||
|
||
msec_per_min=60*1000
|
||
class TDTestCase:
|
||
def init(self, conn, logSql, replicaVar=1):
|
||
self.replicaVar = int(replicaVar)
|
||
tdLog.debug("start to execute %s" % __file__)
|
||
tdSql.init(conn.cursor())
|
||
|
||
def diff_query_form(self, col="c1", alias="", table_expr="db.t1", condition=""):
|
||
|
||
'''
|
||
diff function:
|
||
:param col: string, column name, required parameters;
|
||
:param alias: string, result column another name，or add other funtion;
|
||
:param table_expr: string or expression, data source（eg,table/stable name, result set）, required parameters;
|
||
:param condition: expression；
|
||
:param args: other funtions,like: ', last(col)',or give result column another name, like 'c2'
|
||
:return: diff query statement,default: select diff(c1) from t1
|
||
'''
|
||
|
||
return f"select diff({col}) {alias} from {table_expr} {condition}"
|
||
|
||
def checkdiff(self,col="c1", alias="", table_expr="db.t1", condition="" ):
|
||
line = sys._getframe().f_back.f_lineno
|
||
pre_sql = self.diff_query_form(
|
||
col=col, table_expr=table_expr, condition=condition
|
||
).replace("diff", "count")
|
||
tdSql.query(pre_sql)
|
||
|
||
if tdSql.queryRows == 0:
|
||
tdSql.query(self.diff_query_form(
|
||
col=col, alias=alias, table_expr=table_expr, condition=condition
|
||
))
|
||
print(f"case in {line}: ", end='')
|
||
tdSql.checkRows(0)
|
||
return
|
||
|
||
if "order by tbname" in condition:
|
||
tdSql.query(self.diff_query_form(
|
||
col=col, alias=alias, table_expr=table_expr, condition=condition
|
||
))
|
||
return
|
||
|
||
if "group" in condition:
|
||
|
||
tb_condition = condition.split("group by")[1].split(" ")[1]
|
||
tdSql.query(f"select distinct {tb_condition} from {table_expr}")
|
||
query_result = tdSql.queryResult
|
||
query_rows = tdSql.queryRows
|
||
clear_condition = re.sub('order by [0-9a-z]*|slimit [0-9]*|soffset [0-9]*', "", condition)
|
||
|
||
pre_row = 0
|
||
for i in range(query_rows):
|
||
group_name = query_result[i][0]
|
||
if "where" in clear_condition:
|
||
pre_condition = re.sub('group by [0-9a-z]*', f"{tb_condition}='{group_name}'", clear_condition)
|
||
else:
|
||
pre_condition = "where " + re.sub('group by [0-9a-z]*',f"{tb_condition}='{group_name}'", clear_condition)
|
||
|
||
tdSql.query(f"select {col} {alias} from {table_expr} {pre_condition}")
|
||
pre_data = np.array(tdSql.queryResult)[np.array(tdSql.queryResult) != None]
|
||
if (platform.system().lower() == 'windows' and pre_data.dtype == 'int32'):
|
||
pre_data = np.array(pre_data, dtype = 'int64')
|
||
pre_diff = np.diff(pre_data)
|
||
# trans precision for data
|
||
tdSql.query(self.diff_query_form(
|
||
col=col, alias=alias, table_expr=table_expr, condition=condition
|
||
))
|
||
for j in range(len(pre_diff)):
|
||
print(f"case in {line}:", end='')
|
||
if isinstance(pre_diff[j] , float) :
|
||
pass
|
||
else:
|
||
tdSql.checkData(pre_row+j, 1, pre_diff[j] )
|
||
pre_row += len(pre_diff)
|
||
return
|
||
elif "union" in condition:
|
||
union_sql_0 = self.diff_query_form(
|
||
col=col, alias=alias, table_expr=table_expr, condition=condition
|
||
).split("union all")[0]
|
||
|
||
union_sql_1 = self.diff_query_form(
|
||
col=col, alias=alias, table_expr=table_expr, condition=condition
|
||
).split("union all")[1]
|
||
|
||
tdSql.query(union_sql_0)
|
||
union_diff_0 = tdSql.queryResult
|
||
row_union_0 = tdSql.queryRows
|
||
|
||
tdSql.query(union_sql_1)
|
||
union_diff_1 = tdSql.queryResult
|
||
|
||
tdSql.query(self.diff_query_form(
|
||
col=col, alias=alias, table_expr=table_expr, condition=condition
|
||
))
|
||
for i in range(tdSql.queryRows):
|
||
print(f"case in {line}: ", end='')
|
||
if i < row_union_0:
|
||
tdSql.checkData(i, 0, union_diff_0[i][0])
|
||
else:
|
||
tdSql.checkData(i, 0, union_diff_1[i-row_union_0][0])
|
||
return
|
||
|
||
else:
|
||
sql = f"select {col} from {table_expr} {re.sub('limit [0-9]*|offset [0-9]*','',condition)}"
|
||
tdSql.query(sql)
|
||
offset_val = condition.split("offset")[1].split(" ")[1] if "offset" in condition else 0
|
||
pre_result = np.array(tdSql.queryResult)[np.array(tdSql.queryResult) != None]
|
||
if (platform.system().lower() == 'windows' and pre_result.dtype == 'int32'):
|
||
pre_result = np.array(pre_result, dtype = 'int64')
|
||
pre_diff = np.diff(pre_result)[offset_val:]
|
||
if len(pre_diff) > 0:
|
||
sql =self.diff_query_form(col=col, alias=alias, table_expr=table_expr, condition=condition)
|
||
tdSql.query(sql)
|
||
j = 0
|
||
diff_cnt = len(pre_diff)
|
||
for i in range(tdSql.queryRows):
|
||
print(f"case in {line}: i={i} j={j} pre_diff[j]={pre_diff[j]} ", end='')
|
||
if isinstance(pre_diff[j] , float ):
|
||
if j + 1 < diff_cnt:
|
||
j += 1
|
||
pass
|
||
else:
|
||
if tdSql.getData(i,0) != None:
|
||
tdSql.checkData(i, 0, pre_diff[j])
|
||
if j + 1 < diff_cnt:
|
||
j += 1
|
||
else:
|
||
print(f"getData i={i} is None j={j} ")
|
||
else:
|
||
print("pre_diff len is zero.")
|
||
|
||
pass
|
||
|
||
def diff_current_query(self) :
|
||
|
||
# table schema :ts timestamp, c1 int, c2 float, c3 timestamp, c4 binary(16), c5 double, c6 bool
|
||
# c7 bigint, c8 smallint, c9 tinyint, c10 nchar(16)
|
||
|
||
# case1～6： numeric col:int/bigint/tinyint/smallint/float/double
|
||
self.checkdiff()
|
||
case2 = {"col": "c2"}
|
||
self.checkdiff(**case2)
|
||
case3 = {"col": "c5"}
|
||
self.checkdiff(**case3)
|
||
case4 = {"col": "c7"}
|
||
self.checkdiff(**case4)
|
||
case5 = {"col": "c8"}
|
||
self.checkdiff(**case5)
|
||
case6 = {"col": "c9"}
|
||
self.checkdiff(**case6)
|
||
|
||
# case7~8: nested query
|
||
# case7 = {"table_expr": "(select c1 from db.stb1)"}
|
||
# self.checkdiff(**case7)
|
||
# case8 = {"table_expr": "(select diff(c1) c1 from db.stb1 group by tbname)"}
|
||
# self.checkdiff(**case8)
|
||
|
||
# case9~10: mix with tbname/ts/tag/col
|
||
# case9 = {"alias": ", tbname"}
|
||
# self.checkdiff(**case9)
|
||
# case10 = {"alias": ", _c0"}
|
||
# self.checkdiff(**case10)
|
||
# case11 = {"alias": ", st1"}
|
||
# self.checkdiff(**case11)
|
||
# case12 = {"alias": ", c1"}
|
||
# self.checkdiff(**case12)
|
||
|
||
# case13~15: with single condition
|
||
case13 = {"condition": "where c1 <= 10"}
|
||
self.checkdiff(**case13)
|
||
case14 = {"condition": "where c6 in (0, 1)"}
|
||
self.checkdiff(**case14)
|
||
case15 = {"condition": "where c1 between 1 and 10"}
|
||
self.checkdiff(**case15)
|
||
|
||
# case16: with multi-condition
|
||
case16 = {"condition": "where c6=1 or c6 =0"}
|
||
self.checkdiff(**case16)
|
||
|
||
# case17: only support normal table join
|
||
case17 = {
|
||
"col": "table1.c1 ",
|
||
"table_expr": "db.t1 as table1, db.t2 as table2",
|
||
"condition": "where table1.ts=table2.ts"
|
||
}
|
||
self.checkdiff(**case17)
|
||
# case18~19: with group by , function diff not support group by
|
||
|
||
case19 = {
|
||
"table_expr": "db.stb1 where tbname =='t0' ",
|
||
"condition": "partition by tbname order by tbname" # partition by tbname
|
||
}
|
||
self.checkdiff(**case19)
|
||
|
||
# case20~21: with order by , Not a single-group group function
|
||
|
||
# case22: with union
|
||
# case22 = {
|
||
# "condition": "union all select diff(c1) from db.t2 "
|
||
# }
|
||
# self.checkdiff(**case22)
|
||
tdSql.query("select count(c1) from db.t1 union all select count(c1) from db.t2")
|
||
|
||
# case23: with limit/slimit
|
||
case23 = {
|
||
"condition": "limit 1"
|
||
}
|
||
self.checkdiff(**case23)
|
||
case24 = {
|
||
"table_expr": "db.stb1",
|
||
"condition": "partition by tbname order by tbname slimit 1 soffset 1"
|
||
}
|
||
self.checkdiff(**case24)
|
||
|
||
pass
|
||
|
||
def diff_error_query(self) -> None :
|
||
# unusual test
|
||
#
|
||
# table schema :ts timestamp, c1 int, c2 float, c3 timestamp, c4 binary(16), c5 double, c6 bool
|
||
# c7 bigint, c8 smallint, c9 tinyint, c10 nchar(16)
|
||
#
|
||
# form test
|
||
tdSql.error(self.diff_query_form(col="")) # no col
|
||
tdSql.error("diff(c1) from db.stb1") # no select
|
||
tdSql.error("select diff from db.t1") # no diff condition
|
||
tdSql.error("select diff c1 from db.t1") # no brackets
|
||
tdSql.error("select diff(c1) db.t1") # no from
|
||
tdSql.error("select diff( c1 ) from ") # no table_expr
|
||
# tdSql.error(self.diff_query_form(col="st1")) # tag col
|
||
tdSql.query("select diff(st1) from db.t1")
|
||
# tdSql.error(self.diff_query_form(col=1)) # col is a value
|
||
tdSql.error(self.diff_query_form(col="'c1'")) # col is a string
|
||
tdSql.error(self.diff_query_form(col=None)) # col is NULL 1
|
||
tdSql.error(self.diff_query_form(col="NULL")) # col is NULL 2
|
||
tdSql.error(self.diff_query_form(col='""')) # col is ""
|
||
tdSql.error(self.diff_query_form(col='c%')) # col is spercial char 1
|
||
tdSql.error(self.diff_query_form(col='c_')) # col is spercial char 2
|
||
tdSql.error(self.diff_query_form(col='c.')) # col is spercial char 3
|
||
tdSql.error(self.diff_query_form(col='avg(c1)')) # expr col
|
||
# tdSql.error(self.diff_query_form(col='c6')) # bool col
|
||
tdSql.query("select diff(c6) from db.t1")
|
||
tdSql.error(self.diff_query_form(col='c4')) # binary col
|
||
tdSql.error(self.diff_query_form(col='c10')) # nachr col
|
||
tdSql.error(self.diff_query_form(col='c10')) # not table_expr col
|
||
tdSql.error(self.diff_query_form(col='db.t1')) # tbname
|
||
tdSql.error(self.diff_query_form(col='db.stb1')) # stbname
|
||
tdSql.error(self.diff_query_form(col='db')) # datbasename
|
||
# tdSql.error(self.diff_query_form(col=True)) # col is BOOL 1
|
||
# tdSql.error(self.diff_query_form(col='True')) # col is BOOL 2
|
||
tdSql.error(self.diff_query_form(col='*')) # col is all col
|
||
tdSql.error("select diff[c1] from db.t1") # sql form error 1
|
||
tdSql.error("select diff{c1} from db.t1") # sql form error 2
|
||
tdSql.error(self.diff_query_form(col="[c1]")) # sql form error 3
|
||
# tdSql.error(self.diff_query_form(col="c1, c2")) # sql form error 3
|
||
# tdSql.error(self.diff_query_form(col="c1, 2")) # sql form error 3
|
||
tdSql.error(self.diff_query_form(alias=", count(c1)")) # mix with aggregate function 1
|
||
tdSql.error(self.diff_query_form(alias=", avg(c1)")) # mix with aggregate function 2
|
||
tdSql.error(self.diff_query_form(alias=", min(c1)")) # mix with select function 1
|
||
tdSql.error(self.diff_query_form(alias=", top(c1, 5)")) # mix with select function 2
|
||
tdSql.error(self.diff_query_form(alias=", spread(c1)")) # mix with calculation function 1
|
||
tdSql.query(self.diff_query_form(alias=", diff(c1)")) # mix with calculation function 2
|
||
# tdSql.error(self.diff_query_form(alias=" + 2")) # mix with arithmetic 1
|
||
tdSql.error(self.diff_query_form(alias=" + avg(c1)")) # mix with arithmetic 2
|
||
tdSql.query(self.diff_query_form(alias=", c2")) # mix with other 1
|
||
# tdSql.error(self.diff_query_form(table_expr="db.stb1")) # select stb directly
|
||
stb_join = {
|
||
"col": "stable1.c1",
|
||
"table_expr": "db.stb1 as stable1, db.stb2 as stable2",
|
||
"condition": "where stable1.ts=stable2.ts and stable1.st1=stable2.st2 order by stable1.ts"
|
||
}
|
||
tdSql.error(self.diff_query_form(**stb_join)) # stb join
|
||
interval_sql = {
|
||
"condition": "where ts>0 and ts < now interval(1h) fill(next)"
|
||
}
|
||
tdSql.error(self.diff_query_form(**interval_sql)) # interval
|
||
group_normal_col = {
|
||
"table_expr": "db.t1",
|
||
"condition": "group by c6"
|
||
}
|
||
tdSql.error(self.diff_query_form(**group_normal_col)) # group by normal col
|
||
slimit_soffset_sql = {
|
||
"table_expr": "db.stb1",
|
||
"condition": "group by tbname slimit 1 soffset 1"
|
||
}
|
||
# tdSql.error(self.diff_query_form(**slimit_soffset_sql))
|
||
order_by_tbname_sql = {
|
||
"table_expr": "db.stb1",
|
||
"condition": "group by tbname order by tbname"
|
||
}
|
||
tdSql.error(self.diff_query_form(**order_by_tbname_sql))
|
||
|
||
pass
|
||
|
||
def diff_test_data(self, tbnum:int, data_row:int, basetime:int) -> None :
|
||
for i in range(tbnum):
|
||
for j in range(data_row):
|
||
tdSql.execute(
|
||
f"insert into db.t{i} values ("
|
||
f"{basetime + (j+1)*10 + i* msec_per_min}, {random.randint(-200, -1)}, {random.uniform(200, -1)}, {basetime + random.randint(-200, -1)}, "
|
||
f"'binary_{j}', {random.uniform(-200, -1)}, {random.choice([0,1])}, {random.randint(-200,-1)}, "
|
||
f"{random.randint(-200, -1)}, {random.randint(-127, -1)}, 'nchar_{j}' )"
|
||
)
|
||
|
||
tdSql.execute(
|
||
f"insert into db.t{i} values ("
|
||
f"{basetime - (j+1) * 10 + i* msec_per_min}, {random.randint(1, 200)}, {random.uniform(1, 200)}, {basetime - random.randint(1, 200)}, "
|
||
f"'binary_{j}_1', {random.uniform(1, 200)}, {random.choice([0, 1])}, {random.randint(1,200)}, "
|
||
f"{random.randint(1,200)}, {random.randint(1,127)}, 'nchar_{j}_1' )"
|
||
)
|
||
tdSql.execute(
|
||
f"insert into db.tt{i} values ( {basetime-(j+1) * 10 + i* msec_per_min}, {random.randint(1, 200)} )"
|
||
)
|
||
|
||
pass
|
||
|
||
def diff_test_table(self,tbnum: int) -> None :
|
||
tdSql.execute("drop database if exists db")
|
||
tdSql.execute("create database if not exists db keep 3650")
|
||
tdSql.execute("use db")
|
||
|
||
tdSql.execute(
|
||
"create stable db.stb1 (\
|
||
ts timestamp, c1 int, c2 float, c3 timestamp, c4 binary(16), c5 double, c6 bool, \
|
||
c7 bigint, c8 smallint, c9 tinyint, c10 nchar(16)\
|
||
) \
|
||
tags(st1 int)"
|
||
)
|
||
tdSql.execute(
|
||
"create stable db.stb2 (ts timestamp, c1 int) tags(st2 int)"
|
||
)
|
||
for i in range(tbnum):
|
||
tdSql.execute(f"create table db.t{i} using db.stb1 tags({i})")
|
||
tdSql.execute(f"create table db.tt{i} using db.stb2 tags({i})")
|
||
|
||
pass
|
||
def diff_support_stable(self):
|
||
tdSql.query(" select diff(1) from db.stb1 ")
|
||
tdSql.checkRows(229)
|
||
tdSql.checkData(0,0,0)
|
||
tdSql.query("select diff(c1) from db.stb1 partition by tbname ")
|
||
tdSql.checkRows(220)
|
||
|
||
tdSql.query("select diff(st1+c1) from db.stb1 partition by tbname")
|
||
tdSql.checkRows(220)
|
||
tdSql.query("select diff(st1+c1) from db.stb1 partition by tbname")
|
||
tdSql.checkRows(220)
|
||
tdSql.query("select diff(st1+c1) from db.stb1 partition by tbname")
|
||
tdSql.checkRows(220)
|
||
|
||
# bug need fix
|
||
tdSql.query("select diff(st1+c1) from db.stb1 partition by tbname")
|
||
tdSql.checkRows(220)
|
||
|
||
# bug need fix
|
||
tdSql.query("select tbname , diff(c1) from db.stb1 partition by tbname")
|
||
tdSql.checkRows(220)
|
||
tdSql.query("select tbname , diff(st1) from db.stb1 partition by tbname")
|
||
tdSql.checkRows(220)
|
||
|
||
|
||
# partition by tags
|
||
tdSql.query("select st1 , diff(c1) from db.stb1 partition by st1")
|
||
tdSql.checkRows(220)
|
||
tdSql.query("select diff(c1) from db.stb1 partition by st1")
|
||
tdSql.checkRows(220)
|
||
|
||
|
||
def diff_test_run(self) :
|
||
tdLog.printNoPrefix("==========run test case for diff function==========")
|
||
tbnum = 10
|
||
nowtime = int(round(time.time() * 1000))
|
||
per_table_rows = 10
|
||
self.diff_test_table(tbnum)
|
||
|
||
tdLog.printNoPrefix("######## no data test:")
|
||
self.diff_current_query()
|
||
self.diff_error_query()
|
||
|
||
tdLog.printNoPrefix("######## insert only NULL test:")
|
||
for i in range(tbnum):
|
||
tdSql.execute(f"insert into db.t{i}(ts) values ({nowtime - 5 + i* msec_per_min})")
|
||
tdSql.execute(f"insert into db.t{i}(ts) values ({nowtime + 5 + i* msec_per_min})")
|
||
self.diff_current_query()
|
||
self.diff_error_query()
|
||
|
||
tdLog.printNoPrefix("######## insert data in the range near the max(bigint/double):")
|
||
self.diff_test_table(tbnum)
|
||
tdSql.execute(f"insert into db.t1(ts, c1,c2,c5,c7) values "
|
||
f"({nowtime - (per_table_rows + 1) * 10 + i* msec_per_min}, {2**31-1}, {3.4*10**38}, {1.7*10**308}, {2**63-1})")
|
||
tdSql.execute(f"insert into db.t1(ts, c1,c2,c5,c7) values "
|
||
f"({nowtime - (per_table_rows + 2) * 10 + i* msec_per_min}, {2**31-1}, {3.4*10**38}, {1.7*10**308}, {2**63-1})")
|
||
self.diff_current_query()
|
||
self.diff_error_query()
|
||
|
||
tdLog.printNoPrefix("######## insert data in the range near the min(bigint/double):")
|
||
self.diff_test_table(tbnum)
|
||
tdSql.execute(f"insert into db.t1(ts, c1,c2,c5,c7) values "
|
||
f"({nowtime - (per_table_rows + 1) * 10 + i* msec_per_min}, {1-2**31}, {-3.4*10**38}, {-1.7*10**308}, {1-2**63})")
|
||
tdSql.execute(f"insert into db.t1(ts, c1,c2,c5,c7) values "
|
||
f"({nowtime - (per_table_rows + 2) * 10 + i* msec_per_min}, {1-2**31}, {-3.4*10**38}, {-1.7*10**308}, {512-2**63})")
|
||
self.diff_current_query()
|
||
self.diff_error_query()
|
||
|
||
tdLog.printNoPrefix("######## insert data without NULL data test:")
|
||
self.diff_test_table(tbnum)
|
||
self.diff_test_data(tbnum, per_table_rows, nowtime)
|
||
self.diff_current_query()
|
||
self.diff_error_query()
|
||
|
||
|
||
tdLog.printNoPrefix("######## insert data mix with NULL test:")
|
||
for i in range(tbnum):
|
||
tdSql.execute(f"insert into db.t{i}(ts) values ({nowtime + i* msec_per_min})")
|
||
tdSql.execute(f"insert into db.t{i}(ts) values ({nowtime-(per_table_rows+3)*10 + i* msec_per_min})")
|
||
tdSql.execute(f"insert into db.t{i}(ts) values ({nowtime+(per_table_rows+3)*10 + i* msec_per_min})")
|
||
self.diff_current_query()
|
||
self.diff_error_query()
|
||
|
||
|
||
|
||
tdLog.printNoPrefix("######## check after WAL test:")
|
||
tdSql.query("select * from information_schema.ins_dnodes")
|
||
index = tdSql.getData(0, 0)
|
||
tdDnodes.stop(index)
|
||
tdDnodes.start(index)
|
||
self.diff_current_query()
|
||
self.diff_error_query()
|
||
|
||
def run(self):
|
||
import traceback
|
||
try:
|
||
# run in develop branch
|
||
self.diff_test_run()
|
||
self.diff_support_stable()
|
||
pass
|
||
except Exception as e:
|
||
traceback.print_exc()
|
||
raise e
|
||
|
||
|
||
def stop(self):
|
||
tdSql.close()
|
||
tdLog.success("%s successfully executed" % __file__)
|
||
|
||
tdCases.addWindows(__file__, TDTestCase())
|
||
tdCases.addLinux(__file__, TDTestCase())
|