feat: add new compress ci case compressBasic.py
This commit is contained in:
Alex Duan
parent
a02426fd2f
commit
baecc446fc
|
|
@ -0,0 +1,247 @@
|
|||
###################################################################
|
||||
# Copyright (c) 2016 by TAOS Technologies, Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This file is proprietary and confidential to TAOS Technologies.
|
||||
# No part of this file may be reproduced, stored, transmitted,
|
||||
# disclosed or used in any form or by any means other than as
|
||||
# expressly provided by the written permission from Jianhui Tao
|
||||
#
|
||||
###################################################################
|
||||
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import sys
|
||||
import time
|
||||
import random
|
||||
|
||||
import taos
|
||||
import frame
|
||||
import frame.etool
|
||||
|
||||
|
||||
from frame.log import *
|
||||
from frame.cases import *
|
||||
from frame.sql import *
|
||||
from frame.caseBase import *
|
||||
from frame import *
|
||||
from frame.autogen import *
|
||||
|
||||
|
||||
class TDTestCase(TBase):
|
||||
updatecfgDict = {
|
||||
"compressMsgSize" : "100",
|
||||
}
|
||||
# compress
|
||||
compresses = ["lz4","tsz","zlib","zstd","disabled","xz"]
|
||||
|
||||
# level
|
||||
levels = ["high","medium","low"]
|
||||
|
||||
# default compress
|
||||
defCompress = "lz4"
|
||||
# default level
|
||||
defLevel = "medium"
|
||||
|
||||
# datatype 17
|
||||
dtypes = [ "tinyint","tinyint unsigned","smallint","smallint unsigned","int","int unsigned",
|
||||
"bigint","bigint unsigned","timestamp","bool","float","double","binary(16)","nchar(16)",
|
||||
"varchar(16)","varbinary(16)","geometry(32)"]
|
||||
|
||||
def genAllSqls(self, stbName, max):
|
||||
# encode
|
||||
encodes = [
|
||||
[["tinyint","tinyint unsigned","smallint","smallint unsigned","int","int unsigned","bigint","bigint unsigned"], ["simple8b"]],
|
||||
[["timestamp","bigint","bigint unsigned"], ["delta-i"]],
|
||||
[["bool"], ["bit-packing"]],
|
||||
[["float","double"], ["delta-d"]]
|
||||
]
|
||||
|
||||
c = 0 # column number
|
||||
t = 0 # table number
|
||||
|
||||
sqls = []
|
||||
sql = ""
|
||||
|
||||
# loop append sqls
|
||||
for lines in encodes:
|
||||
print(lines)
|
||||
for datatype in lines[0]:
|
||||
for encode in lines[1]:
|
||||
for compress in self.compresses:
|
||||
for level in self.levels:
|
||||
if sql == "":
|
||||
# first
|
||||
sql = f"create table {self.db}.st{t} (ts timestamp"
|
||||
else:
|
||||
sql += f", c{c} {datatype} ENCODE '{encode}' COMPRESS '{compress}' LEVEL '{level}'"
|
||||
c += 1
|
||||
|
||||
if c >= max:
|
||||
# append sqls
|
||||
sql += f") tags(groupid int) "
|
||||
sqls.append(sql)
|
||||
# reset
|
||||
sql = ""
|
||||
c = 0
|
||||
t += 1
|
||||
|
||||
# break loop
|
||||
if c > 0:
|
||||
# append sqls
|
||||
sql += f") tags(groupid int) "
|
||||
sqls.append(sql)
|
||||
|
||||
return sqls
|
||||
|
||||
# check error create
|
||||
def errorCreate(self):
|
||||
sqls = [
|
||||
f"create table terr(ts timestamp, c0 int ENCODE 'abc') ",
|
||||
f""
|
||||
]
|
||||
|
||||
for dtype in self.dtypes:
|
||||
# encode
|
||||
sql = f"create table terr(ts timestamp, c0 {dtype} ENCODE 'abc') "
|
||||
tdSql.error(sql)
|
||||
# compress
|
||||
sql = f"create table terr(ts timestamp, c0 {dtype} COMPRESS 'def') "
|
||||
tdSql.error(sql)
|
||||
# level
|
||||
sql = f"create table terr(ts timestamp, c0 {dtype} LEVEL 'hig') "
|
||||
tdSql.error(sql)
|
||||
|
||||
# default value correct
|
||||
def defaultCorrect(self):
|
||||
# get default encode compress level
|
||||
sql = f"describe {self.db}.{self.stb}"
|
||||
tdSql.query(sql)
|
||||
|
||||
'''
|
||||
# see AutoGen.types
|
||||
defEncodes = [ "delta-i","delta-i","simple8b","simple8b","simple8b","simple8b","simple8b","simple8b",
|
||||
"simple8b","simple8b","delta-d","delta-d","bit-packing",
|
||||
"disabled","disabled","disabled","disabled","disabled"]
|
||||
'''
|
||||
|
||||
# pass-ci have error
|
||||
defEncodes = [ "delta-i","delta-i","simple8b","simple8b","simple8b","simple8b","simple8b","simple8b",
|
||||
"simple8b","simple8b","delta-d","delta-d","bit-packing",
|
||||
"disabled","disabled","disabled","disabled","simple8b"]
|
||||
|
||||
|
||||
count = tdSql.getRows()
|
||||
for i in range(count):
|
||||
node = tdSql.getData(i, 3)
|
||||
if node == "TAG":
|
||||
break
|
||||
# check
|
||||
tdSql.checkData(i, 4, defEncodes[i])
|
||||
tdSql.checkData(i, 5, self.defCompress)
|
||||
tdSql.checkData(i, 6, self.defLevel)
|
||||
|
||||
tdLog.info("check default encode compress and level successfully.")
|
||||
|
||||
def checkDataDesc(self, tbname, row, col, value):
|
||||
sql = f"describe {tbname}"
|
||||
tdSql.query(sql)
|
||||
tdSql.checkData(row, col, value)
|
||||
|
||||
|
||||
def writeData(self, count):
|
||||
self.autoGen.insert_data(count, True)
|
||||
|
||||
# alter encode compress level
|
||||
def checkAlter(self):
|
||||
tbname = f"{self.db}.{self.stb}"
|
||||
# alter encode 4
|
||||
comp = "delta-i"
|
||||
sql = f"alter table {tbname} modify column c7 ENCODE '{comp}';"
|
||||
tdSql.execute(sql, show=True)
|
||||
self.checkDataDesc(tbname, 8, 4, comp)
|
||||
self.writeData(1000)
|
||||
sql = f"alter table {tbname} modify column c8 ENCODE '{comp}';"
|
||||
tdSql.execute(sql, show=True)
|
||||
self.checkDataDesc(tbname, 9, 4, comp)
|
||||
self.writeData(1000)
|
||||
|
||||
# alter compress 5
|
||||
comps = self.compresses[2:]
|
||||
comps.append(self.compresses[0]) # add lz4
|
||||
for comp in comps:
|
||||
for i in range(self.colCnt - 1):
|
||||
col = f"c{i}"
|
||||
sql = f"alter table {tbname} modify column {col} COMPRESS '{comp}';"
|
||||
tdSql.execute(sql, show=True)
|
||||
self.checkDataDesc(tbname, i + 1, 5, comp)
|
||||
self.writeData(1000)
|
||||
|
||||
# alter float(c9) double(c10) to tsz
|
||||
comp = "tsz"
|
||||
sql = f"alter table {tbname} modify column c9 COMPRESS '{comp}';"
|
||||
tdSql.execute(sql)
|
||||
self.checkDataDesc(tbname, 10, 5, comp)
|
||||
self.writeData(10000)
|
||||
sql = f"alter table {tbname} modify column c10 COMPRESS '{comp}';"
|
||||
tdSql.execute(sql)
|
||||
self.checkDataDesc(tbname, 11, 5, comp)
|
||||
self.writeData(10000)
|
||||
|
||||
# alter level 6
|
||||
for level in self.levels:
|
||||
for i in range(self.colCnt - 1):
|
||||
col = f"c{i}"
|
||||
sql = f"alter table {tbname} modify column {col} LEVEL '{level}';"
|
||||
tdSql.execute(sql)
|
||||
self.writeData(1000)
|
||||
|
||||
|
||||
def validCreate(self):
|
||||
sqls = self.genAllSqls(self.stb, 50)
|
||||
tdSql.executes(sqls, show=True)
|
||||
|
||||
# sql syntax
|
||||
def checkSqlSyntax(self):
|
||||
|
||||
# create tables positive
|
||||
self.validCreate()
|
||||
|
||||
# create table negtive
|
||||
self.errorCreate()
|
||||
|
||||
# check default value corrent
|
||||
self.defaultCorrect()
|
||||
|
||||
# check alter and write
|
||||
self.checkAlter()
|
||||
|
||||
|
||||
# run
|
||||
def run(self):
|
||||
tdLog.debug(f"start to excute {__file__}")
|
||||
|
||||
# create db and stable
|
||||
self.autoGen = AutoGen()
|
||||
self.autoGen.create_db(self.db, 2, 3)
|
||||
tdSql.execute(f"use {self.db}")
|
||||
self.colCnt = 18
|
||||
self.autoGen.create_stable(self.stb, 5, self.colCnt, 8, 8)
|
||||
self.childCnt = 4
|
||||
self.autoGen.create_child(self.stb, "d", self.childCnt)
|
||||
self.autoGen.insert_data(10000)
|
||||
|
||||
# sql syntax
|
||||
self.checkSqlSyntax()
|
||||
|
||||
# operateor
|
||||
self.writeData(10000)
|
||||
self.flushDb()
|
||||
self.writeData(10000)
|
||||
|
||||
tdLog.success(f"{__file__} successfully executed")
|
||||
|
||||
|
||||
|
||||
tdCases.addLinux(__file__, TDTestCase())
|
||||
tdCases.addWindows(__file__, TDTestCase())
|
||||
|
|
@ -14,7 +14,7 @@ import time
|
|||
# Auto Gen class
|
||||
#
|
||||
class AutoGen:
|
||||
def __init__(self, startTs = 1600000000000, step = 1000, batch = 100, fillOne=False):
|
||||
def __init__(self, startTs = 1600000000000, step = 1000, batch = 500, fillOne=False):
|
||||
self.startTs = startTs
|
||||
self.ts = startTs
|
||||
self.step = step
|
||||
|
|
@ -35,20 +35,23 @@ class AutoGen:
|
|||
# _columns_sql
|
||||
def gen_columns_sql(self, pre, cnt, binary_len, nchar_len):
|
||||
types = [
|
||||
'timestamp',
|
||||
'tinyint',
|
||||
'timestamp', # 0
|
||||
'tinyint',
|
||||
'tinyint unsigned', # 3
|
||||
'smallint',
|
||||
'tinyint unsigned',
|
||||
'smallint unsigned',
|
||||
'int',
|
||||
'bigint',
|
||||
'int', # 5
|
||||
'int unsigned',
|
||||
'bigint', # 7
|
||||
'bigint unsigned',
|
||||
'float',
|
||||
'double',
|
||||
'float', # 9
|
||||
'double', # 10
|
||||
'bool',
|
||||
f'varchar({binary_len})',
|
||||
f'nchar({nchar_len})'
|
||||
f'binary({binary_len})', # 12
|
||||
f'varbinary({binary_len})',
|
||||
f'nchar({nchar_len})',
|
||||
f'varchar({nchar_len})',
|
||||
f'geometry(64)' #16
|
||||
]
|
||||
|
||||
sqls = ""
|
||||
|
|
@ -80,9 +83,11 @@ class AutoGen:
|
|||
data = "%f"%(i+i/1000)
|
||||
elif c <= 11 : # bool
|
||||
data = "%d"%(i%2)
|
||||
elif c == 12 : # binary
|
||||
elif c <= 13 : # binary
|
||||
data = '"' + self.random_string(self.bin_len) + '"'
|
||||
elif c == 13 : # binary
|
||||
elif c == 16 : # geometry
|
||||
data = f'"point({i} {i})"'
|
||||
else : # nchar varchar
|
||||
data = '"' + self.random_string(self.nch_len) + '"'
|
||||
|
||||
if datas != "":
|
||||
|
|
@ -142,7 +147,6 @@ class AutoGen:
|
|||
|
||||
def insert_data_child(self, child_name, cnt, batch_size, step):
|
||||
values = ""
|
||||
print("insert child data")
|
||||
ts = self.ts
|
||||
|
||||
# loop do
|
||||
|
|
@ -162,7 +166,6 @@ class AutoGen:
|
|||
if values != "":
|
||||
sql = f"insert into {self.dbname}.{child_name} values {values}"
|
||||
tdSql.execute(sql)
|
||||
tdLog.info(f" insert data i={i}")
|
||||
values = ""
|
||||
|
||||
tdLog.info(f" insert child data {child_name} finished, insert rows={cnt}")
|
||||
|
|
|
|||
|
|
@ -56,6 +56,9 @@ class TBase:
|
|||
def stop(self):
|
||||
tdSql.close()
|
||||
|
||||
def createDb(self, options=""):
|
||||
sql = f"create database {self.db} {options}"
|
||||
tdSql.execute(sql, show=True)
|
||||
|
||||
#
|
||||
# db action
|
||||
|
|
|
|||
Loading…
Reference in New Issue