diff --git a/tests/army/community/storage/compressBasic.py b/tests/army/community/storage/compressBasic.py new file mode 100644 index 0000000000..10366703f4 --- /dev/null +++ b/tests/army/community/storage/compressBasic.py @@ -0,0 +1,247 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import time +import random + +import taos +import frame +import frame.etool + + +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame import * +from frame.autogen import * + + +class TDTestCase(TBase): + updatecfgDict = { + "compressMsgSize" : "100", + } + # compress + compresses = ["lz4","tsz","zlib","zstd","disabled","xz"] + + # level + levels = ["high","medium","low"] + + # default compress + defCompress = "lz4" + # default level + defLevel = "medium" + + # datatype 17 + dtypes = [ "tinyint","tinyint unsigned","smallint","smallint unsigned","int","int unsigned", + "bigint","bigint unsigned","timestamp","bool","float","double","binary(16)","nchar(16)", + "varchar(16)","varbinary(16)","geometry(32)"] + + def genAllSqls(self, stbName, max): + # encode + encodes = [ + [["tinyint","tinyint unsigned","smallint","smallint unsigned","int","int unsigned","bigint","bigint unsigned"], ["simple8b"]], + [["timestamp","bigint","bigint unsigned"], ["delta-i"]], + [["bool"], ["bit-packing"]], + [["float","double"], ["delta-d"]] + ] + + c = 0 # column number + t = 0 # table number + + sqls = [] + sql = "" + + # loop append sqls + for lines in encodes: + print(lines) + for datatype in lines[0]: + for encode in lines[1]: + for compress in self.compresses: + for level in self.levels: + if sql == "": + # first + sql = f"create table {self.db}.st{t} (ts timestamp" + else: + sql += f", c{c} {datatype} ENCODE '{encode}' COMPRESS '{compress}' LEVEL '{level}'" + c += 1 + + if c >= max: + # append sqls + sql += f") tags(groupid int) " + sqls.append(sql) + # reset + sql = "" + c = 0 + t += 1 + + # break loop + if c > 0: + # append sqls + sql += f") tags(groupid int) " + sqls.append(sql) + + return sqls + + # check error create + def errorCreate(self): + sqls = [ + f"create table terr(ts timestamp, c0 int ENCODE 'abc') ", + f"" + ] + + for dtype in self.dtypes: + # encode + sql = f"create table terr(ts timestamp, c0 {dtype} ENCODE 'abc') " + tdSql.error(sql) + # compress + sql = f"create table terr(ts timestamp, c0 {dtype} COMPRESS 'def') " + tdSql.error(sql) + # level + sql = f"create table terr(ts timestamp, c0 {dtype} LEVEL 'hig') " + tdSql.error(sql) + + # default value correct + def defaultCorrect(self): + # get default encode compress level + sql = f"describe {self.db}.{self.stb}" + tdSql.query(sql) + + ''' + # see AutoGen.types + defEncodes = [ "delta-i","delta-i","simple8b","simple8b","simple8b","simple8b","simple8b","simple8b", + "simple8b","simple8b","delta-d","delta-d","bit-packing", + "disabled","disabled","disabled","disabled","disabled"] + ''' + + # pass-ci have error + defEncodes = [ "delta-i","delta-i","simple8b","simple8b","simple8b","simple8b","simple8b","simple8b", + "simple8b","simple8b","delta-d","delta-d","bit-packing", + "disabled","disabled","disabled","disabled","simple8b"] + + + count = tdSql.getRows() + for i in range(count): + node = tdSql.getData(i, 3) + if node == "TAG": + break + # check + tdSql.checkData(i, 4, defEncodes[i]) + tdSql.checkData(i, 5, self.defCompress) + tdSql.checkData(i, 6, self.defLevel) + + tdLog.info("check default encode compress and level successfully.") + + def checkDataDesc(self, tbname, row, col, value): + sql = f"describe {tbname}" + tdSql.query(sql) + tdSql.checkData(row, col, value) + + + def writeData(self, count): + self.autoGen.insert_data(count, True) + + # alter encode compress level + def checkAlter(self): + tbname = f"{self.db}.{self.stb}" + # alter encode 4 + comp = "delta-i" + sql = f"alter table {tbname} modify column c7 ENCODE '{comp}';" + tdSql.execute(sql, show=True) + self.checkDataDesc(tbname, 8, 4, comp) + self.writeData(1000) + sql = f"alter table {tbname} modify column c8 ENCODE '{comp}';" + tdSql.execute(sql, show=True) + self.checkDataDesc(tbname, 9, 4, comp) + self.writeData(1000) + + # alter compress 5 + comps = self.compresses[2:] + comps.append(self.compresses[0]) # add lz4 + for comp in comps: + for i in range(self.colCnt - 1): + col = f"c{i}" + sql = f"alter table {tbname} modify column {col} COMPRESS '{comp}';" + tdSql.execute(sql, show=True) + self.checkDataDesc(tbname, i + 1, 5, comp) + self.writeData(1000) + + # alter float(c9) double(c10) to tsz + comp = "tsz" + sql = f"alter table {tbname} modify column c9 COMPRESS '{comp}';" + tdSql.execute(sql) + self.checkDataDesc(tbname, 10, 5, comp) + self.writeData(10000) + sql = f"alter table {tbname} modify column c10 COMPRESS '{comp}';" + tdSql.execute(sql) + self.checkDataDesc(tbname, 11, 5, comp) + self.writeData(10000) + + # alter level 6 + for level in self.levels: + for i in range(self.colCnt - 1): + col = f"c{i}" + sql = f"alter table {tbname} modify column {col} LEVEL '{level}';" + tdSql.execute(sql) + self.writeData(1000) + + + def validCreate(self): + sqls = self.genAllSqls(self.stb, 50) + tdSql.executes(sqls, show=True) + + # sql syntax + def checkSqlSyntax(self): + + # create tables positive + self.validCreate() + + # create table negtive + self.errorCreate() + + # check default value corrent + self.defaultCorrect() + + # check alter and write + self.checkAlter() + + + # run + def run(self): + tdLog.debug(f"start to excute {__file__}") + + # create db and stable + self.autoGen = AutoGen() + self.autoGen.create_db(self.db, 2, 3) + tdSql.execute(f"use {self.db}") + self.colCnt = 18 + self.autoGen.create_stable(self.stb, 5, self.colCnt, 8, 8) + self.childCnt = 4 + self.autoGen.create_child(self.stb, "d", self.childCnt) + self.autoGen.insert_data(10000) + + # sql syntax + self.checkSqlSyntax() + + # operateor + self.writeData(10000) + self.flushDb() + self.writeData(10000) + + tdLog.success(f"{__file__} successfully executed") + + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/army/frame/autogen.py b/tests/army/frame/autogen.py index cf21977c75..4b5a8e1384 100644 --- a/tests/army/frame/autogen.py +++ b/tests/army/frame/autogen.py @@ -14,7 +14,7 @@ import time # Auto Gen class # class AutoGen: - def __init__(self, startTs = 1600000000000, step = 1000, batch = 100, fillOne=False): + def __init__(self, startTs = 1600000000000, step = 1000, batch = 500, fillOne=False): self.startTs = startTs self.ts = startTs self.step = step @@ -35,20 +35,23 @@ class AutoGen: # _columns_sql def gen_columns_sql(self, pre, cnt, binary_len, nchar_len): types = [ - 'timestamp', - 'tinyint', + 'timestamp', # 0 + 'tinyint', + 'tinyint unsigned', # 3 'smallint', - 'tinyint unsigned', 'smallint unsigned', - 'int', - 'bigint', + 'int', # 5 'int unsigned', + 'bigint', # 7 'bigint unsigned', - 'float', - 'double', + 'float', # 9 + 'double', # 10 'bool', - f'varchar({binary_len})', - f'nchar({nchar_len})' + f'binary({binary_len})', # 12 + f'varbinary({binary_len})', + f'nchar({nchar_len})', + f'varchar({nchar_len})', + f'geometry(64)' #16 ] sqls = "" @@ -80,9 +83,11 @@ class AutoGen: data = "%f"%(i+i/1000) elif c <= 11 : # bool data = "%d"%(i%2) - elif c == 12 : # binary + elif c <= 13 : # binary data = '"' + self.random_string(self.bin_len) + '"' - elif c == 13 : # binary + elif c == 16 : # geometry + data = f'"point({i} {i})"' + else : # nchar varchar data = '"' + self.random_string(self.nch_len) + '"' if datas != "": @@ -142,7 +147,6 @@ class AutoGen: def insert_data_child(self, child_name, cnt, batch_size, step): values = "" - print("insert child data") ts = self.ts # loop do @@ -162,7 +166,6 @@ class AutoGen: if values != "": sql = f"insert into {self.dbname}.{child_name} values {values}" tdSql.execute(sql) - tdLog.info(f" insert data i={i}") values = "" tdLog.info(f" insert child data {child_name} finished, insert rows={cnt}") diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index 2959cf54a1..377ad228e9 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -56,6 +56,9 @@ class TBase: def stop(self): tdSql.close() + def createDb(self, options=""): + sql = f"create database {self.db} {options}" + tdSql.execute(sql, show=True) # # db action