diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 5643067f0c..b5be8123d3 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -4783,7 +4783,6 @@ void valueSetDatum(SValue *pVal, int8_t type, void *pDatum, uint32_t len) { void valueCloneDatum(SValue *pDst, const SValue *pSrc, int8_t type) { if (IS_VAR_DATA_TYPE(type) || type == TSDB_DATA_TYPE_DECIMAL) { - // we assume that pDst->pData not NULL memcpy(pDst->pData, pSrc->pData, pSrc->nData); pDst->nData = pSrc->nData; } else { diff --git a/source/libs/decimal/src/decimal.c b/source/libs/decimal/src/decimal.c index ae55b31e85..1ca2e8fc94 100644 --- a/source/libs/decimal/src/decimal.c +++ b/source/libs/decimal/src/decimal.c @@ -170,6 +170,8 @@ int32_t calcActualWeight(int32_t prec, int32_t scale, int32_t exp, int32_t weigh static int32_t decimalVarFromStr(const char* str, int32_t len, DecimalVar* result) { int32_t code = 0, pos = 0; + int32_t expectPrecision = result->precision; + int32_t expectScale = result->scale; result->precision = 0; result->scale = 0; result->exponent = 0; @@ -231,9 +233,14 @@ static int32_t decimalVarFromStr(const char* str, int32_t len, DecimalVar* resul Decimal64 delta = {0}; if (curPrec > maxPrecision(result->type)) { if (!afterPoint) return TSDB_CODE_DECIMAL_OVERFLOW; - if (rounded || curPrec - 1 != maxPrecision(result->type) || str[pos] < '5') break; + int32_t curScale = result->scale - result->exponent + places; + if (rounded || curScale > expectScale + 1 /*scale already overflowed, no need do rounding*/ || + curPrec - 1 != maxPrecision(result->type) /* not the maxPrecision + 1 digit, no need do rounding*/ || + str[pos] < '5') + break; - // do rounding + // Do rounding for the maxPrecision + 1 digit. + // Here we cannot directly add this digit into the results, because it may cause overflow. DECIMAL64_SET_VALUE(&delta, 1); scaleUp = places - 1; rounded = true; @@ -1585,7 +1592,7 @@ static void decimal64RoundWithPositiveScale(Decimal64* pDec, uint8_t prec, int8_ int32_t decimal64FromStr(const char* str, int32_t len, uint8_t expectPrecision, uint8_t expectScale, Decimal64* pRes) { int32_t code = 0; - DecimalVar var = {.type = DECIMAL_64, .pDec = pRes->words}; + DecimalVar var = {.type = DECIMAL_64, .pDec = pRes->words, .precision = expectPrecision, .scale = expectScale}; DECIMAL64_SET_VALUE(pRes, 0); code = decimalVarFromStr(str, len, &var); if (TSDB_CODE_SUCCESS != code) return code; @@ -1626,7 +1633,7 @@ static void decimal128ScaleTo(Decimal128* pDec, uint8_t oldScale, uint8_t newSca int32_t decimal128FromStr(const char* str, int32_t len, uint8_t expectPrecision, uint8_t expectScale, Decimal128* pRes) { int32_t code = 0; - DecimalVar var = {.type = DECIMAL_128, .pDec = pRes->words}; + DecimalVar var = {.type = DECIMAL_128, .pDec = pRes->words, .precision = expectPrecision, .scale = expectScale}; DECIMAL128_SET_HIGH_WORD(pRes, 0); DECIMAL128_SET_LOW_WORD(pRes, 0); code = decimalVarFromStr(str, len, &var); diff --git a/source/libs/decimal/test/decimalTest.cpp b/source/libs/decimal/test/decimalTest.cpp index 1b21040c5e..d34f0a5940 100644 --- a/source/libs/decimal/test/decimalTest.cpp +++ b/source/libs/decimal/test/decimalTest.cpp @@ -980,39 +980,41 @@ TEST(decimal, decimalFromStr_all) { testDecimalFromStr(units); std::vector> dec128Units = { - {38, 10, "0e-10", "0", false}, - {38, 10, "0e10", "0", false}, - {38, 10, "e-100000", "0", false}, - {38, 10, "e-10", "0", false}, - {38, 10, "e10", "0", false}, - {38, 10, "-1.23456789012300000000000000000099000009e20", "-123456789012300000000.0000000001", false}, - {38, 10, "-1.234567890123e20", "-123456789012300000000.0000000000", false}, - {20, 15, "1234567890.9999999999e-6", "1234.567891000000000", false}, - {20, 15, "1234567890.9999999999999999999e-5", "12345.678910000000000", false}, - {20, 15, "1234567890.99999999999e-5", "12345.678910000000000", false}, - {20, 10, "12345667788.12312e-10", "1.2345667788", false}, - {20, 20, "1.234567123123123e-20", "0.00000000000000000001", false}, - {38, 38, "1.23456789121312312312312312355e-10", "0.00000000012345678912131231231231231236", false}, - {38, 38, "1.23456e-10", "0.00000000012345600000000000000000000000", false}, - {38, 10, "123456789012345678901234567.89012345679", "123456789012345678901234567.8901234568", false}, - {38, 10, "123456789012345678901234567.89012345670", "123456789012345678901234567.8901234567", false}, - {38, 10, "-123456789012345678901234567.89012345671", "-123456789012345678901234567.8901234567", false}, - {38, 10, "-123456789012345678901234567.89012345679", "-123456789012345678901234567.8901234568", false}, - {38, 10, "-9999999999999999999999999999.99999999995", "", true}, - {38, 10, "-9999999999999999999999999999.99999999994", "-9999999999999999999999999999.9999999999", false}, - {38, 10, "9999999999999999999999999999.99999999996", "", true}, - {38, 10, "9999999999999999999999999999.99999999994", "9999999999999999999999999999.9999999999", false}, - {36, 35, "9.99999999999999999999999999999999999", "9.99999999999999999999999999999999999", false}, - {36, 35, "9.999999999999999999999999999999999999111231231", "", true}, - {38, 38, "0.000000000000000000000000000000000000001", "0", false}, - {38, 38, "0.000000000000000000000000000000000000006", "0.00000000000000000000000000000000000001", false}, - {38, 35, "123.000000000000000000000000000000001", "123.00000000000000000000000000000000100", false}, - {38, 5, "123.", "123.00000", false}, - {20, 4, "-.12345", "-0.1235", false}, - {20, 4, "-.", "0", false}, - {30, 10, "1.2345e+20", "", true}, - {38, 38, "1.23456e0", "", true}, - {38, 38, "1.23456e-1", "0.12345600000000000000000000000000000000", false}, + {38, 38, "1.23456789121312312312312312355e-10", "0.00000000012345678912131231231231231236", false}, + {38, 10, "610854818164978322886511028.733672028246706062283745797933332437780013", + "610854818164978322886511028.7336720282", false}, + {38, 10, "0e-10", "0", false}, + {38, 10, "0e10", "0", false}, + {38, 10, "e-100000", "0", false}, + {38, 10, "e-10", "0", false}, + {38, 10, "e10", "0", false}, + {38, 10, "-1.23456789012300000000000000000099000009e20", "-123456789012300000000.0000000001", false}, + {38, 10, "-1.234567890123e20", "-123456789012300000000.0000000000", false}, + {20, 15, "1234567890.9999999999e-6", "1234.567891000000000", false}, + {20, 15, "1234567890.9999999999999999999e-5", "12345.678910000000000", false}, + {20, 15, "1234567890.99999999999e-5", "12345.678910000000000", false}, + {20, 10, "12345667788.12312e-10", "1.2345667788", false}, + {20, 20, "1.234567123123123e-20", "0.00000000000000000001", false}, + {38, 38, "1.23456e-10", "0.00000000012345600000000000000000000000", false}, + {38, 10, "123456789012345678901234567.89012345679", "123456789012345678901234567.8901234568", false}, + {38, 10, "123456789012345678901234567.89012345670", "123456789012345678901234567.8901234567", false}, + {38, 10, "-123456789012345678901234567.89012345671", "-123456789012345678901234567.8901234567", false}, + {38, 10, "-123456789012345678901234567.89012345679", "-123456789012345678901234567.8901234568", false}, + {38, 10, "-9999999999999999999999999999.99999999995", "", true}, + {38, 10, "-9999999999999999999999999999.99999999994", "-9999999999999999999999999999.9999999999", false}, + {38, 10, "9999999999999999999999999999.99999999996", "", true}, + {38, 10, "9999999999999999999999999999.99999999994", "9999999999999999999999999999.9999999999", false}, + {36, 35, "9.99999999999999999999999999999999999", "9.99999999999999999999999999999999999", false}, + {36, 35, "9.999999999999999999999999999999999999111231231", "", true}, + {38, 38, "0.000000000000000000000000000000000000001", "0", false}, + {38, 38, "0.000000000000000000000000000000000000006", "0.00000000000000000000000000000000000001", false}, + {38, 35, "123.000000000000000000000000000000001", "123.00000000000000000000000000000000100", false}, + {38, 5, "123.", "123.00000", false}, + {20, 4, "-.12345", "-0.1235", false}, + {20, 4, "-.", "0", false}, + {30, 10, "1.2345e+20", "", true}, + {38, 38, "1.23456e0", "", true}, + {38, 38, "1.23456e-1", "0.12345600000000000000000000000000000000", false}, }; testDecimalFromStr(dec128Units); } diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 7d086d108d..262b99f50a 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1862,7 +1862,7 @@ static void clearColValArray(SArray* pCols) { int32_t num = taosArrayGetSize(pCols); for (int32_t i = 0; i < num; ++i) { SColVal* pCol = taosArrayGet(pCols, i); - if (IS_VAR_DATA_TYPE(pCol->value.type)) { + if (IS_VAR_DATA_TYPE(pCol->value.type) || pCol->value.type == TSDB_DATA_TYPE_DECIMAL) { taosMemoryFreeClear(pCol->value.pData); } } diff --git a/tests/system-test/2-query/decimal.py b/tests/system-test/2-query/decimal.py index a3dc0127f0..3e7053868f 100644 --- a/tests/system-test/2-query/decimal.py +++ b/tests/system-test/2-query/decimal.py @@ -5,8 +5,8 @@ import time import threading import secrets +from regex import D from sympy import true -from torch import randint import query from tag_lite import column from util.log import * @@ -14,13 +14,16 @@ from util.sql import * from util.cases import * from util.dnodes import * from util.common import * -from decimal import Decimal +from decimal import * + +getcontext().prec = 40 syntax_error = -2147473920 invalid_column = -2147473918 invalid_compress_level = -2147483084 invalid_encode_param = -2147483087 + class DecimalTypeGeneratorConfig: def __init__(self): self.enable_weight_overflow: bool = False @@ -34,71 +37,90 @@ class DecimalTypeGeneratorConfig: self.prec = 38 self.scale = 10 + class DecimalStringRandomGenerator: def __init__(self): self.corner_cases = ["0", "NULL", "0.", ".0", "000.000000"] self.ratio_base: int = 1000000 - + def possible(self, possibility: float) -> bool: return random.randint(0, self.ratio_base) < possibility * self.ratio_base - + def generate_sign(self, positive_ratio: float) -> str: if self.possible(positive_ratio): return "+" return "-" - + def generate_digit(self) -> str: return str(random.randint(0, 9)) def current_should_generate_corner_case(self, corner_case_ratio: float) -> bool: return self.possible(corner_case_ratio) - + def generate_corner_case(self, config: DecimalTypeGeneratorConfig) -> str: if self.possible(0.8): return random.choice(self.corner_cases) else: res = self.generate_digit() * (config.prec - config.scale) if self.possible(0.8): - res += '.' + res += "." if self.possible(0.8): res += self.generate_digit() * config.scale return res - + ## 写入大整数的例子, 如10000000000, scale解析时可能为负数 + ## Generate decimal with E/e def generate_(self, config: DecimalTypeGeneratorConfig) -> str: - ret: str = '' + ret: str = "" sign = self.generate_sign(config.positive_ratio) - if config.with_corner_case and self.current_should_generate_corner_case(config.corner_case_ratio): + if config.with_corner_case and self.current_should_generate_corner_case( + config.corner_case_ratio + ): ret += self.generate_corner_case(config) else: - if config.enable_positive_sign or sign != '+': + if config.enable_positive_sign or sign != "+": ret += sign weight = random.randint(1, config.prec - config.scale) scale = random.randint(1, config.scale) for i in range(weight): ret += self.generate_digit() - - if config.enable_weight_overflow and self.possible(config.weightOverflowRatio): - extra_weight = config.prec - weight + 1 + random.randint(1, self.get_max_prec(config.prec)) + + if config.enable_weight_overflow and self.possible( + config.weightOverflowRatio + ): + extra_weight = ( + config.prec + - weight + + 1 + + random.randint(1, self.get_max_prec(config.prec)) + ) while extra_weight > 0: ret += self.generate_digit() extra_weight -= 1 - ret += '.' + ret += "." for i in range(scale): ret += self.generate_digit() - if config.enable_scale_overflow and self.possible(config.scale_overflow_ratio): - extra_scale = config.scale - scale + 1 + random.randint(1, self.get_max_prec(config.prec)) + if config.enable_scale_overflow and self.possible( + config.scale_overflow_ratio + ): + extra_scale = ( + config.scale + - scale + + 1 + + random.randint(1, self.get_max_prec(config.prec)) + ) while extra_scale > 0: ret += self.generate_digit() extra_scale -= 1 return ret - + def get_max_prec(self, prec): if prec <= 18: return 18 else: return 38 - + + class DecimalColumnAggregator: def __init__(self): self.max: Decimal = Decimal("0") @@ -122,18 +144,19 @@ class DecimalColumnAggregator: if v < self.min: self.min = v + class TaosShell: def __init__(self): self.queryResult = [] self.tmp_file_path = "/tmp/taos_shell_result" def read_result(self): - with open(self.tmp_file_path, 'r') as f: + with open(self.tmp_file_path, "r") as f: lines = f.readlines() lines = lines[1:] for line in lines: col = 0 - vals: List[str] = line.split(',') + vals: List[str] = line.split(",") if len(self.queryResult) == 0: self.queryResult = [[] for i in range(len(vals))] for val in vals: @@ -143,13 +166,16 @@ class TaosShell: def query(self, sql: str): try: command = f'taos -s "{sql} >> {self.tmp_file_path}"' - result = subprocess.run(command, shell=True, check=True, stderr=subprocess.PIPE) + result = subprocess.run( + command, shell=True, check=True, stderr=subprocess.PIPE + ) self.read_result() except subprocess.CalledProcessError as e: - tdLog.error(f"Command '{sql}' failed with error: {e.stderr.decode('utf-8')}") + tdLog.exit(f"Command '{sql}' failed with error: {e.stderr.decode('utf-8')}") self.queryResult = [] return self.queryResult + class TypeEnum: BOOL = 1 TINYINT = 2 @@ -215,16 +241,16 @@ class TypeEnum: else: raise Exception("unknow type") + class DataType: def __init__(self, type: int, length: int = 0, type_mod: int = 0): - self.type : int = type + self.type: int = type self.length = length self.type_mod = type_mod def __str__(self): if self.type_mod != 0: - decimal_type = self.get_decimal_type() - return f"{TypeEnum.get_type_str(self.type)}({decimal_type.precision}, {decimal_type.scale})" + return f"{TypeEnum.get_type_str(self.type)}({self.prec()}, {self.scale()})" if self.length: return f"{TypeEnum.get_type_str(self.type)}({self.length})" return TypeEnum.get_type_str(self.type) @@ -240,12 +266,24 @@ class DataType: def __repr__(self): return f"DataType({self.type}, {self.length}, {self.type_mod})" - + + def prec(self): + return 0 + + def scale(self): + return 0 + def construct_type_value(self, val: str): - if self.type == TypeEnum.BINARY or self.type == TypeEnum.VARCHAR or self.type == TypeEnum.NCHAR or self.type == TypeEnum.VARBINARY: + if ( + self.type == TypeEnum.BINARY + or self.type == TypeEnum.VARCHAR + or self.type == TypeEnum.NCHAR + or self.type == TypeEnum.VARBINARY + ): return f"'{val}'" else: return val + ## TODO generate NULL, None def generate_value(self) -> str: if self.type == TypeEnum.BOOL: @@ -260,7 +298,11 @@ class DataType: return str(secrets.randbelow(9223372036854775808) - 4611686018427387904) if self.type == TypeEnum.FLOAT or self.type == TypeEnum.DOUBLE: return str(random.random()) - if self.type == TypeEnum.VARCHAR or self.type == TypeEnum.NCHAR or self.type == TypeEnum.VARBINARY: + if ( + self.type == TypeEnum.VARCHAR + or self.type == TypeEnum.NCHAR + or self.type == TypeEnum.VARBINARY + ): return f"'{secrets.token_urlsafe(random.randint(0, self.length))[0:random.randint(0, self.length)]}'" if self.type == TypeEnum.TIMESTAMP: return str(secrets.randbelow(9223372036854775808)) @@ -275,13 +317,41 @@ class DataType: if self.type == TypeEnum.JSON: return f'{{"key": "{secrets.token_urlsafe(10)}"}}' raise Exception(f"unsupport type {self.type}") + def check(self, values, offset: int): return True + @staticmethod + def get_all_type_columns() -> List: + other_types = [ + DataType(TypeEnum.BOOL), + DataType(TypeEnum.TINYINT), + DataType(TypeEnum.SMALLINT), + DataType(TypeEnum.INT), + DataType(TypeEnum.BIGINT), + DataType(TypeEnum.FLOAT), + DataType(TypeEnum.DOUBLE), + DataType(TypeEnum.VARCHAR, 255), + DataType(TypeEnum.TIMESTAMP), + DataType(TypeEnum.NCHAR), + DataType(TypeEnum.UTINYINT), + DataType(TypeEnum.USMALLINT), + DataType(TypeEnum.UINT), + DataType(TypeEnum.UBIGINT), + DataType(TypeEnum.JSON), + DataType(TypeEnum.VARBINARY), + DataType(TypeEnum.DECIMAL), + DataType(TypeEnum.BINARY), + DataType(TypeEnum.GEOMETRY), + DataType(TypeEnum.DECIMAL64), + ] + return other_types + + class DecimalType(DataType): def __init__(self, type, precision: int, scale: int): - self.precision = precision - self.scale = scale + self.precision_ = precision + self.scale_ = scale if type == TypeEnum.DECIMAL64: bytes = 8 else: @@ -295,50 +365,82 @@ class DecimalType(DataType): self.values: List[str] = [] def get_decimal_type_mod(self) -> int: - return self.precision * 100 + self.scale + return self.precision_ * 100 + self.scale() + + def prec(self): + return self.precision_ + + def scale(self): + return self.scale_ def __str__(self): - return f"DECIMAL({self.precision}, {self.scale})" + return f"DECIMAL({self.precision_}, {self.scale()})" def __eq__(self, other): - return self.precision == other.precision and self.scale == other.scale + return self.precision_ == other.precision and self.scale() == other.scale def __ne__(self, other): return not self.__eq__(other) def __hash__(self): - return hash((self.precision, self.scale)) + return hash((self.precision_, self.scale())) def __repr__(self): - return f"DecimalType({self.precision}, {self.scale})" - + return f"DecimalType({self.precision_}, {self.scale()})" + def generate_value(self) -> str: val = self.generator.generate_(self.generator_config) self.aggregator.add_value(val) - self.values.append(val) + self.values.append(val) ## save it into files maybe return val - + @staticmethod def default_compression() -> str: return "zstd" + @staticmethod def default_encode() -> str: return "disabled" - + def check(self, values, offset: int): val_from_query = values val_insert = self.values[offset:] for v1, v2 in zip(val_from_query, val_insert): - dec1: Decimal = Decimal(v1) - dec2: Decimal = Decimal(v2) - dec2 = dec2.quantize(Decimal(10) ** -self.scale) - if dec1 != dec2: - tdLog.error(f"check decimal column failed, expect {dec2}, but get {dec1}") + if v2 == "NULL": + if v1.strip() != "NULL": + tdLog.debug( + f"val_insert: {val_insert} val_from_query: {val_from_query}" + ) + tdLog.exit(f"insert NULL, query not NULL: {v1}") + else: + continue + try: + dec1: Decimal = Decimal(v1) + dec2: Decimal = Decimal(v2) + dec2 = dec2.quantize(Decimal("1." + "0" * self.scale()), ROUND_HALF_UP) + except Exception as e: + tdLog.exit(f"failed to convert {v1} or {v2} to decimal, {e}") return False + if dec1 != dec2: + tdLog.exit( + f"check decimal column failed for insert: {v2}, query: {v1}, expect {dec2}, but get {dec1}" + ) + return False + else: + tdLog.debug( + f"check decimal succ, insert:{v2} query:{v1}, py dec: {dec2}" + ) class DecimalColumnTableCreater: - def __init__(self, conn, dbName: str, tbName: str, columns_types: List[DataType], tags_types: List[DataType] = []): + def __init__( + self, + conn, + dbName: str, + tbName: str, + columns_types: List[DataType], + tags_types: List[DataType] = [], + ): self.conn = conn self.dbName = dbName self.tbName = tbName @@ -347,9 +449,9 @@ class DecimalColumnTableCreater: def create(self): if len(self.tags_types) > 0: - table = 'stable' + table = "stable" else: - table = 'table' + table = "table" sql = f"create {table} {self.dbName}.{self.tbName} (ts timestamp" for i, column in enumerate(self.columns_types): sql += f", c{i+1} {column}" @@ -362,7 +464,13 @@ class DecimalColumnTableCreater: sql += ")" self.conn.execute(sql, queryTimes=1) - def create_child_table(self, ctbPrefix: str, ctbNum: int, tag_types: List[DataType], tag_values: List[str]): + def create_child_table( + self, + ctbPrefix: str, + ctbNum: int, + tag_types: List[DataType], + tag_values: List[str], + ): for i in range(ctbNum): sql = f"create table {self.dbName}.{ctbPrefix}{i} using {self.dbName}.{self.tbName} tags(" for j, tag in enumerate(tag_types): @@ -374,7 +482,14 @@ class DecimalColumnTableCreater: class TableInserter: - def __init__(self, conn, dbName: str, tbName: str, columns_types: List[DataType], tags_types: List[DataType] = []): + def __init__( + self, + conn, + dbName: str, + tbName: str, + columns_types: List[DataType], + tags_types: List[DataType] = [], + ): self.conn = conn self.dbName = dbName self.tbName = tbName @@ -393,42 +508,111 @@ class TableInserter: sql += ", " local_flush_database = i % 5000 == 0 if len(sql) > 1000: - #tdLog.debug(f"insert into with sql{sql}") + # tdLog.debug(f"insert into with sql{sql}") if flush_database and local_flush_database: self.conn.execute(f"flush database {self.dbName}", queryTimes=1) self.conn.execute(sql, queryTimes=1) sql = pre_insert if len(sql) > len(pre_insert): - #tdLog.debug(f"insert into with sql{sql}") + # tdLog.debug(f"insert into with sql{sql}") if flush_database: self.conn.execute(f"flush database {self.dbName}", queryTimes=1) self.conn.execute(sql, queryTimes=1) + class TableDataValidator: - def __init__(self, columns: List[DataType], tbName: str, dbName: str, tbIdx: int = 0): + def __init__( + self, columns: List[DataType], tbName: str, dbName: str, tbIdx: int = 0 + ): self.columns = columns self.tbName = tbName self.dbName = dbName self.tbIdx = tbIdx - + def validate(self): sql = f"select * from {self.dbName}.{self.tbName}" res = TaosShell().query(sql) - row_num = len(res) + row_num = len(res[1]) colIdx = 1 for col in self.columns: if col.type == TypeEnum.DECIMAL or col.type == TypeEnum.DECIMAL64: col.check(res[colIdx], row_num * self.tbIdx) colIdx += 1 - + + +class DecimalColumnExpr: + def __init__(self, column: DecimalType): + self.column_ = column + + def execute(self): + pass + + def generate(self): + pass + + +class DecimalBinaryOperator(DecimalColumnExpr): + def __init__(self, op: str, column: DecimalType): + super().__init__(column) + self.op_ = op + + def __str__(self): + return self.op_ + + def generate(self): + pass + + def execute(self, left, right): + if self.op_ == "+": + return left + right + if self.op_ == "-": + return left - right + if self.op_ == "*": + return left * right + if self.op_ == "/": + return left / right + if self.op_ == "%": + return left % right + if self.op_ == "==": + return left == right + if self.op_ == "!=": + return left != right + if self.op_ == ">": + return left > right + if self.op_ == "<": + return left < right + if self.op_ == ">=": + return left >= right + if self.op_ == "<=": + return left <= right + raise Exception(f"unsupport operator {self.op_}") + + +class DecimalBinaryOperatorIn(DecimalBinaryOperator): + def __init__(self, op: str): + super().__init__(op) + + def execute(self, left, right): + if self.op_.lower()() == "in": + return left in right + if self.op_.lower() == "not in": + return left not in right + + class TDTestCase: - updatecfgDict = {'asynclog': 0, 'ttlUnit': 1, 'ttlPushInterval': 5, 'ratioOfVnodeStreamThrea': 4, 'debugFlag': 143} + updatecfgDict = { + "asynclog": 0, + "ttlUnit": 1, + "ttlPushInterval": 5, + "ratioOfVnodeStreamThrea": 4, + "debugFlag": 143, + } def __init__(self): self.vgroups = 4 self.ctbNum = 10 self.rowsPerTbl = 10000 - self.duraion = '1h' + self.duraion = "1h" self.norm_tb_columns = [] self.tags = [] self.stable_name = "meters" @@ -436,55 +620,100 @@ class TDTestCase: self.c_table_prefix = "t" self.db_name = "test" self.c_table_num = 10 - self.no_decimal_col_tb_name = 'tt' + self.no_decimal_col_tb_name = "tt" self.stb_columns = [] - self.stream_name = 'stream1' - self.stream_out_stb = 'stream_out_stb' - self.tsma_name = 'tsma1' + self.stream_name = "stream1" + self.stream_out_stb = "stream_out_stb" + self.tsma_name = "tsma1" + self.query_test_round = 10000 def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) - def create_database(self, tsql, dbName, dropFlag=1, vgroups=2, replica=1, duration: str = '1d'): + def create_database( + self, tsql, dbName, dropFlag=1, vgroups=2, replica=1, duration: str = "1d" + ): if dropFlag == 1: tsql.execute("drop database if exists %s" % (dbName)) - tsql.execute("create database if not exists %s vgroups %d replica %d duration %s" % ( - dbName, vgroups, replica, duration)) + tsql.execute( + "create database if not exists %s vgroups %d replica %d duration %s" + % (dbName, vgroups, replica, duration) + ) tdLog.debug("complete to create database %s" % (dbName)) def create_stable(self, tsql, paraDict): colString = tdCom.gen_column_type_str( - colname_prefix=paraDict["colPrefix"], column_elm_list=paraDict["colSchema"]) + colname_prefix=paraDict["colPrefix"], column_elm_list=paraDict["colSchema"] + ) tagString = tdCom.gen_tag_type_str( - tagname_prefix=paraDict["tagPrefix"], tag_elm_list=paraDict["tagSchema"]) + tagname_prefix=paraDict["tagPrefix"], tag_elm_list=paraDict["tagSchema"] + ) sqlString = f"create table if not exists %s.%s (%s) tags (%s)" % ( - paraDict["dbName"], paraDict["stbName"], colString, tagString) + paraDict["dbName"], + paraDict["stbName"], + colString, + tagString, + ) tdLog.debug("%s" % (sqlString)) tsql.execute(sqlString) - def create_ctable(self, tsql=None, dbName='dbx', stbName='stb', ctbPrefix='ctb', ctbNum=1, ctbStartIdx=0): + def create_ctable( + self, + tsql=None, + dbName="dbx", + stbName="stb", + ctbPrefix="ctb", + ctbNum=1, + ctbStartIdx=0, + ): for i in range(ctbNum): - sqlString = "create table %s.%s%d using %s.%s tags(%d, 'tb%d', 'tb%d', %d, %d, %d)" % (dbName, ctbPrefix, i+ctbStartIdx, dbName, stbName, (i+ctbStartIdx) % 5, i+ctbStartIdx + random.randint( - 1, 100), i+ctbStartIdx + random.randint(1, 100), i+ctbStartIdx + random.randint(1, 100), i+ctbStartIdx + random.randint(1, 100), i+ctbStartIdx + random.randint(1, 100)) + sqlString = ( + "create table %s.%s%d using %s.%s tags(%d, 'tb%d', 'tb%d', %d, %d, %d)" + % ( + dbName, + ctbPrefix, + i + ctbStartIdx, + dbName, + stbName, + (i + ctbStartIdx) % 5, + i + ctbStartIdx + random.randint(1, 100), + i + ctbStartIdx + random.randint(1, 100), + i + ctbStartIdx + random.randint(1, 100), + i + ctbStartIdx + random.randint(1, 100), + i + ctbStartIdx + random.randint(1, 100), + ) + ) tsql.execute(sqlString) - tdLog.debug("complete to create %d child tables by %s.%s" % - (ctbNum, dbName, stbName)) + tdLog.debug( + "complete to create %d child tables by %s.%s" % (ctbNum, dbName, stbName) + ) - def init_normal_tb(self, tsql, db_name: str, tb_name: str, rows: int, start_ts: int, ts_step: int): - sql = 'CREATE TABLE %s.%s (ts timestamp, c1 INT, c2 INT, c3 INT, c4 double, c5 VARCHAR(255))' % ( - db_name, tb_name) + def init_normal_tb( + self, tsql, db_name: str, tb_name: str, rows: int, start_ts: int, ts_step: int + ): + sql = ( + "CREATE TABLE %s.%s (ts timestamp, c1 INT, c2 INT, c3 INT, c4 double, c5 VARCHAR(255))" + % (db_name, tb_name) + ) tsql.execute(sql) - sql = 'INSERT INTO %s.%s values' % (db_name, tb_name) + sql = "INSERT INTO %s.%s values" % (db_name, tb_name) for j in range(rows): - sql += f'(%d, %d,%d,%d,{random.random()},"varchar_%d"),' % (start_ts + j * ts_step + randrange(500), j % - 10 + randrange(200), j % 10, j % 10, j % 10 + randrange(100)) + sql += f'(%d, %d,%d,%d,{random.random()},"varchar_%d"),' % ( + start_ts + j * ts_step + randrange(500), + j % 10 + randrange(200), + j % 10, + j % 10, + j % 10 + randrange(100), + ) tsql.execute(sql) - def insert_data(self, tsql, dbName, ctbPrefix, ctbNum, rowsPerTbl, batchNum, startTs, tsStep): + def insert_data( + self, tsql, dbName, ctbPrefix, ctbNum, rowsPerTbl, batchNum, startTs, tsStep + ): tdLog.debug("start to insert data ............") tsql.execute("use %s" % dbName) pre_insert = "insert into " @@ -494,14 +723,33 @@ class TDTestCase: rowsBatched = 0 sql += " %s.%s%d values " % (dbName, ctbPrefix, i) for j in range(rowsPerTbl): - if (i < ctbNum/2): - sql += "(%d, %d, %d, %d,%d,%d,%d,true,'binary%d', 'nchar%d') " % (startTs + j*tsStep + randrange( - 500), j % 10 + randrange(100), j % 10 + randrange(200), j % 10, j % 10, j % 10, j % 10, j % 10, j % 10) + if i < ctbNum / 2: + sql += "(%d, %d, %d, %d,%d,%d,%d,true,'binary%d', 'nchar%d') " % ( + startTs + j * tsStep + randrange(500), + j % 10 + randrange(100), + j % 10 + randrange(200), + j % 10, + j % 10, + j % 10, + j % 10, + j % 10, + j % 10, + ) else: - sql += "(%d, %d, NULL, %d,NULL,%d,%d,true,'binary%d', 'nchar%d') " % ( - startTs + j*tsStep + randrange(500), j % 10, j % 10, j % 10, j % 10, j % 10, j % 10) + sql += ( + "(%d, %d, NULL, %d,NULL,%d,%d,true,'binary%d', 'nchar%d') " + % ( + startTs + j * tsStep + randrange(500), + j % 10, + j % 10, + j % 10, + j % 10, + j % 10, + j % 10, + ) + ) rowsBatched += 1 - if ((rowsBatched == batchNum) or (j == rowsPerTbl - 1)): + if (rowsBatched == batchNum) or (j == rowsPerTbl - 1): tsql.execute(sql) rowsBatched = 0 if j < rowsPerTbl - 1: @@ -513,84 +761,157 @@ class TDTestCase: tdLog.debug("insert data ............ [OK]") return - def init_data(self, db: str = 'test', ctb_num: int = 10, rows_per_ctb: int = 10000, start_ts: int = 1537146000000, ts_step: int = 500): + def init_data( + self, + db: str = "test", + ctb_num: int = 10, + rows_per_ctb: int = 10000, + start_ts: int = 1537146000000, + ts_step: int = 500, + ): tdLog.printNoPrefix( - "======== prepare test env include database, stable, ctables, and insert data: ") - paraDict = {'dbName': db, - 'dropFlag': 1, - 'vgroups': 4, - 'stbName': 'meters', - 'colPrefix': 'c', - 'tagPrefix': 't', - 'colSchema': [{'type': 'INT', 'count': 1}, {'type': 'BIGINT', 'count': 1}, {'type': 'FLOAT', 'count': 1}, {'type': 'DOUBLE', 'count': 1}, {'type': 'smallint', 'count': 1}, {'type': 'tinyint', 'count': 1}, {'type': 'bool', 'count': 1}, {'type': 'binary', 'len': 10, 'count': 1}, {'type': 'nchar', 'len': 10, 'count': 1}], - 'tagSchema': [{'type': 'INT', 'count': 1}, {'type': 'nchar', 'len': 20, 'count': 1}, {'type': 'binary', 'len': 20, 'count': 1}, {'type': 'BIGINT', 'count': 1}, {'type': 'smallint', 'count': 1}, {'type': 'DOUBLE', 'count': 1}], - 'ctbPrefix': 't', - 'ctbStartIdx': 0, - 'ctbNum': ctb_num, - 'rowsPerTbl': rows_per_ctb, - 'batchNum': 3000, - 'startTs': start_ts, - 'tsStep': ts_step} + "======== prepare test env include database, stable, ctables, and insert data: " + ) + paraDict = { + "dbName": db, + "dropFlag": 1, + "vgroups": 4, + "stbName": "meters", + "colPrefix": "c", + "tagPrefix": "t", + "colSchema": [ + {"type": "INT", "count": 1}, + {"type": "BIGINT", "count": 1}, + {"type": "FLOAT", "count": 1}, + {"type": "DOUBLE", "count": 1}, + {"type": "smallint", "count": 1}, + {"type": "tinyint", "count": 1}, + {"type": "bool", "count": 1}, + {"type": "binary", "len": 10, "count": 1}, + {"type": "nchar", "len": 10, "count": 1}, + ], + "tagSchema": [ + {"type": "INT", "count": 1}, + {"type": "nchar", "len": 20, "count": 1}, + {"type": "binary", "len": 20, "count": 1}, + {"type": "BIGINT", "count": 1}, + {"type": "smallint", "count": 1}, + {"type": "DOUBLE", "count": 1}, + ], + "ctbPrefix": "t", + "ctbStartIdx": 0, + "ctbNum": ctb_num, + "rowsPerTbl": rows_per_ctb, + "batchNum": 3000, + "startTs": start_ts, + "tsStep": ts_step, + } - paraDict['vgroups'] = self.vgroups - paraDict['ctbNum'] = ctb_num - paraDict['rowsPerTbl'] = rows_per_ctb + paraDict["vgroups"] = self.vgroups + paraDict["ctbNum"] = ctb_num + paraDict["rowsPerTbl"] = rows_per_ctb tdLog.info("create database") - self.create_database(tsql=tdSql, dbName=paraDict["dbName"], dropFlag=paraDict["dropFlag"], - vgroups=paraDict["vgroups"], replica=self.replicaVar, duration=self.duraion) + self.create_database( + tsql=tdSql, + dbName=paraDict["dbName"], + dropFlag=paraDict["dropFlag"], + vgroups=paraDict["vgroups"], + replica=self.replicaVar, + duration=self.duraion, + ) tdLog.info("create stb") self.create_stable(tsql=tdSql, paraDict=paraDict) tdLog.info("create child tables") - self.create_ctable(tsql=tdSql, dbName=paraDict["dbName"], - stbName=paraDict["stbName"], ctbPrefix=paraDict["ctbPrefix"], - ctbNum=paraDict["ctbNum"], ctbStartIdx=paraDict["ctbStartIdx"]) - self.insert_data(tsql=tdSql, dbName=paraDict["dbName"], - ctbPrefix=paraDict["ctbPrefix"], ctbNum=paraDict["ctbNum"], - rowsPerTbl=paraDict["rowsPerTbl"], batchNum=paraDict["batchNum"], - startTs=paraDict["startTs"], tsStep=paraDict["tsStep"]) - self.init_normal_tb(tdSql, paraDict['dbName'], 'norm_tb', - paraDict['rowsPerTbl'], paraDict['startTs'], paraDict['tsStep']) + self.create_ctable( + tsql=tdSql, + dbName=paraDict["dbName"], + stbName=paraDict["stbName"], + ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"], + ctbStartIdx=paraDict["ctbStartIdx"], + ) + self.insert_data( + tsql=tdSql, + dbName=paraDict["dbName"], + ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"], + rowsPerTbl=paraDict["rowsPerTbl"], + batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"], + tsStep=paraDict["tsStep"], + ) + self.init_normal_tb( + tdSql, + paraDict["dbName"], + "norm_tb", + paraDict["rowsPerTbl"], + paraDict["startTs"], + paraDict["tsStep"], + ) - def check_desc_for_one_ctb(self, ctbPrefix: str, columns: List[DataType], tags: List[DataType] = []): + def check_desc_for_one_ctb( + self, ctbPrefix: str, columns: List[DataType], tags: List[DataType] = [] + ): ctb_idx = randrange(self.c_table_num) return self.check_desc(f"{ctbPrefix}{ctb_idx}", columns, tags) - def check_desc(self, tbname: str, column_types: List[DataType], tag_types: List[DataType] = []): + def check_desc( + self, tbname: str, column_types: List[DataType], tag_types: List[DataType] = [] + ): sql = f"desc {self.db_name}.{tbname}" tdSql.query(sql, queryTimes=1) results = tdSql.queryResult for i, column_type in enumerate(column_types): if column_type.type == TypeEnum.DECIMAL: - if results[i+1][1] != column_type.__str__(): + if results[i + 1][1] != column_type.__str__(): tdLog.info(str(results)) - tdLog.exit(f"check desc failed for table: {tbname} column {results[i+1][0]} type is {results[i+1][1]}, expect DECIMAL") - if results[i+1][4] != DecimalType.default_encode(): - tdLog.exit(f"check desc failed for table: {tbname} column {results[i+1][0]} encode is {results[i+1][5]}, expect {DecimalType.default_encode()}") - if results[i+1][5] != DecimalType.default_compression(): - tdLog.exit(f"check desc failed for table: {tbname} column {results[i+1][0]} compression is {results[i+1][4]}, expect {DecimalType.default_compression()}") + tdLog.exit( + f"check desc failed for table: {tbname} column {results[i+1][0]} type is {results[i+1][1]}, expect DECIMAL" + ) + if results[i + 1][4] != DecimalType.default_encode(): + tdLog.exit( + f"check desc failed for table: {tbname} column {results[i+1][0]} encode is {results[i+1][5]}, expect {DecimalType.default_encode()}" + ) + if results[i + 1][5] != DecimalType.default_compression(): + tdLog.exit( + f"check desc failed for table: {tbname} column {results[i+1][0]} compression is {results[i+1][4]}, expect {DecimalType.default_compression()}" + ) if tbname == self.stable_name: self.check_desc_for_one_ctb(self.c_table_prefix, column_types, tag_types) - def check_show_create_table(self, tbname: str, column_types: List[DataType], tag_types: List[DataType] = []): + def check_show_create_table( + self, tbname: str, column_types: List[DataType], tag_types: List[DataType] = [] + ): sql = f"show create table {self.db_name}.{tbname}" tdSql.query(sql, queryTimes=1) create_table_sql = tdSql.queryResult[0][1] decimal_idx = 0 results = re.findall(r"DECIMAL\((\d+),(\d+)\)", create_table_sql) for i, column_type in enumerate(column_types): - if column_type.type == TypeEnum.DECIMAL or column_type.type == TypeEnum.DECIMAL64: - result_type = DecimalType(column_type.type, int(results[decimal_idx][0]), int(results[decimal_idx][1])) + if ( + column_type.type == TypeEnum.DECIMAL + or column_type.type == TypeEnum.DECIMAL64 + ): + result_type = DecimalType( + column_type.type, + int(results[decimal_idx][0]), + int(results[decimal_idx][1]), + ) if result_type != column_type: - tdLog.exit(f"check show create table failed for: {tbname} column {i} type is {result_type}, expect {column_type.get_decimal_type()}") + tdLog.exit( + f"check show create table failed for: {tbname} column {i} type is {result_type}, expect {column_type.get_decimal_type()}" + ) decimal_idx += 1 - + def test_add_drop_columns_with_decimal(self, tbname: str, columns: List[DataType]): is_stb = tbname == self.stable_name ## alter table add column - create_c99_sql = f'alter table {self.db_name}.{tbname} add column c99 decimal(37, 19)' + create_c99_sql = ( + f"alter table {self.db_name}.{tbname} add column c99 decimal(37, 19)" + ) columns.append(DecimalType(TypeEnum.DECIMAL, 37, 19)) tdSql.execute(create_c99_sql, queryTimes=1, show=True) self.check_desc(tbname, columns) @@ -601,17 +922,17 @@ class TDTestCase: self.check_desc(tbname, columns) ## drop non decimal column - drop_c6_sql = f'alter table {self.db_name}.{tbname} drop column c6' + drop_c6_sql = f"alter table {self.db_name}.{tbname} drop column c6" tdSql.execute(drop_c6_sql, queryTimes=1, show=True) c6 = columns.pop(5) self.check_desc(tbname, columns) ## drop decimal column and not last column - drop_c99_sql = f'alter table {self.db_name}.{tbname} drop column c99' + drop_c99_sql = f"alter table {self.db_name}.{tbname} drop column c99" tdSql.execute(drop_c99_sql, queryTimes=1, show=True) c99 = columns.pop(len(columns) - 2) self.check_desc(tbname, columns) ## drop decimal column and last column - drop_c100_sql = f'alter table {self.db_name}.{tbname} drop column c100' + drop_c100_sql = f"alter table {self.db_name}.{tbname} drop column c100" tdSql.execute(drop_c100_sql, queryTimes=1, show=True) c100 = columns.pop(len(columns) - 1) self.check_desc(tbname, columns) @@ -638,10 +959,7 @@ class TDTestCase: DataType(TypeEnum.FLOAT), DataType(TypeEnum.VARCHAR, 255), ] - self.tags = [ - DataType(TypeEnum.INT), - DataType(TypeEnum.VARCHAR, 255) - ] + self.tags = [DataType(TypeEnum.INT), DataType(TypeEnum.VARCHAR, 255)] self.stb_columns = [ DecimalType(TypeEnum.DECIMAL, 10, 2), DecimalType(TypeEnum.DECIMAL, 20, 4), @@ -654,36 +972,56 @@ class TDTestCase: DataType(TypeEnum.FLOAT), DataType(TypeEnum.VARCHAR, 255), ] - DecimalColumnTableCreater(tdSql, self.db_name, self.stable_name, self.stb_columns, self.tags).create() + DecimalColumnTableCreater( + tdSql, self.db_name, self.stable_name, self.stb_columns, self.tags + ).create() self.check_show_create_table("meters", self.stb_columns, self.tags) - DecimalColumnTableCreater(tdSql, self.db_name, self.norm_table_name, self.norm_tb_columns).create() + DecimalColumnTableCreater( + tdSql, self.db_name, self.norm_table_name, self.norm_tb_columns + ).create() self.check_desc(self.norm_table_name, self.norm_tb_columns) self.check_show_create_table(self.norm_table_name, self.norm_tb_columns) ## TODO add more values for all rows - tag_values = [ - "1", "t1" - ] - DecimalColumnTableCreater(tdSql, self.db_name, self.stable_name, self.norm_tb_columns).create_child_table(self.c_table_prefix, self.c_table_num, self.tags, tag_values) + tag_values = ["1", "t1"] + DecimalColumnTableCreater( + tdSql, self.db_name, self.stable_name, self.norm_tb_columns + ).create_child_table( + self.c_table_prefix, self.c_table_num, self.tags, tag_values + ) self.check_desc("meters", self.stb_columns, self.tags) self.check_desc("t1", self.norm_tb_columns, self.tags) ## invalid precision/scale - invalid_precision_scale = [("decimal(-1, 2)", syntax_error), ("decimal(39, 2)", invalid_column), ("decimal(10, -1)", syntax_error), - ("decimal(10, 39)", invalid_column), ("decimal(10, 2.5)", syntax_error), ("decimal(10.5, 2)", syntax_error), - ("decimal(10.5, 2.5)", syntax_error), ("decimal(0, 2)", invalid_column), ("decimal(0)", invalid_column), - ("decimal", syntax_error), ("decimal()", syntax_error)] + invalid_precision_scale = [ + ("decimal(-1, 2)", syntax_error), + ("decimal(39, 2)", invalid_column), + ("decimal(10, -1)", syntax_error), + ("decimal(10, 39)", invalid_column), + ("decimal(10, 2.5)", syntax_error), + ("decimal(10.5, 2)", syntax_error), + ("decimal(10.5, 2.5)", syntax_error), + ("decimal(0, 2)", invalid_column), + ("decimal(0)", invalid_column), + ("decimal", syntax_error), + ("decimal()", syntax_error), + ] for i in invalid_precision_scale: sql = f"create table {self.db_name}.invalid_decimal_precision_scale (ts timestamp, c1 {i[0]})" tdSql.error(sql, i[1]) ## can't create decimal tag - sql = 'create stable %s.invalid_decimal_tag (ts timestamp) tags (t1 decimal(10, 2))' % (self.db_name) + sql = ( + "create stable %s.invalid_decimal_tag (ts timestamp) tags (t1 decimal(10, 2))" + % (self.db_name) + ) tdSql.error(sql, invalid_column) ## alter table add/drop column - self.test_add_drop_columns_with_decimal(self.norm_table_name, self.norm_tb_columns) + self.test_add_drop_columns_with_decimal( + self.norm_table_name, self.norm_tb_columns + ) self.test_add_drop_columns_with_decimal(self.stable_name, self.stb_columns) ## drop index from stb @@ -700,12 +1038,36 @@ class TDTestCase: tdSql.error(sql, invalid_encode_param) def test_insert_decimal_values(self): + tdLog.debug("start to insert decimal values") for i in range(self.c_table_num): - TableInserter(tdSql, self.db_name, f"{self.c_table_prefix}{i}", self.stb_columns, self.tags).insert(1000, 1537146000000, 500) + TableInserter( + tdSql, + self.db_name, + f"{self.c_table_prefix}{i}", + self.stb_columns, + self.tags, + ).insert(10000, 1537146000000, 500) - TableInserter(tdSql, self.db_name, self.norm_table_name, self.norm_tb_columns).insert(10, 1537146000000, 500, flush_database=True) - TableDataValidator(self.norm_tb_columns, self.norm_table_name, self.db_name).validate() + for i in range(self.c_table_num): + TableDataValidator( + self.stb_columns, self.c_table_prefix + str(i), self.db_name, i + ).validate() + TableInserter( + tdSql, self.db_name, self.norm_table_name, self.norm_tb_columns + ).insert(100000, 1537146000000, 500, flush_database=True) + TableDataValidator( + self.norm_tb_columns, self.norm_table_name, self.db_name + ).validate() + + tdSql.execute("flush database %s" % (self.db_name), queryTimes=1) + for i in range(self.c_table_num): + TableDataValidator( + self.stb_columns, self.c_table_prefix + str(i), self.db_name, i + ).validate() + TableDataValidator( + self.norm_tb_columns, self.norm_table_name, self.db_name + ).validate() ## insert null/None for decimal type @@ -713,32 +1075,38 @@ class TDTestCase: def no_decimal_table_test(self): columns = [ - DataType(TypeEnum.TINYINT), - DataType(TypeEnum.INT), - DataType(TypeEnum.BIGINT), - DataType(TypeEnum.DOUBLE), - DataType(TypeEnum.FLOAT), - DataType(TypeEnum.VARCHAR, 255), - ] - DecimalColumnTableCreater(tdSql, self.db_name, self.no_decimal_col_tb_name, columns, []).create() - TableInserter(tdSql, self.db_name, self.no_decimal_col_tb_name, columns).insert(10000, 1537146000000, 500, flush_database=True) + DataType(TypeEnum.TINYINT), + DataType(TypeEnum.INT), + DataType(TypeEnum.BIGINT), + DataType(TypeEnum.DOUBLE), + DataType(TypeEnum.FLOAT), + DataType(TypeEnum.VARCHAR, 255), + ] + DecimalColumnTableCreater( + tdSql, self.db_name, self.no_decimal_col_tb_name, columns, [] + ).create() + TableInserter(tdSql, self.db_name, self.no_decimal_col_tb_name, columns).insert( + 10000, 1537146000000, 500, flush_database=True + ) ## TODO wjm test non support decimal version upgrade to decimal support version, and add decimal column ## Test metaentry compatibility problem for decimal type ## How to test it? ## Create table with no decimal type, the metaentries should not have extschma, and add decimal column, the metaentries should have extschema for all columns. - sql = f'ALTER TABLE {self.db_name}.{self.no_decimal_col_tb_name} ADD COLUMN c200 decimal(37, 19)' - tdSql.execute(sql, queryTimes=1) ## now meta entry has ext schemas + sql = f"ALTER TABLE {self.db_name}.{self.no_decimal_col_tb_name} ADD COLUMN c200 decimal(37, 19)" + tdSql.execute(sql, queryTimes=1) ## now meta entry has ext schemas columns.append(DecimalType(TypeEnum.DECIMAL, 37, 19)) self.check_desc(self.no_decimal_col_tb_name, columns) ## After drop this only decimal column, the metaentries should not have extschema for all columns. - sql = f'ALTER TABLE {self.db_name}.{self.no_decimal_col_tb_name} DROP COLUMN c200' - tdSql.execute(sql, queryTimes=1) ## now meta entry has no ext schemas + sql = ( + f"ALTER TABLE {self.db_name}.{self.no_decimal_col_tb_name} DROP COLUMN c200" + ) + tdSql.execute(sql, queryTimes=1) ## now meta entry has no ext schemas columns.pop(len(columns) - 1) self.check_desc(self.no_decimal_col_tb_name, columns) - sql = f'ALTER TABLE {self.db_name}.{self.no_decimal_col_tb_name} ADD COLUMN c200 int' - tdSql.execute(sql, queryTimes=1) ## meta entry has no ext schemas + sql = f"ALTER TABLE {self.db_name}.{self.no_decimal_col_tb_name} ADD COLUMN c200 int" + tdSql.execute(sql, queryTimes=1) ## meta entry has no ext schemas columns.append(DataType(TypeEnum.INT)) self.check_desc(self.no_decimal_col_tb_name, columns) @@ -750,19 +1118,26 @@ class TDTestCase: ## TODO test decimal column for tmq def test_decimal_and_stream(self): - create_stream = f'CREATE STREAM {self.stream_name} FILL_HISTORY 1 INTO {self.db_name}.{self.stream_out_stb} AS SELECT _wstart, count(c1), avg(c2), sum(c3) FROM {self.db_name}.{self.stable_name} INTERVAL(10s)' + create_stream = f"CREATE STREAM {self.stream_name} FILL_HISTORY 1 INTO {self.db_name}.{self.stream_out_stb} AS SELECT _wstart, count(c1), avg(c2), sum(c3) FROM {self.db_name}.{self.stable_name} INTERVAL(10s)" tdSql.execute(create_stream, queryTimes=1, show=True) - self.wait_query_result(f"select count(*) from {self.db_name}.{self.stream_out_stb}", [(50,)], 30) - + self.wait_query_result( + f"select count(*) from {self.db_name}.{self.stream_out_stb}", [(500,)], 30 + ) + def test_decimal_and_tsma(self): create_tsma = f"CREATE TSMA {self.tsma_name} ON {self.db_name}.{self.stable_name} FUNCTION(count(c1), min(c2), max(c3), avg(C3)) INTERVAL(1m)" tdSql.execute(create_tsma, queryTimes=1, show=True) - self.wait_query_result(f"select count(*) from {self.db_name}.{self.tsma_name}_tsma_res_stb_", [(9*self.c_table_num,)], 30) + self.wait_query_result( + f"select count(*) from {self.db_name}.{self.tsma_name}_tsma_res_stb_", + [(9 * self.c_table_num,)], + 30, + ) def run(self): self.test_decimal_ddl() self.no_decimal_table_test() self.test_insert_decimal_values() + self.test_query_query() self.test_decimal_and_stream() self.test_decimal_and_tsma() @@ -779,7 +1154,80 @@ class TDTestCase: time.sleep(1) continue return True - tdLog.exit(f"wait query result timeout for {sql} failed after {times} time, expect {expect_result}, but got {results}") + tdLog.exit( + f"wait query result timeout for {sql} failed after {times} time, expect {expect_result}, but got {results}" + ) + + def a(self, dbname, tbname, columns, operator: DecimalColumnExpr): + ## query + ## build expr, operator.generate() + ## get result + ## check result + ## + pass + + def test_decimal_operators(self): + tdLog.debug("start to test decimal operators") + ## tables: meters, nt + ## columns: c1, c2, c3, c4, c5, c7, c8, c9, c10, c99, c100 + binary_operators = [ + DecimalBinaryOperator("+"), + DecimalBinaryOperator("-"), + DecimalBinaryOperator("*"), + DecimalBinaryOperator("/"), + DecimalBinaryOperator("%"), + DecimalBinaryOperator(">"), + DecimalBinaryOperator("<"), + DecimalBinaryOperator(">="), + DecimalBinaryOperator("<="), + DecimalBinaryOperator("=="), + DecimalBinaryOperator("!="), + DecimalBinaryOperatorIn("in"), + DecimalBinaryOperatorIn("not in"), + ] + all_type_columns = DataType.get_all_type_columns() + + ## decimal operator with constants of all other types + + ## decimal operator with columns of all other types + + unary_operators = ["-"] + + def test_decimal_functions(self): + self.test_decimal_last_first_func() + funcs = ["max", "min", "sum", "avg", "count", "first", "last", "cast"] + + def test_decimal_last_first_func(self): + pass + + def test_query_decimal_with_sma(self): + pass + + def test_query_decimal_where_clause(self): + pass + + def test_query_decimal_order_clause(self): + pass + + def test_query_decimal_group_by_clause(self): + pass + + def test_query_decimal_having_clause(self): + pass + + def test_query_decimal_interval_fill(self): + pass + + def test_query_decimal_partition_by(self): + pass + + def test_query_decimal_case_when(self): + pass + + def test_query_decimal(self): + self.test_decimal_operators() + self.test_decimal_functions() + self.test_query_decimal_with_sma() event = threading.Event()