diff --git a/include/libs/decimal/decimal.h b/include/libs/decimal/decimal.h index a9d845eb13..3319a76ab7 100644 --- a/include/libs/decimal/decimal.h +++ b/include/libs/decimal/decimal.h @@ -34,6 +34,7 @@ typedef struct Decimal64 { #define DECIMAL64_CLONE(pDst, pFrom) ((Decimal64*)(pDst))->words[0] = ((Decimal64*)(pFrom))->words[0] static const Decimal64 decimal64Zero = {0}; +static const Decimal64 decimal64Two = {2}; static const Decimal64 decimal64Min = {(uint64_t)-999999999999999999LL}; static const Decimal64 decimal64Max = {(uint64_t)999999999999999999LL}; #define DECIMAL64_ZERO decimal64Zero diff --git a/include/util/tutil.h b/include/util/tutil.h index b17bdab1ac..18a1ecd72d 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -34,7 +34,7 @@ char **strsplit(char *src, const char *delim, int32_t *num); char *strtolower(char *dst, const char *src); char *strntolower(char *dst, const char *src, int32_t n); char *strntolower_s(char *dst, const char *src, int32_t n); -int64_t strnatoi(char *num, int32_t len); +int64_t strnatoi(const char *num, int32_t len); size_t tstrncspn(const char *str, size_t ssize, const char *reject, size_t rsize); size_t twcsncspn(const TdUcs4 *wcs, size_t size, const TdUcs4 *reject, size_t rsize); diff --git a/source/libs/decimal/src/decimal.c b/source/libs/decimal/src/decimal.c index 28fb9e82c4..287187d408 100644 --- a/source/libs/decimal/src/decimal.c +++ b/source/libs/decimal/src/decimal.c @@ -37,6 +37,27 @@ static SDecimalOps* getDecimalOpsImp(DecimalInternalType t); #define DECIMAL_MIN_ADJUSTED_SCALE 6 +// TODO wjm use uint64_t ??? +static Decimal64 SCALE_MULTIPLIER_64[TSDB_DECIMAL64_MAX_PRECISION + 1] = {1LL, + 10LL, + 100LL, + 1000LL, + 10000LL, + 100000LL, + 1000000LL, + 10000000LL, + 100000000LL, + 1000000000LL, + 10000000000LL, + 100000000000LL, + 1000000000000LL, + 10000000000000LL, + 100000000000000LL, + 1000000000000000LL, + 10000000000000000LL, + 100000000000000000LL, + 1000000000000000000LL}; + typedef struct DecimalVar { DecimalInternalType type; uint8_t precision; @@ -125,13 +146,12 @@ int32_t decimalGetRetType(const SDataType* pLeftT, const SDataType* pRightT, EOp return 0; } -static int32_t decimalVarFromStr(const char* str, int32_t len, DecimalVar* result); - static int32_t decimalVarFromStr(const char* str, int32_t len, DecimalVar* result) { int32_t code = 0, pos = 0; result->precision = 0; result->scale = 0; - bool leadingZeroes = true, afterPoint = false; + result->exponent = 0; + bool leadingZeroes = true, afterPoint = false, rounded = false, stop = false; uint32_t places = 0; result->sign = 1; @@ -148,7 +168,7 @@ static int32_t decimalVarFromStr(const char* str, int32_t len, DecimalVar* resul break; } - for (; pos < len; ++pos) { + for (; pos < len && !stop; ++pos) { switch (str[pos]) { case '.': afterPoint = true; @@ -171,8 +191,24 @@ static int32_t decimalVarFromStr(const char* str, int32_t len, DecimalVar* resul case '9': { leadingZeroes = false; ++places; - if (result->precision + places > maxPrecision(result->type)) { + int32_t curPrec = result->precision + places; + if (curPrec > maxPrecision(result->type)) { if (afterPoint) { + if (!rounded && curPrec - 1 == maxPrecision(result->type) && str[pos] - '0' >= 5) { + Decimal64 delta = {1}; + if (places > 1) { + int32_t scaleUp = places - 1; + while (scaleUp != 0) { + int32_t curScale = TMIN(17, scaleUp); + pOps->multiply(result->pDec, &SCALE_MULTIPLIER_64[curScale], WORD_NUM(Decimal64)); + scaleUp -= curScale; + } + result->precision += places - 1; + result->scale += places - 1; + } + pOps->add(result->pDec, &delta, WORD_NUM(Decimal64)); + rounded = true; + } break; } else { return TSDB_CODE_DECIMAL_OVERFLOW; @@ -181,21 +217,26 @@ static int32_t decimalVarFromStr(const char* str, int32_t len, DecimalVar* resul result->precision += places; if (afterPoint) { result->scale += places; + result->exponent -= places; } - DecimalWord ten = 10, digit = str[pos] - '0'; - while (places-- > 0) { - pOps->multiply(result->pDec, &ten, 1); + Decimal64 digit = {str[pos] - '0'}; + while (places != 0) { + int32_t curScale = TMIN(17, places); + pOps->multiply(result->pDec, &SCALE_MULTIPLIER_64[curScale], WORD_NUM(Decimal64)); + places -= curScale; } - pOps->add(result->pDec, &digit, 1); + pOps->add(result->pDec, &digit, WORD_NUM(Decimal64)); places = 0; break; } } case 'e': - case 'E': - // TODO wjm handle E - break; + case 'E': { + result->exponent += strnatoi(str + pos + 1, len - pos - 1); + stop = true; + } break; default: + stop = true; break; } } @@ -218,26 +259,6 @@ int32_t decimal128ToDataVal(Decimal128* dec, SValue* pVal) { return TSDB_CODE_SUCCESS; } -// TODO wjm use uint64_t ??? -static Decimal64 SCALE_MULTIPLIER_64[TSDB_DECIMAL64_MAX_PRECISION + 1] = {1LL, - 10LL, - 100LL, - 1000LL, - 10000LL, - 100000LL, - 1000000LL, - 10000000LL, - 100000000LL, - 1000000000LL, - 10000000000LL, - 100000000000LL, - 1000000000000LL, - 10000000000000LL, - 100000000000000LL, - 1000000000000000LL, - 10000000000000000LL, - 100000000000000000LL, - 1000000000000000000LL}; #define DECIMAL64_ONE SCALE_MULTIPLIER_64[0] #define DECIMAL64_GET_MAX(precision, pMax) \ @@ -291,6 +312,9 @@ static void decimal64ScaleDown(Decimal64* pDec, uint8_t scaleDown); static void decimal64ScaleUp(Decimal64* pDec, uint8_t scaleUp); static void decimal64ScaleTo(Decimal64* pDec, uint8_t oldScale, uint8_t newScale); +static void decimal64RoundWithPositiveScale(Decimal64* pDec, uint8_t prec, uint8_t scale, uint8_t toPrec, + uint8_t toScale, DecimalRoundType roundType, bool* overflow); + static void decimal128Negate(DecimalType* pInt); static void decimal128Abs(DecimalType* pWord); static void decimal128Add(DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum); @@ -641,8 +665,8 @@ static bool decimal128Eq(const DecimalType* pLeft, const DecimalType* pRight, ui DECIMAL128_LOW_WORD(pLeftDec) == DECIMAL128_LOW_WORD(pRightDec); } -static void extractDecimal128Digits(const Decimal128* pDec, uint64_t* digits, int32_t* digitNum) { #define DIGIT_NUM_ONCE 18 +static void extractDecimal128Digits(const Decimal128* pDec, uint64_t* digits, int32_t* digitNum) { UInt128 a = {0}; UInt128 b = {0}; *digitNum = 0; @@ -677,8 +701,7 @@ static int32_t decimal128ToStr(const DecimalType* pInt, uint8_t scale, char* pBu makeDecimal128(©, DECIMAL128_HIGH_WORD(pDec), DECIMAL128_LOW_WORD(pDec)); decimal128Abs(©); extractDecimal128Digits(©, segments, &digitNum); - buf[0] = '-'; - len = 1; + TAOS_STRNCAT(pBuf, "-", 2); } else { extractDecimal128Digits(pDec, segments, &digitNum); } @@ -1105,9 +1128,25 @@ bool decimalCompare(EOperatorType op, const SDecimalCompareCtx* pLeft, const SDe #define ABS_INT64(v) (v) == INT64_MIN ? (uint64_t)INT64_MAX + 1 : (uint64_t)llabs(v) #define ABS_UINT64(v) (v) -static int64_t int64FromDecimal64(const DecimalType* pDec, uint8_t prec, uint8_t scale) { return 0; } +static int64_t int64FromDecimal64(const DecimalType* pDec, uint8_t prec, uint8_t scale) { + Decimal64 rounded = *(Decimal64*)pDec; + bool overflow = false; + decimal64RoundWithPositiveScale(&rounded, prec, scale, prec, 0, ROUND_TYPE_HALF_ROUND_UP, &overflow); + assert(!overflow); // TODO wjm remove this assert + if (overflow) return 0; -static uint64_t uint64FromDecimal64(const DecimalType* pDec, uint8_t prec, uint8_t scale) { return 0; } + return DECIMAL64_GET_VALUE(&rounded); +} + +static uint64_t uint64FromDecimal64(const DecimalType* pDec, uint8_t prec, uint8_t scale) { + Decimal64 rounded = *(Decimal64*)pDec; + bool overflow = false; + decimal64RoundWithPositiveScale(&rounded, prec, scale, prec, 0, ROUND_TYPE_HALF_ROUND_UP, &overflow); + assert(!overflow); // TODO wjm remove this assert + if (overflow) return 0; + + return DECIMAL64_GET_VALUE(&rounded); +} static int32_t decimal64FromInt64(DecimalType* pDec, uint8_t prec, uint8_t scale, int64_t val) { Decimal64 max = {0}; @@ -1422,16 +1461,108 @@ static void decimal64ScaleTo(Decimal64* pDec, uint8_t oldScale, uint8_t newScale decimal64ScaleDown(pDec, oldScale - newScale); } +static void decimal64ScaleAndCheckOverflow(Decimal64* pDec, uint8_t scale, uint8_t toPrec, uint8_t toScale, + bool* overflow) { + int8_t deltaScale = toScale - scale; + if (deltaScale >= 0) { + Decimal64 max = {0}; + DECIMAL64_GET_MAX(toPrec - deltaScale, &max); + Decimal64 abs = *pDec; + decimal64Abs(&abs); + if (decimal64Gt(&abs, &max, WORD_NUM(Decimal64))) { + if (overflow) *overflow = true; + } else { + decimal64ScaleUp(pDec, deltaScale); + } + } else if (deltaScale < 0) { + Decimal64 res = *pDec, max = {0}; + decimal64ScaleDown(&res, -deltaScale); + DECIMAL64_GET_MAX(toPrec, &max); + if (decimal64Gt(&res, &max, WORD_NUM(Decimal64))) { + if (overflow) *overflow = true; + } else { + *pDec = res; + } + } +} + +static int32_t decimal64CountRoundingDelta(const Decimal64* pDec, int8_t scale, int8_t toScale, + DecimalRoundType roundType) { + if (roundType == ROUND_TYPE_TRUNC || toScale >= scale) return 0; + + Decimal64 dec = *pDec; + int32_t res = 0; + switch (roundType) { + case ROUND_TYPE_HALF_ROUND_UP: { + Decimal64 trailing = dec; + decimal64Mod(&trailing, &SCALE_MULTIPLIER_64[scale - toScale], WORD_NUM(Decimal64)); + if (decimal64Eq(&trailing, &decimal64Zero, WORD_NUM(Decimal64))) { + res = 0; + break; + } + Decimal64 trailingAbs = trailing, baseDiv2 = SCALE_MULTIPLIER_64[scale - toScale]; + decimal64Abs(&trailingAbs); + decimal64divide(&baseDiv2, &decimal64Two, WORD_NUM(Decimal64), NULL); + if (decimal64Lt(&trailingAbs, &baseDiv2, WORD_NUM(Decimal64))) { + res = 0; + break; + } + res = DECIMAL64_SIGN(pDec) == 1 ? 1 : -1; + } break; + default: + break; + } + return res; +} + +static void decimal64RoundWithPositiveScale(Decimal64* pDec, uint8_t prec, uint8_t scale, uint8_t toPrec, + uint8_t toScale, DecimalRoundType roundType, bool* overflow) { + Decimal64 scaled = *pDec; + bool overflowLocal = false; + // scale up or down to toScale + decimal64ScaleAndCheckOverflow(&scaled, scale, toPrec, toScale, &overflowLocal); + if (overflowLocal) { + if (overflow) *overflow = true; + *pDec = decimal64Zero; + return; + } + + // calc rounding delta + int32_t delta = decimal64CountRoundingDelta(pDec, scale, toScale, roundType); + if (delta == 0) { + *pDec = scaled; + return; + } + + Decimal64 deltaDec = {delta}; + // add the delta + decimal64Add(&scaled, &deltaDec, WORD_NUM(Decimal64)); + + // check overflow again + if (toPrec < prec) { + Decimal64 max = {0}; + DECIMAL64_GET_MAX(toPrec, &max); + Decimal64 scaledAbs = scaled; + decimal64Abs(&scaledAbs); + if (decimal64Gt(&scaledAbs, &max, WORD_NUM(Decimal64))) { + if (overflow) *overflow = true; + *pDec = decimal64Zero; + return; + } + } + *pDec = scaled; +} + int32_t decimal64FromStr(const char* str, int32_t len, uint8_t expectPrecision, uint8_t expectScale, Decimal64* pRes) { int32_t code = 0; DecimalVar var = {.type = DECIMAL_64, .pDec = pRes->words}; DECIMAL64_SET_VALUE(pRes, 0); code = decimalVarFromStr(str, len, &var); if (TSDB_CODE_SUCCESS != code) return code; - Decimal64 max = {0}; - DECIMAL64_GET_MAX(expectPrecision, &max); - decimal64ScaleTo(pRes, var.scale, expectScale); - if (decimal64Gt(pRes, &max, 1)) { + bool overflow = false; + decimal64RoundWithPositiveScale(pRes, var.precision, var.scale, expectPrecision, expectScale, + ROUND_TYPE_HALF_ROUND_UP, &overflow); + if (overflow) { return TSDB_CODE_DECIMAL_OVERFLOW; } return code; @@ -1467,10 +1598,10 @@ int32_t decimal128FromStr(const char* str, int32_t len, uint8_t expectPrecision, DECIMAL128_SET_LOW_WORD(pRes, 0); code = decimalVarFromStr(str, len, &var); if (TSDB_CODE_SUCCESS != code) return code; - Decimal128 max = {0}; - DECIMAL128_GET_MAX(expectPrecision, &max); - decimal128ScaleTo(pRes, var.scale, expectScale); - if (decimal128Gt(pRes, &max, 2)) { + bool overflow = false; + decimal128RoundWithPositiveScale(pRes, var.precision, var.scale, expectPrecision, expectScale, + ROUND_TYPE_HALF_ROUND_UP, &overflow); + if (overflow) { return TSDB_CODE_DECIMAL_OVERFLOW; } return code; @@ -1558,6 +1689,7 @@ static void decimal128RoundWithPositiveScale(Decimal128* pDec, uint8_t prec, uin uint8_t toScale, DecimalRoundType roundType, bool* overflow) { Decimal128 scaled = *pDec; bool overflowLocal = false; + // scale up or down to toScale decimal128ModifyScaleAndPrecision(&scaled, scale, toPrec, toScale, &overflowLocal); if (overflowLocal) { if (overflow) *overflow = true; @@ -1565,6 +1697,7 @@ static void decimal128RoundWithPositiveScale(Decimal128* pDec, uint8_t prec, uin return; } + // calc rounding delta, 1 or -1 int32_t delta = decimal128CountRoundingDelta(pDec, scale, toScale, roundType); if (delta == 0) { *pDec = scaled; @@ -1572,17 +1705,22 @@ static void decimal128RoundWithPositiveScale(Decimal128* pDec, uint8_t prec, uin } Decimal64 deltaDec = {delta}; + // add the delta decimal128Add(&scaled, &deltaDec, WORD_NUM(Decimal64)); - Decimal128 max = {0}; - DECIMAL128_GET_MAX(toPrec, &max); - Decimal128 scaledAbs = scaled; - decimal128Abs(&scaledAbs); - if (toPrec < prec && decimal128Gt(&scaledAbs, &max, WORD_NUM(Decimal128))) { - if (overflow) *overflow = true; - *(Decimal128*)pDec = decimal128Zero; - } else { - *(Decimal128*)pDec = scaled; + + // check overflow again + if (toPrec < prec) { + Decimal128 max = {0}; + DECIMAL128_GET_MAX(toPrec, &max); + Decimal128 scaledAbs = scaled; + decimal128Abs(&scaledAbs); + if (decimal128Gt(&scaledAbs, &max, WORD_NUM(Decimal128))) { + if (overflow) *overflow = true; + *(Decimal128*)pDec = decimal128Zero; + return; + } } + *(Decimal128*)pDec = scaled; } static void decimal128ModifyScaleAndPrecision(Decimal128* pDec, uint8_t scale, uint8_t toPrec, int8_t toScale, @@ -1590,7 +1728,7 @@ static void decimal128ModifyScaleAndPrecision(Decimal128* pDec, uint8_t scale, u int8_t deltaScale = toScale - scale; if (deltaScale >= 0) { Decimal128 max = {0}; - DECIMAL128_GET_MAX(toPrec - deltaScale, &max); // TODO wjm test toPrec == 0 + DECIMAL128_GET_MAX(toPrec - deltaScale, &max); // TODO wjm test toPrec == 0, test toPrec - deltaScale Decimal128 abs = *pDec; decimal128Abs(&abs); if (decimal128Gt(&abs, &max, WORD_NUM(Decimal128))) { @@ -1630,7 +1768,7 @@ static int32_t decimal128CountRoundingDelta(const Decimal128* pDec, int8_t scale res = 0; break; } - res = decimal128Lt(pDec, &decimal128Zero, WORD_NUM(Decimal128)) ? -1 : 1; + res = decimal128Lt(pDec, &decimal128Zero, WORD_NUM(Decimal128)) ? -1 : 1; // TODO wjm use sign?? } break; case ROUND_TYPE_TRUNC: default: diff --git a/source/libs/decimal/test/decimalTest.cpp b/source/libs/decimal/test/decimalTest.cpp index 945670153d..f21474a7bd 100644 --- a/source/libs/decimal/test/decimalTest.cpp +++ b/source/libs/decimal/test/decimalTest.cpp @@ -174,8 +174,8 @@ class Numeric { if (prec > NumericType::maxPrec) throw std::string("prec too big") + std::to_string(prec); int32_t code = dec_.fromStr(str, prec, scale) != 0; if (code != 0) { - cout << "failed to init decimal from str: " << str << endl; - throw std::string(tstrerror(code)); + cout << "failed to init decimal from str: " << str << "\t"; + throw std::overflow_error(tstrerror(code)); } } Numeric() = default; @@ -505,7 +505,7 @@ TEST(decimal, decimalFromType) { dec1 = (int64_t)-9999999; ASSERT_EQ(dec1.toString(), "-9999999.0000"); dec1 = "99.99999"; - ASSERT_EQ(dec1.toString(), "99.9999"); + ASSERT_EQ(dec1.toString(), "100.0000"); } TEST(decimal, typeFromDecimal) { @@ -579,7 +579,7 @@ TEST(decimal128, to_string) { ASSERT_STREQ(buf, "1234567890123456789012345678.901234567"); } -TEST(decimal128, divide) { +TEST(decimal, divide) { __int128 i = generate_big_int128(15); int64_t hi = i >> 64; uint64_t lo = i; @@ -666,6 +666,7 @@ TEST(decimal, decimalFromStr) { code = decimal64FromStr(buf, strlen(buf), 6, 3, &dec64); ASSERT_EQ(code, 0); ASSERT_EQ(999999, DECIMAL64_GET_VALUE(&dec64)); + } TEST(decimal, toStr) { @@ -861,10 +862,6 @@ TEST(decimal, randomGenerator) { } } -TEST(deicmal, decimalFromStr_all) { - // TODO test e/E -} - #define ASSERT_OVERFLOW(op) \ do { \ try { \ @@ -878,6 +875,98 @@ TEST(deicmal, decimalFromStr_all) { FAIL(); \ } while (0) +template +struct DecimalFromStrTestUnit { + uint8_t precision; + uint8_t scale; + std::string input; + std::string expect; + bool overflow; +}; + +template +void testDecimalFromStr(std::vector>& units) { + for (auto& unit : units) { + if (unit.overflow) { + auto ff = [&]() { + Numeric dec = {unit.precision, unit.scale, unit.input}; + return dec; + }; + ASSERT_OVERFLOW(ff()); + continue; + } + cout << unit.input << " convert to decimal: (" << (int32_t)unit.precision << "," << (int32_t)unit.scale + << "): " << unit.expect << endl; + Numeric dec = {unit.precision, unit.scale, unit.input}; + ASSERT_EQ(dec.toString(), unit.expect); + } +} + +TEST(decimal, decimalFromStr_all) { + std::vector> units = { + {10, 2, "123.45", "123.45", false}, + {10, 2, "123.456", "123.46", false}, + {10, 2, "123.454", "123.45"}, + {18, 2, "1234567890123456.456", "1234567890123456.46", false}, + {18, 2, "9999999999999999.995", "", true}, + {18, 2, "9999999999999999.994", "9999999999999999.99", false}, + {18, 2, "-9999999999999999.995", "", true}, + {18, 2, "-9999999999999999.994", "-9999999999999999.99", false}, + {18, 2, "-9999999999999999.9999999", "", true}, + {10, 2, "12345678.456", "12345678.46", false}, + {10, 2, "12345678.454", "12345678.45", false}, + {10, 2, "99999999.999", "", true}, + {10, 2, "-99999999.992", "-99999999.99", false}, + {10, 2, "-99999999.999", "", true}, + {10, 2, "-99999989.998", "-99999990.00", false}, + {10, 2, "-99999998.997", "-99999999.00", false}, + {10, 2, "-99999999.009", "-99999999.01", false}, + {18, 17, "-9.99999999999999999999", "", true}, + {18, 16, "-99.999999999999999899999", "-99.9999999999999999", false}, + {18, 16, "-99.999999999999990099999", "-99.9999999999999901", false}, + {18, 18, "0.0000000000000000099", "0.000000000000000010", false}, + {18, 18, "0.0000000000000000001", "0", false}, + {18, 18, "0.0000000000000000005", "0.000000000000000001", false}, + {18, 18, "-0.0000000000000000001", "0", false}, + {18, 18, "-0.00000000000000000019999", "0", false}, + {18, 18, "-0.0000000000000000005", "-0.000000000000000001", false}, + {18, 18, "-0.00000000000000000000000000123123123", "0", false}, + {18, 18, "0.10000000000000000000000000123123123", "0.100000000000000000", false}, + {18, 18, "0.000000000000000000000000000000000000006", "0", false}, + {18, 17, "1.00000000000000000999", "1.00000000000000001", false}, + {18, 17, "1.00000000000000000199", "1.00000000000000000", false}, + {15, 1, "-00000.", "0", false}, + {14, 12, "-.000", "0", false}, + {14, 12, "-.000000000000", "0", false}, + {14, 12, "-.", "0", false}, + //{10, 2, "1.2345e8", "12345000.00", false}, + }; + testDecimalFromStr(units); + + std::vector> dec128Units = { + {38, 10, "123456789012345678901234567.89012345679", "123456789012345678901234567.8901234568", false}, + {38, 10, "123456789012345678901234567.89012345670", "123456789012345678901234567.8901234567", false}, + {38, 10, "-123456789012345678901234567.89012345671", "-123456789012345678901234567.8901234567", false}, + {38, 10, "-123456789012345678901234567.89012345679", "-123456789012345678901234567.8901234568", false}, + {38, 10, "-9999999999999999999999999999.99999999995", "", true}, + {38, 10, "-9999999999999999999999999999.99999999994", "-9999999999999999999999999999.9999999999", false}, + {38, 10, "9999999999999999999999999999.99999999996", "", true}, + {38, 10, "9999999999999999999999999999.99999999994", "9999999999999999999999999999.9999999999", false}, + {36, 35, "9.99999999999999999999999999999999999", "9.99999999999999999999999999999999999", false}, + {36, 35, "9.999999999999999999999999999999999999111231231", "", true}, + {38, 38, "0.000000000000000000000000000000000000001", "0", false}, + {38, 38, "0.000000000000000000000000000000000000006", "0.00000000000000000000000000000000000001", false}, + {38, 35, "123.000000000000000000000000000000001", "123.00000000000000000000000000000000100", false}, + {38, 5, "123.", "123.00000", false}, + {20, 4, "-.12345", "-0.1235", false}, + }; + testDecimalFromStr(dec128Units); + + // TODO test e/E + + // TODO test weight overflow +} + TEST(decimal, op_overflow) { // divide 0 error Numeric<128> dec{38, 2, string(36, '9') + ".99"}; @@ -1174,6 +1263,8 @@ class DecimalTest : public ::testing::Test { static constexpr const char* user = "root"; static constexpr const char* passwd = "taosdata"; static constexpr const char* db = "test"; + DecimalStringRandomGenerator generator_; + DecimalStringRandomGeneratorConfig generator_config_; public: void SetUp() override { @@ -1187,6 +1278,8 @@ class DecimalTest : public ::testing::Test { taos_close(default_conn_); } } + + std::string generate_decimal_str() { return generator_.generate(generator_config_); } }; TEST_F(DecimalTest, insert) { @@ -1293,6 +1386,17 @@ TEST_F(DecimalTest, api_taos_fetch_rows) { taos_close(pTaos); } +TEST_F(DecimalTest, decimalFromStr) { + Numeric<64> numeric64 = {10, 2, "0"}; + + numeric64 = {18, 0, "0"}; + + numeric64 = { 18, 18, "0"}; + + numeric64 = {18, 2, "0"}; + Numeric<128> numeric128 = {38, 10, "0"}; +} + int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); diff --git a/source/util/src/tutil.c b/source/util/src/tutil.c index cdcef4b807..cbceade638 100644 --- a/source/util/src/tutil.c +++ b/source/util/src/tutil.c @@ -278,7 +278,7 @@ char *paGetToken(char *string, char **token, int32_t *tokenLen) { return string; } -int64_t strnatoi(char *num, int32_t len) { +int64_t strnatoi(const char *num, int32_t len) { int64_t ret = 0, i, dig, base = 1; if (len > (int32_t)strlen(num)) { diff --git a/tests/system-test/2-query/decimal.py b/tests/system-test/2-query/decimal.py index 07208ddb3a..a3dc0127f0 100644 --- a/tests/system-test/2-query/decimal.py +++ b/tests/system-test/2-query/decimal.py @@ -1,8 +1,12 @@ +from pydoc import doc from random import randrange from re import A import time import threading import secrets + +from sympy import true +from torch import randint import query from tag_lite import column from util.log import * @@ -10,57 +14,142 @@ from util.sql import * from util.cases import * from util.dnodes import * from util.common import * +from decimal import Decimal syntax_error = -2147473920 invalid_column = -2147473918 invalid_compress_level = -2147483084 invalid_encode_param = -2147483087 -class DecimalType: - def __init__(self, precision: int, scale: int): - self.precision = precision - self.scale = scale - def __str__(self): - return f"DECIMAL({self.precision}, {self.scale})" +class DecimalTypeGeneratorConfig: + def __init__(self): + self.enable_weight_overflow: bool = False + self.weightOverflowRatio: float = 0.001 + self.enable_scale_overflow: bool = True + self.scale_overflow_ratio = 0.1 + self.enable_positive_sign = False + self.with_corner_case = True + self.corner_case_ratio = 0.1 + self.positive_ratio = 0.7 + self.prec = 38 + self.scale = 10 - def __eq__(self, other): - return self.precision == other.precision and self.scale == other.scale - - def __ne__(self, other): - return not self.__eq__(other) - - def __hash__(self): - return hash((self.precision, self.scale)) - - def __repr__(self): - return f"DecimalType({self.precision}, {self.scale})" +class DecimalStringRandomGenerator: + def __init__(self): + self.corner_cases = ["0", "NULL", "0.", ".0", "000.000000"] + self.ratio_base: int = 1000000 - def generate_value(self, allow_weight_overflow = False, allow_scale_overflow = False) -> str: - if allow_weight_overflow: - weight = secrets.randbelow(40) - else: - weight = secrets.randbelow(self.precision - self.scale) - if allow_scale_overflow: - dscale = secrets.randbelow(40 - weight + 1) - else: - dscale = secrets.randbelow(self.precision - weight + 1) - digits :str = '' - for _ in range(weight): - digits += str(secrets.randbelow(10)) - if dscale > 0: - digits += '.' - for _ in range(dscale): - digits += str(secrets.randbelow(10)) - if digits == '': - digits = '0' - return digits + def possible(self, possibility: float) -> bool: + return random.randint(0, self.ratio_base) < possibility * self.ratio_base - @staticmethod - def default_compression() -> str: - return "zstd" - @staticmethod - def default_encode() -> str: - return "disabled" + def generate_sign(self, positive_ratio: float) -> str: + if self.possible(positive_ratio): + return "+" + return "-" + + def generate_digit(self) -> str: + return str(random.randint(0, 9)) + + def current_should_generate_corner_case(self, corner_case_ratio: float) -> bool: + return self.possible(corner_case_ratio) + + def generate_corner_case(self, config: DecimalTypeGeneratorConfig) -> str: + if self.possible(0.8): + return random.choice(self.corner_cases) + else: + res = self.generate_digit() * (config.prec - config.scale) + if self.possible(0.8): + res += '.' + if self.possible(0.8): + res += self.generate_digit() * config.scale + return res + + ## 写入大整数的例子, 如10000000000, scale解析时可能为负数 + def generate_(self, config: DecimalTypeGeneratorConfig) -> str: + ret: str = '' + sign = self.generate_sign(config.positive_ratio) + if config.with_corner_case and self.current_should_generate_corner_case(config.corner_case_ratio): + ret += self.generate_corner_case(config) + else: + if config.enable_positive_sign or sign != '+': + ret += sign + weight = random.randint(1, config.prec - config.scale) + scale = random.randint(1, config.scale) + for i in range(weight): + ret += self.generate_digit() + + if config.enable_weight_overflow and self.possible(config.weightOverflowRatio): + extra_weight = config.prec - weight + 1 + random.randint(1, self.get_max_prec(config.prec)) + while extra_weight > 0: + ret += self.generate_digit() + extra_weight -= 1 + ret += '.' + for i in range(scale): + ret += self.generate_digit() + if config.enable_scale_overflow and self.possible(config.scale_overflow_ratio): + extra_scale = config.scale - scale + 1 + random.randint(1, self.get_max_prec(config.prec)) + while extra_scale > 0: + ret += self.generate_digit() + extra_scale -= 1 + return ret + + def get_max_prec(self, prec): + if prec <= 18: + return 18 + else: + return 38 + +class DecimalColumnAggregator: + def __init__(self): + self.max: Decimal = Decimal("0") + self.min: Decimal = Decimal("0") + self.count: int = 0 + self.sum: Decimal = Decimal("0") + self.null_num: int = 0 + self.none_num: int = 0 + + def add_value(self, value: str): + self.count += 1 + if value == "NULL": + self.null_num += 1 + elif value == "None": + self.none_num += 1 + else: + v: Decimal = Decimal(value) + self.sum += v + if v > self.max: + self.max = v + if v < self.min: + self.min = v + +class TaosShell: + def __init__(self): + self.queryResult = [] + self.tmp_file_path = "/tmp/taos_shell_result" + + def read_result(self): + with open(self.tmp_file_path, 'r') as f: + lines = f.readlines() + lines = lines[1:] + for line in lines: + col = 0 + vals: List[str] = line.split(',') + if len(self.queryResult) == 0: + self.queryResult = [[] for i in range(len(vals))] + for val in vals: + self.queryResult[col].append(val) + col += 1 + + def query(self, sql: str): + try: + command = f'taos -s "{sql} >> {self.tmp_file_path}"' + result = subprocess.run(command, shell=True, check=True, stderr=subprocess.PIPE) + self.read_result() + except subprocess.CalledProcessError as e: + tdLog.error(f"Command '{sql}' failed with error: {e.stderr.decode('utf-8')}") + self.queryResult = [] + return self.queryResult + class TypeEnum: BOOL = 1 TINYINT = 2 @@ -152,13 +241,6 @@ class DataType: def __repr__(self): return f"DataType({self.type}, {self.length}, {self.type_mod})" - @staticmethod - def get_decimal_type_mod(type: DecimalType) -> int: - return type.precision * 100 + type.scale - - def get_decimal_type(self) -> DecimalType: - return DecimalType(self.type_mod // 100, self.type_mod % 100) - def construct_type_value(self, val: str): if self.type == TypeEnum.BINARY or self.type == TypeEnum.VARCHAR or self.type == TypeEnum.NCHAR or self.type == TypeEnum.VARBINARY: return f"'{val}'" @@ -192,8 +274,68 @@ class DataType: return str(secrets.randbelow(9223372036854775808)) if self.type == TypeEnum.JSON: return f'{{"key": "{secrets.token_urlsafe(10)}"}}' - if self.type == TypeEnum.DECIMAL: - return self.get_decimal_type().generate_value() + raise Exception(f"unsupport type {self.type}") + def check(self, values, offset: int): + return True + +class DecimalType(DataType): + def __init__(self, type, precision: int, scale: int): + self.precision = precision + self.scale = scale + if type == TypeEnum.DECIMAL64: + bytes = 8 + else: + bytes = 16 + super().__init__(type, bytes, self.get_decimal_type_mod()) + self.generator: DecimalStringRandomGenerator = DecimalStringRandomGenerator() + self.generator_config: DecimalTypeGeneratorConfig = DecimalTypeGeneratorConfig() + self.generator_config.prec = precision + self.generator_config.scale = scale + self.aggregator: DecimalColumnAggregator = DecimalColumnAggregator() + self.values: List[str] = [] + + def get_decimal_type_mod(self) -> int: + return self.precision * 100 + self.scale + + def __str__(self): + return f"DECIMAL({self.precision}, {self.scale})" + + def __eq__(self, other): + return self.precision == other.precision and self.scale == other.scale + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash((self.precision, self.scale)) + + def __repr__(self): + return f"DecimalType({self.precision}, {self.scale})" + + def generate_value(self) -> str: + val = self.generator.generate_(self.generator_config) + self.aggregator.add_value(val) + self.values.append(val) + return val + + @staticmethod + def default_compression() -> str: + return "zstd" + @staticmethod + def default_encode() -> str: + return "disabled" + + def check(self, values, offset: int): + val_from_query = values + val_insert = self.values[offset:] + for v1, v2 in zip(val_from_query, val_insert): + dec1: Decimal = Decimal(v1) + dec2: Decimal = Decimal(v2) + dec2 = dec2.quantize(Decimal(10) ** -self.scale) + if dec1 != dec2: + tdLog.error(f"check decimal column failed, expect {dec2}, but get {dec1}") + return False + class DecimalColumnTableCreater: def __init__(self, conn, dbName: str, tbName: str, columns_types: List[DataType], tags_types: List[DataType] = []): @@ -262,6 +404,23 @@ class TableInserter: self.conn.execute(f"flush database {self.dbName}", queryTimes=1) self.conn.execute(sql, queryTimes=1) +class TableDataValidator: + def __init__(self, columns: List[DataType], tbName: str, dbName: str, tbIdx: int = 0): + self.columns = columns + self.tbName = tbName + self.dbName = dbName + self.tbIdx = tbIdx + + def validate(self): + sql = f"select * from {self.dbName}.{self.tbName}" + res = TaosShell().query(sql) + row_num = len(res) + colIdx = 1 + for col in self.columns: + if col.type == TypeEnum.DECIMAL or col.type == TypeEnum.DECIMAL64: + col.check(res[colIdx], row_num * self.tbIdx) + colIdx += 1 + class TDTestCase: updatecfgDict = {'asynclog': 0, 'ttlUnit': 1, 'ttlPushInterval': 5, 'ratioOfVnodeStreamThrea': 4, 'debugFlag': 143} @@ -422,9 +581,9 @@ class TDTestCase: decimal_idx = 0 results = re.findall(r"DECIMAL\((\d+),(\d+)\)", create_table_sql) for i, column_type in enumerate(column_types): - if column_type.type == TypeEnum.DECIMAL: - result_type = DecimalType(int(results[decimal_idx][0]), int(results[decimal_idx][1])) - if result_type != column_type.get_decimal_type(): + if column_type.type == TypeEnum.DECIMAL or column_type.type == TypeEnum.DECIMAL64: + result_type = DecimalType(column_type.type, int(results[decimal_idx][0]), int(results[decimal_idx][1])) + if result_type != column_type: tdLog.exit(f"check show create table failed for: {tbname} column {i} type is {result_type}, expect {column_type.get_decimal_type()}") decimal_idx += 1 @@ -432,13 +591,13 @@ class TDTestCase: is_stb = tbname == self.stable_name ## alter table add column create_c99_sql = f'alter table {self.db_name}.{tbname} add column c99 decimal(37, 19)' - columns.append(DataType(TypeEnum.DECIMAL, type_mod=DataType.get_decimal_type_mod(DecimalType(37, 19)))) + columns.append(DecimalType(TypeEnum.DECIMAL, 37, 19)) tdSql.execute(create_c99_sql, queryTimes=1, show=True) self.check_desc(tbname, columns) ## alter table add column with compression create_c100_sql = f'ALTER TABLE {self.db_name}.{tbname} ADD COLUMN c100 decimal(36, 18) COMPRESS "zstd"' tdSql.execute(create_c100_sql, queryTimes=1, show=True) - columns.append(DataType(TypeEnum.DECIMAL, type_mod=DataType.get_decimal_type_mod(DecimalType(36, 18)))) + columns.append(DecimalType(TypeEnum.DECIMAL, 36, 18)) self.check_desc(tbname, columns) ## drop non decimal column @@ -468,10 +627,10 @@ class TDTestCase: ## create decimal type table, normal/super table, decimal64/decimal128 tdLog.printNoPrefix("-------- test create decimal column") self.norm_tb_columns = [ - DataType(TypeEnum.DECIMAL, type_mod=DataType.get_decimal_type_mod(DecimalType(10, 2))), - DataType(TypeEnum.DECIMAL, type_mod=DataType.get_decimal_type_mod(DecimalType(20, 4))), - DataType(TypeEnum.DECIMAL, type_mod=DataType.get_decimal_type_mod(DecimalType(30, 8))), - DataType(TypeEnum.DECIMAL, type_mod=DataType.get_decimal_type_mod(DecimalType(38, 10))), + DecimalType(TypeEnum.DECIMAL, 10, 2), + DecimalType(TypeEnum.DECIMAL, 20, 4), + DecimalType(TypeEnum.DECIMAL, 30, 8), + DecimalType(TypeEnum.DECIMAL, 38, 10), DataType(TypeEnum.TINYINT), DataType(TypeEnum.INT), DataType(TypeEnum.BIGINT), @@ -483,7 +642,18 @@ class TDTestCase: DataType(TypeEnum.INT), DataType(TypeEnum.VARCHAR, 255) ] - self.stb_columns = self.norm_tb_columns.copy() + self.stb_columns = [ + DecimalType(TypeEnum.DECIMAL, 10, 2), + DecimalType(TypeEnum.DECIMAL, 20, 4), + DecimalType(TypeEnum.DECIMAL, 30, 8), + DecimalType(TypeEnum.DECIMAL, 38, 10), + DataType(TypeEnum.TINYINT), + DataType(TypeEnum.INT), + DataType(TypeEnum.BIGINT), + DataType(TypeEnum.DOUBLE), + DataType(TypeEnum.FLOAT), + DataType(TypeEnum.VARCHAR, 255), + ] DecimalColumnTableCreater(tdSql, self.db_name, self.stable_name, self.stb_columns, self.tags).create() self.check_show_create_table("meters", self.stb_columns, self.tags) @@ -533,7 +703,8 @@ class TDTestCase: for i in range(self.c_table_num): TableInserter(tdSql, self.db_name, f"{self.c_table_prefix}{i}", self.stb_columns, self.tags).insert(1000, 1537146000000, 500) - TableInserter(tdSql, self.db_name, self.norm_table_name, self.norm_tb_columns).insert(10000, 1537146000000, 500, flush_database=True) + TableInserter(tdSql, self.db_name, self.norm_table_name, self.norm_tb_columns).insert(10, 1537146000000, 500, flush_database=True) + TableDataValidator(self.norm_tb_columns, self.norm_table_name, self.db_name).validate() ## insert null/None for decimal type @@ -558,7 +729,7 @@ class TDTestCase: ## Create table with no decimal type, the metaentries should not have extschma, and add decimal column, the metaentries should have extschema for all columns. sql = f'ALTER TABLE {self.db_name}.{self.no_decimal_col_tb_name} ADD COLUMN c200 decimal(37, 19)' tdSql.execute(sql, queryTimes=1) ## now meta entry has ext schemas - columns.append(DataType(TypeEnum.DECIMAL, type_mod=DataType.get_decimal_type_mod(DecimalType(37, 19)))) + columns.append(DecimalType(TypeEnum.DECIMAL, 37, 19)) self.check_desc(self.no_decimal_col_tb_name, columns) ## After drop this only decimal column, the metaentries should not have extschema for all columns. diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index ffd8e92ecb..a89ff70757 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -469,6 +469,9 @@ void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, i shellFormatTimestamp(buf, sizeof(buf), *(int64_t *)val, precision); taosFprintfFile(pFile, "%s%s%s", quotationStr, buf, quotationStr); break; + case TSDB_DATA_TYPE_DECIMAL64: + case TSDB_DATA_TYPE_DECIMAL: + taosFprintfFile(pFile, "%s", val); default: break; }