From 8ada049b0a85b00a1e1a28a0e0194051f4f82b52 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 Date: Wed, 19 Feb 2025 16:39:01 +0800 Subject: [PATCH] test parse decimal from string --- source/libs/decimal/src/decimal.c | 100 ++++++++------ source/libs/decimal/test/decimalTest.cpp | 164 ++++++++++++++--------- 2 files changed, 154 insertions(+), 110 deletions(-) diff --git a/source/libs/decimal/src/decimal.c b/source/libs/decimal/src/decimal.c index b37090f228..ae55b31e85 100644 --- a/source/libs/decimal/src/decimal.c +++ b/source/libs/decimal/src/decimal.c @@ -60,11 +60,12 @@ static Decimal64 SCALE_MULTIPLIER_64[TSDB_DECIMAL64_MAX_PRECISION + 1] = {1LL, typedef struct DecimalVar { DecimalInternalType type; - int8_t precision; - int8_t scale; + int8_t precision; + int8_t scale; int32_t exponent; int8_t sign; DecimalType* pDec; + int32_t weight; } DecimalVar; static uint8_t maxPrecision(DecimalInternalType type) { @@ -146,6 +147,27 @@ int32_t decimalGetRetType(const SDataType* pLeftT, const SDataType* pRightT, EOp return 0; } +int32_t calcCurPrec(int32_t prec, int32_t places, int32_t exp, int32_t weight, int32_t firstValidScale) { + if (exp == 0) return prec + places; + if (exp < 0) { + if (weight + exp >= 0) return prec + places; + return prec + places - exp - weight; + } + if (weight > 0) return prec + places; + return prec + places - TMIN(firstValidScale - 1, exp); +} + +int32_t calcActualWeight(int32_t prec, int32_t scale, int32_t exp, int32_t weight, int32_t firstValidScale) { + if (exp == 0) return prec - scale; + if (exp < 0) { + if (weight + exp >= 0) return weight + exp; + return 0; + } + if (weight > 0) return weight + exp; + if (firstValidScale == 0) return 0; + return TMAX(0, exp - firstValidScale); +} + static int32_t decimalVarFromStr(const char* str, int32_t len, DecimalVar* result) { int32_t code = 0, pos = 0; result->precision = 0; @@ -154,6 +176,8 @@ static int32_t decimalVarFromStr(const char* str, int32_t len, DecimalVar* resul bool leadingZeroes = true, afterPoint = false, rounded = false, stop = false; uint32_t places = 0; result->sign = 1; + int32_t weight = 0; + int32_t firstValidScale = 0; if (len == 0) return TSDB_CODE_INVALID_DATA_FMT; SDecimalOps* pOps = getDecimalOpsImp(result->type); @@ -179,6 +203,7 @@ static int32_t decimalVarFromStr(const char* str, int32_t len, DecimalVar* resul for (; pos < len && !stop; ++pos) { switch (str[pos]) { case '.': + weight = result->precision; afterPoint = true; leadingZeroes = false; break; @@ -199,40 +224,33 @@ static int32_t decimalVarFromStr(const char* str, int32_t len, DecimalVar* resul case '9': { leadingZeroes = false; ++places; - int32_t curPrec = result->precision + places - result->exponent; + if (firstValidScale == 0 && afterPoint) firstValidScale = places; + + int32_t curPrec = calcCurPrec(result->precision, places, result->exponent, weight, firstValidScale); + int32_t scaleUp = 0; + Decimal64 delta = {0}; if (curPrec > maxPrecision(result->type)) { - if (afterPoint) { - if (!rounded && curPrec - 1 == maxPrecision(result->type) && str[pos] >= '5') { - Decimal64 delta = {1}; - int32_t scaleUp = places - 1; - while (scaleUp != 0) { - int32_t curScale = TMIN(17, scaleUp); - pOps->multiply(result->pDec, &SCALE_MULTIPLIER_64[curScale], WORD_NUM(Decimal64)); - scaleUp -= curScale; - } - result->precision += places - 1; - result->scale += places - 1; - pOps->add(result->pDec, &delta, WORD_NUM(Decimal64)); - rounded = true; - places = 0; - } - } else { - return TSDB_CODE_DECIMAL_OVERFLOW; - } + if (!afterPoint) return TSDB_CODE_DECIMAL_OVERFLOW; + if (rounded || curPrec - 1 != maxPrecision(result->type) || str[pos] < '5') break; + + // do rounding + DECIMAL64_SET_VALUE(&delta, 1); + scaleUp = places - 1; + rounded = true; } else { - result->precision += places; - if (afterPoint) { - result->scale += places; - } - while (places != 0) { - int32_t curScale = TMIN(17, places); - pOps->multiply(result->pDec, &SCALE_MULTIPLIER_64[curScale], WORD_NUM(Decimal64)); - places -= curScale; - } - Decimal64 digit = {str[pos] - '0'}; - pOps->add(result->pDec, &digit, WORD_NUM(Decimal64)); - places = 0; + scaleUp = places; + DECIMAL64_SET_VALUE(&delta, str[pos] - '0'); } + + result->precision += scaleUp; + if (afterPoint) result->scale += scaleUp; + while (scaleUp != 0) { + int32_t curScale = TMIN(17, scaleUp); + pOps->multiply(result->pDec, &SCALE_MULTIPLIER_64[curScale], WORD_NUM(Decimal64)); + scaleUp -= curScale; + } + pOps->add(result->pDec, &delta, WORD_NUM(Decimal64)); + places = 0; } break; case 'e': case 'E': { @@ -243,6 +261,8 @@ static int32_t decimalVarFromStr(const char* str, int32_t len, DecimalVar* resul break; } } + result->weight = calcActualWeight(result->precision, result->scale, result->exponent, weight, firstValidScale); + if (result->precision + result->scale > 0) result->scale -= result->exponent; if (result->sign < 0) { pOps->negate(result->pDec); } @@ -1569,18 +1589,9 @@ int32_t decimal64FromStr(const char* str, int32_t len, uint8_t expectPrecision, DECIMAL64_SET_VALUE(pRes, 0); code = decimalVarFromStr(str, len, &var); if (TSDB_CODE_SUCCESS != code) return code; - if (var.precision - var.scale + var.exponent - var.scale > expectPrecision - expectScale) { + if (var.weight > (int32_t)expectPrecision - expectScale) { return TSDB_CODE_DECIMAL_OVERFLOW; } - if (var.exponent > var.scale) { - // e + positive, 小数点需要右移, 右移 var.exponent位 - //decimal64Multiply(var.pDec, &SCALE_MULTIPLIER_64[var.exponent - var.scale], WORD_NUM(Decimal64)); - var.scale -= var.exponent; - } else if (var.exponent < var.scale) { - // e + negative, 小数点需要左移, 左移 var.exponent + var.scale 位 - //decimal64divide(var.pDec, &SCALE_MULTIPLIER_64[var.scale - var.exponent], WORD_NUM(Decimal64), NULL); - var.scale -= var.exponent; - } bool overflow = false; decimal64RoundWithPositiveScale(pRes, var.precision, var.scale, expectPrecision, expectScale, ROUND_TYPE_HALF_ROUND_UP, &overflow); @@ -1620,6 +1631,9 @@ int32_t decimal128FromStr(const char* str, int32_t len, uint8_t expectPrecision, DECIMAL128_SET_LOW_WORD(pRes, 0); code = decimalVarFromStr(str, len, &var); if (TSDB_CODE_SUCCESS != code) return code; + if (var.weight > (int32_t)expectPrecision - expectScale) { + return TSDB_CODE_DECIMAL_OVERFLOW; + } bool overflow = false; decimal128RoundWithPositiveScale(pRes, var.precision, var.scale, expectPrecision, expectScale, ROUND_TYPE_HALF_ROUND_UP, &overflow); diff --git a/source/libs/decimal/test/decimalTest.cpp b/source/libs/decimal/test/decimalTest.cpp index fb41136215..5ccbefb47a 100644 --- a/source/libs/decimal/test/decimalTest.cpp +++ b/source/libs/decimal/test/decimalTest.cpp @@ -904,6 +904,18 @@ void testDecimalFromStr(std::vector>& units) { TEST(decimal, decimalFromStr_all) { std::vector> units = { + {10, 5, "1e+2.1", "100.00000", false}, + {10, 5, "0e+1000", "0", false}, + {10, 5, "0e-1000", "0", false}, + {10, 5, "0e-100", "0", false}, + {10, 5, "0e100", "0", false}, + {10, 5, "0e10", "0", false}, + {10, 5, "1.634e-5", "0.00002", false}, + {18, 16, "-0.0009900000000000000009e5", "-99.0000000000000001", false}, + {18, 10, "1.23456e7", "12345600.0000000000", false}, + {10, 5, "0e-10", "0", false}, + {10, 8, "1.000000000000000009e2", "", true}, + {10, 8, "1.2345e2", "", true}, {18, 18, "-0.0000000000000100000010000000900000009e10", "-0.000100000010000001", false}, {18, 18, "-0.1999999999999999999e-1", "-0.020000000000000000", false}, {18, 18, "-0.9999999999999999999e-1", "-0.100000000000000000", false}, @@ -911,80 +923,98 @@ TEST(decimal, decimalFromStr_all) { {10, 10, "-9.9999999e-2", "-0.0999999990", false}, {10, 6, "9.99999e4", "", true}, {18, 4, "9.999999e1", "100.0000", false}, - {18, 10, "1.23456e7", "12345600.0000000000", false}, - {18, 18, "0.0000000000000000000000000010000000000000000199999e26", "0.100000000000000002", false}, - {18, 18, "0.000000000000000000000000001000000000000000009e26", "0.100000000000000001", false}, - {10, 10, "0.000000000010000000009e10", "0.1000000001", false}, - {10, 10, "0.000000000000000000009e10", "0.0000000001", false}, - {10, 10, "0.00000000001e10", "0.1000000000", false}, - {10, 7, "-1234567890e-8", "-12.3456789", false}, - {10, 5, "0e10", "0", true}, - {10, 4, "1e5", "100000.0000", false}, - {10, 3, "123.000E4", "1230000.000", false}, - {10, 3, "123.456E2", "12345.600", false}, - {18, 2, "1.2345e8", "123450000.00", false}, - {10, 2, "1.2345e8", "123450000.00", true}, + {18, 18, "0.0000000000000000000000000010000000000000000199999e26", "0.100000000000000002", false}, + {18, 18, "0.000000000000000000000000001000000000000000009e26", "0.100000000000000001", false}, + {10, 10, "0.000000000010000000009e10", "0.1000000001", false}, + {10, 10, "0.000000000000000000009e10", "0.0000000001", false}, + {10, 10, "0.00000000001e10", "0.1000000000", false}, + {10, 7, "-1234567890e-8", "-12.3456789", false}, + {10, 4, "1e5", "100000.0000", false}, + {10, 3, "123.000E4", "1230000.000", false}, + {10, 3, "123.456E2", "12345.600", false}, + {18, 2, "1.2345e8", "123450000.00", false}, + {10, 2, "1.2345e8", "123450000.00", true}, {18, 4, "9.99999", "10.0000", false}, - {10, 2, "123.45", "123.45", false}, - {10, 2, "123.456", "123.46", false}, - {10, 2, "123.454", "123.45"}, - {18, 2, "1234567890123456.456", "1234567890123456.46", false}, - {18, 2, "9999999999999999.995", "", true}, - {18, 2, "9999999999999999.994", "9999999999999999.99", false}, - {18, 2, "-9999999999999999.995", "", true}, - {18, 2, "-9999999999999999.994", "-9999999999999999.99", false}, - {18, 2, "-9999999999999999.9999999", "", true}, - {10, 2, "12345678.456", "12345678.46", false}, - {10, 2, "12345678.454", "12345678.45", false}, - {10, 2, "99999999.999", "", true}, - {10, 2, "-99999999.992", "-99999999.99", false}, - {10, 2, "-99999999.999", "", true}, - {10, 2, "-99999989.998", "-99999990.00", false}, - {10, 2, "-99999998.997", "-99999999.00", false}, - {10, 2, "-99999999.009", "-99999999.01", false}, - {18, 17, "-9.99999999999999999999", "", true}, - {18, 16, "-99.999999999999999899999", "-99.9999999999999999", false}, - {18, 16, "-99.999999999999990099999", "-99.9999999999999901", false}, - {18, 18, "0.0000000000000000099", "0.000000000000000010", false}, - {18, 18, "0.0000000000000000001", "0", false}, - {18, 18, "0.0000000000000000005", "0.000000000000000001", false}, - {18, 18, "-0.0000000000000000001", "0", false}, - {18, 18, "-0.00000000000000000019999", "0", false}, - {18, 18, "-0.0000000000000000005", "-0.000000000000000001", false}, - {18, 18, "-0.00000000000000000000000000123123123", "0", false}, - {18, 18, "0.10000000000000000000000000123123123", "0.100000000000000000", false}, - {18, 18, "0.000000000000000000000000000000000000006", "0", false}, - {18, 17, "1.00000000000000000999", "1.00000000000000001", false}, - {18, 17, "1.00000000000000000199", "1.00000000000000000", false}, - {15, 1, "-00000.", "0", false}, - {14, 12, "-.000", "0", false}, - {14, 12, "-.000000000000", "0", false}, - {14, 12, "-.", "0", false}, + {10, 2, "123.45", "123.45", false}, + {10, 2, "123.456", "123.46", false}, + {10, 2, "123.454", "123.45"}, + {18, 2, "1234567890123456.456", "1234567890123456.46", false}, + {18, 2, "9999999999999999.995", "", true}, + {18, 2, "9999999999999999.994", "9999999999999999.99", false}, + {18, 2, "-9999999999999999.995", "", true}, + {18, 2, "-9999999999999999.994", "-9999999999999999.99", false}, + {18, 2, "-9999999999999999.9999999", "", true}, + {10, 2, "12345678.456", "12345678.46", false}, + {10, 2, "12345678.454", "12345678.45", false}, + {10, 2, "99999999.999", "", true}, + {10, 2, "-99999999.992", "-99999999.99", false}, + {10, 2, "-99999999.999", "", true}, + {10, 2, "-99999989.998", "-99999990.00", false}, + {10, 2, "-99999998.997", "-99999999.00", false}, + {10, 2, "-99999999.009", "-99999999.01", false}, + {18, 17, "-9.99999999999999999999", "", true}, + {18, 16, "-99.999999999999999899999", "-99.9999999999999999", false}, + {18, 16, "-99.999999999999990099999", "-99.9999999999999901", false}, + {18, 18, "0.0000000000000000099", "0.000000000000000010", false}, + {18, 18, "0.0000000000000000001", "0", false}, + {18, 18, "0.0000000000000000005", "0.000000000000000001", false}, + {18, 18, "-0.0000000000000000001", "0", false}, + {18, 18, "-0.00000000000000000019999", "0", false}, + {18, 18, "-0.0000000000000000005", "-0.000000000000000001", false}, + {18, 18, "-0.00000000000000000000000000123123123", "0", false}, + {18, 18, "0.10000000000000000000000000123123123", "0.100000000000000000", false}, + {18, 18, "0.000000000000000000000000000000000000006", "0", false}, + {18, 17, "1.00000000000000000999", "1.00000000000000001", false}, + {18, 17, "1.00000000000000000199", "1.00000000000000000", false}, + {15, 1, "-00000.", "0", false}, + {14, 12, "-.000", "0", false}, + {14, 12, "-.000000000000", "0", false}, + {14, 12, "-.", "0", false}, + {14, 10, "12345.12345", "", true}, + {14, 0, "123456789012345.123123", "", true}, + {18, 0, "1234567890123456789.123123", "", true}, + {18, 0, "1.23456e18", "", true}, + {18, 18, "1.23456e0", "", true}, + {18, 18, "1.23456e-1", "0.123456000000000000", false}, }; testDecimalFromStr(units); std::vector> dec128Units = { - {38, 10, "123456789012345678901234567.89012345679", "123456789012345678901234567.8901234568", false}, - {38, 10, "123456789012345678901234567.89012345670", "123456789012345678901234567.8901234567", false}, - {38, 10, "-123456789012345678901234567.89012345671", "-123456789012345678901234567.8901234567", false}, - {38, 10, "-123456789012345678901234567.89012345679", "-123456789012345678901234567.8901234568", false}, - {38, 10, "-9999999999999999999999999999.99999999995", "", true}, - {38, 10, "-9999999999999999999999999999.99999999994", "-9999999999999999999999999999.9999999999", false}, - {38, 10, "9999999999999999999999999999.99999999996", "", true}, - {38, 10, "9999999999999999999999999999.99999999994", "9999999999999999999999999999.9999999999", false}, - {36, 35, "9.99999999999999999999999999999999999", "9.99999999999999999999999999999999999", false}, - {36, 35, "9.999999999999999999999999999999999999111231231", "", true}, - {38, 38, "0.000000000000000000000000000000000000001", "0", false}, - {38, 38, "0.000000000000000000000000000000000000006", "0.00000000000000000000000000000000000001", false}, - {38, 35, "123.000000000000000000000000000000001", "123.00000000000000000000000000000000100", false}, - {38, 5, "123.", "123.00000", false}, - {20, 4, "-.12345", "-0.1235", false}, + {38, 10, "0e-10", "0", false}, + {38, 10, "0e10", "0", false}, + {38, 10, "e-100000", "0", false}, + {38, 10, "e-10", "0", false}, + {38, 10, "e10", "0", false}, + {38, 10, "-1.23456789012300000000000000000099000009e20", "-123456789012300000000.0000000001", false}, + {38, 10, "-1.234567890123e20", "-123456789012300000000.0000000000", false}, + {20, 15, "1234567890.9999999999e-6", "1234.567891000000000", false}, + {20, 15, "1234567890.9999999999999999999e-5", "12345.678910000000000", false}, + {20, 15, "1234567890.99999999999e-5", "12345.678910000000000", false}, + {20, 10, "12345667788.12312e-10", "1.2345667788", false}, + {20, 20, "1.234567123123123e-20", "0.00000000000000000001", false}, + {38, 38, "1.23456789121312312312312312355e-10", "0.00000000012345678912131231231231231236", false}, + {38, 38, "1.23456e-10", "0.00000000012345600000000000000000000000", false}, + {38, 10, "123456789012345678901234567.89012345679", "123456789012345678901234567.8901234568", false}, + {38, 10, "123456789012345678901234567.89012345670", "123456789012345678901234567.8901234567", false}, + {38, 10, "-123456789012345678901234567.89012345671", "-123456789012345678901234567.8901234567", false}, + {38, 10, "-123456789012345678901234567.89012345679", "-123456789012345678901234567.8901234568", false}, + {38, 10, "-9999999999999999999999999999.99999999995", "", true}, + {38, 10, "-9999999999999999999999999999.99999999994", "-9999999999999999999999999999.9999999999", false}, + {38, 10, "9999999999999999999999999999.99999999996", "", true}, + {38, 10, "9999999999999999999999999999.99999999994", "9999999999999999999999999999.9999999999", false}, + {36, 35, "9.99999999999999999999999999999999999", "9.99999999999999999999999999999999999", false}, + {36, 35, "9.999999999999999999999999999999999999111231231", "", true}, + {38, 38, "0.000000000000000000000000000000000000001", "0", false}, + {38, 38, "0.000000000000000000000000000000000000006", "0.00000000000000000000000000000000000001", false}, + {38, 35, "123.000000000000000000000000000000001", "123.00000000000000000000000000000000100", false}, + {38, 5, "123.", "123.00000", false}, + {20, 4, "-.12345", "-0.1235", false}, + {20, 4, "-.", "0", false}, + {30, 10, "1.2345e+20", "", true}, + {38, 38, "1.23456e0", "", true}, + {38, 38, "1.23456e-1", "0.12345600000000000000000000000000000000", false}, }; testDecimalFromStr(dec128Units); - - // TODO test e/E - - // TODO test weight overflow } TEST(decimal, op_overflow) {