test insert decimal values

This commit is contained in:
wangjiaming0909 2025-02-18 16:11:39 +08:00
parent b938dbd091
commit 77ed918248
7 changed files with 546 additions and 129 deletions

View File

@ -34,6 +34,7 @@ typedef struct Decimal64 {
#define DECIMAL64_CLONE(pDst, pFrom) ((Decimal64*)(pDst))->words[0] = ((Decimal64*)(pFrom))->words[0]
static const Decimal64 decimal64Zero = {0};
static const Decimal64 decimal64Two = {2};
static const Decimal64 decimal64Min = {(uint64_t)-999999999999999999LL};
static const Decimal64 decimal64Max = {(uint64_t)999999999999999999LL};
#define DECIMAL64_ZERO decimal64Zero

View File

@ -34,7 +34,7 @@ char **strsplit(char *src, const char *delim, int32_t *num);
char *strtolower(char *dst, const char *src);
char *strntolower(char *dst, const char *src, int32_t n);
char *strntolower_s(char *dst, const char *src, int32_t n);
int64_t strnatoi(char *num, int32_t len);
int64_t strnatoi(const char *num, int32_t len);
size_t tstrncspn(const char *str, size_t ssize, const char *reject, size_t rsize);
size_t twcsncspn(const TdUcs4 *wcs, size_t size, const TdUcs4 *reject, size_t rsize);

View File

@ -37,6 +37,27 @@ static SDecimalOps* getDecimalOpsImp(DecimalInternalType t);
#define DECIMAL_MIN_ADJUSTED_SCALE 6
// TODO wjm use uint64_t ???
static Decimal64 SCALE_MULTIPLIER_64[TSDB_DECIMAL64_MAX_PRECISION + 1] = {1LL,
10LL,
100LL,
1000LL,
10000LL,
100000LL,
1000000LL,
10000000LL,
100000000LL,
1000000000LL,
10000000000LL,
100000000000LL,
1000000000000LL,
10000000000000LL,
100000000000000LL,
1000000000000000LL,
10000000000000000LL,
100000000000000000LL,
1000000000000000000LL};
typedef struct DecimalVar {
DecimalInternalType type;
uint8_t precision;
@ -125,13 +146,12 @@ int32_t decimalGetRetType(const SDataType* pLeftT, const SDataType* pRightT, EOp
return 0;
}
static int32_t decimalVarFromStr(const char* str, int32_t len, DecimalVar* result);
static int32_t decimalVarFromStr(const char* str, int32_t len, DecimalVar* result) {
int32_t code = 0, pos = 0;
result->precision = 0;
result->scale = 0;
bool leadingZeroes = true, afterPoint = false;
result->exponent = 0;
bool leadingZeroes = true, afterPoint = false, rounded = false, stop = false;
uint32_t places = 0;
result->sign = 1;
@ -148,7 +168,7 @@ static int32_t decimalVarFromStr(const char* str, int32_t len, DecimalVar* resul
break;
}
for (; pos < len; ++pos) {
for (; pos < len && !stop; ++pos) {
switch (str[pos]) {
case '.':
afterPoint = true;
@ -171,8 +191,24 @@ static int32_t decimalVarFromStr(const char* str, int32_t len, DecimalVar* resul
case '9': {
leadingZeroes = false;
++places;
if (result->precision + places > maxPrecision(result->type)) {
int32_t curPrec = result->precision + places;
if (curPrec > maxPrecision(result->type)) {
if (afterPoint) {
if (!rounded && curPrec - 1 == maxPrecision(result->type) && str[pos] - '0' >= 5) {
Decimal64 delta = {1};
if (places > 1) {
int32_t scaleUp = places - 1;
while (scaleUp != 0) {
int32_t curScale = TMIN(17, scaleUp);
pOps->multiply(result->pDec, &SCALE_MULTIPLIER_64[curScale], WORD_NUM(Decimal64));
scaleUp -= curScale;
}
result->precision += places - 1;
result->scale += places - 1;
}
pOps->add(result->pDec, &delta, WORD_NUM(Decimal64));
rounded = true;
}
break;
} else {
return TSDB_CODE_DECIMAL_OVERFLOW;
@ -181,21 +217,26 @@ static int32_t decimalVarFromStr(const char* str, int32_t len, DecimalVar* resul
result->precision += places;
if (afterPoint) {
result->scale += places;
result->exponent -= places;
}
DecimalWord ten = 10, digit = str[pos] - '0';
while (places-- > 0) {
pOps->multiply(result->pDec, &ten, 1);
Decimal64 digit = {str[pos] - '0'};
while (places != 0) {
int32_t curScale = TMIN(17, places);
pOps->multiply(result->pDec, &SCALE_MULTIPLIER_64[curScale], WORD_NUM(Decimal64));
places -= curScale;
}
pOps->add(result->pDec, &digit, 1);
pOps->add(result->pDec, &digit, WORD_NUM(Decimal64));
places = 0;
break;
}
}
case 'e':
case 'E':
// TODO wjm handle E
break;
case 'E': {
result->exponent += strnatoi(str + pos + 1, len - pos - 1);
stop = true;
} break;
default:
stop = true;
break;
}
}
@ -218,26 +259,6 @@ int32_t decimal128ToDataVal(Decimal128* dec, SValue* pVal) {
return TSDB_CODE_SUCCESS;
}
// TODO wjm use uint64_t ???
static Decimal64 SCALE_MULTIPLIER_64[TSDB_DECIMAL64_MAX_PRECISION + 1] = {1LL,
10LL,
100LL,
1000LL,
10000LL,
100000LL,
1000000LL,
10000000LL,
100000000LL,
1000000000LL,
10000000000LL,
100000000000LL,
1000000000000LL,
10000000000000LL,
100000000000000LL,
1000000000000000LL,
10000000000000000LL,
100000000000000000LL,
1000000000000000000LL};
#define DECIMAL64_ONE SCALE_MULTIPLIER_64[0]
#define DECIMAL64_GET_MAX(precision, pMax) \
@ -291,6 +312,9 @@ static void decimal64ScaleDown(Decimal64* pDec, uint8_t scaleDown);
static void decimal64ScaleUp(Decimal64* pDec, uint8_t scaleUp);
static void decimal64ScaleTo(Decimal64* pDec, uint8_t oldScale, uint8_t newScale);
static void decimal64RoundWithPositiveScale(Decimal64* pDec, uint8_t prec, uint8_t scale, uint8_t toPrec,
uint8_t toScale, DecimalRoundType roundType, bool* overflow);
static void decimal128Negate(DecimalType* pInt);
static void decimal128Abs(DecimalType* pWord);
static void decimal128Add(DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum);
@ -641,8 +665,8 @@ static bool decimal128Eq(const DecimalType* pLeft, const DecimalType* pRight, ui
DECIMAL128_LOW_WORD(pLeftDec) == DECIMAL128_LOW_WORD(pRightDec);
}
static void extractDecimal128Digits(const Decimal128* pDec, uint64_t* digits, int32_t* digitNum) {
#define DIGIT_NUM_ONCE 18
static void extractDecimal128Digits(const Decimal128* pDec, uint64_t* digits, int32_t* digitNum) {
UInt128 a = {0};
UInt128 b = {0};
*digitNum = 0;
@ -677,8 +701,7 @@ static int32_t decimal128ToStr(const DecimalType* pInt, uint8_t scale, char* pBu
makeDecimal128(&copy, DECIMAL128_HIGH_WORD(pDec), DECIMAL128_LOW_WORD(pDec));
decimal128Abs(&copy);
extractDecimal128Digits(&copy, segments, &digitNum);
buf[0] = '-';
len = 1;
TAOS_STRNCAT(pBuf, "-", 2);
} else {
extractDecimal128Digits(pDec, segments, &digitNum);
}
@ -1105,9 +1128,25 @@ bool decimalCompare(EOperatorType op, const SDecimalCompareCtx* pLeft, const SDe
#define ABS_INT64(v) (v) == INT64_MIN ? (uint64_t)INT64_MAX + 1 : (uint64_t)llabs(v)
#define ABS_UINT64(v) (v)
static int64_t int64FromDecimal64(const DecimalType* pDec, uint8_t prec, uint8_t scale) { return 0; }
static int64_t int64FromDecimal64(const DecimalType* pDec, uint8_t prec, uint8_t scale) {
Decimal64 rounded = *(Decimal64*)pDec;
bool overflow = false;
decimal64RoundWithPositiveScale(&rounded, prec, scale, prec, 0, ROUND_TYPE_HALF_ROUND_UP, &overflow);
assert(!overflow); // TODO wjm remove this assert
if (overflow) return 0;
static uint64_t uint64FromDecimal64(const DecimalType* pDec, uint8_t prec, uint8_t scale) { return 0; }
return DECIMAL64_GET_VALUE(&rounded);
}
static uint64_t uint64FromDecimal64(const DecimalType* pDec, uint8_t prec, uint8_t scale) {
Decimal64 rounded = *(Decimal64*)pDec;
bool overflow = false;
decimal64RoundWithPositiveScale(&rounded, prec, scale, prec, 0, ROUND_TYPE_HALF_ROUND_UP, &overflow);
assert(!overflow); // TODO wjm remove this assert
if (overflow) return 0;
return DECIMAL64_GET_VALUE(&rounded);
}
static int32_t decimal64FromInt64(DecimalType* pDec, uint8_t prec, uint8_t scale, int64_t val) {
Decimal64 max = {0};
@ -1422,16 +1461,108 @@ static void decimal64ScaleTo(Decimal64* pDec, uint8_t oldScale, uint8_t newScale
decimal64ScaleDown(pDec, oldScale - newScale);
}
static void decimal64ScaleAndCheckOverflow(Decimal64* pDec, uint8_t scale, uint8_t toPrec, uint8_t toScale,
bool* overflow) {
int8_t deltaScale = toScale - scale;
if (deltaScale >= 0) {
Decimal64 max = {0};
DECIMAL64_GET_MAX(toPrec - deltaScale, &max);
Decimal64 abs = *pDec;
decimal64Abs(&abs);
if (decimal64Gt(&abs, &max, WORD_NUM(Decimal64))) {
if (overflow) *overflow = true;
} else {
decimal64ScaleUp(pDec, deltaScale);
}
} else if (deltaScale < 0) {
Decimal64 res = *pDec, max = {0};
decimal64ScaleDown(&res, -deltaScale);
DECIMAL64_GET_MAX(toPrec, &max);
if (decimal64Gt(&res, &max, WORD_NUM(Decimal64))) {
if (overflow) *overflow = true;
} else {
*pDec = res;
}
}
}
static int32_t decimal64CountRoundingDelta(const Decimal64* pDec, int8_t scale, int8_t toScale,
DecimalRoundType roundType) {
if (roundType == ROUND_TYPE_TRUNC || toScale >= scale) return 0;
Decimal64 dec = *pDec;
int32_t res = 0;
switch (roundType) {
case ROUND_TYPE_HALF_ROUND_UP: {
Decimal64 trailing = dec;
decimal64Mod(&trailing, &SCALE_MULTIPLIER_64[scale - toScale], WORD_NUM(Decimal64));
if (decimal64Eq(&trailing, &decimal64Zero, WORD_NUM(Decimal64))) {
res = 0;
break;
}
Decimal64 trailingAbs = trailing, baseDiv2 = SCALE_MULTIPLIER_64[scale - toScale];
decimal64Abs(&trailingAbs);
decimal64divide(&baseDiv2, &decimal64Two, WORD_NUM(Decimal64), NULL);
if (decimal64Lt(&trailingAbs, &baseDiv2, WORD_NUM(Decimal64))) {
res = 0;
break;
}
res = DECIMAL64_SIGN(pDec) == 1 ? 1 : -1;
} break;
default:
break;
}
return res;
}
static void decimal64RoundWithPositiveScale(Decimal64* pDec, uint8_t prec, uint8_t scale, uint8_t toPrec,
uint8_t toScale, DecimalRoundType roundType, bool* overflow) {
Decimal64 scaled = *pDec;
bool overflowLocal = false;
// scale up or down to toScale
decimal64ScaleAndCheckOverflow(&scaled, scale, toPrec, toScale, &overflowLocal);
if (overflowLocal) {
if (overflow) *overflow = true;
*pDec = decimal64Zero;
return;
}
// calc rounding delta
int32_t delta = decimal64CountRoundingDelta(pDec, scale, toScale, roundType);
if (delta == 0) {
*pDec = scaled;
return;
}
Decimal64 deltaDec = {delta};
// add the delta
decimal64Add(&scaled, &deltaDec, WORD_NUM(Decimal64));
// check overflow again
if (toPrec < prec) {
Decimal64 max = {0};
DECIMAL64_GET_MAX(toPrec, &max);
Decimal64 scaledAbs = scaled;
decimal64Abs(&scaledAbs);
if (decimal64Gt(&scaledAbs, &max, WORD_NUM(Decimal64))) {
if (overflow) *overflow = true;
*pDec = decimal64Zero;
return;
}
}
*pDec = scaled;
}
int32_t decimal64FromStr(const char* str, int32_t len, uint8_t expectPrecision, uint8_t expectScale, Decimal64* pRes) {
int32_t code = 0;
DecimalVar var = {.type = DECIMAL_64, .pDec = pRes->words};
DECIMAL64_SET_VALUE(pRes, 0);
code = decimalVarFromStr(str, len, &var);
if (TSDB_CODE_SUCCESS != code) return code;
Decimal64 max = {0};
DECIMAL64_GET_MAX(expectPrecision, &max);
decimal64ScaleTo(pRes, var.scale, expectScale);
if (decimal64Gt(pRes, &max, 1)) {
bool overflow = false;
decimal64RoundWithPositiveScale(pRes, var.precision, var.scale, expectPrecision, expectScale,
ROUND_TYPE_HALF_ROUND_UP, &overflow);
if (overflow) {
return TSDB_CODE_DECIMAL_OVERFLOW;
}
return code;
@ -1467,10 +1598,10 @@ int32_t decimal128FromStr(const char* str, int32_t len, uint8_t expectPrecision,
DECIMAL128_SET_LOW_WORD(pRes, 0);
code = decimalVarFromStr(str, len, &var);
if (TSDB_CODE_SUCCESS != code) return code;
Decimal128 max = {0};
DECIMAL128_GET_MAX(expectPrecision, &max);
decimal128ScaleTo(pRes, var.scale, expectScale);
if (decimal128Gt(pRes, &max, 2)) {
bool overflow = false;
decimal128RoundWithPositiveScale(pRes, var.precision, var.scale, expectPrecision, expectScale,
ROUND_TYPE_HALF_ROUND_UP, &overflow);
if (overflow) {
return TSDB_CODE_DECIMAL_OVERFLOW;
}
return code;
@ -1558,6 +1689,7 @@ static void decimal128RoundWithPositiveScale(Decimal128* pDec, uint8_t prec, uin
uint8_t toScale, DecimalRoundType roundType, bool* overflow) {
Decimal128 scaled = *pDec;
bool overflowLocal = false;
// scale up or down to toScale
decimal128ModifyScaleAndPrecision(&scaled, scale, toPrec, toScale, &overflowLocal);
if (overflowLocal) {
if (overflow) *overflow = true;
@ -1565,6 +1697,7 @@ static void decimal128RoundWithPositiveScale(Decimal128* pDec, uint8_t prec, uin
return;
}
// calc rounding delta, 1 or -1
int32_t delta = decimal128CountRoundingDelta(pDec, scale, toScale, roundType);
if (delta == 0) {
*pDec = scaled;
@ -1572,17 +1705,22 @@ static void decimal128RoundWithPositiveScale(Decimal128* pDec, uint8_t prec, uin
}
Decimal64 deltaDec = {delta};
// add the delta
decimal128Add(&scaled, &deltaDec, WORD_NUM(Decimal64));
Decimal128 max = {0};
DECIMAL128_GET_MAX(toPrec, &max);
Decimal128 scaledAbs = scaled;
decimal128Abs(&scaledAbs);
if (toPrec < prec && decimal128Gt(&scaledAbs, &max, WORD_NUM(Decimal128))) {
if (overflow) *overflow = true;
*(Decimal128*)pDec = decimal128Zero;
} else {
*(Decimal128*)pDec = scaled;
// check overflow again
if (toPrec < prec) {
Decimal128 max = {0};
DECIMAL128_GET_MAX(toPrec, &max);
Decimal128 scaledAbs = scaled;
decimal128Abs(&scaledAbs);
if (decimal128Gt(&scaledAbs, &max, WORD_NUM(Decimal128))) {
if (overflow) *overflow = true;
*(Decimal128*)pDec = decimal128Zero;
return;
}
}
*(Decimal128*)pDec = scaled;
}
static void decimal128ModifyScaleAndPrecision(Decimal128* pDec, uint8_t scale, uint8_t toPrec, int8_t toScale,
@ -1590,7 +1728,7 @@ static void decimal128ModifyScaleAndPrecision(Decimal128* pDec, uint8_t scale, u
int8_t deltaScale = toScale - scale;
if (deltaScale >= 0) {
Decimal128 max = {0};
DECIMAL128_GET_MAX(toPrec - deltaScale, &max); // TODO wjm test toPrec == 0
DECIMAL128_GET_MAX(toPrec - deltaScale, &max); // TODO wjm test toPrec == 0, test toPrec - deltaScale
Decimal128 abs = *pDec;
decimal128Abs(&abs);
if (decimal128Gt(&abs, &max, WORD_NUM(Decimal128))) {
@ -1630,7 +1768,7 @@ static int32_t decimal128CountRoundingDelta(const Decimal128* pDec, int8_t scale
res = 0;
break;
}
res = decimal128Lt(pDec, &decimal128Zero, WORD_NUM(Decimal128)) ? -1 : 1;
res = decimal128Lt(pDec, &decimal128Zero, WORD_NUM(Decimal128)) ? -1 : 1; // TODO wjm use sign??
} break;
case ROUND_TYPE_TRUNC:
default:

View File

@ -174,8 +174,8 @@ class Numeric {
if (prec > NumericType<BitNum>::maxPrec) throw std::string("prec too big") + std::to_string(prec);
int32_t code = dec_.fromStr(str, prec, scale) != 0;
if (code != 0) {
cout << "failed to init decimal from str: " << str << endl;
throw std::string(tstrerror(code));
cout << "failed to init decimal from str: " << str << "\t";
throw std::overflow_error(tstrerror(code));
}
}
Numeric() = default;
@ -505,7 +505,7 @@ TEST(decimal, decimalFromType) {
dec1 = (int64_t)-9999999;
ASSERT_EQ(dec1.toString(), "-9999999.0000");
dec1 = "99.99999";
ASSERT_EQ(dec1.toString(), "99.9999");
ASSERT_EQ(dec1.toString(), "100.0000");
}
TEST(decimal, typeFromDecimal) {
@ -579,7 +579,7 @@ TEST(decimal128, to_string) {
ASSERT_STREQ(buf, "1234567890123456789012345678.901234567");
}
TEST(decimal128, divide) {
TEST(decimal, divide) {
__int128 i = generate_big_int128(15);
int64_t hi = i >> 64;
uint64_t lo = i;
@ -666,6 +666,7 @@ TEST(decimal, decimalFromStr) {
code = decimal64FromStr(buf, strlen(buf), 6, 3, &dec64);
ASSERT_EQ(code, 0);
ASSERT_EQ(999999, DECIMAL64_GET_VALUE(&dec64));
}
TEST(decimal, toStr) {
@ -861,10 +862,6 @@ TEST(decimal, randomGenerator) {
}
}
TEST(deicmal, decimalFromStr_all) {
// TODO test e/E
}
#define ASSERT_OVERFLOW(op) \
do { \
try { \
@ -878,6 +875,98 @@ TEST(deicmal, decimalFromStr_all) {
FAIL(); \
} while (0)
template <int32_t BitNum>
struct DecimalFromStrTestUnit {
uint8_t precision;
uint8_t scale;
std::string input;
std::string expect;
bool overflow;
};
template <int32_t BitNum>
void testDecimalFromStr(std::vector<DecimalFromStrTestUnit<BitNum>>& units) {
for (auto& unit : units) {
if (unit.overflow) {
auto ff = [&]() {
Numeric<BitNum> dec = {unit.precision, unit.scale, unit.input};
return dec;
};
ASSERT_OVERFLOW(ff());
continue;
}
cout << unit.input << " convert to decimal: (" << (int32_t)unit.precision << "," << (int32_t)unit.scale
<< "): " << unit.expect << endl;
Numeric<BitNum> dec = {unit.precision, unit.scale, unit.input};
ASSERT_EQ(dec.toString(), unit.expect);
}
}
TEST(decimal, decimalFromStr_all) {
std::vector<DecimalFromStrTestUnit<64>> units = {
{10, 2, "123.45", "123.45", false},
{10, 2, "123.456", "123.46", false},
{10, 2, "123.454", "123.45"},
{18, 2, "1234567890123456.456", "1234567890123456.46", false},
{18, 2, "9999999999999999.995", "", true},
{18, 2, "9999999999999999.994", "9999999999999999.99", false},
{18, 2, "-9999999999999999.995", "", true},
{18, 2, "-9999999999999999.994", "-9999999999999999.99", false},
{18, 2, "-9999999999999999.9999999", "", true},
{10, 2, "12345678.456", "12345678.46", false},
{10, 2, "12345678.454", "12345678.45", false},
{10, 2, "99999999.999", "", true},
{10, 2, "-99999999.992", "-99999999.99", false},
{10, 2, "-99999999.999", "", true},
{10, 2, "-99999989.998", "-99999990.00", false},
{10, 2, "-99999998.997", "-99999999.00", false},
{10, 2, "-99999999.009", "-99999999.01", false},
{18, 17, "-9.99999999999999999999", "", true},
{18, 16, "-99.999999999999999899999", "-99.9999999999999999", false},
{18, 16, "-99.999999999999990099999", "-99.9999999999999901", false},
{18, 18, "0.0000000000000000099", "0.000000000000000010", false},
{18, 18, "0.0000000000000000001", "0", false},
{18, 18, "0.0000000000000000005", "0.000000000000000001", false},
{18, 18, "-0.0000000000000000001", "0", false},
{18, 18, "-0.00000000000000000019999", "0", false},
{18, 18, "-0.0000000000000000005", "-0.000000000000000001", false},
{18, 18, "-0.00000000000000000000000000123123123", "0", false},
{18, 18, "0.10000000000000000000000000123123123", "0.100000000000000000", false},
{18, 18, "0.000000000000000000000000000000000000006", "0", false},
{18, 17, "1.00000000000000000999", "1.00000000000000001", false},
{18, 17, "1.00000000000000000199", "1.00000000000000000", false},
{15, 1, "-00000.", "0", false},
{14, 12, "-.000", "0", false},
{14, 12, "-.000000000000", "0", false},
{14, 12, "-.", "0", false},
//{10, 2, "1.2345e8", "12345000.00", false},
};
testDecimalFromStr(units);
std::vector<DecimalFromStrTestUnit<128>> dec128Units = {
{38, 10, "123456789012345678901234567.89012345679", "123456789012345678901234567.8901234568", false},
{38, 10, "123456789012345678901234567.89012345670", "123456789012345678901234567.8901234567", false},
{38, 10, "-123456789012345678901234567.89012345671", "-123456789012345678901234567.8901234567", false},
{38, 10, "-123456789012345678901234567.89012345679", "-123456789012345678901234567.8901234568", false},
{38, 10, "-9999999999999999999999999999.99999999995", "", true},
{38, 10, "-9999999999999999999999999999.99999999994", "-9999999999999999999999999999.9999999999", false},
{38, 10, "9999999999999999999999999999.99999999996", "", true},
{38, 10, "9999999999999999999999999999.99999999994", "9999999999999999999999999999.9999999999", false},
{36, 35, "9.99999999999999999999999999999999999", "9.99999999999999999999999999999999999", false},
{36, 35, "9.999999999999999999999999999999999999111231231", "", true},
{38, 38, "0.000000000000000000000000000000000000001", "0", false},
{38, 38, "0.000000000000000000000000000000000000006", "0.00000000000000000000000000000000000001", false},
{38, 35, "123.000000000000000000000000000000001", "123.00000000000000000000000000000000100", false},
{38, 5, "123.", "123.00000", false},
{20, 4, "-.12345", "-0.1235", false},
};
testDecimalFromStr(dec128Units);
// TODO test e/E
// TODO test weight overflow
}
TEST(decimal, op_overflow) {
// divide 0 error
Numeric<128> dec{38, 2, string(36, '9') + ".99"};
@ -1174,6 +1263,8 @@ class DecimalTest : public ::testing::Test {
static constexpr const char* user = "root";
static constexpr const char* passwd = "taosdata";
static constexpr const char* db = "test";
DecimalStringRandomGenerator generator_;
DecimalStringRandomGeneratorConfig generator_config_;
public:
void SetUp() override {
@ -1187,6 +1278,8 @@ class DecimalTest : public ::testing::Test {
taos_close(default_conn_);
}
}
std::string generate_decimal_str() { return generator_.generate(generator_config_); }
};
TEST_F(DecimalTest, insert) {
@ -1293,6 +1386,17 @@ TEST_F(DecimalTest, api_taos_fetch_rows) {
taos_close(pTaos);
}
TEST_F(DecimalTest, decimalFromStr) {
Numeric<64> numeric64 = {10, 2, "0"};
numeric64 = {18, 0, "0"};
numeric64 = { 18, 18, "0"};
numeric64 = {18, 2, "0"};
Numeric<128> numeric128 = {38, 10, "0"};
}
int main(int argc, char** argv) {
testing::InitGoogleTest(&argc, argv);

View File

@ -278,7 +278,7 @@ char *paGetToken(char *string, char **token, int32_t *tokenLen) {
return string;
}
int64_t strnatoi(char *num, int32_t len) {
int64_t strnatoi(const char *num, int32_t len) {
int64_t ret = 0, i, dig, base = 1;
if (len > (int32_t)strlen(num)) {

View File

@ -1,8 +1,12 @@
from pydoc import doc
from random import randrange
from re import A
import time
import threading
import secrets
from sympy import true
from torch import randint
import query
from tag_lite import column
from util.log import *
@ -10,57 +14,142 @@ from util.sql import *
from util.cases import *
from util.dnodes import *
from util.common import *
from decimal import Decimal
syntax_error = -2147473920
invalid_column = -2147473918
invalid_compress_level = -2147483084
invalid_encode_param = -2147483087
class DecimalType:
def __init__(self, precision: int, scale: int):
self.precision = precision
self.scale = scale
def __str__(self):
return f"DECIMAL({self.precision}, {self.scale})"
class DecimalTypeGeneratorConfig:
def __init__(self):
self.enable_weight_overflow: bool = False
self.weightOverflowRatio: float = 0.001
self.enable_scale_overflow: bool = True
self.scale_overflow_ratio = 0.1
self.enable_positive_sign = False
self.with_corner_case = True
self.corner_case_ratio = 0.1
self.positive_ratio = 0.7
self.prec = 38
self.scale = 10
def __eq__(self, other):
return self.precision == other.precision and self.scale == other.scale
def __ne__(self, other):
return not self.__eq__(other)
def __hash__(self):
return hash((self.precision, self.scale))
def __repr__(self):
return f"DecimalType({self.precision}, {self.scale})"
class DecimalStringRandomGenerator:
def __init__(self):
self.corner_cases = ["0", "NULL", "0.", ".0", "000.000000"]
self.ratio_base: int = 1000000
def generate_value(self, allow_weight_overflow = False, allow_scale_overflow = False) -> str:
if allow_weight_overflow:
weight = secrets.randbelow(40)
else:
weight = secrets.randbelow(self.precision - self.scale)
if allow_scale_overflow:
dscale = secrets.randbelow(40 - weight + 1)
else:
dscale = secrets.randbelow(self.precision - weight + 1)
digits :str = ''
for _ in range(weight):
digits += str(secrets.randbelow(10))
if dscale > 0:
digits += '.'
for _ in range(dscale):
digits += str(secrets.randbelow(10))
if digits == '':
digits = '0'
return digits
def possible(self, possibility: float) -> bool:
return random.randint(0, self.ratio_base) < possibility * self.ratio_base
@staticmethod
def default_compression() -> str:
return "zstd"
@staticmethod
def default_encode() -> str:
return "disabled"
def generate_sign(self, positive_ratio: float) -> str:
if self.possible(positive_ratio):
return "+"
return "-"
def generate_digit(self) -> str:
return str(random.randint(0, 9))
def current_should_generate_corner_case(self, corner_case_ratio: float) -> bool:
return self.possible(corner_case_ratio)
def generate_corner_case(self, config: DecimalTypeGeneratorConfig) -> str:
if self.possible(0.8):
return random.choice(self.corner_cases)
else:
res = self.generate_digit() * (config.prec - config.scale)
if self.possible(0.8):
res += '.'
if self.possible(0.8):
res += self.generate_digit() * config.scale
return res
## 写入大整数的例子, 如10000000000, scale解析时可能为负数
def generate_(self, config: DecimalTypeGeneratorConfig) -> str:
ret: str = ''
sign = self.generate_sign(config.positive_ratio)
if config.with_corner_case and self.current_should_generate_corner_case(config.corner_case_ratio):
ret += self.generate_corner_case(config)
else:
if config.enable_positive_sign or sign != '+':
ret += sign
weight = random.randint(1, config.prec - config.scale)
scale = random.randint(1, config.scale)
for i in range(weight):
ret += self.generate_digit()
if config.enable_weight_overflow and self.possible(config.weightOverflowRatio):
extra_weight = config.prec - weight + 1 + random.randint(1, self.get_max_prec(config.prec))
while extra_weight > 0:
ret += self.generate_digit()
extra_weight -= 1
ret += '.'
for i in range(scale):
ret += self.generate_digit()
if config.enable_scale_overflow and self.possible(config.scale_overflow_ratio):
extra_scale = config.scale - scale + 1 + random.randint(1, self.get_max_prec(config.prec))
while extra_scale > 0:
ret += self.generate_digit()
extra_scale -= 1
return ret
def get_max_prec(self, prec):
if prec <= 18:
return 18
else:
return 38
class DecimalColumnAggregator:
def __init__(self):
self.max: Decimal = Decimal("0")
self.min: Decimal = Decimal("0")
self.count: int = 0
self.sum: Decimal = Decimal("0")
self.null_num: int = 0
self.none_num: int = 0
def add_value(self, value: str):
self.count += 1
if value == "NULL":
self.null_num += 1
elif value == "None":
self.none_num += 1
else:
v: Decimal = Decimal(value)
self.sum += v
if v > self.max:
self.max = v
if v < self.min:
self.min = v
class TaosShell:
def __init__(self):
self.queryResult = []
self.tmp_file_path = "/tmp/taos_shell_result"
def read_result(self):
with open(self.tmp_file_path, 'r') as f:
lines = f.readlines()
lines = lines[1:]
for line in lines:
col = 0
vals: List[str] = line.split(',')
if len(self.queryResult) == 0:
self.queryResult = [[] for i in range(len(vals))]
for val in vals:
self.queryResult[col].append(val)
col += 1
def query(self, sql: str):
try:
command = f'taos -s "{sql} >> {self.tmp_file_path}"'
result = subprocess.run(command, shell=True, check=True, stderr=subprocess.PIPE)
self.read_result()
except subprocess.CalledProcessError as e:
tdLog.error(f"Command '{sql}' failed with error: {e.stderr.decode('utf-8')}")
self.queryResult = []
return self.queryResult
class TypeEnum:
BOOL = 1
TINYINT = 2
@ -152,13 +241,6 @@ class DataType:
def __repr__(self):
return f"DataType({self.type}, {self.length}, {self.type_mod})"
@staticmethod
def get_decimal_type_mod(type: DecimalType) -> int:
return type.precision * 100 + type.scale
def get_decimal_type(self) -> DecimalType:
return DecimalType(self.type_mod // 100, self.type_mod % 100)
def construct_type_value(self, val: str):
if self.type == TypeEnum.BINARY or self.type == TypeEnum.VARCHAR or self.type == TypeEnum.NCHAR or self.type == TypeEnum.VARBINARY:
return f"'{val}'"
@ -192,8 +274,68 @@ class DataType:
return str(secrets.randbelow(9223372036854775808))
if self.type == TypeEnum.JSON:
return f'{{"key": "{secrets.token_urlsafe(10)}"}}'
if self.type == TypeEnum.DECIMAL:
return self.get_decimal_type().generate_value()
raise Exception(f"unsupport type {self.type}")
def check(self, values, offset: int):
return True
class DecimalType(DataType):
def __init__(self, type, precision: int, scale: int):
self.precision = precision
self.scale = scale
if type == TypeEnum.DECIMAL64:
bytes = 8
else:
bytes = 16
super().__init__(type, bytes, self.get_decimal_type_mod())
self.generator: DecimalStringRandomGenerator = DecimalStringRandomGenerator()
self.generator_config: DecimalTypeGeneratorConfig = DecimalTypeGeneratorConfig()
self.generator_config.prec = precision
self.generator_config.scale = scale
self.aggregator: DecimalColumnAggregator = DecimalColumnAggregator()
self.values: List[str] = []
def get_decimal_type_mod(self) -> int:
return self.precision * 100 + self.scale
def __str__(self):
return f"DECIMAL({self.precision}, {self.scale})"
def __eq__(self, other):
return self.precision == other.precision and self.scale == other.scale
def __ne__(self, other):
return not self.__eq__(other)
def __hash__(self):
return hash((self.precision, self.scale))
def __repr__(self):
return f"DecimalType({self.precision}, {self.scale})"
def generate_value(self) -> str:
val = self.generator.generate_(self.generator_config)
self.aggregator.add_value(val)
self.values.append(val)
return val
@staticmethod
def default_compression() -> str:
return "zstd"
@staticmethod
def default_encode() -> str:
return "disabled"
def check(self, values, offset: int):
val_from_query = values
val_insert = self.values[offset:]
for v1, v2 in zip(val_from_query, val_insert):
dec1: Decimal = Decimal(v1)
dec2: Decimal = Decimal(v2)
dec2 = dec2.quantize(Decimal(10) ** -self.scale)
if dec1 != dec2:
tdLog.error(f"check decimal column failed, expect {dec2}, but get {dec1}")
return False
class DecimalColumnTableCreater:
def __init__(self, conn, dbName: str, tbName: str, columns_types: List[DataType], tags_types: List[DataType] = []):
@ -262,6 +404,23 @@ class TableInserter:
self.conn.execute(f"flush database {self.dbName}", queryTimes=1)
self.conn.execute(sql, queryTimes=1)
class TableDataValidator:
def __init__(self, columns: List[DataType], tbName: str, dbName: str, tbIdx: int = 0):
self.columns = columns
self.tbName = tbName
self.dbName = dbName
self.tbIdx = tbIdx
def validate(self):
sql = f"select * from {self.dbName}.{self.tbName}"
res = TaosShell().query(sql)
row_num = len(res)
colIdx = 1
for col in self.columns:
if col.type == TypeEnum.DECIMAL or col.type == TypeEnum.DECIMAL64:
col.check(res[colIdx], row_num * self.tbIdx)
colIdx += 1
class TDTestCase:
updatecfgDict = {'asynclog': 0, 'ttlUnit': 1, 'ttlPushInterval': 5, 'ratioOfVnodeStreamThrea': 4, 'debugFlag': 143}
@ -422,9 +581,9 @@ class TDTestCase:
decimal_idx = 0
results = re.findall(r"DECIMAL\((\d+),(\d+)\)", create_table_sql)
for i, column_type in enumerate(column_types):
if column_type.type == TypeEnum.DECIMAL:
result_type = DecimalType(int(results[decimal_idx][0]), int(results[decimal_idx][1]))
if result_type != column_type.get_decimal_type():
if column_type.type == TypeEnum.DECIMAL or column_type.type == TypeEnum.DECIMAL64:
result_type = DecimalType(column_type.type, int(results[decimal_idx][0]), int(results[decimal_idx][1]))
if result_type != column_type:
tdLog.exit(f"check show create table failed for: {tbname} column {i} type is {result_type}, expect {column_type.get_decimal_type()}")
decimal_idx += 1
@ -432,13 +591,13 @@ class TDTestCase:
is_stb = tbname == self.stable_name
## alter table add column
create_c99_sql = f'alter table {self.db_name}.{tbname} add column c99 decimal(37, 19)'
columns.append(DataType(TypeEnum.DECIMAL, type_mod=DataType.get_decimal_type_mod(DecimalType(37, 19))))
columns.append(DecimalType(TypeEnum.DECIMAL, 37, 19))
tdSql.execute(create_c99_sql, queryTimes=1, show=True)
self.check_desc(tbname, columns)
## alter table add column with compression
create_c100_sql = f'ALTER TABLE {self.db_name}.{tbname} ADD COLUMN c100 decimal(36, 18) COMPRESS "zstd"'
tdSql.execute(create_c100_sql, queryTimes=1, show=True)
columns.append(DataType(TypeEnum.DECIMAL, type_mod=DataType.get_decimal_type_mod(DecimalType(36, 18))))
columns.append(DecimalType(TypeEnum.DECIMAL, 36, 18))
self.check_desc(tbname, columns)
## drop non decimal column
@ -468,10 +627,10 @@ class TDTestCase:
## create decimal type table, normal/super table, decimal64/decimal128
tdLog.printNoPrefix("-------- test create decimal column")
self.norm_tb_columns = [
DataType(TypeEnum.DECIMAL, type_mod=DataType.get_decimal_type_mod(DecimalType(10, 2))),
DataType(TypeEnum.DECIMAL, type_mod=DataType.get_decimal_type_mod(DecimalType(20, 4))),
DataType(TypeEnum.DECIMAL, type_mod=DataType.get_decimal_type_mod(DecimalType(30, 8))),
DataType(TypeEnum.DECIMAL, type_mod=DataType.get_decimal_type_mod(DecimalType(38, 10))),
DecimalType(TypeEnum.DECIMAL, 10, 2),
DecimalType(TypeEnum.DECIMAL, 20, 4),
DecimalType(TypeEnum.DECIMAL, 30, 8),
DecimalType(TypeEnum.DECIMAL, 38, 10),
DataType(TypeEnum.TINYINT),
DataType(TypeEnum.INT),
DataType(TypeEnum.BIGINT),
@ -483,7 +642,18 @@ class TDTestCase:
DataType(TypeEnum.INT),
DataType(TypeEnum.VARCHAR, 255)
]
self.stb_columns = self.norm_tb_columns.copy()
self.stb_columns = [
DecimalType(TypeEnum.DECIMAL, 10, 2),
DecimalType(TypeEnum.DECIMAL, 20, 4),
DecimalType(TypeEnum.DECIMAL, 30, 8),
DecimalType(TypeEnum.DECIMAL, 38, 10),
DataType(TypeEnum.TINYINT),
DataType(TypeEnum.INT),
DataType(TypeEnum.BIGINT),
DataType(TypeEnum.DOUBLE),
DataType(TypeEnum.FLOAT),
DataType(TypeEnum.VARCHAR, 255),
]
DecimalColumnTableCreater(tdSql, self.db_name, self.stable_name, self.stb_columns, self.tags).create()
self.check_show_create_table("meters", self.stb_columns, self.tags)
@ -533,7 +703,8 @@ class TDTestCase:
for i in range(self.c_table_num):
TableInserter(tdSql, self.db_name, f"{self.c_table_prefix}{i}", self.stb_columns, self.tags).insert(1000, 1537146000000, 500)
TableInserter(tdSql, self.db_name, self.norm_table_name, self.norm_tb_columns).insert(10000, 1537146000000, 500, flush_database=True)
TableInserter(tdSql, self.db_name, self.norm_table_name, self.norm_tb_columns).insert(10, 1537146000000, 500, flush_database=True)
TableDataValidator(self.norm_tb_columns, self.norm_table_name, self.db_name).validate()
## insert null/None for decimal type
@ -558,7 +729,7 @@ class TDTestCase:
## Create table with no decimal type, the metaentries should not have extschma, and add decimal column, the metaentries should have extschema for all columns.
sql = f'ALTER TABLE {self.db_name}.{self.no_decimal_col_tb_name} ADD COLUMN c200 decimal(37, 19)'
tdSql.execute(sql, queryTimes=1) ## now meta entry has ext schemas
columns.append(DataType(TypeEnum.DECIMAL, type_mod=DataType.get_decimal_type_mod(DecimalType(37, 19))))
columns.append(DecimalType(TypeEnum.DECIMAL, 37, 19))
self.check_desc(self.no_decimal_col_tb_name, columns)
## After drop this only decimal column, the metaentries should not have extschema for all columns.

View File

@ -469,6 +469,9 @@ void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, i
shellFormatTimestamp(buf, sizeof(buf), *(int64_t *)val, precision);
taosFprintfFile(pFile, "%s%s%s", quotationStr, buf, quotationStr);
break;
case TSDB_DATA_TYPE_DECIMAL64:
case TSDB_DATA_TYPE_DECIMAL:
taosFprintfFile(pFile, "%s", val);
default:
break;
}