xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

decimal test

This commit is contained in:
wangjiaming0909 2025-03-09 22:54:51 +08:00
parent f4dbaf2240
commit c64f28087d
29 changed files with 244 additions and 121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/common/tcommon.h
View File

@ -177,7 +177,7 @@ typedef struct SColumnDataAgg {
struct {
uint64_t decimal128Sum[2];
uint64_t decimal128Max[2];
uint64_t decimal128Min[2]; // TODO wjm 1. use deicmal128Sum for decimal64, 2. add overflow flag
uint64_t decimal128Min[2];
uint8_t overflow;
};
};

1
include/common/tmsg.h
View File

@ -588,7 +588,6 @@ STSRow* tGetSubmitBlkNext(SSubmitBlkIter* pIter);
// for debug
int32_t tPrintFixedSchemaSubmitReq(SSubmitReq* pReq, STSchema* pSchema);
// TODO wjm resolve compatibility problem
struct SSchema {
int8_t type;
int8_t flags;

2
include/libs/decimal/decimal.h
View File

@ -72,7 +72,6 @@ typedef struct SDecimalCompareCtx {
STypeMod typeMod;
} SDecimalCompareCtx;
// TODO wjm check if we need to expose these functions in decimal.h
void makeDecimal64(Decimal64* pDec64, int64_t w);
void makeDecimal128(Decimal128* pDec128, int64_t hi, uint64_t low);
@ -100,7 +99,6 @@ bool decimal128AddCheckOverflow(const Decimal128* pLeft, const DecimalType* pRig
DEFINE_TYPE_FROM_DECIMAL_FUNCS(, Decimal64);
DEFINE_TYPE_FROM_DECIMAL_FUNCS(, Decimal128);
// TODO wjm change rightWordNum to DecimalType??
typedef struct SDecimalOps {
void (*negate)(DecimalType* pWord);
void (*abs)(DecimalType* pWord);

2
include/libs/function/function.h
View File

@ -265,7 +265,7 @@ typedef struct SqlFunctionCtx {
bool bInputFinished;
bool hasWindowOrGroup; // denote that the function is used with time window or group
bool needCleanup; // denote that the function need to be cleaned up
int32_t inputType; // TODO wjm rename it
int32_t inputType; // save the fuction input type funcs like finalize
} SqlFunctionCtx;
typedef struct tExprNode {

1
include/libs/qcom/query.h
View File

@ -134,7 +134,6 @@ typedef struct SViewMeta {
int32_t version;
int32_t numOfCols;
SSchema* pSchema;
// TODO wjm view support decimal
} SViewMeta;
typedef struct SDBVgInfo {

2
source/client/src/clientImpl.c
View File

@ -2162,8 +2162,6 @@ static int32_t convertDecimalType(SReqResultInfo* pResultInfo) {
return code;
}
}
// TODO wjm handle NULL???
// TODO wjm use vardatalen???
pResultInfo->pCol[i].pData = pResultInfo->convertBuf[i];
pResultInfo->row[i] = pResultInfo->pCol[i].pData;
}

1
source/client/src/clientMain.c
View File

@ -762,7 +762,6 @@ int taos_print_row_with_size(char *str, uint32_t size, TAOS_ROW row, TAOS_FIELD
break;
case TSDB_DATA_TYPE_DECIMAL64:
case TSDB_DATA_TYPE_DECIMAL: {
// TODO wjm var header len???
uint32_t decimalLen = strlen(row[i]);
uint32_t copyLen = TMIN(size - len - 1, decimalLen);
(void)memcpy(str + len, row[i], copyLen);

1
source/common/src/tdataformat.c
View File

@ -4788,7 +4788,6 @@ void valueSetDatum(SValue *pVal, int8_t type, void *pDatum, uint32_t len) {
pVal->val = *(uint64_t *)pDatum;
break;
default:
// TODO wjm log some thing???
break;
}
}

5
source/common/src/ttypes.c
View File

@ -94,12 +94,11 @@ tDataTypeCompress tDataCompress[TSDB_DATA_TYPE_MAX] = {
{TSDB_DATA_TYPE_JSON, 4, TSDB_MAX_JSON_TAG_LEN, "JSON", 0, 0, tsCompressString2, tsDecompressString2},
{TSDB_DATA_TYPE_VARBINARY, 9, 1, "VARBINARY", 0, 0, tsCompressString2,
tsDecompressString2}, // placeholder, not implemented
{TSDB_DATA_TYPE_DECIMAL, 7, DECIMAL128_BYTES, "DECIMAL", 0, 0, tsCompressDecimal128, tsDecompressDecimal128}, // placeholder, not implemented
{TSDB_DATA_TYPE_DECIMAL, 7, DECIMAL128_BYTES, "DECIMAL", 0, 0, tsCompressDecimal128, tsDecompressDecimal128},
{TSDB_DATA_TYPE_BLOB, 4, 1, "BLOB", 0, 0, NULL, NULL}, // placeholder, not implemented
{TSDB_DATA_TYPE_MEDIUMBLOB, 10, 1, "MEDIUMBLOB", 0, 0, NULL, NULL}, // placeholder, not implemented
{TSDB_DATA_TYPE_GEOMETRY, 8, 1, "GEOMETRY", 0, 0, tsCompressString2, tsDecompressString2},
{TSDB_DATA_TYPE_DECIMAL64, 9, DECIMAL64_BYTES, "DECIMAL64", 0, 0, tsCompressDecimal64, tsDecompressDecimal64}, // placeholder, not implemented
// TODO wjm decimal compress
{TSDB_DATA_TYPE_DECIMAL64, 9, DECIMAL64_BYTES, "DECIMAL64", 0, 0, tsCompressDecimal64, tsDecompressDecimal64},
};

1
source/dnode/mnode/impl/src/mndStb.c
View File

@ -193,7 +193,6 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) {
}
}
// TODO wjm test it, what if some cols are deleted, maybe rewrite it
if (hasTypeMod) {
for (int32_t i = 0; i < pStb->numOfColumns; ++i) {
SDB_SET_INT32(pRaw, dataPos, pStb->pExtSchemas[i].typeMod, _OVER);

1
source/dnode/vnode/src/meta/metaEntry2.c
View File

@ -1080,7 +1080,6 @@ static int32_t metaHandleSuperTableCreateImpl(SMeta *pMeta, const SMetaEntry *pE
const SMetaHandleParam param = {
.pEntry = pEntry,
};
// TODO wjm debug create/alter stable/ctable logic
code = metaTableOpFn[op->table][op->op](pMeta, &param);
if (TSDB_CODE_SUCCESS != code) {
metaErr(TD_VID(pMeta->pVnode), code);

6
source/dnode/vnode/src/meta/metaTable.c
View File

@ -96,13 +96,9 @@ int32_t dropTableExtSchema(SMetaEntry *pEntry, int32_t dropColId, int32_t newCol
memmove(pEntry->pExtSchemas + dropColId, pEntry->pExtSchemas + dropColId + 1,
(newColNum - dropColId) * sizeof(SExtSchema));
}
for (int32_t i = 0; i < newColNum; i++) { // TODO wjm test it..
for (int32_t i = 0; i < newColNum; i++) {
if (hasExtSchema(pEntry->pExtSchemas + i)) return 0;
}
// if no column has ext schemas, free the memory.
// TODO wjm looks like we can remove it
// Actually it's not necessary, if there's no ext schemas, it will not encode extschemas when encoding meta
// entry
taosMemoryFreeClear(pEntry->pExtSchemas);
return 0;
}

2
source/dnode/vnode/src/meta/metaTable2.c
View File

@ -706,8 +706,6 @@ int32_t metaAddTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, ST
SColCmpr *p = &pEntry->colCmpr.pColCmpr[i];
pRsp->pSchemaExt[i].colId = p->id;
pRsp->pSchemaExt[i].compress = p->alg;
// TODO wjm
// if (pEntry->pExtSchemas) pRsp->pSchemaExt[i].typeMod = pEntry->pExtSchemas[i].typeMod;
}
}

23
source/dnode/vnode/src/tsdb/tsdbCache.c
View File

@ -317,8 +317,12 @@ static int32_t tsdbCacheDeserializeV0(char const *value, SLastCol *pLastCol) {
pLastCol->colVal.value.pData = (uint8_t *)(&pLastColV0[1]);
}
return sizeof(SLastColV0) + pLastColV0->colVal.value.nData;
} else if (pLastCol->colVal.value.type == TSDB_DATA_TYPE_DECIMAL) {
pLastCol->colVal.value.nData = pLastColV0->colVal.value.nData;
pLastCol->colVal.value.pData = (uint8_t*)(&pLastColV0[1]);
return sizeof(SLastColV0) + pLastColV0->colVal.value.nData;
} else {
valueCloneDatum(&pLastCol->colVal.value, &pLastColV0->colVal.value, pLastColV0->colVal.value.type);
pLastCol->colVal.value.val = pLastColV0->colVal.value.val;
return sizeof(SLastColV0);
}
}
@ -409,8 +413,12 @@ static int32_t tsdbCacheSerializeV0(char const *value, SLastCol *pLastCol) {
memcpy(&pLastColV0[1], pLastCol->colVal.value.pData, pLastCol->colVal.value.nData);
}
return sizeof(SLastColV0) + pLastCol->colVal.value.nData;
} else if (pLastCol->colVal.value.type == TSDB_DATA_TYPE_DECIMAL) {
memcpy(&pLastColV0[1], pLastCol->colVal.value.pData, pLastCol->colVal.value.nData);
pLastColV0->colVal.value.nData = pLastCol->colVal.value.nData;
return sizeof(SLastColV0) + pLastCol->colVal.value.nData;
} else {
valueCloneDatum(&pLastColV0->colVal.value, &pLastCol->colVal.value, pLastCol->colVal.value.type);
pLastColV0->colVal.value.val = pLastCol->colVal.value.val;
return sizeof(SLastColV0);
}
@ -422,6 +430,9 @@ static int32_t tsdbCacheSerialize(SLastCol *pLastCol, char **value, size_t *size
if (IS_VAR_DATA_TYPE(pLastCol->colVal.value.type)) {
*size += pLastCol->colVal.value.nData;
}
if (pLastCol->colVal.value.type == TSDB_DATA_TYPE_DECIMAL) {
*size += DECIMAL128_BYTES;
}
*size += sizeof(uint8_t) + sizeof(uint8_t) + sizeof(uint8_t); // version + numOfPKs + cacheStatus
for (int8_t i = 0; i < pLastCol->rowKey.numOfPKs; i++) {
@ -822,6 +833,14 @@ static int32_t tsdbCacheReallocSLastCol(SLastCol *pCol, size_t *pCharge) {
charge += pCol->colVal.value.nData;
}
if (pCol->colVal.value.type == TSDB_DATA_TYPE_DECIMAL) {
void* p = taosMemoryMalloc(pCol->colVal.value.nData);
if (!p) TAOS_CHECK_EXIT(terrno);
(void)memcpy(p, pCol->colVal.value.pData, pCol->colVal.value.nData);
pCol->colVal.value.pData = p;
charge += pCol->colVal.value.nData;
}
if (pCharge) {
*pCharge = charge;
}

15
source/dnode/vnode/src/tsdb/tsdbRead2.c
View File

@ -1354,6 +1354,7 @@ static int32_t copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpI
case TSDB_DATA_TYPE_TIMESTAMP:
case TSDB_DATA_TYPE_DOUBLE:
case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_DECIMAL64:
case TSDB_DATA_TYPE_UBIGINT: {
int32_t mid = dumpedRows >> 1u;
int64_t* pts = (int64_t*)pColData->pData;
@ -1402,6 +1403,20 @@ static int32_t copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpI
}
break;
}
case TSDB_DATA_TYPE_DECIMAL: {
int32_t mid = dumpedRows >> 1u;
DecimalWord* pDec = (DecimalWord*)pColData->pData;
DecimalWord tmp[2] = {0};
for (int32_t j = 0; j < mid; ++j) {
tmp[0] = pDec[2 * j];
tmp[1] = pDec[2 * j + 1];
pDec[2 * j] = pDec[2 * (dumpedRows - j - 1)];
pDec[2 * j + 1] = pDec[2 * (dumpedRows - j - 1) + 1];
pDec[2 * (dumpedRows - j - 1)] = tmp[0];
pDec[2 * (dumpedRows - j - 1) + 1] = tmp[1];
}
break;
}
}
}

18
source/libs/decimal/inc/wideInteger.h
View File

@ -22,28 +22,28 @@
extern "C" {
#endif
typedef struct uint128 {
struct uint128 {
uint64_t low;
uint64_t high;
} uint128;
};
struct int128 {
uint64_t low;
int64_t high;
};
typedef struct uint256 {
uint128 low;
uint128 high;
} uint256; // TODO wjm remove typedef
struct uint256 {
struct uint128 low;
struct uint128 high;
};
struct int256 {
uint128 low;
struct uint128 low;
struct int128 high;
};
#define UInt128 uint128
#define UInt256 uint256
#define UInt128 struct uint128
#define UInt256 struct uint256
#define Int128 struct int128
#define Int256 struct int256

44
source/libs/decimal/src/decimal.c
View File

@ -27,7 +27,7 @@ typedef enum DecimalRoundType {
ROUND_TYPE_CEIL,
ROUND_TYPE_FLOOR,
ROUND_TYPE_TRUNC,
ROUND_TYPE_HALF_ROUND_UP, // TODO wjm use this for scaling down/up
ROUND_TYPE_HALF_ROUND_UP,
} DecimalRoundType;
#define DECIMAL_GET_INTERNAL_TYPE(dataType) ((dataType) == TSDB_DATA_TYPE_DECIMAL ? DECIMAL_128 : DECIMAL_64)
@ -37,7 +37,6 @@ static SDecimalOps* getDecimalOpsImp(DecimalInternalType t);
#define DECIMAL_MIN_ADJUSTED_SCALE 6
// TODO wjm use uint64_t ???
static Decimal64 SCALE_MULTIPLIER_64[TSDB_DECIMAL64_MAX_PRECISION + 1] = {1LL,
10LL,
100LL,
@ -300,31 +299,20 @@ int32_t decimal128ToDataVal(Decimal128* dec, SValue* pVal) {
#define DECIMAL64_SIGN(pDec) (1 | (DECIMAL64_GET_VALUE(pDec) >> 63))
static int32_t decimalGetWhole(const DecimalType* pDec, DecimalInternalType type, int8_t scale, DecimalType* pWhole) {
SDecimalOps* pOps = getDecimalOpsImp(type);
if (type == DECIMAL_64) {
DECIMAL64_CLONE(pWhole, pDec);
Decimal64 scaleMul = SCALE_MULTIPLIER_64[scale];
pOps->divide(pWhole, &scaleMul, 1, NULL);
pOps->abs(pWhole);
} else {
memcpy(pWhole, pDec, DECIMAL_GET_WORD_NUM(type) * sizeof(DecimalWord));
// TODO wjm
// pOps.divide(pWhole, )
}
return 0;
static void decimal64GetWhole(const DecimalType* pDec, int8_t scale, DecimalType* pWhole) {
SDecimalOps* pOps = getDecimalOps(TSDB_DATA_TYPE_DECIMAL64);
DECIMAL64_CLONE(pWhole, pDec);
Decimal64 scaleMul = SCALE_MULTIPLIER_64[scale];
pOps->divide(pWhole, &scaleMul, 1, NULL);
pOps->abs(pWhole);
}
static int32_t decimalGetFrac(const DecimalType* pDec, DecimalInternalType type, int8_t scale, DecimalType* pFrac) {
SDecimalOps* pOps = getDecimalOpsImp(type);
if (type == DECIMAL_64) {
DECIMAL64_CLONE(pFrac, pDec);
Decimal64 scaleMul = SCALE_MULTIPLIER_64[scale];
pOps->mod(pFrac, &scaleMul, 1);
pOps->abs(pFrac);
} else {
}
return 0;
static void decimal64GetFrac(const DecimalType* pDec, int8_t scale, DecimalType* pFrac) {
SDecimalOps* pOps = getDecimalOpsImp(DECIMAL_64);
DECIMAL64_CLONE(pFrac, pDec);
Decimal64 scaleMul = SCALE_MULTIPLIER_64[scale];
pOps->mod(pFrac, &scaleMul, 1);
pOps->abs(pFrac);
}
static void decimal64Negate(DecimalType* pInt);
@ -470,10 +458,10 @@ int32_t decimal64ToStr(const DecimalType* pInt, uint8_t scale, char* pBuf, int32
if (DECIMAL64_SIGN((Decimal64*)pInt) == -1) {
pos = sprintf(pBuf, "-");
}
int32_t code = decimalGetWhole(pInt, DECIMAL_64, scale, &whole);
decimal64GetWhole(pInt, scale, &whole);
pos += snprintf(pBuf + pos, bufLen - pos, "%" PRId64, DECIMAL64_GET_VALUE(&whole));
if (scale > 0) {
(void)decimalGetFrac(pInt, DECIMAL_64, scale, &frac);
decimal64GetFrac(pInt, scale, &frac);
if (DECIMAL64_GET_VALUE(&frac) != 0 || DECIMAL64_GET_VALUE(&whole) != 0) {
TAOS_STRCAT(pBuf + pos, ".");
pos += 1;
@ -1928,7 +1916,7 @@ static int32_t decimal128CountRoundingDelta(const Decimal128* pDec, int8_t scale
res = 0;
break;
}
res = decimal128Lt(pDec, &decimal128Zero, WORD_NUM(Decimal128)) ? -1 : 1; // TODO wjm use sign??
res = DECIMAL128_SIGN(pDec) == -1 ? -1 : 1;
} break;
case ROUND_TYPE_TRUNC:
default:

4
source/libs/decimal/src/detail/wideInteger.cpp
View File

@ -257,6 +257,6 @@ Int256 int256RightShift(const Int256* pLeft, int32_t shift) {
return *(Int256*)&result;
}
const Int256 int256One = {.low = uInt128One, .high = int128Zero};
const Int256 int256One = {uInt128One, int128Zero};
const Int256 int256Zero = {uInt128Zero, int128Zero};
const Int256 int256Two = {.low = uInt128Two, .high = int128Zero};
const Int256 int256Two = {uInt128Two, int128Zero};

4
source/libs/decimal/test/decimalTest.cpp
View File

@ -574,13 +574,13 @@ TEST(decimal, typeFromDecimal) {
uintv = dec1;
doublev = dec1;
ASSERT_EQ(intv, -123);
ASSERT_EQ(uintv, 0);
ASSERT_EQ(uintv, 18446744073709551493ULL);
ASSERT_EQ(doublev, -123.44);
intv = dec1 = "-123.99";
uintv = dec1;
doublev = dec1;
ASSERT_EQ(intv, -124);
ASSERT_EQ(uintv, 0);
ASSERT_EQ(uintv, 18446744073709551492ULL);
ASSERT_EQ(doublev, -123.99);
bool boolv = false;

1
source/libs/executor/src/scanoperator.c
View File

@ -5304,7 +5304,6 @@ int32_t createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* p
nodesRewriteExprPostOrder(&pTagCond, tagScanRewriteTagColumn, (void*)&pInfo->filterCtx);
}
}
// TODO wjm check pInfo->filterCtx.code
__optr_fn_t tagScanNextFn = (pTagScanNode->onlyMetaCtbIdx) ? doTagScanFromCtbIdxNext : doTagScanFromMetaEntryNext;
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, tagScanNextFn, NULL, destroyTagScanOperatorInfo,
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);

2
source/libs/executor/src/timesliceoperator.c
View File

@ -502,8 +502,6 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp
}
code = colDataSetVal(pDst, rows, (char*)&v, isNull);
QUERY_CHECK_CODE(code, lino, _end);
} else if (IS_DECIMAL_TYPE(pDst->info.type)) {
// TODO wjm
}
++fillColIndex;

3
source/libs/function/inc/functionResInfoInt.h
View File

@ -53,7 +53,7 @@ typedef struct SDecimalSumRes {
int16_t type;
int64_t prevTs;
bool isPrevTsSet;
bool overflow; // if overflow is true, dsum to be used for any type;
bool overflow;
uint32_t flag; // currently not used
} SDecimalSumRes;
@ -86,7 +86,6 @@ typedef struct SDecimalSumRes {
#define SUM_RES_INC_DSUM(pSumRes, val) ((SSumRes*)(pSumRes))->dsum += val
#define SUM_RES_GET_DECIMAL_SUM(pSumRes) ((SDecimalSumRes*)(pSumRes))->sum
// TODO wjm check for overflow
#define SUM_RES_INC_DECIMAL_SUM(pSumRes, pVal, type) \
do { \
const SDecimalOps* pOps = getDecimalOps(TSDB_DATA_TYPE_DECIMAL); \

2
source/libs/function/src/detail/tavgfunction.c
View File

@ -427,7 +427,6 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) {
case TSDB_DATA_TYPE_DECIMAL:
case TSDB_DATA_TYPE_DECIMAL64: {
const char* pDec = pCol->pData;
// TODO wjm check for overflow
for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) {
bool overflow = false;
if (type == TSDB_DATA_TYPE_DECIMAL64) {
@ -607,7 +606,6 @@ int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
if (AVG_RES_GET_COUNT(pRes, true, pCtx->inputType) > 0) {
if(AVG_RES_GET_SUM_OVERFLOW(pRes, true, pCtx->inputType)) {
// overflow flag set , use dsum TODO wjm check deicmal overflow and return error
AVG_RES_GET_AVG(pRes) = SUM_RES_GET_DSUM(&AVG_RES_GET_SUM(pRes)) / ((double)AVG_RES_GET_COUNT(pRes, false, 0));
}else if (IS_SIGNED_NUMERIC_TYPE(type)) {
AVG_RES_GET_AVG(pRes) = SUM_RES_GET_ISUM(&AVG_RES_GET_SUM(pRes)) / ((double)AVG_RES_GET_COUNT(pRes, false, 0));

5
source/libs/parser/src/parInsertSql.c
View File

@ -1612,7 +1612,7 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql,
break;
}
case TSDB_DATA_TYPE_UTINYINT: {
int32_t code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &VALUE_GET_TRIVIAL_DATUM(&pVal->value));
int32_t code = toUIntegerEx(pToken->z, pToken->n, pToken->type, (uint64_t*)&VALUE_GET_TRIVIAL_DATUM(&pVal->value));
if (TSDB_CODE_SUCCESS != code) {
return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned tinyint data", pToken->z);
} else if (VALUE_GET_TRIVIAL_DATUM(&pVal->value) > UINT8_MAX) {
@ -1819,9 +1819,6 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql,
if (TSDB_CODE_SUCCESS != code) {
return code;
}
// TODO wjm
// precision check
// scale auto fit
code = decimal64ToDataVal(&dec, &pVal->value);
if (TSDB_CODE_SUCCESS != code) {
return code;

11
source/libs/parser/src/parTranslater.c
View File

@ -3437,6 +3437,12 @@ static int32_t selectCommonType(SDataType* commonType, const SDataType* newType)
}
if (commonType->type == newType->type) {
if (IS_DECIMAL_TYPE(commonType->type)) {
if ((commonType->precision - commonType->scale) < (newType->precision - newType->scale)) {
commonType->precision = newType->precision;
commonType->scale = newType->scale;
}
}
commonType->bytes = TMAX(commonType->bytes, newType->bytes);
return TSDB_CODE_SUCCESS;
}
@ -3447,6 +3453,11 @@ static int32_t selectCommonType(SDataType* commonType, const SDataType* newType)
} else if ((resultType == TSDB_DATA_TYPE_NCHAR) &&
(IS_MATHABLE_TYPE(commonType->type) || IS_MATHABLE_TYPE(newType->type))) {
commonType->bytes = TMAX(TMAX(commonType->bytes, newType->bytes), QUERY_NUMBER_MAX_DISPLAY_LEN * TSDB_NCHAR_SIZE);
} else if (IS_DECIMAL_TYPE(resultType)) {
if ((commonType->precision - commonType->scale) < (newType->precision - newType->scale)) {
commonType->precision = newType->precision;
commonType->scale = newType->scale;
}
} else {
commonType->bytes = TMAX(TMAX(commonType->bytes, newType->bytes), TYPE_BYTES[resultType]);
}

43
source/libs/scalar/src/filter.c
View File

@ -3935,6 +3935,7 @@ typedef struct {
uint64_t u; // for uint
double d; // for double
uint8_t *pData; // for varchar, nchar, len prefixed
Decimal dec; // for decimal
};
SDataType type; // TODO: original data type, may not be used?
} SFltSclDatum;
@ -3960,7 +3961,7 @@ int32_t fltSclCompareWithFloat64(SFltSclDatum *val1, SFltSclDatum *val2) {
return compareDoubleVal(&d, &val2->d);
}
case FLT_SCL_DATUM_KIND_DECIMAL: {
double d = doubleFromDecimal128(val1->pData, val1->type.precision, val1->type.scale);
double d = doubleFromDecimal128(&val1->dec, val1->type.precision, val1->type.scale);
return compareDoubleVal(&d, &val2->d);
}
// TODO: varchar, nchar
@ -4034,12 +4035,12 @@ int32_t fltSclCompareDatum(SFltSclDatum *val1, SFltSclDatum *val2) {
return fltSclCompareWithFloat64(val1, val2);
}
case FLT_SCL_DATUM_KIND_DECIMAL64: {
void* pData1 = val1->kind == FLT_SCL_DATUM_KIND_DECIMAL64 ? (void*)&val1->i : (void*)val1->pData;
void* pData1 = val1->kind == FLT_SCL_DATUM_KIND_DECIMAL64 ? (void*)&val1->i : (void*)&val1->dec;
return fltSclCompareWithDecimal(pData1, &val1->type, &val2->i, &val2->type);
}
case FLT_SCL_DATUM_KIND_DECIMAL: {
void* pData1 = val1->kind == FLT_SCL_DATUM_KIND_DECIMAL64 ? (void*)&val1->i : (void*)val1->pData;
return fltSclCompareWithDecimal(pData1, &val1->type, val2->pData, &val2->type);
void* pData1 = val1->kind == FLT_SCL_DATUM_KIND_DECIMAL64 ? (void*)&val1->i : (void*)&val1->dec;
return fltSclCompareWithDecimal(pData1, &val1->type, &val2->dec, &val2->type);
}
default:
qError("not supported kind when compare datum. kind2 : %d", val2->kind);
@ -4235,7 +4236,7 @@ static int32_t fltSclBuildDecimalDatumFromValueNode(SFltSclDatum* datum, SColumn
case TSDB_DATA_TYPE_DECIMAL:
datum->kind = FLT_SCL_DATUM_KIND_DECIMAL;
datum->type = valDt;
datum->pData = (void*)valNode->datum.p;
datum->dec = *(Decimal*)valNode->datum.p;
FLT_RET(0);
default:
qError("not supported type %d when build decimal datum from value node", valNode->node.resType.type);
@ -4247,15 +4248,12 @@ static int32_t fltSclBuildDecimalDatumFromValueNode(SFltSclDatum* datum, SColumn
pData = &datum->i;
datum->kind = FLT_SCL_DATUM_KIND_DECIMAL64;
} else if (datum->type.type == TSDB_DATA_TYPE_DECIMAL) {
pData = taosMemoryCalloc(1, pColNode->node.resType.bytes);
if (!pData) FLT_ERR_RET(terrno);
datum->pData = pData;
pData = &datum->dec;
datum->kind = FLT_SCL_DATUM_KIND_DECIMAL;
}
if (datum->kind == FLT_SCL_DATUM_KIND_DECIMAL64 || datum->kind == FLT_SCL_DATUM_KIND_DECIMAL) {
int32_t code = convertToDecimal(pInput, &valDt, pData, &datum->type);
if (TSDB_CODE_SUCCESS != code) return code; // TODO wjm handle overflow error
//valNode->node.resType = datum->type;
if (TSDB_CODE_SUCCESS != code) return code;
}
}
FLT_RET(0);
@ -4345,8 +4343,7 @@ int32_t fltSclBuildDatumFromBlockSmaValue(SFltSclDatum *datum, uint8_t type, voi
break;
case TSDB_DATA_TYPE_DECIMAL:
datum->kind = FLT_SCL_DATUM_KIND_DECIMAL;
datum->pData = taosMemoryCalloc(1, tDataTypes[type].bytes);
memcpy(datum->pData, val, tDataTypes[type].bytes);
datum->dec = *(Decimal *)val;
break;
// TODO:varchar/nchar/json
@ -4850,6 +4847,10 @@ EDealRes fltReviseRewriter(SNode **pNode, void *pContext) {
stat->scalarMode = true;
return DEAL_RES_CONTINUE;
}
if (IS_DECIMAL_TYPE(valueNode->node.resType.type)) {
stat->scalarMode = true;
return DEAL_RES_CONTINUE;
}
return DEAL_RES_CONTINUE;
}
@ -4873,7 +4874,8 @@ EDealRes fltReviseRewriter(SNode **pNode, void *pContext) {
uint8_t type = valueNode->node.resType.type;
SNode *node = NULL;
FOREACH(node, listNode->pNodeList) {
if (type != ((SValueNode *)node)->node.resType.type) {
uint8_t nodeT = ((SExprNode*)node)->resType.type;
if (type != nodeT || IS_DECIMAL_TYPE(nodeT)) {
stat->scalarMode = true;
return DEAL_RES_CONTINUE;
}
@ -5037,7 +5039,6 @@ _return:
FLT_RET(code);
}
// TODO wjm start from here, check why 这里将double赋值给整数?????
static int32_t fltSclBuildRangePointsForInOper(SFltSclOperator* oper, SArray* points) {
SNodeListNode *listNode = (SNodeListNode *)oper->valNode;
SFltSclDatum minDatum = {.kind = FLT_SCL_DATUM_KIND_INT64, .i = INT64_MAX, .type = oper->colNode->node.resType};
@ -5054,7 +5055,7 @@ static int32_t fltSclBuildRangePointsForInOper(SFltSclOperator* oper, SArray* po
if (IS_DECIMAL_TYPE(valDatum.type.type)) {
double v = valDatum.type.type == TSDB_DATA_TYPE_DECIMAL64
? doubleFromDecimal64(&valDatum.i, valDatum.type.precision, valDatum.type.scale)
: doubleFromDecimal128(valDatum.pData, valDatum.type.precision, valDatum.type.scale);
: doubleFromDecimal128(&valDatum.dec, valDatum.type.precision, valDatum.type.scale);
if (minDatum.kind == FLT_SCL_DATUM_KIND_FLOAT64) {
minDatum.d = TMIN(v, minDatum.d);
maxDatum.d = TMAX(v, maxDatum.d);
@ -5369,16 +5370,24 @@ int32_t fltOptimizeNodes(SFilterInfo *pInfo, SNode **pNode, SFltTreeStat *pStat)
}
FLT_ERR_JRET(fltSclProcessCNF(pInfo, sclOpList, colRangeList));
pInfo->sclCtx.fltSclRange = colRangeList;
colRangeList = NULL;
_return:
for (int32_t i = 0; i < taosArrayGetSize(sclOpList); ++i) {
SFltSclOperator *sclOp = taosArrayGet(sclOpList, i);
if (NULL == sclOp) {
FLT_ERR_JRET(TSDB_CODE_OUT_OF_RANGE);
code = TSDB_CODE_OUT_OF_RANGE;
break;
}
nodesDestroyNode((SNode *)sclOp->colNode);
nodesDestroyNode((SNode *)sclOp->valNode);
}
_return:
for (int32_t i = 0; i < taosArrayGetSize(colRangeList); ++i) {
SFltSclColumnRange *colRange = taosArrayGet(colRangeList, i);
nodesDestroyNode((SNode *)colRange->colNode);
taosArrayDestroy(colRange->points);
}
taosArrayDestroy(sclOpList);
return code;
}

4
source/libs/scalar/src/scalar.c
View File

@ -168,10 +168,6 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type, SType
if (overflow) {
continue;
}
// TODO wjm For decimal types, after conversion, check if we lose some scale to ignore values with larger scale
// e.g. convert decimal(18, 4) to decimal(18, 2) with value:
// 1.2345 -> 1.23. 1.23 != 1.2345, ignore this value, can't be the same as any decimal(18, 2)
// 1.2300 -> 1.23. 1.2300 == 1.23, take this value.
if (IS_VAR_DATA_TYPE(type)) {
buf = colDataGetVarData(out.columnData, 0);

10
source/libs/scalar/src/sclvector.c
View File

@ -254,7 +254,7 @@ static FORCE_INLINE int32_t varToDecimal(char* buf, SScalarParam* pOut, int32_t
Decimal *pDec = (Decimal *)colDataGetData(pOut->columnData, rowIndex);
int32_t code = decimalFromStr(buf, strlen(buf), pOut->columnData->info.precision, pOut->columnData->info.scale, pDec);
if (TSDB_CODE_SUCCESS != code) {
// TODO wjm set overflow???
if (overflow) *overflow = code == TSDB_CODE_DECIMAL_OVERFLOW;
SCL_RET(code);
}
SCL_RET(code);
@ -1020,7 +1020,7 @@ int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut,
Decimal value = {0};
SDataType inputType = GET_COL_DATA_TYPE(pInputCol->info), outputType = GET_COL_DATA_TYPE(pOutputCol->info);
int32_t code = convertToDecimal(colDataGetData(pInputCol, i), &inputType, &value, &outputType);
if (TSDB_CODE_SUCCESS != code) return code; // TODO wjm handle overflow
if (TSDB_CODE_SUCCESS != code) return code;
code = colDataSetVal(pOutputCol, i, (const char*)&value, false);
if (TSDB_CODE_SUCCESS != code) return code;
}
@ -1147,10 +1147,6 @@ int32_t vectorConvertCols(SScalarParam *pLeft, SScalarParam *pRight, SScalarPara
int32_t leftType = GET_PARAM_TYPE(pLeft);
int32_t rightType = GET_PARAM_TYPE(pRight);
if (leftType == rightType) {
if (IS_DECIMAL_TYPE(leftType)) {
//TODO wjm force do conversion for decimal type, do not convert any more, do conversion inside decimal.c
//TODO wjm where c1 = "999999999999999.99999"; this str will be converted to double and do comapre, add doc in TS
}
return TSDB_CODE_SUCCESS;
}
@ -1944,7 +1940,7 @@ int32_t doVectorCompare(SScalarParam *pLeft, SScalarParam *pLeftVar, SScalarPara
fp = filterGetCompFuncEx(lType, rType, optr);
}
if (pLeftVar != NULL) {// TODO wjm test when pLeftVar is not NULL
if (pLeftVar != NULL) {
SCL_ERR_RET(filterGetCompFunc(&fpVar, GET_PARAM_TYPE(pLeftVar), optr));
}
if (startIndex < 0) {

149
tests/system-test/2-query/decimal.py
View File

@ -41,13 +41,14 @@ invalid_operation = -2147483136
scalar_convert_err = -2147470768
decimal_insert_validator_test = False
decimal_test_query = True
decimal_insert_validator_test = True
operator_test_round = 1
tb_insert_rows = 1000
binary_op_with_const_test = False
binary_op_with_col_test = False
unary_op_test = False
binary_op_in_where_test = False
binary_op_with_const_test = True
binary_op_with_col_test = True
unary_op_test = True
binary_op_in_where_test = True
test_decimal_funcs = True
cast_func_test_round = 10
@ -156,6 +157,8 @@ class DecimalColumnAggregator:
self.sum: Decimal = Decimal("0")
self.null_num: int = 0
self.none_num: int = 0
self.first = None
self.last = None
def add_value(self, value: str, scale: int):
self.count += 1
@ -165,6 +168,9 @@ class DecimalColumnAggregator:
self.none_num += 1
else:
v: Decimal = get_decimal(value, scale)
if self.first is None:
self.first = v
self.last = v
self.sum += v
if v > self.max:
self.max = v
@ -520,6 +526,7 @@ class DecimalType(DataType):
super().__init__(type, bytes, self.get_decimal_type_mod())
self.decimal_generator: DecimalStringRandomGenerator = DecimalStringRandomGenerator()
self.generator_config: DecimalTypeGeneratorConfig = DecimalTypeGeneratorConfig()
self.generator_config.with_corner_case = False
self.generator_config.prec = precision
self.generator_config.scale = scale
self.aggregator: DecimalColumnAggregator = DecimalColumnAggregator()
@ -625,13 +632,13 @@ class Column:
def get_typed_val(self, val):
return self.type_.get_typed_val(val)
def get_typed_val_for_execute(self, val, const_col = False):
return self.type_.get_typed_val_for_execute(val, const_col)
def get_constant_val(self):
return self.get_typed_val(self.saved_vals[''][0])
def get_constant_val_for_execute(self):
return self.get_typed_val_for_execute(self.saved_vals[''][0], const_col=True)
@ -651,7 +658,7 @@ class Column:
else:
idx -= l
return self.get_typed_val_for_execute(self.saved_vals[tbname][idx])
def get_cardinality(self, tbname):
if self.is_constant_col():
return 1
@ -660,6 +667,23 @@ class Column:
else:
return len(self.saved_vals[tbname])
def get_ordered_result(self, tbname: str, asc: bool) -> list:
if tbname in self.saved_vals:
return sorted(
[
get_decimal(val, self.type_.scale())
for val in self.saved_vals[tbname]
],
reverse=not asc,
)
else:
res = []
for val in self.saved_vals.values():
res.extend(val)
return sorted(
[get_decimal(val, self.type_.scale()) for val in res], reverse=not asc
)
## tbName: for normal table, pass the tbname, for child table, pass the child table name
def generate_value(self, tbName: str = '', save: bool = True):
val = self.type_.generate_value()
@ -718,7 +742,7 @@ class Column:
+ Column.get_decimal_types()
+ types_unable_to_be_const
)
@staticmethod
def get_decimal_types() -> List:
return [TypeEnum.DECIMAL, TypeEnum.DECIMAL64]
@ -930,6 +954,9 @@ class DecimalFunction(DecimalColumnExpr):
DecimalSumFunction(),
DecimalAvgFunction(),
DecimalCountFunction(),
#DecimalLastRowFunction(),
#DecimalLastFunction(),
#DecimalFirstFunction(),
]
def check_results(self, query_col_res: List) -> bool:
@ -1066,12 +1093,14 @@ class DecimalAggFunction(DecimalFunction):
class DecimalLastRowFunction(DecimalAggFunction):
def __init__(self):
super().__init__("last_row({0})", DecimalLastRowFunction.execute_last_row, "last_row")
self.res_ = None
def get_func_res(self):
return 1
decimal_type:DecimalType = self.query_col.type_
return decimal_type.aggregator.last
def generate_res_type(self):
self.res_type_ = self.query_col.type_
def execute_last_row(self, params):
return 1
self.res_ = Decimal(params[0])
class DecimalCacheLastRowFunction(DecimalAggFunction):
def __init__(self):
@ -1087,10 +1116,28 @@ class DecimalCacheLastFunction(DecimalAggFunction):
pass
class DecimalFirstFunction(DecimalAggFunction):
pass
def __init__(self):
super().__init__("first({0})", DecimalFirstFunction.execute_first, "first")
self.res_ = None
def get_func_res(self):
decimal_type: DecimalType = self.query_col.type_
return decimal_type.aggregator.first
def generate_res_type(self):
self.res_type_ = self.query_col.type_
def execute_first(self, params):
pass
class DecimalLastFunction(DecimalAggFunction):
pass
def __init__(self):
super().__init__("last({0})", DecimalLastFunction.execute_last, "last")
self.res_ = None
def get_func_res(self):
decimal_type:DecimalType = self.query_col.type_
return decimal_type.aggregator.last
def generate_res_type(self):
self.res_type_ = self.query_col.type_
def execute_last(self, params):
pass
class DecimalHyperloglogFunction(DecimalAggFunction):
pass
@ -1542,7 +1589,7 @@ class TDTestCase:
if results[i + 1][1] != col.type_.__str__():
tdLog.info(str(results))
tdLog.exit(
f"check desc failed for table: {tbname} column {results[i+1][0]} type is {results[i+1][1]}, expect DECIMAL"
f"check desc failed for table: {tbname} column {results[i+1][0]} type is {results[i+1][1]}, expect {col.type_}"
)
if results[i + 1][4] != DecimalType.default_encode():
tdLog.exit(
@ -1795,7 +1842,7 @@ class TDTestCase:
self.test_add_drop_columns_with_decimal(self.no_decimal_col_tb_name, columns)
def test_decimal_ddl(self):
tdSql.execute("create database test", queryTimes=1)
tdSql.execute("create database test cachemodel 'both'", queryTimes=1)
self.test_decimal_column_ddl()
## TODO test decimal column for tmq
@ -1814,6 +1861,20 @@ class TDTestCase:
[(9 * self.c_table_num,)],
30,
)
def test_decimal_and_view(self):
c1 = self.norm_tb_columns[0]
create_view_sql = f'create view {self.db_name}.view1 as select {c1} as c1, cast({c1} as decimal(38, 10)) as c2 from {self.db_name}.{self.norm_table_name}'
tdSql.execute(create_view_sql)
res = TaosShell().query(f'select c1 from {self.db_name}.view1')
if len(res[0]) != c1.get_cardinality(self.norm_table_name):
tdLog.exit(f"query from view1 got rows: {len(res)} expect: {c1.get_cardinality(self.norm_table_name)}")
for i in range(len(res[0])):
v_query = res[0][i]
v_insert = c1.get_val_for_execute(self.norm_table_name, i)
if Decimal(v_query) != v_insert:
tdLog.exit(f"query from view got different results: {v_query}, expect: {v_insert}")
#self.check_desc("view1", [c1, Column(DecimalType(TypeEnum.DECIMAL, 38, 10))])
def run(self):
self.test_decimal_ddl()
@ -1822,6 +1883,7 @@ class TDTestCase:
self.test_query_decimal()
self.test_decimal_and_stream()
self.test_decimal_and_tsma()
self.test_decimal_and_view()
def stop(self):
tdSql.close()
@ -1850,6 +1912,8 @@ class TDTestCase:
if col.name_ == '':
continue
for col2 in tb_cols:
if col2.name_ =='':
continue
if expr.should_skip_for_decimal([col, col2]):
continue
select_expr = expr.generate((col, col2))
@ -2124,9 +2188,24 @@ class TDTestCase:
## 4. (dec op dec) op const col
## 5. (dec op const col) op dec
## 6. (dec op dec) op dec
def test_query_with_order_by_for_tb(self, tbname: str, cols: list[Column]):
for col in cols:
if col.type_.is_decimal_type() and col.name_ != '':
self.test_query_with_order_by(col, tbname)
def test_query_with_order_by(self, order_col: Column, tbname):
sql = f"select {order_col} from {self.db_name}.{tbname} order by {order_col} asc"
query_res = TaosShell().query(sql)[0]
calculated_ordered_res = order_col.get_ordered_result(tbname, True)
for v_from_query, v_from_calc in zip(query_res, calculated_ordered_res):
if Decimal(v_from_query) != v_from_calc:
tdLog.exit(f"query result: {v_from_query} not equal to calculated result: {v_from_calc}")
def test_query_decimal_order_clause(self):
pass
self.test_query_with_order_by_for_tb(self.norm_table_name, self.norm_tb_columns)
self.test_query_with_order_by_for_tb(self.stable_name, self.stb_columns)
def test_query_decimal_group_by_clause(self):
pass
@ -2141,7 +2220,39 @@ class TDTestCase:
pass
def test_query_decimal_case_when(self):
pass
sql = "select case when cast(1 as decimal(10, 4)) >= 1 then cast(88888888.88 as decimal(10,2)) else cast(3.333 as decimal(10,3)) end"
res = TaosShell().query(sql)[0]
if res[0] != "88888888.88":
tdLog.exit(f"query result for sql: {sql}: {res[0]} not equal to expected result: 88888888.88")
sql = "select case when cast(1 as decimal(10, 4)) > 1 then cast(88888888.88 as decimal(10,2)) else cast(3.333 as decimal(10,3)) end"
res = TaosShell().query(sql)[0]
if res[0] != "3.33":
tdLog.exit(f"query result for sql: {sql}: {res[0]} not equal to expected result: 3.33")
sql = "select case when cast(1 as decimal(10, 4)) > 1 then cast(88888888.88 as decimal(10,2)) else 1.23 end"
res = TaosShell().query(sql)[0]
if float(res[0]) != float(1.23):
tdLog.exit(f"query result for sql: {sql}: {res[0]} not equal to expected result: 1.23")
sql = "select case when cast(1 as decimal(10, 4)) >= 1 then cast(88888888.88 as decimal(10,2)) else 1.23 end"
res = TaosShell().query(sql)[0]
if float(res[0]) != float(88888888.88):
tdLog.exit(f"query result for sql: {sql}: {res[0]} not equal to expected result: 88888888.88")
sql = "select case when cast(1 as decimal(10, 4)) >= 1 then cast(88888888.88 as decimal(10,2)) else '1.23' end"
res = TaosShell().query(sql)[0]
if float(res[0]) != 88888888.88:
tdLog.exit(f"query result for sql: {sql}: {res[0]} not equal to expected result: 88888888.88")
sql = "select case when cast(1 as decimal(10, 4)) > 1 then cast(88888888.88 as decimal(10,2)) else '1.23' end"
res = TaosShell().query(sql)[0]
if float(res[0]) != 1.23:
tdLog.exit(f"query result for sql: {sql}: {res[0]} not equal to expected result: 88888888.88")
sql = "select case when cast(1 as decimal(10, 4)) > 1 then cast(88888888.88 as decimal(10,2)) else 'abcd' end"
res = TaosShell().query(sql)[0]
if float(res[0]) != 0:
tdLog.exit(f"query result for sql: {sql}: {res[0]} not equal to expected result: 0")
def test_decimal_agg_funcs(self, dbname, tbname, tb_cols: List[Column], get_agg_funcs_func):
agg_funcs: List[DecimalFunction] = get_agg_funcs_func()
@ -2186,9 +2297,13 @@ class TDTestCase:
self.test_decimal_cast_func(self.db_name, self.norm_table_name, self.norm_tb_columns)
def test_query_decimal(self):
if not decimal_test_query:
return
self.test_decimal_operators()
self.test_decimal_functions()
self.test_query_decimal_with_sma()
self.test_query_decimal_order_clause()
self.test_query_decimal_case_when()
event = threading.Event()