From 310f81f5e6aa16b752831d750d992e0952a8290a Mon Sep 17 00:00:00 2001 From: wangjiaming0909 Date: Tue, 31 Dec 2024 18:45:29 +0800 Subject: [PATCH] scalar decimal --- include/libs/decimal/decimal.h | 14 ++- source/libs/decimal/src/decimal.c | 153 ++++++++++++++++++++++---- source/libs/parser/src/parInsertSql.c | 4 +- source/libs/scalar/CMakeLists.txt | 1 + source/libs/scalar/src/scalar.c | 7 +- source/libs/scalar/src/sclvector.c | 49 ++++++++- 6 files changed, 200 insertions(+), 28 deletions(-) diff --git a/include/libs/decimal/decimal.h b/include/libs/decimal/decimal.h index 5a8561bdff..df175cfb40 100644 --- a/include/libs/decimal/decimal.h +++ b/include/libs/decimal/decimal.h @@ -33,6 +33,10 @@ typedef struct Decimal128 { DecimalWord words[2]; } Decimal128; +#define Decimal Decimal128 +#define decimalFromStr decimal128FromStr +#define makeDecimal makeDecimal128 + void makeDecimal64(Decimal64* pDec64, DecimalWord w); void makeDecimal128(Decimal128* pDec128, int64_t hi, uint64_t low); @@ -41,14 +45,20 @@ void makeDecimal128(Decimal128* pDec128, int64_t hi, uint64_t low); int32_t decimalCalcTypeMod(const SDataType* pType); void decimalFromTypeMod(STypeMod typeMod, uint8_t* precision, uint8_t* scale); -int32_t decimal64FromStr(const char* str, int32_t len, uint8_t* precision, uint8_t* scale, Decimal64* result); -int32_t decimal128FromStr(const char* str, int32_t len, uint8_t* precision, uint8_t* scale, Decimal128* result); +int32_t decimal64FromStr(const char* str, int32_t len, uint8_t expectPrecision, uint8_t expectScale, Decimal64* result); +int32_t decimal128FromStr(const char* str, int32_t len, uint8_t expectPrecision, uint8_t expectScale, + Decimal128* result); int32_t decimal64ToDataVal(const Decimal64* dec, SValue* pVal); int32_t decimal128ToDataVal(Decimal128* dec, SValue* pVal); int32_t decimalToStr(const DecimalWord* pDec, int8_t type, int8_t precision, int8_t scale, char* pBuf, int32_t bufLen); +int32_t decimalGetRetType(const SDataType* pLeftT, const SDataType* pRightT, EOperatorType opType, SDataType* pOutType); +int32_t decimalOp(EOperatorType op, const SDataType* pLeftT, const SDataType* pRightT, const SDataType* pOutT, + const void* pLeftData, const void* pRightData, void* pOutputData); +int32_t convertToDecimal(const void* pData, const SDataType* pInputType, void* pOut, const SDataType* poutType); + typedef struct SDecimalOps { void (*negate)(DecimalWord* pWord); void (*abs)(DecimalWord* pWord); diff --git a/source/libs/decimal/src/decimal.c b/source/libs/decimal/src/decimal.c index 956eea1de0..8e3293bec8 100644 --- a/source/libs/decimal/src/decimal.c +++ b/source/libs/decimal/src/decimal.c @@ -26,9 +26,10 @@ typedef enum DecimalInternalType { #define DECIMAL_GET_INTERNAL_TYPE(dataType) ((dataType) == TSDB_DATA_TYPE_DECIMAL ? DECIMAL_128 : DECIMAL_64) #define DECIMAL_GET_WORD_NUM(decimalInternalType) \ ((decimalInternalType) == DECIMAL_64 ? DECIMAL_WORD_NUM(Decimal64) : DECIMAL_WORD_NUM(Decimal128)) -#define DecimalMax Decimal128 static SDecimalOps* getDecimalOpsImp(DecimalInternalType t); +#define DECIMAL_MIN_ADJUSTED_SCALE 6 + typedef struct DecimalVar { DecimalInternalType type; uint8_t precision; @@ -49,6 +50,51 @@ static uint8_t maxPrecision(DecimalInternalType type) { } } +int32_t decimalGetRetType(const SDataType* pLeftT, const SDataType* pRightT, EOperatorType opType, + SDataType* pOutType) { + if (IS_FLOAT_TYPE(pLeftT->type) || IS_FLOAT_TYPE(pRightT->type)) { + pOutType->type = TSDB_DATA_TYPE_DOUBLE; + pOutType->bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; + return 0; + } + + // TODO wjm check not supported types + uint8_t p1 = pLeftT->precision, s1 = pLeftT->scale, p2 = pRightT->precision, s2 = pRightT->scale; + + if (IS_DECIMAL_TYPE(pLeftT->type)) { + p2 = TSDB_DECIMAL128_MAX_PRECISION; + s1 = s2; // TODO wjm take which scale? Maybe use default DecimalMax + } else { + p1 = TSDB_DECIMAL128_MAX_PRECISION; + s2 = s1; + } + + switch (opType) { + case OP_TYPE_ADD: + case OP_TYPE_SUB: + pOutType->scale = TMAX(s1, s2); + pOutType->precision = TMAX(p1 - s1, p2 - s2) + pOutType->scale + 1; + break; + case OP_TYPE_MULTI: + pOutType->scale = s1 + s2; + pOutType->precision = p1 + p2 + 1; + break; + case OP_TYPE_DIV: + pOutType->scale = TMAX(s1 + p2 + 1, DECIMAL_MIN_ADJUSTED_SCALE); + pOutType->precision = p1 - s1 + s2 + pOutType->scale; + break; + case OP_TYPE_REM: + pOutType->scale = TMAX(s1, s2); + pOutType->precision = TMIN(p1 - s1, p2 - s2) + pOutType->scale; + break; + default: + return TSDB_CODE_TSC_INVALID_OPERATION; + } + pOutType->type = TSDB_DATA_TYPE_DECIMAL; + pOutType->bytes = tDataTypes[TSDB_DATA_TYPE_DECIMAL].bytes; + return 0; +} + static int32_t decimalVarFromStr(const char* str, int32_t len, DecimalVar* result); int32_t decimalCalcTypeMod(const SDataType* pType) { @@ -63,20 +109,24 @@ void decimalFromTypeMod(STypeMod typeMod, uint8_t* precision, uint8_t* scale) { *scale = (uint8_t)(typeMod & 0xFF); } -int32_t decimal64FromStr(const char* str, int32_t len, uint8_t* precision, uint8_t* scale, Decimal64* result) { +int32_t decimal64FromStr(const char* str, int32_t len, uint8_t expectPrecision, uint8_t expectScale, + Decimal64* result) { int32_t code = 0; DecimalVar var = {.type = DECIMAL_64, .words = result->words}; code = decimalVarFromStr(str, len, &var); - *precision = var.precision; - *scale = var.scale; + if (TSDB_CODE_SUCCESS != code) return code; + + // TODO wjm precision check + // scale auto fit return code; } -int32_t decimal128FromStr(const char* str, int32_t len, uint8_t* precision, uint8_t* scale, Decimal128* result) { +int32_t decimal128FromStr(const char* str, int32_t len, uint8_t expectPrecision, uint8_t expectScale, + Decimal128* result) { int32_t code = 0; DecimalVar var = {.type = DECIMAL_128, .words = result->words}; - *precision = var.precision; - *scale = var.scale; + code = decimalVarFromStr(str, len, &var); + if (TSDB_CODE_SUCCESS != code) return code; return code; } @@ -224,7 +274,8 @@ static void decimal64Abs(DecimalWord* pInt); static void decimal64Add(DecimalWord* pLeft, const DecimalWord* pRight, uint8_t rightWordNum); static void decimal64Subtract(DecimalWord* pLeft, const DecimalWord* pRight, uint8_t rightWordNum); static void decimal64Multiply(DecimalWord* pLeft, const DecimalWord* pRight, uint8_t rightWordNum); -static void decimal64divide(DecimalWord* pLeft, const DecimalWord* pRight, uint8_t rightWordNum, DecimalWord* pRemainder); +static void decimal64divide(DecimalWord* pLeft, const DecimalWord* pRight, uint8_t rightWordNum, + DecimalWord* pRemainder); static void decimal64Mod(DecimalWord* pLeft, const DecimalWord* pRight, uint8_t rightWordNum); static bool decimal64Lt(const DecimalWord* pLeft, const DecimalWord* pRight, uint8_t rightWordNum); static bool decimal64Gt(const DecimalWord* pLeft, const DecimalWord* pRight, uint8_t rightWordNum); @@ -236,7 +287,8 @@ static void decimal128Abs(DecimalWord* pWord); static void decimal128Add(DecimalWord* pLeft, const DecimalWord* pRight, uint8_t rightWordNum); static void decimal128Subtract(DecimalWord* pLeft, const DecimalWord* pRight, uint8_t rightWordNum); static void decimal128Multiply(DecimalWord* pLeft, const DecimalWord* pRight, uint8_t rightWordNum); -static void decimal128divide(DecimalWord* pLeft, const DecimalWord* pRight, uint8_t rightWordNum, DecimalWord* pRemainder); +static void decimal128divide(DecimalWord* pLeft, const DecimalWord* pRight, uint8_t rightWordNum, + DecimalWord* pRemainder); static void decimal128Mod(DecimalWord* pLeft, const DecimalWord* pRight, uint8_t rightWordNum); static bool decimal128Lt(const DecimalWord* pLeft, const DecimalWord* pRight, uint8_t rightWordNum); static bool decimal128Gt(const DecimalWord* pLeft, const DecimalWord* pRight, uint8_t rightWordNum); @@ -380,13 +432,14 @@ static bool decimal128Lt(const DecimalWord* pLeft, const DecimalWord* pRight, ui DECIMAL128_LOW_WORDS(pLeftDec) < DECIMAL128_LOW_WORDS(pRightDec)); } -static void decimal128divide(DecimalWord* pLeft, const DecimalWord* pRight, uint8_t rightWordNum, DecimalWord* pRemainder) { - Decimal128* pLeftDec = (Decimal128*)pLeft, *pRightDec = (Decimal128*)pRight, *pRemainderDec = (Decimal128*)pRemainder; - Decimal128 right = {0}; +static void decimal128divide(DecimalWord* pLeft, const DecimalWord* pRight, uint8_t rightWordNum, + DecimalWord* pRemainder) { + Decimal128 *pLeftDec = (Decimal128*)pLeft, *pRightDec = (Decimal128*)pRight, *pRemainderDec = (Decimal128*)pRemainder; + Decimal128 right = {0}; DECIMAL128_CHECK_RIGHT_WORD_NUM(rightWordNum, pRightDec, right, pRight); - bool negate = DECIMAL128_SIGN(pLeftDec) != DECIMAL128_SIGN(pRightDec); - UInt128 a = {0}, b = {0}, c = {0}, d = {0}; + bool negate = DECIMAL128_SIGN(pLeftDec) != DECIMAL128_SIGN(pRightDec); + UInt128 a = {0}, b = {0}, c = {0}, d = {0}; Decimal128 x = *pLeftDec, y = *pRightDec; decimal128Abs(x.words); decimal128Abs(y.words); @@ -402,9 +455,7 @@ static void decimal128divide(DecimalWord* pLeft, const DecimalWord* pRight, uint if (DECIMAL128_SIGN(pLeftDec) == -1) decimal128Negate(pRemainderDec->words); } -static void decimal128Mod(DecimalWord* pLeft, const DecimalWord* pRight, uint8_t rightWordNum) { - -} +static void decimal128Mod(DecimalWord* pLeft, const DecimalWord* pRight, uint8_t rightWordNum) {} static bool decimal128Gt(const DecimalWord* pLeft, const DecimalWord* pRight, uint8_t rightWordNum) { Decimal128 *pLeftDec = (Decimal128*)pLeft, *pRightDec = (Decimal128*)pRight; @@ -476,14 +527,15 @@ static int32_t decimal128ToStr(const DecimalWord* pInt, uint8_t scale, char* pBu return len; } -int32_t decimalToStr(const DecimalWord* pDec, int8_t dataType, int8_t precision, int8_t scale, char* pBuf, int32_t bufLen) { +int32_t decimalToStr(const DecimalWord* pDec, int8_t dataType, int8_t precision, int8_t scale, char* pBuf, + int32_t bufLen) { pBuf[0] = '\0'; DecimalInternalType iType = DECIMAL_GET_INTERNAL_TYPE(dataType); switch (iType) { case DECIMAL_64: { int32_t wordNum = DECIMAL_GET_WORD_NUM(iType); SDecimalOps* pOps = getDecimalOpsImp(iType); - DecimalMax whole = {0}, frac = {0}; + Decimal whole = {0}, frac = {0}; DecimalWord zero = 0; int32_t pos = 0; @@ -506,3 +558,66 @@ int32_t decimalToStr(const DecimalWord* pDec, int8_t dataType, int8_t precision, } return 0; } + +int32_t decimalOp(EOperatorType op, const SDataType* pLeftT, const SDataType* pRightT, const SDataType* pOutT, + const void* pLeftData, const void* pRightData, void* pOutputData) { + int32_t code = 0; + if (pOutT->type != TSDB_DATA_TYPE_DECIMAL) return TSDB_CODE_INTERNAL_ERROR; + + Decimal pLeft = *(Decimal*)pLeftData, pRight = *(Decimal*)pRightData; + if (TSDB_DATA_TYPE_DECIMAL != pLeftT->type || pLeftT->scale != pOutT->scale) { + code = convertToDecimal(pLeftData, pLeftT, &pLeft, pOutT); + if (TSDB_CODE_SUCCESS != code) return code; + } + if (pRightT && (TSDB_DATA_TYPE_DECIMAL != pRightT->type || pRightT->scale != pOutT->scale)) { + code = convertToDecimal(pRightData, pRightT, &pRight, pOutT); + if (TSDB_CODE_SUCCESS != code) return code; + } + + SDecimalOps* pOps = getDecimalOps(TSDB_DATA_TYPE_DECIMAL); + switch (op) { + case OP_TYPE_ADD: + pOps->add(pLeft.words, pRight.words, WORD_NUM(Decimal)); + break; + default: + break; + } + return code; +} + +#define MAKE_DECIMAL_64(pDec, hi, lo) makeDecimal64(pDec, (int64_t)lo) +#define MAKE_DECIMAL_128(pDec, hi, lo) makeDecimal128(pDec, (int64_t)hi, (uint64_t)lo) + +#define CONVERT_TO_DECIMAL(TYPE, BitSize, inputType, pInputData, pOut, MAKE_DECIMAL) \ + switch (inputType) { \ + case TSDB_DATA_TYPE_BOOL: { \ + TYPE* pDec = (TYPE*)pOut; \ + MAKE_DECIMAL(pDec, 0, *(bool*)pInputData); \ + } break; \ + case TSDB_DATA_TYPE_TINYINT: { \ + TYPE* pDec = (TYPE*)pOut; \ + int8_t* pV = (int8_t*)pInputData; \ + MAKE_DECIMAL(pDec, *pV < 0 ? -1 : 0, abs(*(int8_t*)pInputData)); \ + } \ + case TSDB_DATA_TYPE_NULL: \ + default: \ + break; \ + } + +int32_t convertToDecimal(const void* pData, const SDataType* pInputType, void* pOut, const SDataType* pOutType) { + if (pInputType->type == pOutType->type) return 0; + int32_t code = 0; + + switch (pOutType->type) { + case TSDB_DATA_TYPE_DECIMAL64: + CONVERT_TO_DECIMAL(Decimal64, 64, pInputType->type, pData, pOut, MAKE_DECIMAL_64); + break; + case TSDB_DATA_TYPE_DECIMAL: + CONVERT_TO_DECIMAL(Decimal128, 128, pInputType->type, pData, pOut, MAKE_DECIMAL_128); + break; + default: + code = TSDB_CODE_INTERNAL_ERROR; + break; + } + return code; +} diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index abea6e7af2..9382180a5a 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1788,7 +1788,7 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, case TSDB_DATA_TYPE_DECIMAL: { uint8_t precision = 0, scale = 0; Decimal128 dec = {0}; - int32_t code = decimal128FromStr(pToken->z, pToken->n, &precision, &scale, &dec); + int32_t code = decimal128FromStr(pToken->z, pToken->n, precision, scale, &dec); if (TSDB_CODE_SUCCESS != code) { return code; } @@ -1805,7 +1805,7 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, case TSDB_DATA_TYPE_DECIMAL64: { uint8_t precision = 0, scale = 0; Decimal64 dec = {0}; - int32_t code = decimal64FromStr(pToken->z, pToken->n, &precision, &scale, &dec); + int32_t code = decimal64FromStr(pToken->z, pToken->n, precision, scale, &dec); if (TSDB_CODE_SUCCESS != code) { return code; } diff --git a/source/libs/scalar/CMakeLists.txt b/source/libs/scalar/CMakeLists.txt index 1fe0f9a18d..985fb2c2d3 100644 --- a/source/libs/scalar/CMakeLists.txt +++ b/source/libs/scalar/CMakeLists.txt @@ -16,6 +16,7 @@ target_link_libraries(scalar PRIVATE qcom PRIVATE parser PRIVATE geometry + PRIVATE decimal ) if(${BUILD_TEST}) diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 46a7f9b772..b200191dd5 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -10,6 +10,7 @@ #include "tdatablock.h" #include "ttime.h" #include "tudf.h" +#include "decimal.h" int32_t scalarGetOperatorParamNum(EOperatorType type) { if (OP_TYPE_IS_NULL == type || OP_TYPE_IS_NOT_NULL == type || OP_TYPE_IS_TRUE == type || @@ -1721,7 +1722,6 @@ static int32_t sclGetMathOperatorResType(SOperatorNode *pOp) { SDataType ldt = ((SExprNode *)(pOp->pLeft))->resType; SDataType rdt = ((SExprNode *)(pOp->pRight))->resType; - bool hasFloatType = IS_FLOAT_TYPE(ldt.type) || IS_FLOAT_TYPE(rdt.type); bool hasDecimalType = IS_DECIMAL_TYPE(ldt.type) || IS_DECIMAL_TYPE(rdt.type); if ((TSDB_DATA_TYPE_TIMESTAMP == ldt.type && TSDB_DATA_TYPE_TIMESTAMP == rdt.type) || @@ -1735,9 +1735,8 @@ static int32_t sclGetMathOperatorResType(SOperatorNode *pOp) { pOp->node.resType.type = TSDB_DATA_TYPE_TIMESTAMP; pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes; } else { - if (hasDecimalType && !hasFloatType) { - pOp->node.resType.type = TSDB_DATA_TYPE_DECIMAL; - pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_DECIMAL].bytes; + if (hasDecimalType) { + decimalGetRetType(&ldt, &rdt, pOp->opType, &pOp->node.resType); } else { pOp->node.resType.type = TSDB_DATA_TYPE_DOUBLE; pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 4239b8c940..209857df12 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -28,6 +28,7 @@ #include "ttime.h" #include "ttypes.h" #include "geosWrapper.h" +#include "decimal.h" #define LEFT_COL ((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void *)pLeftCol : pLeftCol->pData)) #define RIGHT_COL ((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void *)pRightCol : pRightCol->pData)) @@ -38,6 +39,8 @@ IS_JSON_NULL(pRight->columnData->info.type, colDataGetVarData(pRight->columnData, i)) #define IS_HELPER_NULL(col, i) colDataIsNull_s(col, i) || IS_JSON_NULL(col->info.type, colDataGetVarData(col, i)) +#define GET_COL_DATA_TYPE(col) \ + { .type = (col).type, .precision = (col).precision, .bytes = (col).bytes, .scale = (col).scale } bool noConvertBeforeCompare(int32_t leftType, int32_t rightType, int32_t optr) { return IS_NUMERIC_TYPE(leftType) && IS_NUMERIC_TYPE(rightType) && @@ -239,6 +242,16 @@ static FORCE_INLINE int32_t varToTimestamp(char *buf, SScalarParam *pOut, int32_ SCL_RET(code); } +static FORCE_INLINE int32_t varToDecimal(char* buf, SScalarParam* pOut, int32_t rowIndex, int32_t* overflow) { + Decimal *pDec = (Decimal *)colDataGetData(pOut->columnData, rowIndex); + int32_t code = decimalFromStr(buf, strlen(buf), pOut->columnData->info.precision, pOut->columnData->info.scale, pDec); + if (TSDB_CODE_SUCCESS != code) { + // TODO wjm set overflow??? + SCL_RET(code); + } + SCL_RET(code); +} + static FORCE_INLINE int32_t varToSigned(char *buf, SScalarParam *pOut, int32_t rowIndex, int32_t *overflow) { if (overflow) { int64_t minValue = tDataTypes[pOut->columnData->info.type].minValue; @@ -495,6 +508,8 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) { } else if (TSDB_DATA_TYPE_VARBINARY == pCtx->outType) { func = varToVarbinary; vton = true; + } else if (IS_DECIMAL_TYPE(pCtx->outType)) { + func = varToDecimal; } else { sclError("invalid convert outType:%d, inType:%d", pCtx->outType, pCtx->inType); SCL_ERR_RET(TSDB_CODE_APP_ERROR); @@ -987,6 +1002,22 @@ int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut, case TSDB_DATA_TYPE_GEOMETRY: { return vectorConvertToVarData(&cCtx); } + case TSDB_DATA_TYPE_DECIMAL: { + for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { + if (colDataIsNull_f(pInputCol->nullbitmap, i)) { + colDataSetNULL(pOutputCol, i); + continue; + } + + Decimal value = {0}; + SDataType inputType = GET_COL_DATA_TYPE(pInputCol->info), outputType = GET_COL_DATA_TYPE(pOutputCol->info); + int32_t code = convertToDecimal(colDataGetData(pInputCol, i), &inputType, &value, &outputType); + if (TSDB_CODE_SUCCESS != code) return code; + code = colDataSetVal(pOutputCol, i, (const char*)&value, false); + if (TSDB_CODE_SUCCESS != code) return code; + } + break; + } default: sclError("invalid convert output type:%d", cCtx.outType); return TSDB_CODE_APP_ERROR; @@ -1278,7 +1309,23 @@ int32_t vectorMathAdd(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *p SCL_ERR_JRET(vectorMathAddHelper(pLeftCol, pRightCol, pOutputCol, pLeft->numOfRows, step, i)); } } else if (IS_DECIMAL_TYPE(pOutputCol->info.type)) { - + Decimal *output = (Decimal *)pOutputCol->pData; + if (pLeft->numOfRows == pRight->numOfRows) { + for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { + if (IS_NULL) { + colDataSetNULL(pOutputCol, i); + continue; + } + SDataType leftType = GET_COL_DATA_TYPE(pLeft->columnData->info), + rightType = GET_COL_DATA_TYPE(pRight->columnData->info), + outType = GET_COL_DATA_TYPE(pOutputCol->info); + SCL_ERR_JRET(decimalOp(OP_TYPE_ADD, &leftType, &rightType, &outType, colDataGetData(pLeftCol, i), + colDataGetData(pRightCol, i), output)); + } + } else if (pLeft->numOfRows == 1) { + // TODO wjm + } else if (pRight->numOfRows == 1) { + } } _return: