diff --git a/docs/en/14-reference/09-error-code.md b/docs/en/14-reference/09-error-code.md index 725f5a22f5..ab967de4e1 100644 --- a/docs/en/14-reference/09-error-code.md +++ b/docs/en/14-reference/09-error-code.md @@ -73,6 +73,8 @@ This document details the server error codes that may be encountered when using | 0x80000134 | Invalid value | Invalid value | Preserve the scene and logs, report issue on github | | 0x80000135 | Invalid fqdn | Invalid FQDN | Check if the configured or input FQDN value is correct | | 0x8000013C | Invalid disk id | Invalid disk id | Check users whether the mounted disk is invalid or use the parameter diskIDCheckEnabled to skip the disk check. | +| 0x8000013D | Decimal value overflow | Decimal value overflow | Check query expression and decimal values | +| 0x8000013E | Division by zero error | Division by zero | Check division expression | ## tsc diff --git a/docs/zh/14-reference/09-error-code.md b/docs/zh/14-reference/09-error-code.md index 87809022dd..3f632e7829 100644 --- a/docs/zh/14-reference/09-error-code.md +++ b/docs/zh/14-reference/09-error-code.md @@ -76,6 +76,8 @@ description: TDengine 服务端的错误码列表和详细说明 | 0x80000134 | Invalid value | 无效值 | 保留现场和日志，github 上报 issue | | 0x80000135 | Invalid fqdn | 无效 FQDN | 检查配置或输入的 FQDN 值是否正确 | | 0x8000013C | Invalid disk id | 不合法的 disk id | 建议用户检查挂载磁盘是否失效或者使用参数 diskIDCheckEnabled 来跳过磁盘检查 | +| 0x8000013D | Decimal value overflow | Decimal 计算溢出 | 检查计算表达式和参数值是否计算结果导致类型溢出 | +| 0x8000013E | Division by zero error | Division by zero | 检查除法操作是否除以0 | diff --git a/include/client/taos.h b/include/client/taos.h index 433779f811..919ace9fc4 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -52,7 +52,8 @@ typedef void TAOS_SUB; #define TSDB_DATA_TYPE_MEDIUMBLOB 19 #define TSDB_DATA_TYPE_BINARY TSDB_DATA_TYPE_VARCHAR // string #define TSDB_DATA_TYPE_GEOMETRY 20 // geometry -#define TSDB_DATA_TYPE_MAX 21 +#define TSDB_DATA_TYPE_DECIMAL64 21 // decimal64 +#define TSDB_DATA_TYPE_MAX 22 typedef enum { TSDB_OPTION_LOCALE, @@ -270,6 +271,7 @@ DLL_EXPORT int taos_affected_rows(TAOS_RES *res); DLL_EXPORT int64_t taos_affected_rows64(TAOS_RES *res); DLL_EXPORT TAOS_FIELD *taos_fetch_fields(TAOS_RES *res); +DLL_EXPORT TAOS_FIELD_E *taos_fetch_fields_e(TAOS_RES *res); DLL_EXPORT int taos_select_db(TAOS *taos, const char *db); DLL_EXPORT int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields); DLL_EXPORT int taos_print_row_with_size(char *str, uint32_t size, TAOS_ROW row, TAOS_FIELD *fields, int num_fields); diff --git a/include/common/tcol.h b/include/common/tcol.h index cbe72bbc94..fb70119dda 100644 --- a/include/common/tcol.h +++ b/include/common/tcol.h @@ -79,7 +79,7 @@ uint8_t columnLevelVal(const char* level); uint8_t columnEncodeVal(const char* encode); uint16_t columnCompressVal(const char* compress); -bool useCompress(uint8_t tableType); +bool withExtSchema(uint8_t tableType); bool checkColumnEncode(char encode[TSDB_CL_COMPRESS_OPTION_LEN]); bool checkColumnEncodeOrSetDefault(uint8_t type, char encode[TSDB_CL_COMPRESS_OPTION_LEN]); bool checkColumnCompress(char compress[TSDB_CL_COMPRESS_OPTION_LEN]); diff --git a/include/common/tcommon.h b/include/common/tcommon.h index c2043fba3a..40c6cd2b05 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -166,14 +166,35 @@ typedef enum EStreamType { #pragma pack(push, 1) typedef struct SColumnDataAgg { - int16_t colId; + int32_t colId; int16_t numOfNull; - int64_t sum; - int64_t max; - int64_t min; + union { + struct { + int64_t sum; + int64_t max; + int64_t min; + }; + struct { + uint64_t decimal128Sum[2]; + uint64_t decimal128Max[2]; + uint64_t decimal128Min[2]; + uint8_t overflow; + }; + }; } SColumnDataAgg; #pragma pack(pop) +#define DECIMAL_AGG_FLAG 0x80000000 + +#define COL_AGG_GET_SUM_PTR(pAggs, dataType) \ + (!IS_DECIMAL_TYPE(dataType) ? (void*)&pAggs->sum : (void*)pAggs->decimal128Sum) + +#define COL_AGG_GET_MAX_PTR(pAggs, dataType) \ + (!IS_DECIMAL_TYPE(dataType) ? (void*)&pAggs->max : (void*)pAggs->decimal128Max) + +#define COL_AGG_GET_MIN_PTR(pAggs, dataType) \ + (!IS_DECIMAL_TYPE(dataType) ? (void*)&pAggs->min : (void*)pAggs->decimal128Min) + typedef struct SBlockID { // The uid of table, from which current data block comes. And it is always 0, if current block is the // result of calculation. @@ -430,6 +451,14 @@ static inline bool isTsmaResSTb(const char* stbName) { return false; } +static inline STypeMod typeGetTypeModFromColInfo(const SColumnInfo* pCol) { + return typeGetTypeMod(pCol->type, pCol->precision, pCol->scale, pCol->bytes); +} + +static inline STypeMod typeGetTypeModFromCol(const SColumn* pCol) { + return typeGetTypeMod(pCol->type, pCol->precision, pCol->scale, pCol->bytes); +} + #ifdef __cplusplus } #endif diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 519614b94e..55546fdd73 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -80,6 +80,9 @@ typedef struct SBlockOrderInfo { #define IS_JSON_NULL(type, data) \ ((type) == TSDB_DATA_TYPE_JSON && (*(data) == TSDB_DATA_TYPE_NULL || tTagIsJsonNull(data))) +#define GET_COL_DATA_TYPE(col) \ + { .type = (col).type, .precision = (col).precision, .bytes = (col).bytes, .scale = (col).scale } + static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, uint32_t row) { if (!pColumnInfoData->hasNull) { return false; diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 7b4b8c33ce..afc7b1d916 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -43,6 +43,8 @@ typedef struct SColData SColData; typedef struct SRowKey SRowKey; typedef struct SValueColumn SValueColumn; +struct SColumnDataAgg; +typedef struct SColumnDataAgg* SColumnDataAggPtr; #define HAS_NONE ((uint8_t)0x1) #define HAS_NULL ((uint8_t)0x2) @@ -187,7 +189,7 @@ uint8_t tColDataGetBitValue(const SColData *pColData, int32_t iVal); int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMalloc, void *arg); void tColDataArrGetRowKey(SColData *aColData, int32_t nColData, int32_t iRow, SRowKey *key); -extern void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, int16_t *numOfNull); +extern void (*tColDataCalcSMA[])(SColData *pColData, SColumnDataAggPtr pAggs); int32_t tColDataCompress(SColData *colData, SColDataCompressInfo *info, SBuffer *output, SBuffer *assist); int32_t tColDataDecompress(void *input, SColDataCompressInfo *info, SColData *colData, SBuffer *assist); @@ -244,6 +246,8 @@ typedef struct { uint32_t offset; } SPrimaryKeyIndex; +#define DATUM_MAX_SIZE 16 + struct SValue { int8_t type; union { @@ -255,6 +259,16 @@ struct SValue { }; }; +#define VALUE_GET_DATUM(pVal, type) \ + (IS_VAR_DATA_TYPE(type) || type == TSDB_DATA_TYPE_DECIMAL) ? (pVal)->pData : (void*)&(pVal)->val + +#define VALUE_GET_TRIVIAL_DATUM(pVal) ((pVal)->val) +#define VALUE_SET_TRIVIAL_DATUM(pVal, v) (pVal)->val = v + +void valueSetDatum(SValue *pVal, int8_t type, void *pDatum, uint32_t len); +void valueCloneDatum(SValue *pDst, const SValue *pSrc, int8_t type); +void valueClearDatum(SValue *pVal, int8_t type); + #define TD_MAX_PK_COLS 2 struct SRowKey { TSKEY ts; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 12de5479a6..ff56c7b33f 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -515,6 +515,7 @@ typedef struct SFieldWithOptions { int8_t flags; int32_t bytes; uint32_t compress; + STypeMod typeMod; } SFieldWithOptions; typedef struct SRetention { @@ -596,6 +597,7 @@ struct SSchema { struct SSchemaExt { col_id_t colId; uint32_t compress; + STypeMod typeMod; }; // @@ -659,6 +661,7 @@ void tFreeSSubmitRsp(SSubmitRsp* pRsp); #define COL_SET_NULL ((int8_t)0x10) #define COL_SET_VAL ((int8_t)0x20) #define COL_IS_SYSINFO ((int8_t)0x40) +#define COL_HAS_TYPE_MOD ((int8_t)0x80) #define COL_IS_SET(FLG) (((FLG) & (COL_SET_VAL | COL_SET_NULL)) != 0) #define COL_CLR_SET(FLG) ((FLG) &= (~(COL_SET_VAL | COL_SET_NULL))) @@ -677,6 +680,13 @@ void tFreeSSubmitRsp(SSubmitRsp* pRsp); (s)->flags &= (~COL_IDX_ON); \ } while (0) +#define SSCHEMA_SET_TYPE_MOD(s) \ + do { \ + (s)->flags |= COL_HAS_TYPE_MOD; \ + } while (0) + +#define HAS_TYPE_MOD(s) (((s)->flags & COL_HAS_TYPE_MOD)) + #define SSCHMEA_TYPE(s) ((s)->type) #define SSCHMEA_FLAGS(s) ((s)->flags) #define SSCHMEA_COLID(s) ((s)->colId) @@ -693,6 +703,12 @@ typedef struct { char tsSlowLogExceptDb[TSDB_DB_NAME_LEN]; } SMonitorParas; +typedef struct { + STypeMod typeMod; +} SExtSchema; + +bool hasExtSchema(const SExtSchema* pExtSchema); + typedef struct { int32_t nCols; int32_t version; @@ -843,12 +859,14 @@ static FORCE_INLINE int32_t tDecodeSSchema(SDecoder* pDecoder, SSchema* pSchema) static FORCE_INLINE int32_t tEncodeSSchemaExt(SEncoder* pEncoder, const SSchemaExt* pSchemaExt) { TAOS_CHECK_RETURN(tEncodeI16v(pEncoder, pSchemaExt->colId)); TAOS_CHECK_RETURN(tEncodeU32(pEncoder, pSchemaExt->compress)); + TAOS_CHECK_RETURN(tEncodeI32(pEncoder, pSchemaExt->typeMod)); return 0; } static FORCE_INLINE int32_t tDecodeSSchemaExt(SDecoder* pDecoder, SSchemaExt* pSchemaExt) { TAOS_CHECK_RETURN(tDecodeI16v(pDecoder, &pSchemaExt->colId)); TAOS_CHECK_RETURN(tDecodeU32(pDecoder, &pSchemaExt->compress)); + TAOS_CHECK_RETURN(tDecodeI32(pDecoder, &pSchemaExt->typeMod)); return 0; } @@ -881,6 +899,7 @@ static FORCE_INLINE void* taosDecodeSSchemaWrapper(const void* buf, SSchemaWrapp } static FORCE_INLINE int32_t tEncodeSSchemaWrapper(SEncoder* pEncoder, const SSchemaWrapper* pSW) { + if (pSW == NULL) {return TSDB_CODE_INVALID_PARA;} TAOS_CHECK_RETURN(tEncodeI32v(pEncoder, pSW->nCols)); TAOS_CHECK_RETURN(tEncodeI32v(pEncoder, pSW->version)); for (int32_t i = 0; i < pSW->nCols; i++) { @@ -890,6 +909,7 @@ static FORCE_INLINE int32_t tEncodeSSchemaWrapper(SEncoder* pEncoder, const SSch } static FORCE_INLINE int32_t tDecodeSSchemaWrapper(SDecoder* pDecoder, SSchemaWrapper* pSW) { + if (pSW == NULL) {return TSDB_CODE_INVALID_PARA;} TAOS_CHECK_RETURN(tDecodeI32v(pDecoder, &pSW->nCols)); TAOS_CHECK_RETURN(tDecodeI32v(pDecoder, &pSW->version)); @@ -988,6 +1008,7 @@ typedef struct { int32_t sqlLen; char* sql; int64_t keep; + SArray* pTypeMods; } SMAlterStbReq; int32_t tSerializeSMAlterStbReq(void* buf, int32_t bufLen, SMAlterStbReq* pReq); @@ -3243,6 +3264,7 @@ typedef struct SVCreateStbReq { int8_t colCmpred; SColCmprWrapper colCmpr; int64_t keep; + SExtSchema* pExtSchemas; } SVCreateStbReq; int tEncodeSVCreateStbReq(SEncoder* pCoder, const SVCreateStbReq* pReq); @@ -3283,6 +3305,7 @@ typedef struct SVCreateTbReq { int32_t sqlLen; char* sql; SColCmprWrapper colCmpr; + SExtSchema* pExtSchemas; } SVCreateTbReq; int tEncodeSVCreateTbReq(SEncoder* pCoder, const SVCreateTbReq* pReq); @@ -3306,6 +3329,7 @@ static FORCE_INLINE void tdDestroySVCreateTbReq(SVCreateTbReq* req) { taosMemoryFreeClear(req->ntb.schemaRow.pSchema); } taosMemoryFreeClear(req->colCmpr.pColCmpr); + taosMemoryFreeClear(req->pExtSchemas); } typedef struct { @@ -3422,6 +3446,8 @@ typedef struct { int8_t source; // TD_REQ_FROM_TAOX-taosX or TD_REQ_FROM_APP-taosClient uint32_t compress; // TSDB_ALTER_TABLE_UPDATE_COLUMN_COMPRESS SArray* pMultiTag; // TSDB_ALTER_TABLE_ADD_MULTI_TAGS + // for Add column + STypeMod typeMod; } SVAlterTbReq; int32_t tEncodeSVAlterTbReq(SEncoder* pEncoder, const SVAlterTbReq* pReq); @@ -4316,7 +4342,6 @@ typedef struct { }; void* data; //for free in client, only effected if type is data or metadata. raw data not effected bool blockDataElementFree; // if true, free blockDataElement in blockData,(true in server, false in client) - } SMqDataRsp; int32_t tEncodeMqDataRsp(SEncoder* pEncoder, const SMqDataRsp* pObj); diff --git a/include/common/ttypes.h b/include/common/ttypes.h index 95fe14e572..f081eef40c 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -45,6 +45,9 @@ typedef struct { } SNCharNullT; #pragma pack(pop) +#define STypeMod int32_t +void extractTypeFromTypeMod(uint8_t type, STypeMod typeMod, uint8_t *prec, uint8_t* scale, int32_t *bytes); + #define varDataTLen(v) (sizeof(VarDataLenT) + varDataLen(v)) #define varDataCopy(dst, v) (void)memcpy((dst), (void *)(v), varDataTLen(v)) #define varDataLenByData(v) (*(VarDataLenT *)(((char *)(v)) - VARSTR_HEADER_SIZE)) @@ -53,45 +56,74 @@ typedef struct { #define varDataNetLen(v) (htons(((VarDataLenT *)(v))[0])) #define varDataNetTLen(v) (sizeof(VarDataLenT) + varDataNetLen(v)) -#define GET_TYPED_DATA(_v, _finalType, _type, _data) \ - do { \ - switch (_type) { \ - case TSDB_DATA_TYPE_BOOL: \ - case TSDB_DATA_TYPE_TINYINT: \ - (_v) = (_finalType)GET_INT8_VAL(_data); \ - break; \ - case TSDB_DATA_TYPE_UTINYINT: \ - (_v) = (_finalType)GET_UINT8_VAL(_data); \ - break; \ - case TSDB_DATA_TYPE_SMALLINT: \ - (_v) = (_finalType)GET_INT16_VAL(_data); \ - break; \ - case TSDB_DATA_TYPE_USMALLINT: \ - (_v) = (_finalType)GET_UINT16_VAL(_data); \ - break; \ - case TSDB_DATA_TYPE_TIMESTAMP: \ - case TSDB_DATA_TYPE_BIGINT: \ - (_v) = (_finalType)(GET_INT64_VAL(_data)); \ - break; \ - case TSDB_DATA_TYPE_UBIGINT: \ - (_v) = (_finalType)(GET_UINT64_VAL(_data)); \ - break; \ - case TSDB_DATA_TYPE_FLOAT: \ - (_v) = (_finalType)GET_FLOAT_VAL(_data); \ - break; \ - case TSDB_DATA_TYPE_DOUBLE: \ - (_v) = (_finalType)GET_DOUBLE_VAL(_data); \ - break; \ - case TSDB_DATA_TYPE_UINT: \ - (_v) = (_finalType)GET_UINT32_VAL(_data); \ - break; \ - case TSDB_DATA_TYPE_INT: \ - (_v) = (_finalType)GET_INT32_VAL(_data); \ - break; \ - default: \ - (_v) = (_finalType)varDataLen(_data); \ - break; \ - } \ +#define DEFINE_TYPE_FROM_DECIMAL_FUNC(oType, decimalType) \ + oType oType##From##decimalType(const void* pDec, uint8_t prec, uint8_t scale) + +#define DEFINE_TYPE_FROM_DECIMAL_FUNCS(prefix, decimalType) \ + prefix DEFINE_TYPE_FROM_DECIMAL_FUNC(bool, decimalType); \ + prefix DEFINE_TYPE_FROM_DECIMAL_FUNC(int8_t, decimalType); \ + prefix DEFINE_TYPE_FROM_DECIMAL_FUNC(uint8_t, decimalType); \ + prefix DEFINE_TYPE_FROM_DECIMAL_FUNC(int16_t, decimalType); \ + prefix DEFINE_TYPE_FROM_DECIMAL_FUNC(uint16_t, decimalType); \ + prefix DEFINE_TYPE_FROM_DECIMAL_FUNC(int32_t, decimalType); \ + prefix DEFINE_TYPE_FROM_DECIMAL_FUNC(uint32_t, decimalType); \ + prefix DEFINE_TYPE_FROM_DECIMAL_FUNC(int64_t, decimalType); \ + prefix DEFINE_TYPE_FROM_DECIMAL_FUNC(uint64_t, decimalType); \ + prefix DEFINE_TYPE_FROM_DECIMAL_FUNC(float, decimalType); \ + prefix DEFINE_TYPE_FROM_DECIMAL_FUNC(double, decimalType); + +DEFINE_TYPE_FROM_DECIMAL_FUNCS(extern, Decimal64); +DEFINE_TYPE_FROM_DECIMAL_FUNCS(extern, Decimal128); + +#define GET_TYPED_DATA(_v, _finalType, _type, _data, inputTypeMod) \ + do { \ + switch (_type) { \ + case TSDB_DATA_TYPE_BOOL: \ + case TSDB_DATA_TYPE_TINYINT: \ + (_v) = (_finalType)GET_INT8_VAL(_data); \ + break; \ + case TSDB_DATA_TYPE_UTINYINT: \ + (_v) = (_finalType)GET_UINT8_VAL(_data); \ + break; \ + case TSDB_DATA_TYPE_SMALLINT: \ + (_v) = (_finalType)GET_INT16_VAL(_data); \ + break; \ + case TSDB_DATA_TYPE_USMALLINT: \ + (_v) = (_finalType)GET_UINT16_VAL(_data); \ + break; \ + case TSDB_DATA_TYPE_TIMESTAMP: \ + case TSDB_DATA_TYPE_BIGINT: \ + (_v) = (_finalType)(GET_INT64_VAL(_data)); \ + break; \ + case TSDB_DATA_TYPE_UBIGINT: \ + (_v) = (_finalType)(GET_UINT64_VAL(_data)); \ + break; \ + case TSDB_DATA_TYPE_FLOAT: \ + (_v) = (_finalType)GET_FLOAT_VAL(_data); \ + break; \ + case TSDB_DATA_TYPE_DOUBLE: \ + (_v) = (_finalType)GET_DOUBLE_VAL(_data); \ + break; \ + case TSDB_DATA_TYPE_UINT: \ + (_v) = (_finalType)GET_UINT32_VAL(_data); \ + break; \ + case TSDB_DATA_TYPE_INT: \ + (_v) = (_finalType)GET_INT32_VAL(_data); \ + break; \ + case TSDB_DATA_TYPE_DECIMAL: { \ + uint8_t prec = 0, scale = 0; \ + extractTypeFromTypeMod(_type, inputTypeMod, &prec, &scale, NULL); \ + (_v) = _finalType##FromDecimal128(_data, prec, scale); \ + } break; \ + case TSDB_DATA_TYPE_DECIMAL64: { \ + uint8_t prec = 0, scale = 0; \ + extractTypeFromTypeMod(_type, inputTypeMod, &prec, &scale, NULL); \ + (_v) = _finalType##FromDecimal64(_data, prec, scale); \ + } break; \ + default: \ + (_v) = (_finalType)varDataLen(_data); \ + break; \ + } \ } while (0) #define SET_TYPED_DATA(_v, _type, _data) \ @@ -265,8 +297,9 @@ typedef struct { #define IS_INTEGER_TYPE(_t) ((IS_SIGNED_NUMERIC_TYPE(_t)) || (IS_UNSIGNED_NUMERIC_TYPE(_t))) #define IS_TIMESTAMP_TYPE(_t) ((_t) == TSDB_DATA_TYPE_TIMESTAMP) #define IS_BOOLEAN_TYPE(_t) ((_t) == TSDB_DATA_TYPE_BOOL) +#define IS_DECIMAL_TYPE(_t) ((_t) == TSDB_DATA_TYPE_DECIMAL || (_t) == TSDB_DATA_TYPE_DECIMAL64) -#define IS_NUMERIC_TYPE(_t) ((IS_SIGNED_NUMERIC_TYPE(_t)) || (IS_UNSIGNED_NUMERIC_TYPE(_t)) || (IS_FLOAT_TYPE(_t))) +#define IS_NUMERIC_TYPE(_t) ((IS_SIGNED_NUMERIC_TYPE(_t)) || (IS_UNSIGNED_NUMERIC_TYPE(_t)) || (IS_FLOAT_TYPE(_t)) || (IS_DECIMAL_TYPE(_t))) #define IS_MATHABLE_TYPE(_t) \ (IS_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL) || (_t) == (TSDB_DATA_TYPE_TIMESTAMP)) @@ -364,6 +397,13 @@ typedef struct tDataTypeCompress { extern tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX]; extern tDataTypeCompress tDataCompress[TSDB_DATA_TYPE_MAX]; +typedef struct SDataType { + uint8_t type; + uint8_t precision; + uint8_t scale; + int32_t bytes; +} SDataType; + bool isValidDataType(int32_t type); int32_t operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type); @@ -371,6 +411,21 @@ void assignVal(char *val, const char *src, int32_t len, int32_t type); void *getDataMin(int32_t type, void *value); void *getDataMax(int32_t type, void *value); +STypeMod typeGetTypeMod(uint8_t type, uint8_t prec, uint8_t scale, int32_t bytes); +STypeMod typeGetTypeModFromDataType(const SDataType* pDataType); +uint8_t decimalTypeFromPrecision(uint8_t precision); +STypeMod decimalCalcTypeMod(uint8_t prec, uint8_t scale); +void decimalFromTypeMod(STypeMod typeMod, uint8_t *precision, uint8_t *scale); +// pType->type should has been set +void fillTypeFromTypeMod(SDataType *pType, STypeMod mod); +uint8_t getScaleFromTypeMod(int32_t type, STypeMod mod); +// TODO fix me!! for compatibility issue, save precision in scale in bytes, move it to somewhere else +void fillBytesForDecimalType(int32_t *pBytes, int32_t type, uint8_t precision, uint8_t scale); +void extractDecimalTypeInfoFromBytes(int32_t *pBytes, uint8_t *precision, uint8_t *scale); + +int32_t calcTypeBytesFromSchemaBytes(int32_t type, int32_t schemaBytes, bool isStmt); +int32_t calcSchemaBytesFromTypeBytes(int32_t type, int32_t varTypeBytes, bool isStmt); + #ifdef __cplusplus } #endif diff --git a/include/libs/decimal/decimal.h b/include/libs/decimal/decimal.h new file mode 100644 index 0000000000..e5cdcf814f --- /dev/null +++ b/include/libs/decimal/decimal.h @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DECIMAL_H_ +#define _TD_DECIMAL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "tdef.h" +#include "ttypes.h" +typedef struct SValue SValue; +typedef void DecimalType; + +typedef struct Decimal64 { + DecimalWord words[1]; // do not touch it directly, use DECIMAL64_GET_VALUE MACRO +} Decimal64; + +#define DECIMAL64_GET_VALUE(pDec) (int64_t)((pDec)->words[0]) +#define DECIMAL64_SET_VALUE(pDec, val) (*(int64_t*)((pDec)->words)) = (int64_t)(val) +#define DECIMAL64_CLONE(pDst, pFrom) ((Decimal64*)(pDst))->words[0] = ((Decimal64*)(pFrom))->words[0] + +static const Decimal64 decimal64Zero = {0}; +static const Decimal64 decimal64Two = {2}; +static const Decimal64 decimal64Min = {(uint64_t)-999999999999999999LL}; +static const Decimal64 decimal64Max = {(uint64_t)999999999999999999LL}; +#define DECIMAL64_ZERO decimal64Zero +#define DECIMAL64_MAX decimal64Max +#define DECIMAL64_MIN decimal64Min + +typedef struct Decimal128 { + DecimalWord words[2]; // do not touch it directly, use DECIMAL128_HIGH_WORD/DECIMAL128_LOW_WORD +} Decimal128; + +#define Decimal Decimal128 +#define decimalFromStr decimal128FromStr +#define makeDecimal makeDecimal128 + +#define DEFINE_DECIMAL128(lo, hi) {lo, hi} +static const Decimal128 decimal128Zero = DEFINE_DECIMAL128(0, 0); +static const Decimal128 decimal128Two = DEFINE_DECIMAL128(2, 0); +static const Decimal128 decimal128Max = DEFINE_DECIMAL128(687399551400673280ULL - 1, 5421010862427522170LL); +static const Decimal128 decimal128Min = DEFINE_DECIMAL128(17759344522308878337ULL, 13025733211282029445ULL); +#define DECIMAL128_LOW_WORD(pDec) (uint64_t)((pDec)->words[0]) +#define DECIMAL128_SET_LOW_WORD(pDec, val) (pDec)->words[0] = val +#define DECIMAL128_HIGH_WORD(pDec) (int64_t)((pDec)->words[1]) +#define DECIMAL128_SET_HIGH_WORD(pDec, val) *(int64_t*)((pDec)->words + 1) = val + +#define DECIMAL128_ZERO decimal128Zero +#define DECIMAL128_MAX decimal128Max +#define DECIMAL128_MIN decimal128Min +#define DECIMAL128_CLONE(pDst, pFrom) makeDecimal128(pDst, DECIMAL128_HIGH_WORD(pFrom), DECIMAL128_LOW_WORD(pFrom)) + +typedef struct SDecimalCompareCtx { + const void* pData; + int8_t type; + STypeMod typeMod; +} SDecimalCompareCtx; + +void makeDecimal64(Decimal64* pDec64, int64_t w); +void makeDecimal128(Decimal128* pDec128, int64_t hi, uint64_t low); + +void decimalFromTypeMod(STypeMod typeMod, uint8_t* precision, uint8_t* scale); + +int32_t decimal64FromStr(const char* str, int32_t len, uint8_t expectPrecision, uint8_t expectScale, Decimal64* result); +int32_t decimal128FromStr(const char* str, int32_t len, uint8_t expectPrecision, uint8_t expectScale, + Decimal128* result); + +int32_t decimal64ToDataVal(const Decimal64* dec, SValue* pVal); +int32_t decimal128ToDataVal(Decimal128* dec, SValue* pVal); + +int32_t decimalToStr(const DecimalType* pDec, int8_t type, int8_t precision, int8_t scale, char* pBuf, int32_t bufLen); + +int32_t decimalGetRetType(const SDataType* pLeftT, const SDataType* pRightT, EOperatorType opType, SDataType* pOutType); +bool decimal64Compare(EOperatorType op, const SDecimalCompareCtx* pLeft, const SDecimalCompareCtx* pRight); +bool decimalCompare(EOperatorType op, const SDecimalCompareCtx* pLeft, const SDecimalCompareCtx* pRight); +int32_t decimalOp(EOperatorType op, const SDataType* pLeftT, const SDataType* pRightT, const SDataType* pOutT, + const void* pLeftData, const void* pRightData, void* pOutputData); +int32_t convertToDecimal(const void* pData, const SDataType* pInputType, void* pOut, const SDataType* pOutType); +bool decimal128AddCheckOverflow(const Decimal128* pLeft, const DecimalType* pRight, uint8_t rightWordNum); +void encodeDecimal(const DecimalType* pDec, int8_t type, void* pBuf); +void decodeDecimal(const void* pBuf, int8_t type, DecimalType* pDec); + +DEFINE_TYPE_FROM_DECIMAL_FUNCS(, Decimal64); +DEFINE_TYPE_FROM_DECIMAL_FUNCS(, Decimal128); + +// word num use DECIMAL_WORD_NUM(Decimal64) or DECIMAL_WORD_NUM(Decimal128) +typedef struct SDecimalOps { + void (*negate)(DecimalType* pWord); + void (*abs)(DecimalType* pWord); + void (*add)(DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum); + void (*subtract)(DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum); + void (*multiply)(DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum); + void (*divide)(DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum, DecimalType* pRemainder); + void (*mod)(DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum); + bool (*lt)(const DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum); + bool (*gt)(const DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum); + bool (*eq)(const DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum); + int32_t (*toStr)(const DecimalType* pInt, uint8_t scale, char* pBuf, int32_t bufLen); +} SDecimalOps; + +// all these ops only used for operations on decimal types with same scale +const SDecimalOps* getDecimalOps(int8_t dataType); + +#if 0 +__int128 decimal128ToInt128(const Decimal128* pDec); +#endif +int32_t TEST_decimal64From_int64_t(Decimal64* pDec, uint8_t prec, uint8_t scale, int64_t v); +int32_t TEST_decimal64From_uint64_t(Decimal64* pDec, uint8_t prec, uint8_t scale, uint64_t v); +int32_t TEST_decimal64From_double(Decimal64* pDec, uint8_t prec, uint8_t scale, double v); +double TEST_decimal64ToDouble(Decimal64* pDec, uint8_t prec, uint8_t scale); +int32_t TEST_decimal64FromDecimal64(const Decimal64* pInput, uint8_t inputPrec, uint8_t inputScale, Decimal64* pOutput, + uint8_t outputPrec, uint8_t outputScale); +int32_t TEST_decimal64FromDecimal128(const Decimal128* pInput, uint8_t prec, uint8_t scale, Decimal64* pOutput, + uint8_t outputPrec, uint8_t outputScale); + +int32_t TEST_decimal128From_int64_t(Decimal128* pDec, uint8_t prec, uint8_t scale, int64_t v); +int32_t TEST_decimal128From_uint64_t(Decimal128* pDec, uint8_t prec, uint8_t scale, uint64_t v); +int32_t TEST_decimal128From_double(Decimal128* pDec, uint8_t prec, uint8_t scale, double v); +double TEST_decimal128ToDouble(Decimal128* pDec, uint8_t prec, uint8_t scale); +int32_t TEST_decimal128FromDecimal64(const Decimal64* pInput, uint8_t inputPrec, uint8_t inputScale, + Decimal128* pOutput, uint8_t outputPrec, uint8_t outputScale); +int32_t TEST_decimal128FromDecimal128(const Decimal128* pDec, uint8_t prec, uint8_t scale, Decimal128* pOutput, + uint8_t outputPrec, uint8_t outputScale); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DECIMAL_H_*/ diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index cff08e5d41..01c3252f91 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -84,6 +84,7 @@ typedef struct SMetaEntry { uint8_t* pBuf; SColCmprWrapper colCmpr; // col compress alg + SExtSchema* pExtSchemas; } SMetaEntry; typedef struct SMetaReader { @@ -136,6 +137,11 @@ typedef struct SMetaTableInfo { char tbName[TSDB_TABLE_NAME_LEN]; } SMetaTableInfo; +static FORCE_INLINE void destroyMetaTableInfo(SMetaTableInfo* mtInfo){ + if (mtInfo == NULL) return; + tDeleteSchemaWrapper(mtInfo->schema); +} + typedef struct SSnapContext { struct SMeta* pMeta; int64_t snapVersion; diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 4cd946ee35..9588384730 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -265,6 +265,7 @@ typedef struct SqlFunctionCtx { bool bInputFinished; bool hasWindowOrGroup; // denote that the function is used with time window or group bool needCleanup; // denote that the function need to be cleaned up + int32_t inputType; // save the fuction input type funcs like finalize } SqlFunctionCtx; typedef struct tExprNode { @@ -291,12 +292,14 @@ struct SScalarParam { SColumnInfoData *columnData; SHashObj *pHashFilter; SHashObj *pHashFilterOthers; - int32_t hashValueType; + int32_t filterValueType; void *param; // other parameter, such as meta handle from vnode, to extract table name/tag value int32_t numOfRows; int32_t numOfQualified; // number of qualified elements in the final results timezone_t tz; void *charsetCxt; + SArray *pFilterArr; // for types that can't filter with hash + STypeMod filterValueTypeMod; }; static inline void setTzCharset(SScalarParam* param, timezone_t tz, void* charsetCxt){ @@ -315,7 +318,7 @@ typedef struct SPoint { } SPoint; void taosGetLinearInterpolationVal(SPoint *point, int32_t outputType, SPoint *point1, SPoint *point2, - int32_t inputType); + int32_t inputType, STypeMod inputTypeMod); #define LEASTSQUARES_DOUBLE_ITEM_LENGTH 25 #define LEASTSQUARES_BUFF_LENGTH 128 diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 662c01a718..fe6436f79e 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -25,6 +25,7 @@ extern "C" { #include "tmsg.h" #include "tsimplehash.h" #include "tvariant.h" +#include "ttypes.h" #define TABLE_TOTAL_COL_NUM(pMeta) ((pMeta)->tableInfo.numOfColumns + (pMeta)->tableInfo.numOfTags) #define TABLE_META_SIZE(pMeta) \ @@ -45,13 +46,6 @@ typedef struct SRawExprNode { bool isPseudoColumn; } SRawExprNode; -typedef struct SDataType { - uint8_t type; - uint8_t precision; - uint8_t scale; - int32_t bytes; -} SDataType; - typedef struct SExprNode { ENodeType type; SDataType resType; @@ -203,6 +197,8 @@ typedef struct SFunctionNode { bool dual; // whether select stmt without from stmt, true for without. timezone_t tz; void *charsetCxt; + const struct SFunctionNode* pSrcFuncRef; + SDataType srcFuncInputType; } SFunctionNode; typedef struct STableNode { @@ -655,6 +651,7 @@ typedef struct SQuery { SArray* pDbList; SArray* pPlaceholderValues; SNode* pPrepareRoot; + SExtSchema* pResExtSchema; } SQuery; void nodesWalkSelectStmtImpl(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext); diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index e8a3ef7601..9483f8fae2 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -189,7 +189,7 @@ int32_t smlBuildOutputRaw(SQuery* handle, SHashObj* pVgHash); int rawBlockBindRawData(SHashObj* pVgroupHash, SArray* pVgroupList, STableMeta* pTableMeta, void* data); int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, void* fields, int numFields, bool needChangeLength, char* errstr, int32_t errstrLen, bool raw); -int32_t checkSchema(SSchema* pColSchema, int8_t* fields, char* errstr, int32_t errstrLen); +int32_t checkSchema(SSchema* pColSchema, SSchemaExt* pColExtSchema, int8_t* fields, char* errstr, int32_t errstrLen); int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray); int32_t serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap, SArray** pOut); diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index 3a1a14023c..ef2337d76d 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -41,11 +41,12 @@ pDst need to freed in caller int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst); int32_t scalarGetOperatorParamNum(EOperatorType type); -int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type, int8_t processType); +int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type, STypeMod typeMod, int8_t processType); int32_t vectorGetConvertType(int32_t type1, int32_t type2); int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut, int32_t *overflow, int32_t startIndex, int32_t numOfRows); -int32_t vectorConvertSingleCol(SScalarParam *input, SScalarParam *output, int32_t type, int32_t startIndex, int32_t numOfRows); +int32_t vectorConvertSingleCol(SScalarParam *input, SScalarParam *output, int32_t type, STypeMod typeMod, int32_t startIndex, int32_t numOfRows); +STypeMod getConvertTypeMod(int32_t type, const SColumnInfo *pCol1, const SColumnInfo *pCol2); /* Math functions */ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 674fbfa772..89016fd01f 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -166,6 +166,8 @@ int32_t taosGetErrSize(); #define TSDB_CODE_UNSUPPORT_OS TAOS_DEF_ERROR_CODE(0, 0x013A) #define TSDB_CODE_TIME_ERROR TAOS_DEF_ERROR_CODE(0, 0x013B) #define TSDB_CODE_INVALID_DISK_ID TAOS_DEF_ERROR_CODE(0, 0x013C) +#define TSDB_CODE_DECIMAL_OVERFLOW TAOS_DEF_ERROR_CODE(0, 0x013D) +#define TSDB_CODE_DIVISION_BY_ZERO TAOS_DEF_ERROR_CODE(0, 0x013E) //client #define TSDB_CODE_TSC_INVALID_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0200) diff --git a/include/util/tcompare.h b/include/util/tcompare.h index 09b35bbc8c..41bedc1a19 100644 --- a/include/util/tcompare.h +++ b/include/util/tcompare.h @@ -60,11 +60,13 @@ int32_t setChkInBytes1(const void *pLeft, const void *pRight); int32_t setChkInBytes2(const void *pLeft, const void *pRight); int32_t setChkInBytes4(const void *pLeft, const void *pRight); int32_t setChkInBytes8(const void *pLeft, const void *pRight); +int32_t setChkInDecimalHash(const void* pLeft, const void* pRight); int32_t setChkNotInBytes1(const void *pLeft, const void *pRight); int32_t setChkNotInBytes2(const void *pLeft, const void *pRight); int32_t setChkNotInBytes4(const void *pLeft, const void *pRight); int32_t setChkNotInBytes8(const void *pLeft, const void *pRight); +int32_t setChkNotInDecimalHash(const void* pLeft, const void* pRight); int32_t compareChkInString(const void *pLeft, const void *pRight); int32_t compareChkNotInString(const void *pLeft, const void *pRight); @@ -86,6 +88,12 @@ int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight); int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight); int32_t compareLenBinaryVal(const void *pLeft, const void *pRight); +int32_t compareDecimal64(const void* pleft, const void* pright); +int32_t compareDecimal128(const void* pleft, const void* pright); + +int32_t compareDecimal64SameScale(const void* pleft, const void* pright); +int32_t compareDecimal128SameScale(const void* pleft, const void* pright); + int32_t comparestrRegexMatch(const void *pLeft, const void *pRight); int32_t comparestrRegexNMatch(const void *pLeft, const void *pRight); diff --git a/include/util/tcompression.h b/include/util/tcompression.h index 140b7fe392..63740e1826 100644 --- a/include/util/tcompression.h +++ b/include/util/tcompression.h @@ -198,6 +198,14 @@ int32_t tsCompressBigint2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int3 int32_t nBuf); int32_t tsDecompressBigint2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg, void *pBuf, int32_t nBuf); +int32_t tsCompressDecimal64(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg, + void *pBuf, int32_t nBuf); +int32_t tsDecompressDecimal64(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg, + void *pBuf, int32_t nBuf); +int32_t tsCompressDecimal128(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg, + void *pBuf, int32_t nBuf); +int32_t tsDecompressDecimal128(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg, + void *pBuf, int32_t nBuf); /************************************************************************* * STREAM COMPRESSION diff --git a/include/util/tdef.h b/include/util/tdef.h index a77ad5dbf0..399804f6da 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -47,7 +47,7 @@ extern "C" { #define TD_VER_MAX UINT64_MAX // TODO: use the real max version from query handle // Bytes for each type. -extern const int32_t TYPE_BYTES[21]; +extern const int32_t TYPE_BYTES[22]; #define CHAR_BYTES sizeof(char) #define SHORT_BYTES sizeof(int16_t) @@ -60,6 +60,9 @@ extern const int32_t TYPE_BYTES[21]; #define TSDB_KEYSIZE sizeof(TSKEY) #define TSDB_NCHAR_SIZE sizeof(TdUcs4) +#define DECIMAL64_BYTES 8 +#define DECIMAL128_BYTES 16 + // NULL definition #define TSDB_DATA_BOOL_NULL 0x02 #define TSDB_DATA_TINYINT_NULL 0x80 @@ -694,6 +697,24 @@ typedef enum { #define MIN_RESERVE_MEM_SIZE 1024 // MB +// Decimal +#define TSDB_DECIMAL64_MAX_PRECISION 18 +#define TSDB_DECIMAL64_MAX_SCALE TSDB_DECIMAL64_MAX_PRECISION + +#define TSDB_DECIMAL128_MAX_PRECISION 38 +#define TSDB_DECIMAL128_MAX_SCALE TSDB_DECIMAL128_MAX_PRECISION + +#define TSDB_DECIMAL_MIN_PRECISION 1 +#define TSDB_DECIMAL_MAX_PRECISION TSDB_DECIMAL128_MAX_PRECISION +#define TSDB_DECIMAL_MIN_SCALE 0 +#define TSDB_DECIMAL_MAX_SCALE TSDB_DECIMAL_MAX_PRECISION +#define GET_DEICMAL_MAX_PRECISION(type) (type) == TSDB_DATA_TYPE_DECIMAL64 ? TSDB_DECIMAL64_MAX_PRECISION : TSDB_DECIMAL_MAX_SCALE + +typedef uint64_t DecimalWord; +#define DECIMAL_WORD_NUM(TYPE) (sizeof(TYPE) / sizeof(DecimalWord)) + +#define COMPILE_TIME_ASSERT(pred) switch(0) {case 0: case pred:;} + #ifdef __cplusplus } #endif diff --git a/include/util/tutil.h b/include/util/tutil.h index 09c53981ce..d216bd00a1 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -34,7 +34,7 @@ char **strsplit(char *src, const char *delim, int32_t *num); char *strtolower(char *dst, const char *src); char *strntolower(char *dst, const char *src, int32_t n); char *strntolower_s(char *dst, const char *src, int32_t n); -int64_t strnatoi(char *num, int32_t len); +int64_t strnatoi(const char *num, int32_t len); size_t tstrncspn(const char *str, size_t ssize, const char *reject, size_t rsize); size_t twcsncspn(const TdUcs4 *wcs, size_t size, const TdUcs4 *reject, size_t rsize); diff --git a/source/client/CMakeLists.txt b/source/client/CMakeLists.txt index 2519893a4b..1b38ecd7ad 100644 --- a/source/client/CMakeLists.txt +++ b/source/client/CMakeLists.txt @@ -28,7 +28,7 @@ endif() target_link_libraries( ${TAOS_LIB} INTERFACE api - PRIVATE os util common transport monitor nodes parser command planner catalog scheduler function qcom geometry ${TAOSD_MODULE} + PRIVATE os util common transport monitor nodes parser command planner catalog scheduler function qcom geometry ${TAOSD_MODULE} decimal ) if(TD_WINDOWS) @@ -67,7 +67,7 @@ target_include_directories( target_link_libraries( ${TAOS_LIB_STATIC} INTERFACE api - PRIVATE os util common transport monitor nodes parser command planner catalog scheduler function qcom geometry + PRIVATE os util common transport monitor nodes parser command planner catalog scheduler function qcom geometry decimal ) if(${BUILD_TEST}) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 8cffc6c056..1464c3580e 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -205,7 +205,7 @@ typedef struct SReqResultInfo { SExecResult execRes; const char* pRspMsg; const char* pData; - TAOS_FIELD* fields; // todo, column names are not needed. + TAOS_FIELD_E* fields; // todo, column names are not needed. TAOS_FIELD* userFields; // the fields info that return to user uint32_t numOfCols; int32_t* length; @@ -313,11 +313,11 @@ typedef struct SSyncQueryParam { void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4); void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4); -void doSetOneRowPtr(SReqResultInfo* pResultInfo); +void doSetOneRowPtr(SReqResultInfo* pResultInfo, bool isStmt); void setResPrecision(SReqResultInfo* pResInfo, int32_t precision); -int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4); -int32_t setResultDataPtr(SReqResultInfo* pResultInfo, bool convertUcs4); -int32_t setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols); +int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4, bool isStmt); +int32_t setResultDataPtr(SReqResultInfo* pResultInfo, bool convertUcs4, bool isStmt); +int32_t setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols, const SExtSchema* pExtSchema, bool isStmt); void doFreeReqResultInfo(SReqResultInfo* pResInfo); int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName, SArray** pReq); void syncCatalogFn(SMetaData* pResult, void* param, int32_t code); diff --git a/source/client/inc/clientStmt.h b/source/client/inc/clientStmt.h index 662ffc33ee..74260c42e0 100644 --- a/source/client/inc/clientStmt.h +++ b/source/client/inc/clientStmt.h @@ -48,7 +48,7 @@ typedef struct SStmtTableCache { } SStmtTableCache; typedef struct SStmtQueryResInfo { - TAOS_FIELD *fields; + TAOS_FIELD_E *fields; TAOS_FIELD *userFields; uint32_t numOfCols; int32_t precision; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 7f8d8f4193..128b98ffff 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -28,6 +28,8 @@ #include "tref.h" #include "tsched.h" #include "tversion.h" +#include "decimal.h" + static int32_t initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet); static int32_t buildConnectMsg(SRequestObj* pRequest, SMsgSendInfo** pMsgSendInfo); @@ -313,7 +315,7 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtC code = qParseSql(&cxt, pQuery); if (TSDB_CODE_SUCCESS == code) { if ((*pQuery)->haveResultSet) { - code = setResSchemaInfo(&pRequest->body.resInfo, (*pQuery)->pResSchema, (*pQuery)->numOfResCols); + code = setResSchemaInfo(&pRequest->body.resInfo, (*pQuery)->pResSchema, (*pQuery)->numOfResCols, (*pQuery)->pResExtSchema, pRequest->isStmtBind); setResPrecision(&pRequest->body.resInfo, (*pQuery)->precision); } } @@ -335,7 +337,7 @@ int32_t execLocalCmd(SRequestObj* pRequest, SQuery* pQuery) { int8_t biMode = atomic_load_8(&pRequest->pTscObj->biMode); int32_t code = qExecCommand(&pRequest->pTscObj->id, pRequest->pTscObj->sysInfo, pQuery->pRoot, &pRsp, biMode, pRequest->pTscObj->optionInfo.charsetCxt); if (TSDB_CODE_SUCCESS == code && NULL != pRsp) { - code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, pRequest->body.resInfo.convertUcs4); + code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, pRequest->body.resInfo.convertUcs4, pRequest->isStmtBind); } return code; @@ -373,7 +375,7 @@ void asyncExecLocalCmd(SRequestObj* pRequest, SQuery* pQuery) { int32_t code = qExecCommand(&pRequest->pTscObj->id, pRequest->pTscObj->sysInfo, pQuery->pRoot, &pRsp, atomic_load_8(&pRequest->pTscObj->biMode), pRequest->pTscObj->optionInfo.charsetCxt); if (TSDB_CODE_SUCCESS == code && NULL != pRsp) { - code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, pRequest->body.resInfo.convertUcs4); + code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, pRequest->body.resInfo.convertUcs4, pRequest->isStmtBind); } SReqResultInfo* pResultInfo = &pRequest->body.resInfo; @@ -515,7 +517,7 @@ int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArra return qCreateQueryPlan(&cxt, pPlan, pNodeList); } -int32_t setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols) { +int32_t setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols, const SExtSchema* pExtSchema, bool isStmt) { if (pResInfo == NULL || pSchema == NULL || numOfCols <= 0) { tscError("invalid paras, pResInfo == NULL || pSchema == NULL || numOfCols <= 0"); return TSDB_CODE_INVALID_PARA; @@ -528,7 +530,7 @@ int32_t setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32 if (pResInfo->userFields != NULL) { taosMemoryFree(pResInfo->userFields); } - pResInfo->fields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD)); + pResInfo->fields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD_E)); if (NULL == pResInfo->fields) return terrno; pResInfo->userFields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD)); if (NULL == pResInfo->userFields) { @@ -541,17 +543,14 @@ int32_t setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32 } for (int32_t i = 0; i < pResInfo->numOfCols; ++i) { - pResInfo->fields[i].bytes = pSchema[i].bytes; pResInfo->fields[i].type = pSchema[i].type; - pResInfo->userFields[i].bytes = pSchema[i].bytes; pResInfo->userFields[i].type = pSchema[i].type; - - if (pSchema[i].type == TSDB_DATA_TYPE_VARCHAR || pSchema[i].type == TSDB_DATA_TYPE_VARBINARY || - pSchema[i].type == TSDB_DATA_TYPE_GEOMETRY) { - pResInfo->userFields[i].bytes -= VARSTR_HEADER_SIZE; - } else if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR || pSchema[i].type == TSDB_DATA_TYPE_JSON) { - pResInfo->userFields[i].bytes = (pResInfo->userFields[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + // userFields must convert to type bytes, no matter isStmt or not + pResInfo->userFields[i].bytes = calcTypeBytesFromSchemaBytes(pSchema[i].type, pSchema[i].bytes, false); + pResInfo->fields[i].bytes = calcTypeBytesFromSchemaBytes(pSchema[i].type, pSchema[i].bytes, isStmt); + if (IS_DECIMAL_TYPE(pSchema[i].type) && pExtSchema) { + decimalFromTypeMod(pExtSchema[i].typeMod, &pResInfo->fields[i].precision, &pResInfo->fields[i].scale); } tstrncpy(pResInfo->fields[i].name, pSchema[i].name, tListLen(pResInfo->fields[i].name)); @@ -1942,15 +1941,15 @@ TAOS* taos_connect_auth(const char* ip, const char* user, const char* auth, cons // return taos_connect(ipStr, userStr, passStr, dbStr, port); // } -void doSetOneRowPtr(SReqResultInfo* pResultInfo) { +void doSetOneRowPtr(SReqResultInfo* pResultInfo, bool isStmt) { for (int32_t i = 0; i < pResultInfo->numOfCols; ++i) { SResultColumn* pCol = &pResultInfo->pCol[i]; int32_t type = pResultInfo->fields[i].type; - int32_t bytes = pResultInfo->fields[i].bytes; + int32_t schemaBytes = calcSchemaBytesFromTypeBytes(type, pResultInfo->fields[i].bytes, isStmt); if (IS_VAR_DATA_TYPE(type)) { - if (!IS_VAR_NULL_TYPE(type, bytes) && pCol->offset[pResultInfo->current] != -1) { + if (!IS_VAR_NULL_TYPE(type, schemaBytes) && pCol->offset[pResultInfo->current] != -1) { char* pStart = pResultInfo->pCol[i].offset[pResultInfo->current] + pResultInfo->pCol[i].pData; pResultInfo->length[i] = varDataLen(pStart); @@ -1961,8 +1960,8 @@ void doSetOneRowPtr(SReqResultInfo* pResultInfo) { } } else { if (!colDataIsNull_f(pCol->nullbitmap, pResultInfo->current)) { - pResultInfo->row[i] = pResultInfo->pCol[i].pData + bytes * pResultInfo->current; - pResultInfo->length[i] = bytes; + pResultInfo->row[i] = pResultInfo->pCol[i].pData + schemaBytes * pResultInfo->current; + pResultInfo->length[i] = schemaBytes; } else { pResultInfo->row[i] = NULL; pResultInfo->length[i] = 0; @@ -1994,7 +1993,7 @@ void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4) } pRequest->code = - setQueryResultFromRsp(&pRequest->body.resInfo, (const SRetrieveTableRsp*)pResInfo->pData, convertUcs4); + setQueryResultFromRsp(&pRequest->body.resInfo, (const SRetrieveTableRsp*)pResInfo->pData, convertUcs4, pRequest->isStmtBind); if (pRequest->code != TSDB_CODE_SUCCESS) { pResultInfo->numOfRows = 0; return NULL; @@ -2013,7 +2012,7 @@ void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4) } if (setupOneRowPtr) { - doSetOneRowPtr(pResultInfo); + doSetOneRowPtr(pResultInfo, pRequest->isStmtBind); pResultInfo->current += 1; } @@ -2060,7 +2059,7 @@ void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertU return NULL; } else { if (setupOneRowPtr) { - doSetOneRowPtr(pResultInfo); + doSetOneRowPtr(pResultInfo, pRequest->isStmtBind); pResultInfo->current += 1; } @@ -2087,14 +2086,14 @@ static int32_t doPrepareResPtr(SReqResultInfo* pResInfo) { return TSDB_CODE_SUCCESS; } -static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t* colLength) { +static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t* colLength, bool isStmt) { int32_t idx = -1; iconv_t conv = taosAcquireConv(&idx, C2M, pResultInfo->charsetCxt); if (conv == (iconv_t)-1) return TSDB_CODE_TSC_INTERNAL_ERROR; for (int32_t i = 0; i < pResultInfo->numOfCols; ++i) { int32_t type = pResultInfo->fields[i].type; - int32_t bytes = pResultInfo->fields[i].bytes; + int32_t schemaBytes = calcSchemaBytesFromTypeBytes(pResultInfo->fields[i].type, pResultInfo->fields[i].bytes, isStmt); if (type == TSDB_DATA_TYPE_NCHAR && colLength[i] > 0) { char* p = taosMemoryRealloc(pResultInfo->convertBuf[i], colLength[i]); @@ -2111,11 +2110,11 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t* colLength) { char* pStart = pCol->offset[j] + pCol->pData; int32_t len = taosUcs4ToMbsEx((TdUcs4*)varDataVal(pStart), varDataLen(pStart), varDataVal(p), conv); - if (len < 0 || len > bytes || (p + len) >= (pResultInfo->convertBuf[i] + colLength[i])) { + if (len < 0 || len > schemaBytes || (p + len) >= (pResultInfo->convertBuf[i] + colLength[i])) { tscError( "doConvertUCS4 error, invalid data. len:%d, bytes:%d, (p + len):%p, (pResultInfo->convertBuf[i] + " "colLength[i]):%p", - len, bytes, (p + len), (pResultInfo->convertBuf[i] + colLength[i])); + len, schemaBytes, (p + len), (pResultInfo->convertBuf[i] + colLength[i])); taosReleaseConv(idx, conv, C2M, pResultInfo->charsetCxt); return TSDB_CODE_TSC_INTERNAL_ERROR; } @@ -2134,6 +2133,36 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t* colLength) { return TSDB_CODE_SUCCESS; } +static int32_t convertDecimalType(SReqResultInfo* pResultInfo) { + for (int32_t i = 0; i < pResultInfo->numOfCols; ++i) { + TAOS_FIELD_E* pField = pResultInfo->fields + i; + int32_t type = pField->type; + int32_t bufLen = 0; + char* p = NULL; + if (!IS_DECIMAL_TYPE(type) || !pResultInfo->pCol[i].pData) { + continue; + } else { + bufLen = 64; + p = taosMemoryRealloc(pResultInfo->convertBuf[i], bufLen * pResultInfo->numOfRows); + pField->bytes = bufLen; + } + if (!p) return terrno; + pResultInfo->convertBuf[i] = p; + + for (int32_t j = 0; j < pResultInfo->numOfRows; ++j) { + int32_t code = decimalToStr((DecimalWord*)(pResultInfo->pCol[i].pData + j * tDataTypes[type].bytes), type, + pField->precision, pField->scale, p, bufLen); + p += bufLen; + if (TSDB_CODE_SUCCESS != code) { + return code; + } + } + pResultInfo->pCol[i].pData = pResultInfo->convertBuf[i]; + pResultInfo->row[i] = pResultInfo->pCol[i].pData; + } + return 0; +} + int32_t getVersion1BlockMetaSize(const char* p, int32_t numOfCols) { return sizeof(int32_t) + sizeof(int32_t) + sizeof(int32_t) * 3 + sizeof(uint64_t) + numOfCols * (sizeof(int8_t) + sizeof(int32_t)); @@ -2365,7 +2394,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo) { return TSDB_CODE_SUCCESS; } -int32_t setResultDataPtr(SReqResultInfo* pResultInfo, bool convertUcs4) { +int32_t setResultDataPtr(SReqResultInfo* pResultInfo, bool convertUcs4, bool isStmt) { if (pResultInfo == NULL || pResultInfo->numOfCols <= 0 || pResultInfo->fields == NULL) { tscError("setResultDataPtr paras error"); return TSDB_CODE_TSC_INTERNAL_ERROR; @@ -2423,6 +2452,10 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, bool convertUcs4) { int32_t bytes = *(int32_t*)p; p += sizeof(int32_t); + + if (IS_DECIMAL_TYPE(type) && pResultInfo->fields[i].precision == 0) { + extractDecimalTypeInfoFromBytes(&bytes, &pResultInfo->fields[i].precision, &pResultInfo->fields[i].scale); + } } int32_t* colLength = (int32_t*)p; @@ -2454,7 +2487,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, bool convertUcs4) { } pResultInfo->pCol[i].pData = pStart; - pResultInfo->length[i] = pResultInfo->fields[i].bytes; + pResultInfo->length[i] = calcSchemaBytesFromTypeBytes(pResultInfo->fields[i].type, pResultInfo->fields[i].bytes, isStmt); pResultInfo->row[i] = pResultInfo->pCol[i].pData; pStart += colLength[i]; @@ -2471,9 +2504,12 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, bool convertUcs4) { #ifndef DISALLOW_NCHAR_WITHOUT_ICONV if (convertUcs4) { - code = doConvertUCS4(pResultInfo, colLength); + code = doConvertUCS4(pResultInfo, colLength, isStmt); } #endif + if (TSDB_CODE_SUCCESS == code && convertUcs4) { + code = convertDecimalType(pResultInfo); + } return code; } @@ -2514,7 +2550,7 @@ void resetConnectDB(STscObj* pTscObj) { (void)taosThreadMutexUnlock(&pTscObj->mutex); } -int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4) { +int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4, bool isStmt) { if (pResultInfo == NULL || pRsp == NULL) { tscError("setQueryResultFromRsp paras is null"); return TSDB_CODE_TSC_INTERNAL_ERROR; @@ -2583,7 +2619,7 @@ int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableR // TODO handle the compressed case pResultInfo->totalRows += pResultInfo->numOfRows; - int32_t code = setResultDataPtr(pResultInfo, convertUcs4); + int32_t code = setResultDataPtr(pResultInfo, convertUcs4, isStmt); return code; } @@ -3046,7 +3082,7 @@ static void fetchCallback(void* pResult, void* param, int32_t code) { } pRequest->code = - setQueryResultFromRsp(pResultInfo, (const SRetrieveTableRsp*)pResultInfo->pData, pResultInfo->convertUcs4); + setQueryResultFromRsp(pResultInfo, (const SRetrieveTableRsp*)pResultInfo->pData, pResultInfo->convertUcs4, pRequest->isStmtBind); if (pRequest->code != TSDB_CODE_SUCCESS) { pResultInfo->numOfRows = 0; tscError("req:0x%" PRIx64 ", fetch results failed, code:%s, QID:0x%" PRIx64, pRequest->self, tstrerror(pRequest->code), diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 0873bd5619..dc67d1cf71 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -608,6 +608,14 @@ TAOS_RES *taos_query_with_reqid(TAOS *taos, const char *sql, int64_t reqid) { return taosQueryImplWithReqid(taos, sql, false, reqid); } +TAOS_FIELD_E *taos_fetch_fields_e(TAOS_RES *res) { + if (taos_num_fields(res) == 0 || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { + return NULL; + } + SReqResultInfo* pResInfo = tscGetCurResInfo(res); + return pResInfo->fields; +} + TAOS_ROW taos_fetch_row(TAOS_RES *res) { if (res == NULL) { return NULL; @@ -639,7 +647,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { } if (pResultInfo->current < pResultInfo->numOfRows) { - doSetOneRowPtr(pResultInfo); + doSetOneRowPtr(pResultInfo, false); pResultInfo->current += 1; return pResultInfo->row; } else { @@ -647,7 +655,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { return NULL; } - doSetOneRowPtr(pResultInfo); + doSetOneRowPtr(pResultInfo, false); pResultInfo->current += 1; return pResultInfo->row; } @@ -760,6 +768,14 @@ int taos_print_row_with_size(char *str, uint32_t size, TAOS_ROW row, TAOS_FIELD case TSDB_DATA_TYPE_BOOL: len += tsnprintf(str + len, size - len, "%d", *((int8_t *)row[i])); + break; + case TSDB_DATA_TYPE_DECIMAL64: + case TSDB_DATA_TYPE_DECIMAL: { + uint32_t decimalLen = strlen(row[i]); + uint32_t copyLen = TMIN(size - len - 1, decimalLen); + (void)memcpy(str + len, row[i], copyLen); + len += copyLen; + } break; default: break; } @@ -1269,7 +1285,7 @@ void handleQueryAnslyseRes(SSqlCallbackWrapper *pWrapper, SMetaData *pResultMeta } if (pQuery->haveResultSet) { - code = setResSchemaInfo(&pRequest->body.resInfo, pQuery->pResSchema, pQuery->numOfResCols); + code = setResSchemaInfo(&pRequest->body.resInfo, pQuery->pResSchema, pQuery->numOfResCols, pQuery->pResExtSchema, pRequest->isStmtBind); setResPrecision(&pRequest->body.resInfo, pQuery->precision); } } diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 7e47adedbe..f86f984e0c 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -682,7 +682,7 @@ int32_t processShowVariablesRsp(void* param, SDataBuf* pMsg, int32_t code) { code = buildShowVariablesRsp(rsp.variables, &pRes); } if (TSDB_CODE_SUCCESS == code) { - code = setQueryResultFromRsp(&pRequest->body.resInfo, pRes, false); + code = setQueryResultFromRsp(&pRequest->body.resInfo, pRes, false, pRequest->isStmtBind); } if (code != 0) { @@ -837,7 +837,7 @@ int32_t processCompactDbRsp(void* param, SDataBuf* pMsg, int32_t code) { code = buildRetriveTableRspForCompactDb(&rsp, &pRes); } if (TSDB_CODE_SUCCESS == code) { - code = setQueryResultFromRsp(&pRequest->body.resInfo, pRes, false); + code = setQueryResultFromRsp(&pRequest->body.resInfo, pRes, false, pRequest->isStmtBind); } if (code != 0) { diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index b43507c2a6..55f6f85347 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -516,7 +516,7 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) { RAW_NULL_CHECK(tvalue); } else { double val = 0; - GET_TYPED_DATA(val, double, pTagVal->type, &pTagVal->i64); + GET_TYPED_DATA(val, double, pTagVal->type, &pTagVal->i64, 0); // currently tag type can't be decimal, so pass 0 as typeMod tvalue = cJSON_CreateNumber(val); RAW_NULL_CHECK(tvalue); } @@ -1060,6 +1060,7 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, uint32_t metaLen) { SColCmpr* pCmp = &req.colCmpr.pColCmpr[i]; field.compress = pCmp->alg; } + if (req.pExtSchemas) field.typeMod = req.pExtSchemas[i].typeMod; RAW_NULL_CHECK(taosArrayPush(pReq.pColumns, &field)); } pReq.pTags = taosArrayInit(req.schemaTag.nCols, sizeof(SField)); @@ -1974,10 +1975,11 @@ static bool needRefreshMeta(void* rawData, STableMeta* pTableMeta, SSchemaWrappe int j = 0; for (; j < pTableMeta->tableInfo.numOfColumns; j++) { SSchema* pColSchema = &pTableMeta->schema[j]; + SSchemaExt* pColExtSchema = &pTableMeta->schemaExt[j]; char* fieldName = pSW->pSchema[i].name; if (strcmp(pColSchema->name, fieldName) == 0) { - if (checkSchema(pColSchema, fields, NULL, 0) != 0){ + if (checkSchema(pColSchema, pColExtSchema, fields, NULL, 0) != 0){ return true; } break; @@ -2714,4 +2716,4 @@ static int32_t tmqWriteBatchMetaDataImpl(TAOS* taos, void* meta, uint32_t metaLe end: tDeleteMqBatchMetaRsp(&rsp); return code; -} \ No newline at end of file +} diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 5187a76e82..fe9388db51 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -213,7 +213,7 @@ int32_t stmtBackupQueryFields(STscStmt* pStmt) { int32_t stmtRestoreQueryFields(STscStmt* pStmt) { SStmtQueryResInfo* pRes = &pStmt->sql.queryRes; - int32_t size = pRes->numOfCols * sizeof(TAOS_FIELD); + int32_t size = pRes->numOfCols * sizeof(TAOS_FIELD_E); pStmt->exec.pRequest->body.resInfo.numOfCols = pRes->numOfCols; pStmt->exec.pRequest->body.resInfo.precision = pRes->precision; @@ -1270,8 +1270,9 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) { if (pStmt->sql.pQuery->haveResultSet) { STMT_ERR_RET(setResSchemaInfo(&pStmt->exec.pRequest->body.resInfo, pStmt->sql.pQuery->pResSchema, - pStmt->sql.pQuery->numOfResCols)); + pStmt->sql.pQuery->numOfResCols, pStmt->sql.pQuery->pResExtSchema, true)); taosMemoryFreeClear(pStmt->sql.pQuery->pResSchema); + taosMemoryFreeClear(pStmt->sql.pQuery->pResExtSchema); setResPrecision(&pStmt->exec.pRequest->body.resInfo, pStmt->sql.pQuery->precision); } @@ -1861,7 +1862,7 @@ int stmtGetParam(TAOS_STMT* stmt, int idx, int* type, int* bytes) { } *type = pField[idx].type; - *bytes = pField[idx].bytes; + *bytes = calcSchemaBytesFromTypeBytes(pField[idx].type, pField[idx].bytes, true); _return: diff --git a/source/client/src/clientStmt2.c b/source/client/src/clientStmt2.c index ab5fd52ee3..c8b6abfbd1 100644 --- a/source/client/src/clientStmt2.c +++ b/source/client/src/clientStmt2.c @@ -1498,8 +1498,9 @@ int stmtBindBatch2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* bind, int32_t colIdx, SVCr if (pStmt->sql.pQuery->haveResultSet) { STMT_ERR_RET(setResSchemaInfo(&pStmt->exec.pRequest->body.resInfo, pStmt->sql.pQuery->pResSchema, - pStmt->sql.pQuery->numOfResCols)); + pStmt->sql.pQuery->numOfResCols, pStmt->sql.pQuery->pResExtSchema, true)); taosMemoryFreeClear(pStmt->sql.pQuery->pResSchema); + taosMemoryFreeClear(pStmt->sql.pQuery->pResExtSchema); setResPrecision(&pStmt->exec.pRequest->body.resInfo, pStmt->sql.pQuery->precision); } diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index ad1f7f028e..31bd9af3c6 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -3012,7 +3012,7 @@ int32_t tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4, SReqResultInfo** pRes doFreeReqResultInfo(&pRspObj->resInfo); SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(data->blockSchema, pRspObj->resIter); if (pSW) { - TAOS_CHECK_RETURN(setResSchemaInfo(&pRspObj->resInfo, pSW->pSchema, pSW->nCols)); + TAOS_CHECK_RETURN(setResSchemaInfo(&pRspObj->resInfo, pSW->pSchema, pSW->nCols, NULL, false)); } } @@ -3028,7 +3028,7 @@ int32_t tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4, SReqResultInfo** pRes pRspObj->resInfo.precision = precision; pRspObj->resInfo.totalRows += pRspObj->resInfo.numOfRows; - int32_t code = setResultDataPtr(&pRspObj->resInfo, convertUcs4); + int32_t code = setResultDataPtr(&pRspObj->resInfo, convertUcs4, false); if (code != 0) { return code; } diff --git a/source/common/src/msg/tmsg.c b/source/common/src/msg/tmsg.c index 25f15b0408..66d6bcfb1e 100644 --- a/source/common/src/msg/tmsg.c +++ b/source/common/src/msg/tmsg.c @@ -673,6 +673,7 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq TAOS_CHECK_EXIT(tEncodeI32(&encoder, pField->bytes)); TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pField->name)); TAOS_CHECK_EXIT(tEncodeU32(&encoder, pField->compress)); + TAOS_CHECK_EXIT(tEncodeI32(&encoder, pField->typeMod)); } for (int32_t i = 0; i < pReq->numOfTags; ++i) { @@ -760,6 +761,7 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR TAOS_CHECK_EXIT(tDecodeI32(&decoder, &field.bytes)); TAOS_CHECK_EXIT(tDecodeCStrTo(&decoder, field.name)); TAOS_CHECK_EXIT(tDecodeU32(&decoder, &field.compress)); + TAOS_CHECK_EXIT(tDecodeI32(&decoder, &field.typeMod)); if (taosArrayPush(pReq->pColumns, &field) == NULL) { TAOS_CHECK_EXIT(terrno); } @@ -922,6 +924,19 @@ int32_t tSerializeSMAlterStbReq(void *buf, int32_t bufLen, SMAlterStbReq *pReq) } TAOS_CHECK_EXIT(tEncodeI64(&encoder, pReq->keep)); ENCODESQL(); + if (pReq->alterType == TSDB_ALTER_TABLE_ADD_COLUMN || + pReq->alterType == TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION) { + if (taosArrayGetSize(pReq->pTypeMods) > 0) { + int8_t hasTypeMod = 1; + TAOS_CHECK_EXIT(tEncodeI8(&encoder, hasTypeMod)); + for (int32_t i = 0; i < pReq->pTypeMods->size; ++i) { + const STypeMod *pTypeMod = taosArrayGet(pReq->pTypeMods, i); + TAOS_CHECK_ERRNO(tEncodeI32(&encoder, *pTypeMod)); + } + } else { + TAOS_CHECK_EXIT(tEncodeI8(&encoder, 0)); + } + } tEndEncode(&encoder); _exit: @@ -987,7 +1002,24 @@ int32_t tDeserializeSMAlterStbReq(void *buf, int32_t bufLen, SMAlterStbReq *pReq TAOS_CHECK_EXIT(tDecodeI64(&decoder, &pReq->keep)); } DECODESQL(); - + if (!tDecodeIsEnd(&decoder) && (pReq->alterType == TSDB_ALTER_TABLE_ADD_COLUMN || + pReq->alterType == TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION)) { + int8_t hasTypeMod = 0; + TAOS_CHECK_EXIT(tDecodeI8(&decoder, &hasTypeMod)); + if (hasTypeMod == 1) { + pReq->pTypeMods = taosArrayInit(pReq->numOfFields, sizeof(STypeMod)); + if (!pReq->pTypeMods) { + TAOS_CHECK_EXIT(terrno); + } + for (int32_t i = 0; i < pReq->numOfFields; ++i) { + STypeMod typeMod = 0; + TAOS_CHECK_EXIT(tDecodeI32(&decoder, &typeMod)); + if (taosArrayPush(pReq->pTypeMods, &typeMod) == NULL) { + TAOS_CHECK_EXIT(terrno); + } + } + } + } tEndDecode(&decoder); _exit: @@ -1000,6 +1032,7 @@ void tFreeSMAltertbReq(SMAlterStbReq *pReq) { pReq->pFields = NULL; taosMemoryFreeClear(pReq->comment); FREESQL(); + taosArrayDestroy(pReq->pTypeMods); } int32_t tSerializeSEpSet(void *buf, int32_t bufLen, const SEpSet *pEpset) { @@ -3914,7 +3947,7 @@ int32_t tSerializeSTableCfgRsp(void *buf, int32_t bufLen, STableCfgRsp *pRsp) { TAOS_CHECK_EXIT(tEncodeI32(&encoder, pRsp->tagsLen)); TAOS_CHECK_EXIT(tEncodeBinary(&encoder, pRsp->pTags, pRsp->tagsLen)); - if (useCompress(pRsp->tableType)) { + if (withExtSchema(pRsp->tableType)) { for (int32_t i = 0; i < pRsp->numOfColumns; ++i) { SSchemaExt *pSchemaExt = &pRsp->pSchemaExt[i]; TAOS_CHECK_EXIT(tEncodeSSchemaExt(&encoder, pSchemaExt)); @@ -3990,7 +4023,7 @@ int32_t tDeserializeSTableCfgRsp(void *buf, int32_t bufLen, STableCfgRsp *pRsp) TAOS_CHECK_EXIT(tDecodeBinaryAlloc(&decoder, (void **)&pRsp->pTags, NULL)); if (!tDecodeIsEnd(&decoder)) { - if (useCompress(pRsp->tableType) && pRsp->numOfColumns > 0) { + if (withExtSchema(pRsp->tableType) && pRsp->numOfColumns > 0) { pRsp->pSchemaExt = taosMemoryMalloc(sizeof(SSchemaExt) * pRsp->numOfColumns); if (pRsp->pSchemaExt == NULL) { TAOS_CHECK_EXIT(terrno); @@ -6085,7 +6118,7 @@ static int32_t tEncodeSTableMetaRsp(SEncoder *pEncoder, STableMetaRsp *pRsp) { TAOS_CHECK_RETURN(tEncodeSSchema(pEncoder, pSchema)); } - if (useCompress(pRsp->tableType)) { + if (withExtSchema(pRsp->tableType)) { for (int32_t i = 0; i < pRsp->numOfColumns; ++i) { SSchemaExt *pSchemaExt = &pRsp->pSchemaExt[i]; TAOS_CHECK_RETURN(tEncodeSSchemaExt(pEncoder, pSchemaExt)); @@ -6126,7 +6159,7 @@ static int32_t tDecodeSTableMetaRsp(SDecoder *pDecoder, STableMetaRsp *pRsp) { } if (!tDecodeIsEnd(pDecoder)) { - if (useCompress(pRsp->tableType) && pRsp->numOfColumns > 0) { + if (withExtSchema(pRsp->tableType) && pRsp->numOfColumns > 0) { pRsp->pSchemaExt = taosMemoryMalloc(sizeof(SSchemaExt) * pRsp->numOfColumns); if (pRsp->pSchemaExt == NULL) { TAOS_CHECK_RETURN(terrno); @@ -10124,7 +10157,13 @@ int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateS SFieldWithOptions *pField = taosArrayGet(pReq->pCols, i); TAOS_CHECK_EXIT(tEncodeI8(&encoder, pField->type)); TAOS_CHECK_EXIT(tEncodeI8(&encoder, pField->flags)); - TAOS_CHECK_EXIT(tEncodeI32(&encoder, pField->bytes)); + int32_t bytes = pField->bytes; + if (IS_DECIMAL_TYPE(pField->type)) { + uint8_t prec = 0, scale = 0; + extractTypeFromTypeMod(pField->type, pField->typeMod, &prec, &scale, NULL); + fillBytesForDecimalType(&bytes, pField->type, prec, scale); + } + TAOS_CHECK_EXIT(tEncodeI32(&encoder, bytes)); TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pField->name)); } @@ -10437,6 +10476,44 @@ _exit: return code; } +static int32_t tEncodeSExtSchema(SEncoder* pCoder, const SExtSchema* pExtSchema) { + int32_t code = 0, lino; + TAOS_CHECK_EXIT(tEncodeI32v(pCoder, pExtSchema->typeMod)); + +_exit: + return code; +} + +int32_t tDecodeSExtSchema(SDecoder* pCoder, SExtSchema* pExtSchema) { + int32_t code = 0, lino; + TAOS_CHECK_EXIT(tDecodeI32v(pCoder, &pExtSchema->typeMod)); + +_exit: + return code; +} + +static int32_t tEncodeSExtSchemas(SEncoder* pCoder, const SExtSchema* pExtSchemas, int32_t nCol) { + int32_t code = 0, lino; + for (int32_t i = 0; i < nCol; ++i) { + TAOS_CHECK_EXIT(tEncodeSExtSchema(pCoder, pExtSchemas + i)); + } + +_exit: + return code; +} + +static int32_t tDecodeSExtSchemas(SDecoder* pCoder, SExtSchema** ppExtSchema, int32_t nCol) { + int32_t code = 0, lino; + *ppExtSchema = tDecoderMalloc(pCoder, sizeof(SExtSchema) * nCol); + if (!*ppExtSchema) TAOS_CHECK_EXIT(terrno); + for (int32_t i = 0; i < nCol; ++i) { + TAOS_CHECK_EXIT(tDecodeSExtSchema(pCoder, (*ppExtSchema) + i)); + } + +_exit: + return code; +} + int tEncodeSVCreateStbReq(SEncoder *pCoder, const SVCreateStbReq *pReq) { int32_t code = 0; int32_t lino; @@ -10461,6 +10538,12 @@ int tEncodeSVCreateStbReq(SEncoder *pCoder, const SVCreateStbReq *pReq) { TAOS_CHECK_EXIT(tEncodeI8(pCoder, pReq->colCmpred)); TAOS_CHECK_EXIT(tEncodeSColCmprWrapper(pCoder, &pReq->colCmpr)); TAOS_CHECK_EXIT(tEncodeI64(pCoder, pReq->keep)); + if (pReq->pExtSchemas) { + TAOS_CHECK_EXIT(tEncodeI8(pCoder, 1)); + TAOS_CHECK_EXIT(tEncodeSExtSchemas(pCoder, pReq->pExtSchemas, pReq->schemaRow.nCols)); + } else { + TAOS_CHECK_EXIT(tEncodeI8(pCoder, 0)); + } tEndEncode(pCoder); _exit: @@ -10498,6 +10581,13 @@ int tDecodeSVCreateStbReq(SDecoder *pCoder, SVCreateStbReq *pReq) { if (!tDecodeIsEnd(pCoder)) { TAOS_CHECK_EXIT(tDecodeI64(pCoder, &pReq->keep)); } + if (!tDecodeIsEnd(pCoder)) { + int8_t hasExtSchema = 0; + TAOS_CHECK_EXIT(tDecodeI8(pCoder, &hasExtSchema)); + if (hasExtSchema) { + TAOS_CHECK_EXIT(tDecodeSExtSchemas(pCoder, &pReq->pExtSchemas, pReq->schemaRow.nCols)); + } + } } tEndDecode(pCoder); @@ -10547,6 +10637,12 @@ int tEncodeSVCreateTbReq(SEncoder *pCoder, const SVCreateTbReq *pReq) { // Encode Column Options: encode compress level if (pReq->type == TSDB_SUPER_TABLE || pReq->type == TSDB_NORMAL_TABLE) { TAOS_CHECK_EXIT(tEncodeSColCmprWrapper(pCoder, &pReq->colCmpr)); + if (pReq->pExtSchemas) { + TAOS_CHECK_EXIT(tEncodeI8(pCoder, 1)); + TAOS_CHECK_EXIT(tEncodeSExtSchemas(pCoder, pReq->pExtSchemas, pReq->ntb.schemaRow.nCols)); + } else { + TAOS_CHECK_EXIT(tEncodeI8(pCoder, 0)); + } } tEndEncode(pCoder); @@ -10611,6 +10707,14 @@ int tDecodeSVCreateTbReq(SDecoder *pCoder, SVCreateTbReq *pReq) { if (!tDecodeIsEnd(pCoder)) { TAOS_CHECK_EXIT(tDecodeSColCmprWrapperEx(pCoder, &pReq->colCmpr)); } + + if (!tDecodeIsEnd(pCoder)) { + int8_t hasExtSchema = 0; + TAOS_CHECK_EXIT(tDecodeI8(pCoder, &hasExtSchema)); + if (hasExtSchema) { + TAOS_CHECK_EXIT(tDecodeSExtSchemas(pCoder, &pReq->pExtSchemas, pReq->ntb.schemaRow.nCols)); + } + } } tEndDecode(pCoder); @@ -10968,6 +11072,9 @@ int32_t tEncodeSVAlterTbReq(SEncoder *pEncoder, const SVAlterTbReq *pReq) { } TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->ctimeMs)); TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pReq->source)); + if (pReq->action == TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION || pReq->action == TSDB_ALTER_TABLE_ADD_COLUMN) { + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->typeMod)); + } tEndEncode(pEncoder); _exit: @@ -11071,6 +11178,11 @@ int32_t tDecodeSVAlterTbReq(SDecoder *pDecoder, SVAlterTbReq *pReq) { if (!tDecodeIsEnd(pDecoder)) { TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pReq->source)); } + if (pReq->action == TSDB_ALTER_TABLE_ADD_COLUMN || pReq->action == TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION) { + if (!tDecodeIsEnd(pDecoder)) { + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->typeMod)); + } + } tEndDecode(pDecoder); _exit: @@ -11250,7 +11362,7 @@ int32_t tEncodeSTqOffsetVal(SEncoder *pEncoder, const STqOffsetVal *pOffsetVal) if (IS_VAR_DATA_TYPE(pOffsetVal->primaryKey.type)) { TAOS_CHECK_EXIT(tEncodeBinary(pEncoder, pOffsetVal->primaryKey.pData, pOffsetVal->primaryKey.nData)); } else { - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pOffsetVal->primaryKey.val)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, VALUE_GET_TRIVIAL_DATUM(&pOffsetVal->primaryKey))); } } else if (pOffsetVal->type == TMQ_OFFSET__LOG) { @@ -11281,7 +11393,7 @@ int32_t tDecodeSTqOffsetVal(SDecoder *pDecoder, STqOffsetVal *pOffsetVal) { TAOS_CHECK_EXIT( tDecodeBinaryAlloc32(pDecoder, (void **)&pOffsetVal->primaryKey.pData, &pOffsetVal->primaryKey.nData)); } else { - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pOffsetVal->primaryKey.val)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &VALUE_GET_TRIVIAL_DATUM(&pOffsetVal->primaryKey))); } } } else if (pOffsetVal->type == TMQ_OFFSET__LOG) { @@ -11312,7 +11424,7 @@ void tFormatOffset(char *buf, int32_t maxLen, const STqOffsetVal *pVal) { taosMemoryFree(tmp); } else { (void)snprintf(buf, maxLen, "tsdb:%" PRId64 "|%" PRId64 ",pk type:%d,val:%" PRId64, pVal->uid, pVal->ts, - pVal->primaryKey.type, pVal->primaryKey.val); + pVal->primaryKey.type, VALUE_GET_TRIVIAL_DATUM(&pVal->primaryKey)); } } } @@ -11694,7 +11806,7 @@ void tDeleteMqDataRsp(SMqDataRsp *rsp) { tDeleteMqDataRspCommon(rsp); } int32_t tEncodeSTaosxRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) { int32_t code = 0; - int32_t lino; + int32_t lino = 0; TAOS_CHECK_EXIT(tEncodeMqDataRspCommon(pEncoder, pRsp)); TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pRsp->createTableNum)); @@ -11736,7 +11848,6 @@ int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) { } } } - _exit: return code; } @@ -13403,3 +13514,7 @@ void tDeleteMqBatchMetaRsp(SMqBatchMetaRsp *pRsp) { pRsp->batchMetaReq = NULL; pRsp->batchMetaLen = NULL; } + +bool hasExtSchema(const SExtSchema *pExtSchema) { + return pExtSchema->typeMod != 0; +} \ No newline at end of file diff --git a/source/common/src/tcol.c b/source/common/src/tcol.c index 32e0109b2b..2903ef5a3a 100644 --- a/source/common/src/tcol.c +++ b/source/common/src/tcol.c @@ -62,8 +62,9 @@ uint8_t getDefaultEncode(uint8_t type) { return TSDB_COLVAL_ENCODE_DISABLED; case TSDB_DATA_TYPE_VARBINARY: return TSDB_COLVAL_ENCODE_DISABLED; + case TSDB_DATA_TYPE_DECIMAL64: case TSDB_DATA_TYPE_DECIMAL: - return TSDB_COLVAL_ENCODE_DELTAD; + return TSDB_COLVAL_ENCODE_DISABLED; case TSDB_DATA_TYPE_BLOB: return TSDB_COLVAL_ENCODE_SIMPLE8B; case TSDB_DATA_TYPE_MEDIUMBLOB: @@ -110,8 +111,9 @@ uint16_t getDefaultCompress(uint8_t type) { return TSDB_COLVAL_COMPRESS_LZ4; case TSDB_DATA_TYPE_VARBINARY: return TSDB_COLVAL_COMPRESS_ZSTD; + case TSDB_DATA_TYPE_DECIMAL64: case TSDB_DATA_TYPE_DECIMAL: - return TSDB_COLVAL_COMPRESS_LZ4; + return TSDB_COLVAL_COMPRESS_ZSTD; case TSDB_DATA_TYPE_BLOB: return TSDB_COLVAL_COMPRESS_LZ4; case TSDB_DATA_TYPE_MEDIUMBLOB: @@ -348,7 +350,7 @@ int32_t setColCompressByOption(uint8_t type, uint8_t encode, uint16_t compressTy return TSDB_CODE_SUCCESS; } -bool useCompress(uint8_t tableType) { +bool withExtSchema(uint8_t tableType) { return TSDB_SUPER_TABLE == tableType || TSDB_NORMAL_TABLE == tableType || TSDB_CHILD_TABLE == tableType; } @@ -413,6 +415,8 @@ int8_t validColEncode(uint8_t type, uint8_t l1) { return TSDB_COLVAL_ENCODE_SIMPLE8B == l1 || TSDB_COLVAL_ENCODE_XOR == l1 ? 1 : 0; } else if (type == TSDB_DATA_TYPE_GEOMETRY) { return 1; + } else if (type == TSDB_DATA_TYPE_DECIMAL64 || type == TSDB_DATA_TYPE_DECIMAL) { + return l1 == TSDB_COLVAL_ENCODE_DISABLED ? 1 : 0; } return 0; } diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 7b995b62bf..e7ccd2289f 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -725,10 +725,13 @@ int32_t blockDataUpdatePkRange(SSDataBlock* pDataBlock, int32_t pkColumnIndex, b void* skey = colDataGetData(pColInfoData, 0); void* ekey = colDataGetData(pColInfoData, (pInfo->rows - 1)); + int64_t val = 0; if (asc) { if (IS_NUMERIC_TYPE(pColInfoData->info.type)) { - GET_TYPED_DATA(pInfo->pks[0].val, int64_t, pColInfoData->info.type, skey); - GET_TYPED_DATA(pInfo->pks[1].val, int64_t, pColInfoData->info.type, ekey); + GET_TYPED_DATA(val, int64_t, pColInfoData->info.type, skey, typeGetTypeModFromColInfo(&pColInfoData->info)); + VALUE_SET_TRIVIAL_DATUM(&pInfo->pks[0], val); + GET_TYPED_DATA(val, int64_t, pColInfoData->info.type, ekey, typeGetTypeModFromColInfo(&pColInfoData->info)); + VALUE_SET_TRIVIAL_DATUM(&pInfo->pks[1], val); } else { // todo refactor memcpy(pInfo->pks[0].pData, varDataVal(skey), varDataLen(skey)); pInfo->pks[0].nData = varDataLen(skey); @@ -738,8 +741,10 @@ int32_t blockDataUpdatePkRange(SSDataBlock* pDataBlock, int32_t pkColumnIndex, b } } else { if (IS_NUMERIC_TYPE(pColInfoData->info.type)) { - GET_TYPED_DATA(pInfo->pks[0].val, int64_t, pColInfoData->info.type, ekey); - GET_TYPED_DATA(pInfo->pks[1].val, int64_t, pColInfoData->info.type, skey); + GET_TYPED_DATA(val, int64_t, pColInfoData->info.type, ekey, typeGetTypeModFromColInfo(&pColInfoData->info)); + VALUE_SET_TRIVIAL_DATUM(&pInfo->pks[0], val); + GET_TYPED_DATA(val, int64_t, pColInfoData->info.type, skey, typeGetTypeModFromColInfo(&pColInfoData->info)); + VALUE_SET_TRIVIAL_DATUM(&pInfo->pks[1], val); } else { // todo refactor memcpy(pInfo->pks[0].pData, varDataVal(ekey), varDataLen(ekey)); pInfo->pks[0].nData = varDataLen(ekey); @@ -2803,7 +2808,9 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq2** ppReq, const SSDataBlock* pDat terrno = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; return terrno; } - SColVal cv = COL_VAL_VALUE(pCol->colId, ((SValue){.type = pCol->type, .val = *(TSKEY*)var})); + SValue val = {.type = pCol->type}; + VALUE_SET_TRIVIAL_DATUM(&val, *(TSKEY*)var); + SColVal cv = COL_VAL_VALUE(pCol->colId, val); void* px = taosArrayPush(pVals, &cv); if (px == NULL) { return terrno; @@ -2816,7 +2823,9 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq2** ppReq, const SSDataBlock* pDat return terrno; } } else { - SColVal cv = COL_VAL_VALUE(pCol->colId, ((SValue){.type = pCol->type, .val = *(int64_t*)var})); + SValue val = {.type = pCol->type}; + VALUE_SET_TRIVIAL_DATUM(&val, *(int64_t*)var); + SColVal cv = COL_VAL_VALUE(pCol->colId, val); void* px = taosArrayPush(pVals, &cv); if (px == NULL) { return terrno; @@ -2869,31 +2878,31 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq2** ppReq, const SSDataBlock* pDat } else { SValue sv = {.type = pCol->type}; if (pCol->type == pColInfoData->info.type) { - memcpy(&sv.val, var, tDataTypes[pCol->type].bytes); + valueSetDatum(&sv, sv.type, var, tDataTypes[pCol->type].bytes); } else { /** * 1. sum/avg would convert to int64_t/uint64_t/double during aggregation * 2. below conversion may lead to overflow or loss, the app should select the right data type. */ - char tv[8] = {0}; + char tv[DATUM_MAX_SIZE] = {0}; if (pColInfoData->info.type == TSDB_DATA_TYPE_FLOAT) { float v = 0; - GET_TYPED_DATA(v, float, pColInfoData->info.type, var); + GET_TYPED_DATA(v, float, pColInfoData->info.type, var, typeGetTypeModFromColInfo(&pColInfoData->info)); SET_TYPED_DATA(&tv, pCol->type, v); } else if (pColInfoData->info.type == TSDB_DATA_TYPE_DOUBLE) { double v = 0; - GET_TYPED_DATA(v, double, pColInfoData->info.type, var); + GET_TYPED_DATA(v, double, pColInfoData->info.type, var, typeGetTypeModFromColInfo(&pColInfoData->info)); SET_TYPED_DATA(&tv, pCol->type, v); } else if (IS_SIGNED_NUMERIC_TYPE(pColInfoData->info.type)) { int64_t v = 0; - GET_TYPED_DATA(v, int64_t, pColInfoData->info.type, var); + GET_TYPED_DATA(v, int64_t, pColInfoData->info.type, var, typeGetTypeModFromColInfo(&pColInfoData->info)); SET_TYPED_DATA(&tv, pCol->type, v); } else { uint64_t v = 0; - GET_TYPED_DATA(v, uint64_t, pColInfoData->info.type, var); + GET_TYPED_DATA(v, uint64_t, pColInfoData->info.type, var, typeGetTypeModFromColInfo(&pColInfoData->info)); SET_TYPED_DATA(&tv, pCol->type, v); } - memcpy(&sv.val, tv, tDataTypes[pCol->type].bytes); + valueSetDatum(&sv, sv.type, tv, tDataTypes[pCol->type].bytes); } SColVal cv = COL_VAL_VALUE(pCol->colId, sv); void* px = taosArrayPush(pVals, &cv); @@ -3187,7 +3196,12 @@ int32_t blockEncode(const SSDataBlock* pBlock, char* data, size_t dataBuflen, in *((int8_t*)data) = pColInfoData->info.type; data += sizeof(int8_t); - *((int32_t*)data) = pColInfoData->info.bytes; + int32_t bytes = pColInfoData->info.bytes; + *((int32_t*)data) = bytes; + if (IS_DECIMAL_TYPE(pColInfoData->info.type)) { + fillBytesForDecimalType((int32_t*)data, pColInfoData->info.type, pColInfoData->info.precision, + pColInfoData->info.scale); + } data += sizeof(int32_t); } @@ -3328,6 +3342,10 @@ int32_t blockDecode(SSDataBlock* pBlock, const char* pData, const char** pEndPos pStart += sizeof(int8_t); pColInfoData->info.bytes = *(int32_t*)pStart; + if (IS_DECIMAL_TYPE(pColInfoData->info.type)) { + extractDecimalTypeInfoFromBytes(&pColInfoData->info.bytes, &pColInfoData->info.precision, + &pColInfoData->info.scale); + } pStart += sizeof(int32_t); if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { @@ -3574,6 +3592,22 @@ int32_t trimDataBlock(SSDataBlock* pBlock, int32_t totalRows, const bool* pBoolL j += 1; } break; + case TSDB_DATA_TYPE_DECIMAL64: + case TSDB_DATA_TYPE_DECIMAL: + while (j < totalRows) { + if (pBoolList[j] == 0) { + j += 1; + continue; + } + if (colDataIsNull_f(pBitmap, j)) { + colDataSetNull_f(pDst->nullbitmap, numOfRows); + } else { + memcpy(pDst->pData + numOfRows * pDst->info.bytes, pDst->pData + j * pDst->info.bytes, pDst->info.bytes); + } + numOfRows += 1; + j += 1; + } + break; } } @@ -3700,12 +3734,14 @@ int32_t blockDataCheck(const SSDataBlock* pDataBlock) { } else { if (TSDB_DATA_TYPE_FLOAT == pCol->info.type) { float v = 0; - GET_TYPED_DATA(v, float, pCol->info.type, colDataGetNumData(pCol, r)); + GET_TYPED_DATA(v, float, pCol->info.type, colDataGetNumData(pCol, r), typeGetTypeModFromColInfo(&pCol->info)); } else if (TSDB_DATA_TYPE_DOUBLE == pCol->info.type) { double v = 0; - GET_TYPED_DATA(v, double, pCol->info.type, colDataGetNumData(pCol, r)); + GET_TYPED_DATA(v, double, pCol->info.type, colDataGetNumData(pCol, r), typeGetTypeModFromColInfo(&pCol->info)); + } else if (IS_DECIMAL_TYPE(pCol->info.type)) { + // SKIP for decimal types } else { - GET_TYPED_DATA(typeValue, int64_t, pCol->info.type, colDataGetNumData(pCol, r)); + GET_TYPED_DATA(typeValue, int64_t, pCol->info.type, colDataGetNumData(pCol, r), typeGetTypeModFromColInfo(&pCol->info)); } } } diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 32b8ac3074..ada69fd350 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -18,6 +18,7 @@ #include "tRealloc.h" #include "tdatablock.h" #include "tlog.h" +#include "decimal.h" static int32_t (*tColDataAppendValueImpl[8][3])(SColData *pColData, uint8_t *pData, uint32_t nData); static int32_t (*tColDataUpdateValueImpl[8][3])(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward); @@ -277,7 +278,7 @@ static int32_t tRowBuildTupleRow(SArray *aColVal, const SRowBuildScanInfo *sinfo (*ppRow)->numOfPKs = sinfo->numOfPKs; (*ppRow)->sver = schema->version; (*ppRow)->len = sinfo->tupleRowSize; - (*ppRow)->ts = colValArray[0].value.val; + (*ppRow)->ts = VALUE_GET_TRIVIAL_DATUM(&colValArray[0].value); if (sinfo->tupleFlag == HAS_NONE || sinfo->tupleFlag == HAS_NULL) { return 0; @@ -315,7 +316,8 @@ static int32_t tRowBuildTupleRow(SArray *aColVal, const SRowBuildScanInfo *sinfo varlen += colValArray[colValIndex].value.nData; } } else { - (void)memcpy(fixed + schema->columns[i].offset, &colValArray[colValIndex].value.val, + (void)memcpy(fixed + schema->columns[i].offset, + VALUE_GET_DATUM(&colValArray[colValIndex].value, schema->columns[i].type), tDataTypes[schema->columns[i].type].bytes); } } else if (COL_VAL_IS_NULL(&colValArray[colValIndex])) { // NULL @@ -360,7 +362,7 @@ static int32_t tRowBuildKVRow(SArray *aColVal, const SRowBuildScanInfo *sinfo, c (*ppRow)->numOfPKs = sinfo->numOfPKs; (*ppRow)->sver = schema->version; (*ppRow)->len = sinfo->kvRowSize; - (*ppRow)->ts = colValArray[0].value.val; + (*ppRow)->ts = VALUE_GET_TRIVIAL_DATUM(&colValArray[0].value); if (!(sinfo->flag != HAS_NONE && sinfo->flag != HAS_NULL)) { return TSDB_CODE_INVALID_PARA; @@ -397,7 +399,7 @@ static int32_t tRowBuildKVRow(SArray *aColVal, const SRowBuildScanInfo *sinfo, c payloadSize += colValArray[colValIndex].value.nData; } else { payloadSize += tPutI16v(payload + payloadSize, colValArray[colValIndex].cid); - (void)memcpy(payload + payloadSize, &colValArray[colValIndex].value.val, + (void)memcpy(payload + payloadSize, VALUE_GET_DATUM(&colValArray[colValIndex].value, schema->columns[i].type), tDataTypes[schema->columns[i].type].bytes); payloadSize += tDataTypes[schema->columns[i].type].bytes; } @@ -490,8 +492,9 @@ int32_t tRowBuildFromBind(SBindInfo *infos, int32_t numOfInfos, bool infoSorted, } value.pData = (uint8_t *)infos[iInfo].bind->buffer + infos[iInfo].bind->buffer_length * iRow; } else { - (void)memcpy(&value.val, (uint8_t *)infos[iInfo].bind->buffer + infos[iInfo].bind->buffer_length * iRow, - infos[iInfo].bind->buffer_length); + valueSetDatum(&value, infos[iInfo].type, + (uint8_t *)infos[iInfo].bind->buffer + infos[iInfo].bind->buffer_length * iRow, + infos[iInfo].bind->buffer_length); } colVal = COL_VAL_VALUE(infos[iInfo].columnId, value); } @@ -543,7 +546,7 @@ int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) pColVal->cid = pTColumn->colId; pColVal->value.type = pTColumn->type; pColVal->flag = CV_FLAG_VALUE; - (void)memcpy(&pColVal->value.val, &pRow->ts, sizeof(TSKEY)); + VALUE_SET_TRIVIAL_DATUM(&pColVal->value, pRow->ts); return 0; } @@ -607,7 +610,7 @@ int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) pColVal->value.pData = NULL; } } else { - (void)memcpy(&pColVal->value.val, pData, pTColumn->bytes); + valueSetDatum(&pColVal->value, pTColumn->type, pData, pTColumn->bytes); } } return 0; @@ -658,7 +661,7 @@ int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) pColVal->value.pData = varlen + *(int32_t *)(fixed + pTColumn->offset); pColVal->value.pData += tGetU32v(pColVal->value.pData, &pColVal->value.nData); } else { - (void)memcpy(&pColVal->value.val, fixed + pTColumn->offset, TYPE_BYTES[pTColumn->type]); + valueSetDatum(&pColVal->value, pTColumn->type, fixed + pTColumn->offset, TYPE_BYTES[pTColumn->type]); } } @@ -902,7 +905,7 @@ SColVal *tRowIterNext(SRowIter *pIter) { pIter->cv.cid = pTColumn->colId; pIter->cv.value.type = pTColumn->type; pIter->cv.flag = CV_FLAG_VALUE; - (void)memcpy(&pIter->cv.value.val, &pIter->pRow->ts, sizeof(TSKEY)); + VALUE_SET_TRIVIAL_DATUM(&pIter->cv.value, pIter->pRow->ts); goto _exit; } @@ -947,7 +950,7 @@ SColVal *tRowIterNext(SRowIter *pIter) { pIter->cv.value.pData = NULL; } } else { - (void)memcpy(&pIter->cv.value.val, pData, pTColumn->bytes); + valueSetDatum(&pIter->cv.value, pTColumn->type, pData, pTColumn->bytes); } } @@ -1006,7 +1009,7 @@ SColVal *tRowIterNext(SRowIter *pIter) { pIter->cv.value.pData = NULL; } } else { - (void)memcpy(&pIter->cv.value.val, pIter->pf + pTColumn->offset, TYPE_BYTES[pTColumn->type]); + valueSetDatum(&pIter->cv.value, pTColumn->type, pIter->pf + pTColumn->offset, TYPE_BYTES[pTColumn->type]); } goto _exit; } @@ -1326,7 +1329,7 @@ void tRowGetPrimaryKey(SRow *row, SRowKey *key) { key->pks[i].pData = tdata; key->pks[i].pData += tGetU32v(key->pks[i].pData, &key->pks[i].nData); } else { - (void)memcpy(&key->pks[i].val, tdata, tDataTypes[indices[i].type].bytes); + valueSetDatum(key->pks + i, indices[i].type, tdata, tDataTypes[indices[i].type].bytes); } } } @@ -1346,26 +1349,26 @@ int32_t tValueCompare(const SValue *tv1, const SValue *tv2) { switch (tv1->type) { case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: - T_COMPARE_SCALAR_VALUE(int8_t, &tv1->val, &tv2->val); + T_COMPARE_SCALAR_VALUE(int8_t, &VALUE_GET_TRIVIAL_DATUM(tv1), &VALUE_GET_TRIVIAL_DATUM(tv2)); case TSDB_DATA_TYPE_SMALLINT: - T_COMPARE_SCALAR_VALUE(int16_t, &tv1->val, &tv2->val); + T_COMPARE_SCALAR_VALUE(int16_t, &VALUE_GET_TRIVIAL_DATUM(tv1), &VALUE_GET_TRIVIAL_DATUM(tv2)); case TSDB_DATA_TYPE_INT: - T_COMPARE_SCALAR_VALUE(int32_t, &tv1->val, &tv2->val); + T_COMPARE_SCALAR_VALUE(int32_t, &VALUE_GET_TRIVIAL_DATUM(tv1), &VALUE_GET_TRIVIAL_DATUM(tv2)); case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_TIMESTAMP: - T_COMPARE_SCALAR_VALUE(int64_t, &tv1->val, &tv2->val); + T_COMPARE_SCALAR_VALUE(int64_t, &VALUE_GET_TRIVIAL_DATUM(tv1), &VALUE_GET_TRIVIAL_DATUM(tv2)); case TSDB_DATA_TYPE_FLOAT: - T_COMPARE_SCALAR_VALUE(float, &tv1->val, &tv2->val); + T_COMPARE_SCALAR_VALUE(float, &VALUE_GET_TRIVIAL_DATUM(tv1), &VALUE_GET_TRIVIAL_DATUM(tv2)); case TSDB_DATA_TYPE_DOUBLE: - T_COMPARE_SCALAR_VALUE(double, &tv1->val, &tv2->val); + T_COMPARE_SCALAR_VALUE(double, &VALUE_GET_TRIVIAL_DATUM(tv1), &VALUE_GET_TRIVIAL_DATUM(tv2)); case TSDB_DATA_TYPE_UTINYINT: - T_COMPARE_SCALAR_VALUE(uint8_t, &tv1->val, &tv2->val); + T_COMPARE_SCALAR_VALUE(uint8_t, &VALUE_GET_TRIVIAL_DATUM(tv1), &VALUE_GET_TRIVIAL_DATUM(tv2)); case TSDB_DATA_TYPE_USMALLINT: - T_COMPARE_SCALAR_VALUE(uint16_t, &tv1->val, &tv2->val); + T_COMPARE_SCALAR_VALUE(uint16_t, &VALUE_GET_TRIVIAL_DATUM(tv1), &VALUE_GET_TRIVIAL_DATUM(tv2)); case TSDB_DATA_TYPE_UINT: - T_COMPARE_SCALAR_VALUE(uint32_t, &tv1->val, &tv2->val); + T_COMPARE_SCALAR_VALUE(uint32_t, &VALUE_GET_TRIVIAL_DATUM(tv1), &VALUE_GET_TRIVIAL_DATUM(tv2)); case TSDB_DATA_TYPE_UBIGINT: - T_COMPARE_SCALAR_VALUE(uint64_t, &tv1->val, &tv2->val); + T_COMPARE_SCALAR_VALUE(uint64_t, &VALUE_GET_TRIVIAL_DATUM(tv1), &VALUE_GET_TRIVIAL_DATUM(tv2)); case TSDB_DATA_TYPE_GEOMETRY: case TSDB_DATA_TYPE_BINARY: { int32_t ret = strncmp((const char *)tv1->pData, (const char *)tv2->pData, TMIN(tv1->nData, tv2->nData)); @@ -1420,12 +1423,7 @@ void tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc) { SValue *pVal = &pDst->pks[i]; pVal->type = pSrc->pks[i].type; - if (IS_NUMERIC_TYPE(pVal->type)) { - pVal->val = pSrc->pks[i].val; - } else { - pVal->nData = pSrc->pks[i].nData; - (void)memcpy(pVal->pData, pSrc->pks[i].pData, pVal->nData); - } + valueCloneDatum(pVal, pSrc->pks + i, pVal->type); } } } @@ -2267,8 +2265,7 @@ int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal) { return TSDB_CODE_INVALID_PARA; } return tColDataAppendValueImpl[pColData->flag][pColVal->flag]( - pColData, IS_VAR_DATA_TYPE(pColData->type) ? pColVal->value.pData : (uint8_t *)&pColVal->value.val, - pColVal->value.nData); + pColData, VALUE_GET_DATUM(&pColVal->value, pColData->type), pColVal->value.nData); } static FORCE_INLINE int32_t tColDataUpdateValue10(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) { @@ -2581,8 +2578,7 @@ int32_t tColDataUpdateValue(SColData *pColData, SColVal *pColVal, bool forward) if (tColDataUpdateValueImpl[pColData->flag][pColVal->flag] == NULL) return 0; return tColDataUpdateValueImpl[pColData->flag][pColVal->flag]( - pColData, IS_VAR_DATA_TYPE(pColData->type) ? pColVal->value.pData : (uint8_t *)&pColVal->value.val, - pColVal->value.nData, forward); + pColData, VALUE_GET_DATUM(&pColVal->value, pColData->type), pColVal->value.nData, forward); } static FORCE_INLINE void tColDataGetValue1(SColData *pColData, int32_t iVal, SColVal *pColVal) { // HAS_NONE @@ -2614,8 +2610,8 @@ static FORCE_INLINE void tColDataGetValue4(SColData *pColData, int32_t iVal, SCo } value.pData = pColData->pData + pColData->aOffset[iVal]; } else { - (void)memcpy(&value.val, pColData->pData + tDataTypes[pColData->type].bytes * iVal, - tDataTypes[pColData->type].bytes); + valueSetDatum(&value, pColData->type, pColData->pData + tDataTypes[pColData->type].bytes * iVal, + tDataTypes[pColData->type].bytes); } *pColVal = COL_VAL_VALUE(pColData->cid, value); } @@ -3323,9 +3319,7 @@ int32_t tRowBuildFromBind2(SBindInfo2 *infos, int32_t numOfInfos, bool infoSorte if (TSDB_DATA_TYPE_BOOL == value.type && *val > 1) { *val = 1; } - (void)memcpy(&value.val, val, - /*(uint8_t *)infos[iInfo].bind->buffer + infos[iInfo].bind->buffer_length * iRow,*/ - infos[iInfo].bytes /*bind->buffer_length*/); + valueSetDatum(&value, infos[iInfo].type, val, infos[iInfo].bytes); } colVal = COL_VAL_VALUE(infos[iInfo].columnId, value); } @@ -3839,8 +3833,9 @@ int32_t tDecodeRow(SDecoder *pDecoder, SRow **ppRow) { if ((MIN) > (VAL)) (MIN) = (VAL); \ } while (0) -static FORCE_INLINE void tColDataCalcSMABool(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, - int16_t *numOfNull) { +static FORCE_INLINE void tColDataCalcSMABool(SColData *pColData, SColumnDataAgg* pAggs) { + int64_t *sum = &pAggs->sum, *max = &pAggs->max, *min = &pAggs->min; + int16_t*numOfNull = &pAggs->numOfNull; *sum = 0; *max = 0; *min = 1; @@ -3870,8 +3865,9 @@ static FORCE_INLINE void tColDataCalcSMABool(SColData *pColData, int64_t *sum, i } } -static FORCE_INLINE void tColDataCalcSMATinyInt(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, - int16_t *numOfNull) { +static FORCE_INLINE void tColDataCalcSMATinyInt(SColData *pColData, SColumnDataAgg* pAggs) { + int64_t *sum = &pAggs->sum, *max = &pAggs->max, *min = &pAggs->min; + int16_t *numOfNull = &pAggs->numOfNull; *sum = 0; *max = INT8_MIN; *min = INT8_MAX; @@ -3901,8 +3897,9 @@ static FORCE_INLINE void tColDataCalcSMATinyInt(SColData *pColData, int64_t *sum } } -static FORCE_INLINE void tColDataCalcSMATinySmallInt(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, - int16_t *numOfNull) { +static FORCE_INLINE void tColDataCalcSMATinySmallInt(SColData *pColData, SColumnDataAgg* pAggs) { + int64_t *sum = &pAggs->sum, *max = &pAggs->max, *min = &pAggs->min; + int16_t *numOfNull = &pAggs->numOfNull; *sum = 0; *max = INT16_MIN; *min = INT16_MAX; @@ -3932,8 +3929,9 @@ static FORCE_INLINE void tColDataCalcSMATinySmallInt(SColData *pColData, int64_t } } -static FORCE_INLINE void tColDataCalcSMAInt(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, - int16_t *numOfNull) { +static FORCE_INLINE void tColDataCalcSMAInt(SColData *pColData, SColumnDataAgg* pAggs) { + int64_t *sum = &pAggs->sum, *max = &pAggs->max, *min = &pAggs->min; + int16_t *numOfNull = &pAggs->numOfNull; *sum = 0; *max = INT32_MIN; *min = INT32_MAX; @@ -3963,8 +3961,9 @@ static FORCE_INLINE void tColDataCalcSMAInt(SColData *pColData, int64_t *sum, in } } -static FORCE_INLINE void tColDataCalcSMABigInt(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, - int16_t *numOfNull) { +static FORCE_INLINE void tColDataCalcSMABigInt(SColData *pColData, SColumnDataAgg* pAggs) { + int64_t *sum = &pAggs->sum, *max = &pAggs->max, *min = &pAggs->min; + int16_t *numOfNull = &pAggs->numOfNull; *sum = 0; *max = INT64_MIN; *min = INT64_MAX; @@ -3994,8 +3993,9 @@ static FORCE_INLINE void tColDataCalcSMABigInt(SColData *pColData, int64_t *sum, } } -static FORCE_INLINE void tColDataCalcSMAFloat(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, - int16_t *numOfNull) { +static FORCE_INLINE void tColDataCalcSMAFloat(SColData *pColData, SColumnDataAgg* pAggs) { + int64_t *sum = &pAggs->sum, *max = &pAggs->max, *min = &pAggs->min; + int16_t *numOfNull = &pAggs->numOfNull; *(double *)sum = 0; *(double *)max = -FLT_MAX; *(double *)min = FLT_MAX; @@ -4025,8 +4025,9 @@ static FORCE_INLINE void tColDataCalcSMAFloat(SColData *pColData, int64_t *sum, } } -static FORCE_INLINE void tColDataCalcSMADouble(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, - int16_t *numOfNull) { +static FORCE_INLINE void tColDataCalcSMADouble(SColData *pColData, SColumnDataAgg* pAggs) { + int64_t *sum = &pAggs->sum, *max = &pAggs->max, *min = &pAggs->min; + int16_t *numOfNull = &pAggs->numOfNull; *(double *)sum = 0; *(double *)max = -DBL_MAX; *(double *)min = DBL_MAX; @@ -4056,8 +4057,9 @@ static FORCE_INLINE void tColDataCalcSMADouble(SColData *pColData, int64_t *sum, } } -static FORCE_INLINE void tColDataCalcSMAUTinyInt(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, - int16_t *numOfNull) { +static FORCE_INLINE void tColDataCalcSMAUTinyInt(SColData *pColData, SColumnDataAgg* pAggs) { + int64_t *sum = &pAggs->sum, *max = &pAggs->max, *min = &pAggs->min; + int16_t *numOfNull = &pAggs->numOfNull; *(uint64_t *)sum = 0; *(uint64_t *)max = 0; *(uint64_t *)min = UINT8_MAX; @@ -4087,8 +4089,9 @@ static FORCE_INLINE void tColDataCalcSMAUTinyInt(SColData *pColData, int64_t *su } } -static FORCE_INLINE void tColDataCalcSMATinyUSmallInt(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, - int16_t *numOfNull) { +static FORCE_INLINE void tColDataCalcSMATinyUSmallInt(SColData *pColData, SColumnDataAgg* pAggs) { + int64_t *sum = &pAggs->sum, *max = &pAggs->max, *min = &pAggs->min; + int16_t *numOfNull = &pAggs->numOfNull; *(uint64_t *)sum = 0; *(uint64_t *)max = 0; *(uint64_t *)min = UINT16_MAX; @@ -4118,8 +4121,9 @@ static FORCE_INLINE void tColDataCalcSMATinyUSmallInt(SColData *pColData, int64_ } } -static FORCE_INLINE void tColDataCalcSMAUInt(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, - int16_t *numOfNull) { +static FORCE_INLINE void tColDataCalcSMAUInt(SColData *pColData, SColumnDataAgg* pAggs) { + int64_t *sum = &pAggs->sum, *max = &pAggs->max, *min = &pAggs->min; + int16_t *numOfNull = &pAggs->numOfNull; *(uint64_t *)sum = 0; *(uint64_t *)max = 0; *(uint64_t *)min = UINT32_MAX; @@ -4149,8 +4153,9 @@ static FORCE_INLINE void tColDataCalcSMAUInt(SColData *pColData, int64_t *sum, i } } -static FORCE_INLINE void tColDataCalcSMAUBigInt(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, - int16_t *numOfNull) { +static FORCE_INLINE void tColDataCalcSMAUBigInt(SColData *pColData, SColumnDataAgg* pAggs) { + int64_t *sum = &pAggs->sum, *max = &pAggs->max, *min = &pAggs->min; + int16_t *numOfNull = &pAggs->numOfNull; *(uint64_t *)sum = 0; *(uint64_t *)max = 0; *(uint64_t *)min = UINT64_MAX; @@ -4180,8 +4185,9 @@ static FORCE_INLINE void tColDataCalcSMAUBigInt(SColData *pColData, int64_t *sum } } -static FORCE_INLINE void tColDataCalcSMAVarType(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, - int16_t *numOfNull) { +static FORCE_INLINE void tColDataCalcSMAVarType(SColData *pColData, SColumnDataAgg* pAggs) { + int64_t *sum = &pAggs->sum, *max = &pAggs->max, *min = &pAggs->min; + int16_t *numOfNull = &pAggs->numOfNull; *(uint64_t *)sum = 0; *(uint64_t *)max = 0; *(uint64_t *)min = 0; @@ -4216,7 +4222,90 @@ static FORCE_INLINE void tColDataCalcSMAVarType(SColData *pColData, int64_t *sum } } -void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, int16_t *numOfNull) = { +#define CALC_DECIMAL_SUM_MAX_MIN(TYPE, pSumOp, pCompOp, pColData, pSum, pMax, pMin) \ + do { \ + if (decimal128AddCheckOverflow((Decimal *)pSum, pVal, DECIMAL_WORD_NUM(TYPE))) *pOverflow = true; \ + pSumOp->add(pSum, pVal, DECIMAL_WORD_NUM(TYPE)); \ + if (pCompOp->gt(pVal, pMax, DECIMAL_WORD_NUM(TYPE))) { \ + *(pMax) = *pVal; \ + } \ + if (pCompOp->lt(pVal, pMin, DECIMAL_WORD_NUM(TYPE))) { \ + *(pMin) = *pVal; \ + } \ + } while (0) + +static FORCE_INLINE void tColDataCalcSMADecimal64Type(SColData* pColData, SColumnDataAgg* pAggs) { + Decimal128 *pSum = (Decimal128 *)pAggs->decimal128Sum; + Decimal64 *pMax = (Decimal64 *)pAggs->decimal128Max, *pMin = (Decimal64 *)pAggs->decimal128Min; + uint8_t *pOverflow = &pAggs->overflow; + *pSum = DECIMAL128_ZERO; + *pMax = DECIMAL64_MIN; + *pMin = DECIMAL64_MAX; + pAggs->numOfNull = 0; + pAggs->colId |= DECIMAL_AGG_FLAG; + + Decimal64 *pVal = NULL; + const SDecimalOps *pSumOps = getDecimalOps(TSDB_DATA_TYPE_DECIMAL); + const SDecimalOps *pCompOps = getDecimalOps(TSDB_DATA_TYPE_DECIMAL64); + if (HAS_VALUE == pColData->flag) { + for (int32_t iVal = 0; iVal < pColData->nVal; ++iVal) { + pVal = ((Decimal64*)pColData->pData) + iVal; + CALC_DECIMAL_SUM_MAX_MIN(Decimal64, pSumOps, pCompOps, pColData, pSum, pMax, pMin); + } + } else { + for (int32_t iVal = 0; iVal < pColData->nVal; ++iVal) { + switch (tColDataGetBitValue(pColData, iVal)) { + case 0: + case 1: + pAggs->numOfNull++; + break; + case 2: + pVal = ((Decimal64 *)pColData->pData) + iVal; + CALC_DECIMAL_SUM_MAX_MIN(Decimal64, pSumOps, pCompOps, pColData, pSum, pMax, pMin); + break; + default: + break; + } + } + } +} + +static FORCE_INLINE void tColDataCalcSMADecimal128Type(SColData* pColData, SColumnDataAgg* pAggs) { + Decimal128 *pSum = (Decimal128 *)pAggs->decimal128Sum, *pMax = (Decimal128 *)pAggs->decimal128Max, + *pMin = (Decimal128 *)pAggs->decimal128Min; + uint8_t *pOverflow = &pAggs->overflow; + *pSum = DECIMAL128_ZERO; + *pMax = DECIMAL128_MIN; + *pMin = DECIMAL128_MAX; + pAggs->numOfNull = 0; + pAggs->colId |= DECIMAL_AGG_FLAG; + + Decimal128 *pVal = NULL; + const SDecimalOps *pOps = getDecimalOps(TSDB_DATA_TYPE_DECIMAL); + if (HAS_VALUE == pColData->flag) { + for (int32_t iVal = 0; iVal < pColData->nVal; ++iVal) { + pVal = ((Decimal128*)pColData->pData) + iVal; + CALC_DECIMAL_SUM_MAX_MIN(Decimal128, pOps, pOps, pColData, pSum, pMax, pMin); + } + } else { + for (int32_t iVal = 0; iVal < pColData->nVal; ++iVal) { + switch (tColDataGetBitValue(pColData, iVal)) { + case 0: + case 1: + pAggs->numOfNull++; + break; + case 2: + pVal = ((Decimal128*)pColData->pData) + iVal; + CALC_DECIMAL_SUM_MAX_MIN(Decimal128, pOps, pOps, pColData, pSum, pMax, pMin); + break; + default: + break; + } + } + } +} + +void (*tColDataCalcSMA[])(SColData *pColData, SColumnDataAgg* pAggs) = { NULL, tColDataCalcSMABool, // TSDB_DATA_TYPE_BOOL tColDataCalcSMATinyInt, // TSDB_DATA_TYPE_TINYINT @@ -4234,10 +4323,11 @@ void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_ tColDataCalcSMAUBigInt, // TSDB_DATA_TYPE_UBIGINT tColDataCalcSMAVarType, // TSDB_DATA_TYPE_JSON tColDataCalcSMAVarType, // TSDB_DATA_TYPE_VARBINARY - tColDataCalcSMAVarType, // TSDB_DATA_TYPE_DECIMAL + tColDataCalcSMADecimal128Type, // TSDB_DATA_TYPE_DECIMAL tColDataCalcSMAVarType, // TSDB_DATA_TYPE_BLOB NULL, // TSDB_DATA_TYPE_MEDIUMBLOB - tColDataCalcSMAVarType // TSDB_DATA_TYPE_GEOMETRY + tColDataCalcSMAVarType, // TSDB_DATA_TYPE_GEOMETRY + tColDataCalcSMADecimal64Type, // TSDB_DATA_TYPE_DECIMAL64 }; // SValueColumn ================================ @@ -4284,7 +4374,7 @@ int32_t tValueColumnAppend(SValueColumn *valCol, const SValue *value) { return code; } } else { - code = tBufferPut(&valCol->data, &value->val, tDataTypes[value->type].bytes); + code = tBufferPut(&valCol->data, VALUE_GET_DATUM(value, value->type), tDataTypes[value->type].bytes); if (code) return code; } valCol->numOfValues++; @@ -4317,7 +4407,7 @@ int32_t tValueColumnUpdate(SValueColumn *valCol, int32_t idx, const SValue *valu } return tBufferPutAt(&valCol->data, offsets[idx], value->pData, value->nData); } else { - return tBufferPutAt(&valCol->data, idx * tDataTypes[valCol->type].bytes, &value->val, + return tBufferPutAt(&valCol->data, idx * tDataTypes[valCol->type].bytes, VALUE_GET_DATUM(value, valCol->type), tDataTypes[valCol->type].bytes); } return 0; @@ -4343,7 +4433,7 @@ int32_t tValueColumnGet(SValueColumn *valCol, int32_t idx, SValue *value) { value->pData = (uint8_t *)tBufferGetDataAt(&valCol->data, offset); } else { SBufferReader reader = BUFFER_READER_INITIALIZER(idx * tDataTypes[value->type].bytes, &valCol->data); - TAOS_CHECK_RETURN(tBufferGet(&reader, tDataTypes[value->type].bytes, &value->val)); + TAOS_CHECK_RETURN(tBufferGet(&reader, tDataTypes[value->type].bytes, VALUE_GET_DATUM(value, value->type))); } return 0; } @@ -4677,3 +4767,45 @@ int32_t tDecompressDataToBuffer(void *input, SCompressInfo *info, SBuffer *outpu output->size += info->originalSize; return 0; } + +// handle all types, including var data +void valueSetDatum(SValue *pVal, int8_t type, void *pDatum, uint32_t len) { + if (IS_VAR_DATA_TYPE(type) || type == TSDB_DATA_TYPE_DECIMAL) { + pVal->pData = pDatum; + pVal->nData = len; + } else { + switch (len) { + case sizeof(uint8_t): + pVal->val = *(uint8_t *)pDatum; + break; + case sizeof(uint16_t): + pVal->val = *(uint16_t *)pDatum; + break; + case sizeof(uint32_t): + pVal->val = *(uint32_t *)pDatum; + break; + case sizeof(uint64_t): + pVal->val = *(uint64_t *)pDatum; + break; + default: + break; + } + } +} + +void valueCloneDatum(SValue *pDst, const SValue *pSrc, int8_t type) { + if (IS_VAR_DATA_TYPE(type) || type == TSDB_DATA_TYPE_DECIMAL) { + memcpy(pDst->pData, pSrc->pData, pSrc->nData); + pDst->nData = pSrc->nData; + } else { + pDst->val = pSrc->val; + } +} +void valueClearDatum(SValue *pVal, int8_t type) { + if (IS_VAR_DATA_TYPE(type) || type == TSDB_DATA_TYPE_DECIMAL) { + taosMemoryFreeClear(pVal->pData); + pVal->nData = 0; + } else { + pVal->val = 0; + } +} diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 2b95e96130..ee82a13584 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -517,7 +517,7 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow, int8_t r val = varBuf; ++iBound; } else { - val = (const void *)&pColVal->value.val; + val = VALUE_GET_DATUM(&pColVal->value, pTColumn->type); ++iBound; } } else { @@ -972,7 +972,7 @@ int32_t tTSRowGetVal(STSRow *pRow, STSchema *pTSchema, int16_t iCol, SColVal *pC pColVal->value.nData = varDataLen(cv.val); pColVal->value.pData = varDataVal(cv.val); } else { - (void)memcpy(&pColVal->value.val, cv.val, tDataTypes[pTColumn->type].bytes); + valueSetDatum(&pColVal->value, pTColumn->type, cv.val, tDataTypes[pTColumn->type].bytes); } } return 0; diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index 1a0740b2b9..bd5ea3a33f 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -17,7 +17,7 @@ #include "ttypes.h" #include "tcompression.h" -const int32_t TYPE_BYTES[21] = { +const int32_t TYPE_BYTES[TSDB_DATA_TYPE_MAX] = { 2, // TSDB_DATA_TYPE_NULL CHAR_BYTES, // TSDB_DATA_TYPE_BOOL CHAR_BYTES, // TSDB_DATA_TYPE_TINYINT @@ -35,10 +35,11 @@ const int32_t TYPE_BYTES[21] = { sizeof(uint64_t), // TSDB_DATA_TYPE_UBIGINT TSDB_MAX_JSON_TAG_LEN, // TSDB_DATA_TYPE_JSON sizeof(VarDataOffsetT), // TSDB_DATA_TYPE_VARBINARY - TSDB_MAX_TAGS_LEN, // TSDB_DATA_TYPE_DECIMAL: placeholder, not implemented + DECIMAL128_BYTES, // TSDB_DATA_TYPE_DECIMAL: placeholder, not implemented TSDB_MAX_TAGS_LEN, // TSDB_DATA_TYPE_BLOB: placeholder, not implemented TSDB_MAX_TAGS_LEN, // TSDB_DATA_TYPE_MEDIUMBLOB: placeholder, not implemented sizeof(VarDataOffsetT), // TSDB_DATA_TYPE_GEOMETRY + DECIMAL64_BYTES, // TSDB_DATA_TYPE_DECIMAL64 }; tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX] = { @@ -62,11 +63,12 @@ tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX] = { {TSDB_DATA_TYPE_UBIGINT, 15, LONG_BYTES, "BIGINT UNSIGNED", 0, UINT64_MAX, tsCompressBigint, tsDecompressBigint}, {TSDB_DATA_TYPE_JSON, 4, TSDB_MAX_JSON_TAG_LEN, "JSON", 0, 0, tsCompressString, tsDecompressString}, {TSDB_DATA_TYPE_VARBINARY, 9, 1, "VARBINARY", 0, 0, tsCompressString, - tsDecompressString}, // placeholder, not implemented - {TSDB_DATA_TYPE_DECIMAL, 7, 1, "DECIMAL", 0, 0, NULL, NULL}, // placeholder, not implemented - {TSDB_DATA_TYPE_BLOB, 4, 1, "BLOB", 0, 0, NULL, NULL}, // placeholder, not implemented - {TSDB_DATA_TYPE_MEDIUMBLOB, 10, 1, "MEDIUMBLOB", 0, 0, NULL, NULL}, // placeholder, not implemented + tsDecompressString}, // placeholder, not implemented + {TSDB_DATA_TYPE_DECIMAL, 7, DECIMAL128_BYTES, "DECIMAL", 0, 0, NULL, NULL}, // placeholder, not implemented + {TSDB_DATA_TYPE_BLOB, 4, 1, "BLOB", 0, 0, NULL, NULL}, // placeholder, not implemented + {TSDB_DATA_TYPE_MEDIUMBLOB, 10, 1, "MEDIUMBLOB", 0, 0, NULL, NULL}, // placeholder, not implemented {TSDB_DATA_TYPE_GEOMETRY, 8, 1, "GEOMETRY", 0, 0, tsCompressString, tsDecompressString}, + {TSDB_DATA_TYPE_DECIMAL64, 7, DECIMAL64_BYTES, "DECIMAL", 0, 0, NULL, NULL}, }; tDataTypeCompress tDataCompress[TSDB_DATA_TYPE_MAX] = { @@ -92,10 +94,11 @@ tDataTypeCompress tDataCompress[TSDB_DATA_TYPE_MAX] = { {TSDB_DATA_TYPE_JSON, 4, TSDB_MAX_JSON_TAG_LEN, "JSON", 0, 0, tsCompressString2, tsDecompressString2}, {TSDB_DATA_TYPE_VARBINARY, 9, 1, "VARBINARY", 0, 0, tsCompressString2, tsDecompressString2}, // placeholder, not implemented - {TSDB_DATA_TYPE_DECIMAL, 7, 1, "DECIMAL", 0, 0, NULL, NULL}, // placeholder, not implemented + {TSDB_DATA_TYPE_DECIMAL, 7, DECIMAL128_BYTES, "DECIMAL", 0, 0, tsCompressDecimal128, tsDecompressDecimal128}, {TSDB_DATA_TYPE_BLOB, 4, 1, "BLOB", 0, 0, NULL, NULL}, // placeholder, not implemented {TSDB_DATA_TYPE_MEDIUMBLOB, 10, 1, "MEDIUMBLOB", 0, 0, NULL, NULL}, // placeholder, not implemented {TSDB_DATA_TYPE_GEOMETRY, 8, 1, "GEOMETRY", 0, 0, tsCompressString2, tsDecompressString2}, + {TSDB_DATA_TYPE_DECIMAL64, 9, DECIMAL64_BYTES, "DECIMAL64", 0, 0, tsCompressDecimal64, tsDecompressDecimal64}, }; @@ -229,3 +232,83 @@ int32_t operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type) { return 0; } + +uint8_t decimalTypeFromPrecision(uint8_t precision) { + return precision > TSDB_DECIMAL64_MAX_PRECISION ? TSDB_DATA_TYPE_DECIMAL : TSDB_DATA_TYPE_DECIMAL64; +} + +STypeMod decimalCalcTypeMod(uint8_t prec, uint8_t scale) { + return ((STypeMod)prec << 8) + scale; +} + +void decimalFromTypeMod(STypeMod typeMod, uint8_t* precision, uint8_t* scale) { + if (precision) *precision = (uint8_t)((typeMod >> 8) & 0xFF); + if (scale) *scale = (uint8_t)(typeMod & 0xFF); +} + +STypeMod typeGetTypeModFromDataType(const SDataType* pDataType) { + if (IS_DECIMAL_TYPE(pDataType->type)) return decimalCalcTypeMod(pDataType->precision, pDataType->scale); + return 0; +} + +STypeMod typeGetTypeMod(uint8_t type, uint8_t prec, uint8_t scale, int32_t bytes) { + if (IS_DECIMAL_TYPE(type)) { + return decimalCalcTypeMod(prec, scale); + } + return 0; +} + +void fillTypeFromTypeMod(SDataType* pType, STypeMod mod) { + if (IS_DECIMAL_TYPE(pType->type)) { + decimalFromTypeMod(mod, &pType->precision, &pType->scale); + } +} + +void extractTypeFromTypeMod(uint8_t type, STypeMod typeMod, uint8_t *prec, uint8_t *scale, int32_t *bytes) { + if (IS_DECIMAL_TYPE(type)) { + decimalFromTypeMod(typeMod, prec, scale); + } else { + if (prec) *prec = 0; + if (scale) *scale = 0; + } + if (bytes) *bytes = tDataTypes[type].bytes; +} + +uint8_t getScaleFromTypeMod(int32_t type, STypeMod mod) { + if (IS_DECIMAL_TYPE(type)) return (uint8_t)(mod & 0xFF); + return 0; +} + +// bytes, 0, prec, scale +void fillBytesForDecimalType(int32_t *pBytes, int32_t type, uint8_t precision, uint8_t scale) { + *pBytes = 0; + *pBytes = tDataTypes[type].bytes << 24; + *pBytes |= (uint32_t)precision << 8; + *pBytes |= scale; +} + +void extractDecimalTypeInfoFromBytes(int32_t *pBytes, uint8_t *precision, uint8_t *scale) { + *precision = (uint8_t)((*pBytes >> 8) & 0xFF); + *scale = (uint8_t)(*pBytes & 0xFF); + *pBytes >>= 24; +} + +int32_t calcTypeBytesFromSchemaBytes(int32_t type, int32_t schemaBytes, bool isStmt) { + if (isStmt) return schemaBytes; + if (type == TSDB_DATA_TYPE_VARCHAR || type == TSDB_DATA_TYPE_VARBINARY || type == TSDB_DATA_TYPE_GEOMETRY) { + return schemaBytes - VARSTR_HEADER_SIZE; + } else if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_JSON) { + return (schemaBytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + } + return schemaBytes; +} + +int32_t calcSchemaBytesFromTypeBytes(int32_t type, int32_t varTypeBytes, bool isStmt) { + if (isStmt) return varTypeBytes; + if (type == TSDB_DATA_TYPE_VARCHAR || type == TSDB_DATA_TYPE_VARBINARY || type == TSDB_DATA_TYPE_GEOMETRY) { + return varTypeBytes + VARSTR_HEADER_SIZE; + } else if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_JSON) { + return varTypeBytes * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; + } + return varTypeBytes; +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 08f6da5ecb..1ad14f1fc9 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -561,6 +561,7 @@ typedef struct { col_id_t colId; int32_t cmprAlg; } SCmprObj; + typedef struct { char name[TSDB_TABLE_FNAME_LEN]; char db[TSDB_DB_FNAME_LEN]; @@ -591,6 +592,7 @@ typedef struct { int8_t source; SColCmpr* pCmpr; int64_t keep; + SExtSchema* pExtSchemas; } SStbObj; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndIndex.c b/source/dnode/mnode/impl/src/mndIndex.c index abc2e7bdcf..e670ca90a3 100644 --- a/source/dnode/mnode/impl/src/mndIndex.c +++ b/source/dnode/mnode/impl/src/mndIndex.c @@ -664,6 +664,7 @@ static int32_t mndSetUpdateIdxStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStb pNew->pColumns = NULL; pNew->pCmpr = NULL; pNew->pTags = NULL; + pNew->pExtSchemas = NULL; pNew->updateTime = taosGetTimestampMs(); pNew->lock = 0; @@ -733,6 +734,7 @@ _OVER: taosMemoryFree(newStb.pTags); taosMemoryFree(newStb.pColumns); taosMemoryFree(newStb.pCmpr); + taosMemoryFreeClear(newStb.pExtSchemas); } mndTransDrop(pTrans); TAOS_RETURN(code); @@ -847,6 +849,7 @@ _OVER: taosMemoryFree(newObj.pTags); taosMemoryFree(newObj.pColumns); taosMemoryFree(newObj.pCmpr); + taosMemoryFreeClear(newObj.pExtSchemas); mndTransDrop(pTrans); mndReleaseStb(pMnode, pStb); diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 81d1f7677e..918143642b 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -1595,8 +1595,8 @@ static int32_t mndCreateTSMABuildCreateStreamReq(SCreateTSMACxt *pCxt) { if (!pCxt->pCreateStreamReq->pTags) { return terrno; } - SField f = {0}; - int32_t code = 0; + SFieldWithOptions f = {0}; + int32_t code = 0; if (pCxt->pSrcStb) { for (int32_t idx = 0; idx < pCxt->pCreateStreamReq->numOfTags - 1; ++idx) { SSchema *pSchema = &pCxt->pSrcStb->pTags[idx]; @@ -1630,6 +1630,10 @@ static int32_t mndCreateTSMABuildCreateStreamReq(SCreateTSMACxt *pCxt) { f.type = pExprNode->resType.type; f.flags = COL_SMA_ON; tstrncpy(f.name, pExprNode->userAlias, TSDB_COL_NAME_LEN); + if (IS_DECIMAL_TYPE(f.type)) { + f.typeMod = decimalCalcTypeMod(pExprNode->resType.precision, pExprNode->resType.scale); + f.flags |= COL_HAS_TYPE_MOD; + } if (NULL == taosArrayPush(pCxt->pCreateStreamReq->pCols, &f)) { code = terrno; break; @@ -1797,7 +1801,7 @@ static int32_t mndCreateTSMA(SCreateTSMACxt *pCxt) { } } if (LIST_LENGTH(pProjects) > 0) { - createStreamReq.pCols = taosArrayInit(LIST_LENGTH(pProjects), sizeof(SField)); + createStreamReq.pCols = taosArrayInit(LIST_LENGTH(pProjects), sizeof(SFieldWithOptions)); if (!createStreamReq.pCols) { code = terrno; goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 05956108a4..eb57fe8700 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -117,11 +117,12 @@ void mndCleanupStb(SMnode *pMnode) {} SSdbRaw *mndStbActionEncode(SStbObj *pStb) { int32_t code = 0; int32_t lino = 0; + bool hasTypeMod = false; terrno = TSDB_CODE_OUT_OF_MEMORY; int32_t size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema) + pStb->commentLen + pStb->ast1Len + pStb->ast2Len + pStb->numOfColumns * sizeof(SColCmpr) + STB_RESERVE_SIZE + - taosArrayGetSize(pStb->pFuncs) * TSDB_FUNC_NAME_LEN; + taosArrayGetSize(pStb->pFuncs) * TSDB_FUNC_NAME_LEN + sizeof(int32_t) * pStb->numOfColumns; SSdbRaw *pRaw = sdbAllocRaw(SDB_STB, STB_VER_NUMBER, size); if (pRaw == NULL) goto _OVER; @@ -155,6 +156,7 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) { SDB_SET_INT16(pRaw, dataPos, pSchema->colId, _OVER) SDB_SET_INT32(pRaw, dataPos, pSchema->bytes, _OVER) SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER) + hasTypeMod = hasTypeMod || HAS_TYPE_MOD(pSchema); } for (int32_t i = 0; i < pStb->numOfTags; ++i) { @@ -192,6 +194,13 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) { } SDB_SET_INT64(pRaw, dataPos, pStb->keep, _OVER) + + if (hasTypeMod) { + for (int32_t i = 0; i < pStb->numOfColumns; ++i) { + SDB_SET_INT32(pRaw, dataPos, pStb->pExtSchemas[i].typeMod, _OVER); + } + } + SDB_SET_RESERVE(pRaw, dataPos, STB_RESERVE_SIZE, _OVER) SDB_SET_DATALEN(pRaw, dataPos, _OVER) @@ -214,6 +223,7 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_OUT_OF_MEMORY; SSdbRow *pRow = NULL; SStbObj *pStb = NULL; + bool hasExtSchemas = false; int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; @@ -266,6 +276,7 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { SDB_GET_INT16(pRaw, dataPos, &pSchema->colId, _OVER) SDB_GET_INT32(pRaw, dataPos, &pSchema->bytes, _OVER) SDB_GET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER) + hasExtSchemas = hasExtSchemas || HAS_TYPE_MOD(pSchema); } for (int32_t i = 0; i < pStb->numOfTags; ++i) { @@ -320,6 +331,16 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { } SDB_GET_INT64(pRaw, dataPos, &pStb->keep, _OVER) + // type mod + if (hasExtSchemas) { + pStb->pExtSchemas = taosMemoryCalloc(pStb->numOfColumns, sizeof(SExtSchema)); + if (!pStb->pExtSchemas) goto _OVER; + for (int32_t i = 0; i < pStb->numOfColumns; ++i) { + SSchema *pSchema = &pStb->pColumns[i]; + SDB_GET_INT32(pRaw, dataPos, &pStb->pExtSchemas[i].typeMod, _OVER) + } + } + SDB_GET_RESERVE(pRaw, dataPos, STB_RESERVE_SIZE, _OVER) terrno = 0; @@ -332,6 +353,7 @@ _OVER: taosMemoryFreeClear(pStb->pTags); taosMemoryFreeClear(pStb->comment); taosMemoryFree(pStb->pCmpr); + taosMemoryFreeClear(pStb->pExtSchemas); } taosMemoryFreeClear(pRow); return NULL; @@ -349,6 +371,7 @@ void mndFreeStb(SStbObj *pStb) { taosMemoryFreeClear(pStb->pAst1); taosMemoryFreeClear(pStb->pAst2); taosMemoryFreeClear(pStb->pCmpr); + taosMemoryFreeClear(pStb->pExtSchemas); } static int32_t mndStbActionInsert(SSdb *pSdb, SStbObj *pStb) { @@ -363,69 +386,55 @@ static int32_t mndStbActionDelete(SSdb *pSdb, SStbObj *pStb) { } static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) { + terrno = 0; mTrace("stb:%s, perform update action, old row:%p new row:%p", pOld->name, pOld, pNew); taosWLockLatch(&pOld->lock); int32_t numOfColumns = pOld->numOfColumns; if (pOld->numOfColumns < pNew->numOfColumns) { void *pColumns = taosMemoryMalloc(pNew->numOfColumns * sizeof(SSchema)); - if (pColumns != NULL) { - taosMemoryFree(pOld->pColumns); - pOld->pColumns = pColumns; - } else { - terrno = TSDB_CODE_OUT_OF_MEMORY; - mTrace("stb:%s, failed to perform update action since %s", pOld->name, terrstr()); - taosWUnLockLatch(&pOld->lock); + if (pColumns == NULL) { + goto END; } + taosMemoryFree(pOld->pColumns); + pOld->pColumns = pColumns; } if (pOld->numOfTags < pNew->numOfTags) { void *pTags = taosMemoryMalloc(pNew->numOfTags * sizeof(SSchema)); - if (pTags != NULL) { - taosMemoryFree(pOld->pTags); - pOld->pTags = pTags; - } else { - terrno = TSDB_CODE_OUT_OF_MEMORY; - mTrace("stb:%s, failed to perform update action since %s", pOld->name, terrstr()); - taosWUnLockLatch(&pOld->lock); + if (pTags == NULL) { + goto END; } + taosMemoryFree(pOld->pTags); + pOld->pTags = pTags; } if (pOld->commentLen < pNew->commentLen && pNew->commentLen > 0) { void *comment = taosMemoryMalloc(pNew->commentLen + 1); - if (comment != NULL) { - taosMemoryFree(pOld->comment); - pOld->comment = comment; - } else { - terrno = TSDB_CODE_OUT_OF_MEMORY; - mTrace("stb:%s, failed to perform update action since %s", pOld->name, terrstr()); - taosWUnLockLatch(&pOld->lock); + if (comment == NULL) { + goto END; } + taosMemoryFree(pOld->comment); + pOld->comment = comment; } pOld->commentLen = pNew->commentLen; if (pOld->ast1Len < pNew->ast1Len) { void *pAst1 = taosMemoryMalloc(pNew->ast1Len + 1); - if (pAst1 != NULL) { - taosMemoryFree(pOld->pAst1); - pOld->pAst1 = pAst1; - } else { - terrno = TSDB_CODE_OUT_OF_MEMORY; - mTrace("stb:%s, failed to perform update action since %s", pOld->name, terrstr()); - taosWUnLockLatch(&pOld->lock); + if (pAst1 == NULL) { + goto END; } + taosMemoryFree(pOld->pAst1); + pOld->pAst1 = pAst1; } if (pOld->ast2Len < pNew->ast2Len) { void *pAst2 = taosMemoryMalloc(pNew->ast2Len + 1); - if (pAst2 != NULL) { - taosMemoryFree(pOld->pAst2); - pOld->pAst2 = pAst2; - } else { - terrno = TSDB_CODE_OUT_OF_MEMORY; - mTrace("stb:%s, failed to perform update action since %s", pOld->name, terrstr()); - taosWUnLockLatch(&pOld->lock); + if (pAst2 == NULL) { + goto END; } + taosMemoryFree(pOld->pAst2); + pOld->pAst2 = pAst2; } pOld->updateTime = pNew->updateTime; @@ -459,13 +468,26 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) { if (numOfColumns < pNew->numOfColumns) { taosMemoryFree(pOld->pCmpr); pOld->pCmpr = taosMemoryCalloc(pNew->numOfColumns, sizeof(SColCmpr)); + if (pOld->pCmpr == NULL){ + goto END; + } memcpy(pOld->pCmpr, pNew->pCmpr, pNew->numOfColumns * sizeof(SColCmpr)); } else { memcpy(pOld->pCmpr, pNew->pCmpr, pNew->numOfColumns * sizeof(SColCmpr)); } + if (pNew->pExtSchemas) { + taosMemoryFreeClear(pOld->pExtSchemas); + pOld->pExtSchemas = taosMemoryCalloc(pNew->numOfColumns, sizeof(SExtSchema)); + if (pOld->pExtSchemas == NULL){ + goto END; + } + memcpy(pOld->pExtSchemas, pNew->pExtSchemas, pNew->numOfColumns * sizeof(SExtSchema)); + } + +END: taosWUnLockLatch(&pOld->lock); - return 0; + return terrno; } SStbObj *mndAcquireStb(SMnode *pMnode, char *stbName) { @@ -563,6 +585,7 @@ void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int3 } } } + req.pExtSchemas = pStb->pExtSchemas; // only reference to it. // get length int32_t ret = 0; tEncodeSize(tEncodeSVCreateStbReq, &req, contLen, ret); @@ -861,6 +884,7 @@ static SSchema *mndFindStbColumns(const SStbObj *pStb, const char *colName) { int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreate, SDbObj *pDb) { int32_t code = 0; + bool hasTypeMods = false; memcpy(pDst->name, pCreate->name, TSDB_TABLE_FNAME_LEN); memcpy(pDst->db, pDb->name, TSDB_DB_FNAME_LEN); pDst->createdTime = taosGetTimestampMs(); @@ -937,6 +961,7 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat memcpy(pSchema->name, pField->name, TSDB_COL_NAME_LEN); pSchema->colId = pDst->nextColId; pDst->nextColId++; + hasTypeMods = hasTypeMods || HAS_TYPE_MOD(pSchema); } for (int32_t i = 0; i < pDst->numOfTags; ++i) { @@ -961,6 +986,18 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat pColCmpr->id = pSchema->colId; pColCmpr->alg = pField->compress; } + + if (hasTypeMods) { + pDst->pExtSchemas = taosMemoryCalloc(pDst->numOfColumns, sizeof(SExtSchema)); + if (!pDst->pExtSchemas) { + code = terrno; + TAOS_RETURN(code); + } + for (int32_t i = 0; i < pDst->numOfColumns; ++i) { + SFieldWithOptions * pField = taosArrayGet(pCreate->pColumns, i); + pDst->pExtSchemas[i].typeMod = pField->typeMod; + } + } TAOS_RETURN(code); } static int32_t mndGenIdxNameForFirstTag(char *fullname, char *dbname, char *stbname, char *tagname) { @@ -1204,8 +1241,9 @@ static int32_t mndBuildStbFromAlter(SStbObj *pStb, SStbObj *pDst, SMCreateStbReq pDst->pColumns = taosMemoryCalloc(1, pDst->numOfColumns * sizeof(SSchema)); pDst->pTags = taosMemoryCalloc(1, pDst->numOfTags * sizeof(SSchema)); pDst->pCmpr = taosMemoryCalloc(1, pDst->numOfColumns * sizeof(SColCmpr)); + pDst->pExtSchemas = taosMemoryCalloc(pDst->numOfColumns, sizeof(SExtSchema)); - if (pDst->pColumns == NULL || pDst->pTags == NULL || pDst->pCmpr == NULL) { + if (pDst->pColumns == NULL || pDst->pTags == NULL || pDst->pCmpr == NULL || pDst->pExtSchemas == NULL) { code = terrno; TAOS_RETURN(code); } @@ -1253,6 +1291,9 @@ static int32_t mndBuildStbFromAlter(SStbObj *pStb, SStbObj *pDst, SMCreateStbReq } else { p->alg = pField->compress; } + if (pField->flags & COL_HAS_TYPE_MOD) { + pDst->pExtSchemas[i].typeMod = pField->typeMod; + } } pDst->tagVer = createReq->tagVer; pDst->colVer = createReq->colVer; @@ -1385,6 +1426,7 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) { taosMemoryFreeClear(pDst.pTags); taosMemoryFreeClear(pDst.pColumns); taosMemoryFreeClear(pDst.pCmpr); + taosMemoryFreeClear(pDst.pExtSchemas); goto _OVER; } @@ -1392,6 +1434,7 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) { taosMemoryFreeClear(pDst.pTags); taosMemoryFreeClear(pDst.pColumns); taosMemoryFreeClear(pDst.pCmpr); + taosMemoryFreeClear(pDst.pExtSchemas); } else { code = mndCreateStb(pMnode, pReq, &createReq, pDb); } @@ -1458,6 +1501,13 @@ int32_t mndAllocStbSchemas(const SStbObj *pOld, SStbObj *pNew) { memcpy(pNew->pColumns, pOld->pColumns, sizeof(SSchema) * pOld->numOfColumns); memcpy(pNew->pTags, pOld->pTags, sizeof(SSchema) * pOld->numOfTags); memcpy(pNew->pCmpr, pOld->pCmpr, sizeof(SColCmpr) * pOld->numOfColumns); + if (pOld->pExtSchemas) { + pNew->pExtSchemas = taosMemoryCalloc(pNew->numOfColumns, sizeof(SExtSchema)); + if (pNew->pExtSchemas == NULL) { + TAOS_RETURN(terrno); + } + memcpy(pNew->pExtSchemas, pOld->pExtSchemas, sizeof(SExtSchema) * pOld->numOfColumns); + } TAOS_RETURN(0); } @@ -1880,7 +1930,7 @@ static int32_t mndUpdateSuperTableColumnCompress(SMnode *pMnode, const SStbObj * TAOS_RETURN(code); } -static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray *pFields, int32_t ncols, +static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, const SMAlterStbReq* pReq, int32_t ncols, int8_t withCompress) { int32_t code = 0; if (pOld->numOfColumns + ncols + pOld->numOfTags > TSDB_MAX_COLUMNS) { @@ -1892,7 +1942,7 @@ static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray TAOS_RETURN(code); } - if (!mndValidateSchema(pOld->pColumns, pOld->numOfColumns, pFields, TSDB_MAX_BYTES_PER_ROW)) { + if (!mndValidateSchema(pOld->pColumns, pOld->numOfColumns, pReq->pFields, TSDB_MAX_BYTES_PER_ROW)) { code = TSDB_CODE_PAR_INVALID_ROW_LENGTH; TAOS_RETURN(code); } @@ -1908,7 +1958,7 @@ static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray for (int32_t i = 0; i < ncols; i++) { if (withCompress) { - SFieldWithOptions *pField = taosArrayGet(pFields, i); + SFieldWithOptions *pField = taosArrayGet(pReq->pFields, i); if (mndFindSuperTableColumnIndex(pOld, pField->name) >= 0) { code = TSDB_CODE_MND_COLUMN_ALREADY_EXIST; TAOS_RETURN(code); @@ -1931,7 +1981,7 @@ static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray pCmpr->alg = pField->compress; mInfo("stb:%s, start to add column %s", pNew->name, pSchema->name); } else { - SField *pField = taosArrayGet(pFields, i); + SField *pField = taosArrayGet(pReq->pFields, i); if (mndFindSuperTableColumnIndex(pOld, pField->name) >= 0) { code = TSDB_CODE_MND_COLUMN_ALREADY_EXIST; TAOS_RETURN(code); @@ -1955,6 +2005,25 @@ static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray mInfo("stb:%s, start to add column %s", pNew->name, pSchema->name); } } + // 1. old schema already has extschemas + // 2. new schema has extschemas + if (pReq->pTypeMods || pOld->pExtSchemas) { + if (!pNew->pExtSchemas) { + // all ext schemas reset to zero + pNew->pExtSchemas = taosMemoryCalloc(pNew->numOfColumns, sizeof(SExtSchema)); + if (!pNew->pExtSchemas) TAOS_RETURN(terrno); + } + if (pOld->pExtSchemas) { + memcpy(pNew->pExtSchemas, pOld->pExtSchemas, pOld->numOfColumns * sizeof(SExtSchema)); + } + if (taosArrayGetSize(pReq->pTypeMods) > 0) { + // copy added column ext schema + for (int32_t i = 0; i < ncols; ++i) { + pNew->pColumns[pOld->numOfColumns + i].flags |= COL_HAS_TYPE_MOD; + pNew->pExtSchemas[pOld->numOfColumns + i].typeMod = *(STypeMod *)taosArrayGet(pReq->pTypeMods, i); + } + } + } pNew->colVer++; TAOS_RETURN(code); @@ -1986,6 +2055,9 @@ static int32_t mndDropSuperTableColumn(SMnode *pMnode, const SStbObj *pOld, SStb int32_t sz = pNew->numOfColumns - col - 1; memmove(pNew->pColumns + col, pNew->pColumns + col + 1, sizeof(SSchema) * sz); memmove(pNew->pCmpr + col, pNew->pCmpr + col + 1, sizeof(SColCmpr) * sz); + if (pOld->pExtSchemas) { + memmove(pNew->pExtSchemas + col, pNew->pExtSchemas + col + 1, sizeof(SExtSchema) * sz); + } pNew->numOfColumns--; pNew->colVer++; @@ -2206,6 +2278,9 @@ static int32_t mndBuildStbSchemaImp(SDbObj *pDb, SStbObj *pStb, const char *tbNa SSchemaExt *pSchEx = &pRsp->pSchemaExt[i]; pSchEx->colId = pCmpr->id; pSchEx->compress = pCmpr->alg; + if (pStb->pExtSchemas) { + pSchEx->typeMod = pStb->pExtSchemas[i].typeMod; + } } taosRUnLockLatch(&pStb->lock); @@ -2271,6 +2346,9 @@ static int32_t mndBuildStbCfgImp(SDbObj *pDb, SStbObj *pStb, const char *tbName, SSchemaExt *pSchExt = &pRsp->pSchemaExt[i]; pSchExt->colId = pCmpr->id; pSchExt->compress = pCmpr->alg; + if (pStb->pExtSchemas) { + pSchExt->typeMod = pStb->pExtSchemas[i].typeMod; + } } taosRUnLockLatch(&pStb->lock); @@ -2604,6 +2682,7 @@ static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *p stbObj.pTags = NULL; stbObj.pFuncs = NULL; stbObj.pCmpr = NULL; + stbObj.pExtSchemas = NULL; stbObj.updateTime = taosGetTimestampMs(); stbObj.lock = 0; bool updateTagIndex = false; @@ -2625,7 +2704,7 @@ static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *p code = mndAlterStbTagBytes(pMnode, pOld, &stbObj, pField0); break; case TSDB_ALTER_TABLE_ADD_COLUMN: - code = mndAddSuperTableColumn(pOld, &stbObj, pAlter->pFields, pAlter->numOfFields, 0); + code = mndAddSuperTableColumn(pOld, &stbObj, pAlter, pAlter->numOfFields, 0); break; case TSDB_ALTER_TABLE_DROP_COLUMN: pField0 = taosArrayGet(pAlter->pFields, 0); @@ -2643,7 +2722,7 @@ static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *p code = mndUpdateSuperTableColumnCompress(pMnode, pOld, &stbObj, pAlter->pFields, pAlter->numOfFields); break; case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION: - code = mndAddSuperTableColumn(pOld, &stbObj, pAlter->pFields, pAlter->numOfFields, 1); + code = mndAddSuperTableColumn(pOld, &stbObj, pAlter, pAlter->numOfFields, 1); break; default: needRsp = false; @@ -2665,6 +2744,7 @@ _OVER: if (pAlter->commentLen > 0) { taosMemoryFreeClear(stbObj.comment); } + taosMemoryFreeClear(stbObj.pExtSchemas); TAOS_RETURN(code); } @@ -3646,6 +3726,11 @@ static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB colTypeLen += tsnprintf(varDataVal(colTypeStr) + colTypeLen, sizeof(colTypeStr) - colTypeLen - VARSTR_HEADER_SIZE, "(%d)", (int32_t)((pStb->pColumns[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); + } else if (IS_DECIMAL_TYPE(colType)) { + STypeMod typeMod = pStb->pExtSchemas[i].typeMod; + uint8_t prec = 0, scale = 0; + decimalFromTypeMod(typeMod, &prec, &scale); + colTypeLen += sprintf(varDataVal(colTypeStr) + colTypeLen, "(%d,%d)", prec, scale); } varDataSetLen(colTypeStr, colTypeLen); RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (char *)colTypeStr, false), pStb, &lino, _OVER); diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 483108e8f4..040532d951 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -642,6 +642,11 @@ static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStre pField->type = pStream->outputSchema.pSchema[i].type; pField->bytes = pStream->outputSchema.pSchema[i].bytes; pField->compress = createDefaultColCmprByType(pField->type); + if (IS_DECIMAL_TYPE(pField->type)) { + uint8_t prec = 0, scale = 0; + extractDecimalTypeInfoFromBytes(&pField->bytes, &prec, &scale); + pField->typeMod = decimalCalcTypeMod(prec, scale); + } } if (pStream->tagSchema.nCols == 0) { diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index cc3cb79cfa..e6c2ab0937 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -221,6 +221,7 @@ typedef struct STqReader { SSDataBlock *pResBlock; int64_t lastTs; bool hasPrimaryKey; + SExtSchema *extSchema; } STqReader; STqReader *tqReaderOpen(SVnode *pVnode); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 8658e2df4c..2ba0a1e207 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -169,8 +169,9 @@ int32_t metaDropMultipleTables(SMeta* pMeta, int64_t version, SArray* tb int metaTtlFindExpired(SMeta* pMeta, int64_t timePointMs, SArray* tbUids, int32_t ttlDropMaxCount); int metaAlterTable(SMeta* pMeta, int64_t version, SVAlterTbReq* pReq, STableMetaRsp* pMetaRsp); int metaUpdateChangeTimeWithLock(SMeta* pMeta, tb_uid_t uid, int64_t changeTimeMs); -SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock); -int64_t metaGetTableCreateTime(SMeta* pMeta, tb_uid_t uid, int lock); +SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock, SExtSchema** extSchema); +int64_t metaGetTableCreateTime(SMeta *pMeta, tb_uid_t uid, int lock); +SExtSchema* metaGetSExtSchema(const SMetaEntry *pME); int32_t metaGetTbTSchemaNotNull(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock, STSchema** ppTSchema); int32_t metaGetTbTSchemaMaybeNull(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock, STSchema** ppTSchema); STSchema* metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock); diff --git a/source/dnode/vnode/src/meta/metaEntry.c b/source/dnode/vnode/src/meta/metaEntry.c index bd52d38a4b..4bb2e5c3e0 100644 --- a/source/dnode/vnode/src/meta/metaEntry.c +++ b/source/dnode/vnode/src/meta/metaEntry.c @@ -15,6 +15,83 @@ #include "meta.h" +static bool schemasHasTypeMod(const SSchema *pSchema, int32_t nCols) { + for (int32_t i = 0; i < nCols; i++) { + if (HAS_TYPE_MOD(pSchema + i)) { + return true; + } + } + return false; +} + +static int32_t metaEncodeExtSchema(SEncoder* pCoder, const SMetaEntry* pME) { + if (pME->pExtSchemas) { + const SSchemaWrapper *pSchWrapper = NULL; + bool hasTypeMods = false; + if (pME->type == TSDB_SUPER_TABLE) { + pSchWrapper = &pME->stbEntry.schemaRow; + } else if (pME->type == TSDB_NORMAL_TABLE) { + pSchWrapper = &pME->ntbEntry.schemaRow; + } else { + return 0; + } + hasTypeMods = schemasHasTypeMod(pSchWrapper->pSchema, pSchWrapper->nCols); + + for (int32_t i = 0; i < pSchWrapper->nCols && hasTypeMods; ++i) { + TAOS_CHECK_RETURN(tEncodeI32v(pCoder, pME->pExtSchemas[i].typeMod)); + } + } + return 0; +} + +static int32_t metaDecodeExtSchemas(SDecoder* pDecoder, SMetaEntry* pME) { + bool hasExtSchema = false; + SSchemaWrapper* pSchWrapper = NULL; + if (pME->type == TSDB_SUPER_TABLE) { + pSchWrapper = &pME->stbEntry.schemaRow; + } else if (pME->type == TSDB_NORMAL_TABLE) { + pSchWrapper = &pME->ntbEntry.schemaRow; + } else { + return 0; + } + + hasExtSchema = schemasHasTypeMod(pSchWrapper->pSchema, pSchWrapper->nCols); + if (hasExtSchema && pSchWrapper->nCols > 0) { + pME->pExtSchemas = (SExtSchema*)tDecoderMalloc(pDecoder, sizeof(SExtSchema) * pSchWrapper->nCols); + if (pME->pExtSchemas == NULL) { + return terrno; + } + + for (int32_t i = 0; i < pSchWrapper->nCols && hasExtSchema; i++) { + TAOS_CHECK_RETURN(tDecodeI32v(pDecoder, &pME->pExtSchemas[i].typeMod)); + } + } + + return 0; +} + +SExtSchema* metaGetSExtSchema(const SMetaEntry *pME) { + const SSchemaWrapper *pSchWrapper = NULL; + bool hasTypeMods = false; + if (pME->type == TSDB_SUPER_TABLE) { + pSchWrapper = &pME->stbEntry.schemaRow; + } else if (pME->type == TSDB_NORMAL_TABLE) { + pSchWrapper = &pME->ntbEntry.schemaRow; + } else { + return NULL; + } + hasTypeMods = schemasHasTypeMod(pSchWrapper->pSchema, pSchWrapper->nCols); + + if (hasTypeMods) { + SExtSchema* ret = taosMemoryMalloc(sizeof(SExtSchema) * pSchWrapper->nCols); + if (ret != NULL){ + memcpy(ret, pME->pExtSchemas, pSchWrapper->nCols * sizeof(SExtSchema)); + } + return ret; + } + return NULL; +} + int meteEncodeColCmprEntry(SEncoder *pCoder, const SMetaEntry *pME) { const SColCmprWrapper *pw = &pME->colCmpr; TAOS_CHECK_RETURN(tEncodeI32v(pCoder, pw->nCols)); @@ -129,6 +206,7 @@ int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) { return TSDB_CODE_INVALID_PARA; } TAOS_CHECK_RETURN(meteEncodeColCmprEntry(pCoder, pME)); + TAOS_CHECK_RETURN(metaEncodeExtSchema(pCoder, pME)); } if (pME->type == TSDB_SUPER_TABLE) { TAOS_CHECK_RETURN(tEncodeI64(pCoder, pME->stbEntry.keep)); @@ -211,6 +289,9 @@ int metaDecodeEntryImpl(SDecoder *pCoder, SMetaEntry *pME, bool headerOnly) { } TABLE_SET_COL_COMPRESSED(pME->flags); } + if (!tDecodeIsEnd(pCoder)) { + TAOS_CHECK_RETURN(metaDecodeExtSchemas(pCoder, pME)); + } } if (pME->type == TSDB_SUPER_TABLE) { if (!tDecodeIsEnd(pCoder)) { @@ -218,6 +299,7 @@ int metaDecodeEntryImpl(SDecoder *pCoder, SMetaEntry *pME, bool headerOnly) { } } + tEndDecode(pCoder); return 0; } @@ -270,6 +352,7 @@ void metaCloneEntryFree(SMetaEntry **ppEntry) { return; } metaCloneColCmprFree(&(*ppEntry)->colCmpr); + taosMemoryFreeClear((*ppEntry)->pExtSchemas); taosMemoryFreeClear(*ppEntry); return; @@ -377,6 +460,15 @@ int32_t metaCloneEntry(const SMetaEntry *pEntry, SMetaEntry **ppEntry) { metaCloneEntryFree(ppEntry); return code; } + if (pEntry->pExtSchemas && pEntry->colCmpr.nCols > 0) { + (*ppEntry)->pExtSchemas = taosMemoryCalloc(pEntry->colCmpr.nCols, sizeof(SExtSchema)); + if (!(*ppEntry)->pExtSchemas) { + code = terrno; + metaCloneEntryFree(ppEntry); + return code; + } + memcpy((*ppEntry)->pExtSchemas, pEntry->pExtSchemas, sizeof(SExtSchema) * pEntry->colCmpr.nCols); + } return code; } diff --git a/source/dnode/vnode/src/meta/metaEntry2.c b/source/dnode/vnode/src/meta/metaEntry2.c index a5af974d93..d244209702 100644 --- a/source/dnode/vnode/src/meta/metaEntry2.c +++ b/source/dnode/vnode/src/meta/metaEntry2.c @@ -1080,7 +1080,6 @@ static int32_t metaHandleSuperTableCreateImpl(SMeta *pMeta, const SMetaEntry *pE const SMetaHandleParam param = { .pEntry = pEntry, }; - code = metaTableOpFn[op->table][op->op](pMeta, ¶m); if (TSDB_CODE_SUCCESS != code) { metaErr(TD_VID(pMeta->pVnode), code); diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 7548c526d9..c2bdbd32bb 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -378,7 +378,7 @@ int32_t metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType) { return 0; } -SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, int lock) { +SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, int lock, SExtSchema** extSchema) { void *pData = NULL; int nData = 0; int64_t version; @@ -409,6 +409,7 @@ _query: if (me.type == TSDB_SUPER_TABLE) { if (sver == -1 || sver == me.stbEntry.schemaRow.version) { pSchema = tCloneSSchemaWrapper(&me.stbEntry.schemaRow); + if (extSchema != NULL) *extSchema = metaGetSExtSchema(&me); tDecoderClear(&dc); goto _exit; } @@ -419,10 +420,12 @@ _query: } else { if (sver == -1 || sver == me.ntbEntry.schemaRow.version) { pSchema = tCloneSSchemaWrapper(&me.ntbEntry.schemaRow); + if (extSchema != NULL) *extSchema = metaGetSExtSchema(&me); tDecoderClear(&dc); goto _exit; } } + if (extSchema != NULL) *extSchema = metaGetSExtSchema(&me); tDecoderClear(&dc); // query from skm db @@ -664,7 +667,7 @@ STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, int lock) { STSchema *pTSchema = NULL; SSchemaWrapper *pSW = NULL; - pSW = metaGetTableSchema(pMeta, uid, sver, lock); + pSW = metaGetTableSchema(pMeta, uid, sver, lock, NULL); if (!pSW) return NULL; pTSchema = tBuildTSchema(pSW->pSchema, pSW->nCols, pSW->version); @@ -1465,6 +1468,7 @@ END: if (pCursor->pMeta) metaULock(pCursor->pMeta); if (pCursor->pCur) tdbTbcClose(pCursor->pCur); if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); + taosMemoryFreeClear(oStbEntry.pExtSchemas); tDecoderClear(&dc); tdbFree(pData); diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c index 64693274f4..ce934694e0 100644 --- a/source/dnode/vnode/src/meta/metaSnapshot.c +++ b/source/dnode/vnode/src/meta/metaSnapshot.c @@ -593,7 +593,7 @@ int32_t setForSnapShot(SSnapContext* ctx, int64_t uid) { void taosXSetTablePrimaryKey(SSnapContext* ctx, int64_t uid) { bool ret = false; - SSchemaWrapper* schema = metaGetTableSchema(ctx->pMeta, uid, -1, 1); + SSchemaWrapper* schema = metaGetTableSchema(ctx->pMeta, uid, -1, 1, NULL); if (schema && schema->nCols >= 2 && schema->pSchema[1].flags & COL_IS_KEY) { ret = true; } diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index d856e9e36d..c54c8d2ab3 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -69,6 +69,40 @@ int32_t updataTableColCmpr(SColCmprWrapper *pWp, SSchema *pSchema, int8_t add, u return 0; } +int32_t addTableExtSchema(SMetaEntry *pEntry, const SSchema *pColumn, int32_t newColNum, SExtSchema *pExtSchema) { + // no need to add ext schema when no column needs ext schemas + if (!HAS_TYPE_MOD(pColumn) && !pEntry->pExtSchemas) return 0; + if (!pEntry->pExtSchemas) { + // add a column which needs ext schema + // set all extschemas to zero for all columns alrady existed + pEntry->pExtSchemas = (SExtSchema *)taosMemoryCalloc(newColNum, sizeof(SExtSchema)); + } else { + // already has columns with ext schema + pEntry->pExtSchemas = (SExtSchema *)taosMemoryRealloc(pEntry->pExtSchemas, sizeof(SExtSchema) * newColNum); + } + if (!pEntry->pExtSchemas) return terrno; + pEntry->pExtSchemas[newColNum - 1] = *pExtSchema; + return 0; +} + +int32_t dropTableExtSchema(SMetaEntry *pEntry, int32_t dropColId, int32_t newColNum) { + // no ext schema, no need to drop + if (!pEntry->pExtSchemas) return 0; + if (dropColId == newColNum) { + // drop the last column + pEntry->pExtSchemas[dropColId - 1] = (SExtSchema){0}; + } else { + // drop a column in the middle + memmove(pEntry->pExtSchemas + dropColId, pEntry->pExtSchemas + dropColId + 1, + (newColNum - dropColId) * sizeof(SExtSchema)); + } + for (int32_t i = 0; i < newColNum; i++) { + if (hasExtSchema(pEntry->pExtSchemas + i)) return 0; + } + taosMemoryFreeClear(pEntry->pExtSchemas); + return 0; +} + int metaUpdateMetaRsp(tb_uid_t uid, char *tbName, SSchemaWrapper *pSchema, STableMetaRsp *pMetaRsp) { pMetaRsp->pSchemas = taosMemoryMalloc(pSchema->nCols * sizeof(SSchema)); if (NULL == pMetaRsp->pSchemas) { @@ -812,7 +846,7 @@ int32_t metaGetColCmpr(SMeta *pMeta, tb_uid_t uid, SHashObj **ppColCmprObj) { taosHashClear(pColCmprObj); return rc; } - if (useCompress(e.type)) { + if (withExtSchema(e.type)) { SColCmprWrapper *p = &e.colCmpr; for (int32_t i = 0; i < p->nCols; i++) { SColCmpr *pCmpr = &p->pColCmpr[i]; diff --git a/source/dnode/vnode/src/meta/metaTable2.c b/source/dnode/vnode/src/meta/metaTable2.c index 8c72a1e8cf..23812fccbc 100644 --- a/source/dnode/vnode/src/meta/metaTable2.c +++ b/source/dnode/vnode/src/meta/metaTable2.c @@ -21,6 +21,8 @@ extern int32_t metaFetchEntryByUid(SMeta *pMeta, int64_t uid, SMetaEntry **ppEnt extern int32_t metaFetchEntryByName(SMeta *pMeta, const char *name, SMetaEntry **ppEntry); extern void metaFetchEntryFree(SMetaEntry **ppEntry); extern int32_t updataTableColCmpr(SColCmprWrapper *pWp, SSchema *pSchema, int8_t add, uint32_t compress); +extern int32_t addTableExtSchema(SMetaEntry* pEntry, const SSchema* pColumn, int32_t newColNum, SExtSchema* pExtSchema); +extern int32_t dropTableExtSchema(SMetaEntry* pEntry, int32_t dropColId, int32_t newColNum); static int32_t metaCheckCreateSuperTableReq(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { int32_t vgId = TD_VID(pMeta->pVnode); @@ -190,6 +192,7 @@ int32_t metaCreateSuperTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq TABLE_SET_COL_COMPRESSED(entry.flags); entry.colCmpr = pReq->colCmpr; } + entry.pExtSchemas = pReq->pExtSchemas; code = metaHandleEntry2(pMeta, &entry); if (TSDB_CODE_SUCCESS == code) { @@ -436,6 +439,9 @@ static int32_t metaBuildCreateNormalTableRsp(SMeta *pMeta, SMetaEntry *pEntry, S SColCmpr *p = &pEntry->colCmpr.pColCmpr[i]; (*ppRsp)->pSchemaExt[i].colId = p->id; (*ppRsp)->pSchemaExt[i].compress = p->alg; + if (pEntry->pExtSchemas) { + (*ppRsp)->pSchemaExt[i].typeMod = pEntry->pExtSchemas[i].typeMod; + } } return code; @@ -455,17 +461,18 @@ static int32_t metaCreateNormalTable(SMeta *pMeta, int64_t version, SVCreateTbRe } SMetaEntry entry = { - .version = version, - .type = TSDB_NORMAL_TABLE, - .uid = pReq->uid, - .name = pReq->name, - .ntbEntry.btime = pReq->btime, - .ntbEntry.ttlDays = pReq->ttl, - .ntbEntry.commentLen = pReq->commentLen, - .ntbEntry.comment = pReq->comment, - .ntbEntry.schemaRow = pReq->ntb.schemaRow, - .ntbEntry.ncid = pReq->ntb.schemaRow.pSchema[pReq->ntb.schemaRow.nCols - 1].colId + 1, - .colCmpr = pReq->colCmpr, + .version = version, + .type = TSDB_NORMAL_TABLE, + .uid = pReq->uid, + .name = pReq->name, + .ntbEntry.btime = pReq->btime, + .ntbEntry.ttlDays = pReq->ttl, + .ntbEntry.commentLen = pReq->commentLen, + .ntbEntry.comment = pReq->comment, + .ntbEntry.schemaRow = pReq->ntb.schemaRow, + .ntbEntry.ncid = pReq->ntb.schemaRow.pSchema[pReq->ntb.schemaRow.nCols - 1].colId + 1, + .colCmpr = pReq->colCmpr, + .pExtSchemas = pReq->pExtSchemas, }; TABLE_SET_COL_COMPRESSED(entry.flags); @@ -621,6 +628,7 @@ int32_t metaAddTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, ST int32_t rowSize = 0; SSchemaWrapper *pSchema = &pEntry->ntbEntry.schemaRow; SSchema *pColumn; + SExtSchema extSchema = {0}; pEntry->version = version; for (int32_t i = 0; i < pSchema->nCols; i++) { pColumn = &pSchema->pSchema[i]; @@ -655,6 +663,7 @@ int32_t metaAddTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, ST pColumn->type = pReq->type; pColumn->flags = pReq->flags; pColumn->colId = pEntry->ntbEntry.ncid++; + extSchema.typeMod = pReq->typeMod; tstrncpy(pColumn->name, pReq->colName, TSDB_COL_NAME_LEN); uint32_t compress; if (TSDB_ALTER_TABLE_ADD_COLUMN == pReq->action) { @@ -669,6 +678,13 @@ int32_t metaAddTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, ST metaFetchEntryFree(&pEntry); TAOS_RETURN(code); } + code = addTableExtSchema(pEntry, pColumn, pSchema->nCols, &extSchema); + if (code) { + metaError("vgId:%d, %s failed to add ext schema at %s:%d since %s, version:%" PRId64, TD_VID(pMeta->pVnode), + __func__, __FILE__, __LINE__, tstrerror(code), version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(code); + } // do handle entry code = metaHandleEntry2(pMeta, pEntry); @@ -777,6 +793,15 @@ int32_t metaDropTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, S TAOS_RETURN(TSDB_CODE_VND_INVALID_TABLE_ACTION); } + // update column extschema + code = dropTableExtSchema(pEntry, iColumn, pSchema->nCols); + if (code) { + metaError("vgId:%d, %s failed to remove extschema at %s:%d since %s, version:%" PRId64, TD_VID(pMeta->pVnode), + __func__, __FILE__, __LINE__, tstrerror(code), version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(code); + } + // do handle entry code = metaHandleEntry2(pMeta, pEntry); if (code) { @@ -1763,6 +1788,7 @@ int32_t metaAlterSuperTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) .stbEntry.schemaTag = pReq->schemaTag, .stbEntry.keep = pReq->keep, .colCmpr = pReq->colCmpr, + .pExtSchemas = pReq->pExtSchemas, }; TABLE_SET_COL_COMPRESSED(entry.flags); @@ -1819,4 +1845,4 @@ int32_t metaDropMultipleTables(SMeta *pMeta, int64_t version, SArray *uidArray) } } return code; -} \ No newline at end of file +} diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 6d41426d08..8dccd2af94 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -283,7 +283,7 @@ void tqSetTablePrimaryKey(STqReader* pReader, int64_t uid) { return; } bool ret = false; - SSchemaWrapper* schema = metaGetTableSchema(pReader->pVnodeMeta, uid, -1, 1); + SSchemaWrapper* schema = metaGetTableSchema(pReader->pVnodeMeta, uid, -1, 1, NULL); if (schema && schema->nCols >= 2 && schema->pSchema[1].flags & COL_IS_KEY) { ret = true; } @@ -336,6 +336,7 @@ void tqReaderClose(STqReader* pReader) { tDeleteSchemaWrapper(pReader->pSchemaWrapper); } + taosMemoryFree(pReader->extSchema); if (pReader->pColIdList) { taosArrayDestroy(pReader->pColIdList); } @@ -597,7 +598,7 @@ END: return code; } -int32_t tqMaskBlock(SSchemaWrapper* pDst, SSDataBlock* pBlock, const SSchemaWrapper* pSrc, char* mask) { +int32_t tqMaskBlock(SSchemaWrapper* pDst, SSDataBlock* pBlock, const SSchemaWrapper* pSrc, char* mask, SExtSchema* extSrc) { if (pDst == NULL || pBlock == NULL || pSrc == NULL || mask == NULL) { return TSDB_CODE_INVALID_PARA; } @@ -620,6 +621,9 @@ int32_t tqMaskBlock(SSchemaWrapper* pDst, SSDataBlock* pBlock, const SSchemaWrap pDst->pSchema[j++] = pSrc->pSchema[i]; SColumnInfoData colInfo = createColumnInfoData(pSrc->pSchema[i].type, pSrc->pSchema[i].bytes, pSrc->pSchema[i].colId); + if (extSrc != NULL) { + decimalFromTypeMod(extSrc[i].typeMod, &colInfo.info.precision, &colInfo.info.scale); + } code = blockDataAppendColInfo(pBlock, &colInfo); if (code != 0) { return code; @@ -653,6 +657,9 @@ static int32_t buildResSDataBlock(STqReader* pReader, SSchemaWrapper* pSchema, c SSchema* pColSchema = &pSchema->pSchema[i]; SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId); + if (IS_DECIMAL_TYPE(pColSchema->type) && pReader->extSchema != NULL) { + decimalFromTypeMod(pReader->extSchema[i].typeMod, &colInfo.info.precision, &colInfo.info.scale); + } int32_t code = blockDataAppendColInfo(pBlock, &colInfo); if (code != TSDB_CODE_SUCCESS) { blockDataFreeRes(pBlock); @@ -680,6 +687,9 @@ static int32_t buildResSDataBlock(STqReader* pReader, SSchemaWrapper* pSchema, c j++; } else { SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId); + if (IS_DECIMAL_TYPE(pColSchema->type) && pReader->extSchema != NULL) { + decimalFromTypeMod(pReader->extSchema[i].typeMod, &colInfo.info.precision, &colInfo.info.scale); + } int32_t code = blockDataAppendColInfo(pBlock, &colInfo); if (code != TSDB_CODE_SUCCESS) { return -1; @@ -708,7 +718,7 @@ static int32_t doSetVal(SColumnInfoData* pColumnInfoData, int32_t rowIndex, SCol colDataSetNULL(pColumnInfoData, rowIndex); } } else { - code = colDataSetVal(pColumnInfoData, rowIndex, (void*)&pColVal->value.val, !COL_VAL_IS_VALUE(pColVal)); + code = colDataSetVal(pColumnInfoData, rowIndex, VALUE_GET_DATUM(&pColVal->value, pColVal->value.type), !COL_VAL_IS_VALUE(pColVal)); } return code; @@ -741,8 +751,8 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char* if ((suid != 0 && pReader->cachedSchemaSuid != suid) || (suid == 0 && pReader->cachedSchemaUid != uid) || (pReader->cachedSchemaVer != sversion)) { tDeleteSchemaWrapper(pReader->pSchemaWrapper); - - pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1); + taosMemoryFree(pReader->extSchema); + pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1, &pReader->extSchema); if (pReader->pSchemaWrapper == NULL) { tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", uid:%" PRId64 "version %d, possibly dropped table", @@ -883,7 +893,7 @@ END: } static int32_t processBuildNew(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArray* blocks, SArray* schemas, - SSchemaWrapper* pSchemaWrapper, char* assigned, int32_t numOfRows, int32_t curRow, + char* assigned, int32_t numOfRows, int32_t curRow, int32_t* lastRow) { int32_t code = 0; SSchemaWrapper* pSW = NULL; @@ -901,7 +911,7 @@ static int32_t processBuildNew(STqReader* pReader, SSubmitTbData* pSubmitTbData, pSW = taosMemoryCalloc(1, sizeof(SSchemaWrapper)); TQ_NULL_GO_TO_END(pSW); - TQ_ERR_GO_TO_END(tqMaskBlock(pSW, block, pSchemaWrapper, assigned)); + TQ_ERR_GO_TO_END(tqMaskBlock(pSW, block, pReader->pSchemaWrapper, assigned, pReader->extSchema)); tqTrace("vgId:%d, build new block, col %d", pReader->pWalReader->pWal->cfg.vgId, (int32_t)taosArrayGetSize(block->pDataBlock)); @@ -911,6 +921,7 @@ static int32_t processBuildNew(STqReader* pReader, SSubmitTbData* pSubmitTbData, TQ_NULL_GO_TO_END(taosArrayPush(blocks, block)); TQ_NULL_GO_TO_END(taosArrayPush(schemas, &pSW)); pSW = NULL; + taosMemoryFreeClear(block); END: @@ -949,7 +960,7 @@ static int32_t tqProcessColData(STqReader* pReader, SSubmitTbData* pSubmitTbData } if (buildNew) { - TQ_ERR_GO_TO_END(processBuildNew(pReader, pSubmitTbData, blocks, schemas, pSchemaWrapper, assigned, numOfRows, + TQ_ERR_GO_TO_END(processBuildNew(pReader, pSubmitTbData, blocks, schemas, assigned, numOfRows, curRow, &lastRow)); } @@ -1013,7 +1024,7 @@ int32_t tqProcessRowData(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArra } if (buildNew) { - TQ_ERR_GO_TO_END(processBuildNew(pReader, pSubmitTbData, blocks, schemas, pSchemaWrapper, assigned, numOfRows, + TQ_ERR_GO_TO_END(processBuildNew(pReader, pSubmitTbData, blocks, schemas, assigned, numOfRows, curRow, &lastRow)); } @@ -1119,7 +1130,8 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SMqDataRsp* pRsp, SArray* block pReader->lastBlkUid = uid; tDeleteSchemaWrapper(pReader->pSchemaWrapper); - pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1); + taosMemoryFree(pReader->extSchema); + pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1, &pReader->extSchema); if (pReader->pSchemaWrapper == NULL) { tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table", pReader->pWalReader->pWal->cfg.vgId, uid, pReader->cachedSchemaVer); diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index 549a47d006..9b242e4c84 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -313,7 +313,7 @@ END: if (code != 0){ tqError("%s failed at %d, vgId:%d, task exec error since %s", __FUNCTION__ , lino, pTq->pVnode->config.vgId, tstrerror(code)); } - taosMemoryFree(pSW); + tDeleteSchemaWrapper(pSW); taosMemoryFree(tbName); return code; } diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index c1c3623bde..ec643d81d1 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -823,7 +823,7 @@ int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDat } } else { SValue sv = {.type = pCol->type}; - memcpy(&sv.val, colData, tDataTypes[pCol->type].bytes); + valueSetDatum(&sv, pCol->type, colData, tDataTypes[pCol->type].bytes); SColVal cv = COL_VAL_VALUE(pCol->colId, sv); void* p = taosArrayPush(pVals, &cv); if (p == NULL) { diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index de21f6eb0b..4811836a6c 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -75,7 +75,6 @@ static int32_t tqInitTaosxRsp(SMqDataRsp* pRsp, STqOffsetVal pOffset) { pRsp->blockSchema = taosArrayInit(0, sizeof(void*)); TSDB_CHECK_NULL(pRsp->blockSchema, code, lino, END, terrno); - END: if (code != 0){ tqError("%s failed at:%d, code:%s", __FUNCTION__ , lino, tstrerror(code)); diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 637e07e618..e1895d4dc4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -334,6 +334,10 @@ static int32_t tsdbCacheDeserializeV0(char const *value, SLastCol *pLastCol) { pLastCol->colVal.value.pData = (uint8_t *)(&pLastColV0[1]); } return sizeof(SLastColV0) + pLastColV0->colVal.value.nData; + } else if (pLastCol->colVal.value.type == TSDB_DATA_TYPE_DECIMAL) { + pLastCol->colVal.value.nData = pLastColV0->colVal.value.nData; + pLastCol->colVal.value.pData = (uint8_t*)(&pLastColV0[1]); + return sizeof(SLastColV0) + pLastColV0->colVal.value.nData; } else { pLastCol->colVal.value.val = pLastColV0->colVal.value.val; return sizeof(SLastColV0); @@ -426,6 +430,12 @@ static int32_t tsdbCacheSerializeV0(char const *value, SLastCol *pLastCol) { memcpy(&pLastColV0[1], pLastCol->colVal.value.pData, pLastCol->colVal.value.nData); } return sizeof(SLastColV0) + pLastCol->colVal.value.nData; + } else if (pLastCol->colVal.value.type == TSDB_DATA_TYPE_DECIMAL) { + pLastColV0->colVal.value.nData = pLastCol->colVal.value.nData; + if (pLastCol->colVal.value.nData > 0) { + memcpy(&pLastColV0[1], pLastCol->colVal.value.pData, pLastCol->colVal.value.nData); + } + return sizeof(SLastColV0) + pLastCol->colVal.value.nData; } else { pLastColV0->colVal.value.val = pLastCol->colVal.value.val; return sizeof(SLastColV0); @@ -439,6 +449,9 @@ static int32_t tsdbCacheSerialize(SLastCol *pLastCol, char **value, size_t *size if (IS_VAR_DATA_TYPE(pLastCol->colVal.value.type)) { *size += pLastCol->colVal.value.nData; } + if (pLastCol->colVal.value.type == TSDB_DATA_TYPE_DECIMAL) { + *size += DECIMAL128_BYTES; + } *size += sizeof(uint8_t) + sizeof(uint8_t) + sizeof(uint8_t); // version + numOfPKs + cacheStatus for (int8_t i = 0; i < pLastCol->rowKey.numOfPKs; i++) { @@ -840,6 +853,16 @@ static int32_t tsdbCacheReallocSLastCol(SLastCol *pCol, size_t *pCharge) { charge += pCol->colVal.value.nData; } + if (pCol->colVal.value.type == TSDB_DATA_TYPE_DECIMAL) { + if (pCol->colVal.value.nData > 0) { + void *p = taosMemoryMalloc(pCol->colVal.value.nData); + if (!p) TAOS_CHECK_EXIT(terrno); + (void)memcpy(p, pCol->colVal.value.pData, pCol->colVal.value.nData); + pCol->colVal.value.pData = p; + } + charge += pCol->colVal.value.nData; + } + if (pCharge) { *pCharge = charge; } @@ -866,7 +889,8 @@ void tsdbCacheFreeSLastColItem(void *pItem) { } } - if (IS_VAR_DATA_TYPE(pCol->colVal.value.type) && pCol->colVal.value.pData) { + if ((IS_VAR_DATA_TYPE(pCol->colVal.value.type) || pCol->colVal.value.type == TSDB_DATA_TYPE_DECIMAL) && + pCol->colVal.value.pData) { taosMemoryFree(pCol->colVal.value.pData); } } @@ -888,7 +912,8 @@ static void tsdbCacheDeleter(const void *key, size_t klen, void *value, void *ud } } - if (IS_VAR_DATA_TYPE(pLastCol->colVal.value.type) /* && pLastCol->colVal.value.nData > 0*/) { + if (IS_VAR_DATA_TYPE(pLastCol->colVal.value.type) || + pLastCol->colVal.value.type == TSDB_DATA_TYPE_DECIMAL /* && pLastCol->colVal.value.nData > 0*/) { taosMemoryFree(pLastCol->colVal.value.pData); } @@ -1341,7 +1366,7 @@ static void tsdbCacheUpdateLastColToNone(SLastCol *pLastCol, ELastCacheStatus ca taosMemoryFreeClear(pPKValue->pData); pPKValue->nData = 0; } else { - pPKValue->val = 0; + valueClearDatum(pPKValue, pPKValue->type); } } pLastCol->rowKey.numOfPKs = 0; @@ -1351,7 +1376,7 @@ static void tsdbCacheUpdateLastColToNone(SLastCol *pLastCol, ELastCacheStatus ca taosMemoryFreeClear(pLastCol->colVal.value.pData); pLastCol->colVal.value.nData = 0; } else { - pLastCol->colVal.value.val = 0; + valueClearDatum(&pLastCol->colVal.value, pLastCol->colVal.value.type); } pLastCol->colVal = COL_VAL_NONE(pLastCol->colVal.cid, pLastCol->colVal.value.type); @@ -1708,11 +1733,12 @@ int32_t tsdbCacheColFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SBlo tsdbRowGetKey(&lRow, &tsdbRowKey); { + SValue tsVal = {.type = TSDB_DATA_TYPE_TIMESTAMP}; + VALUE_SET_TRIVIAL_DATUM(&tsVal, lRow.pBlockData->aTSKEY[lRow.iRow]); SLastUpdateCtx updateCtx = { .lflag = LFLAG_LAST, .tsdbRowKey = tsdbRowKey, - .colVal = COL_VAL_VALUE(PRIMARYKEY_TIMESTAMP_COL_ID, ((SValue){.type = TSDB_DATA_TYPE_TIMESTAMP, - .val = lRow.pBlockData->aTSKEY[lRow.iRow]}))}; + .colVal = COL_VAL_VALUE(PRIMARYKEY_TIMESTAMP_COL_ID, tsVal)}; if (!taosArrayPush(ctxArray, &updateCtx)) { TAOS_CHECK_GOTO(terrno, &lino, _exit); } @@ -3857,7 +3883,9 @@ static int32_t mergeLastCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SC } if (slotIds[iCol] == 0) { STColumn *pTColumn = &pTSchema->columns[0]; - *pColVal = COL_VAL_VALUE(pTColumn->colId, ((SValue){.type = pTColumn->type, .val = rowKey.key.ts})); + SValue val = {.type = pTColumn->type}; + VALUE_SET_TRIVIAL_DATUM(&val, rowKey.key.ts); + *pColVal = COL_VAL_VALUE(pTColumn->colId, val); SLastCol colTmp = {.rowKey = rowKey.key, .colVal = *pColVal, .cacheStatus = TSDB_LAST_CACHE_VALID}; TAOS_CHECK_GOTO(tsdbCacheReallocSLastCol(&colTmp, NULL), &lino, _err); @@ -4025,7 +4053,9 @@ static int32_t mergeLastRowCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, } if (slotIds[iCol] == 0) { STColumn *pTColumn = &pTSchema->columns[0]; - *pColVal = COL_VAL_VALUE(pTColumn->colId, ((SValue){.type = pTColumn->type, .val = rowKey.key.ts})); + SValue val = {.type = pTColumn->type}; + VALUE_SET_TRIVIAL_DATUM(&val, rowKey.key.ts); + *pColVal = COL_VAL_VALUE(pTColumn->colId, val); SLastCol colTmp = {.rowKey = rowKey.key, .colVal = *pColVal, .cacheStatus = TSDB_LAST_CACHE_VALID}; TAOS_CHECK_GOTO(tsdbCacheReallocSLastCol(&colTmp, NULL), &lino, _err); diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index f5aeb609d5..9daa4650ca 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -77,7 +77,8 @@ static int32_t saveOneRowForLastRaw(SLastCol* pColVal, SCacheRowsReader* pReader TSDB_CHECK_CODE(code, lino, _end); } } else { - code = colDataSetVal(pColInfoData, numOfRows, (const char*)&pVal->value.val, !COL_VAL_IS_VALUE(pVal)); + code = colDataSetVal(pColInfoData, numOfRows, VALUE_GET_DATUM(&pVal->value, pColVal->colVal.value.type), + !COL_VAL_IS_VALUE(pVal)); TSDB_CHECK_CODE(code, lino, _end); } @@ -166,7 +167,8 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p memcpy(varDataVal(p->buf), pColVal->colVal.value.pData, pColVal->colVal.value.nData); p->bytes = pColVal->colVal.value.nData + VARSTR_HEADER_SIZE; // binary needs to plus the header size } else { - memcpy(p->buf, &pColVal->colVal.value.val, pReader->pSchema->columns[slotId].bytes); + memcpy(p->buf, VALUE_GET_DATUM(&pColVal->colVal.value, pColVal->colVal.value.type), + pReader->pSchema->columns[slotId].bytes); p->bytes = pReader->pSchema->columns[slotId].bytes; } } @@ -543,7 +545,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 } } - if (IS_VAR_DATA_TYPE(pCol->type)) { + if (IS_VAR_DATA_TYPE(pCol->type) || pCol->type == TSDB_DATA_TYPE_DECIMAL) { p.colVal.value.pData = taosMemoryCalloc(pCol->bytes, sizeof(char)); TSDB_CHECK_NULL(p.colVal.value.pData, code, lino, _end, terrno); } @@ -601,7 +603,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 memcpy(p->rowKey.pks[j].pData, pColVal->rowKey.pks[j].pData, pColVal->rowKey.pks[j].nData); p->rowKey.pks[j].nData = pColVal->rowKey.pks[j].nData; } else { - p->rowKey.pks[j].val = pColVal->rowKey.pks[j].val; + valueCloneDatum(p->rowKey.pks + j, pColVal->rowKey.pks + j, p->rowKey.pks[j].type); } } @@ -628,7 +630,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 goto _end; } - if (!IS_VAR_DATA_TYPE(pColVal->colVal.value.type)) { + if (!IS_VAR_DATA_TYPE(pColVal->colVal.value.type) && pColVal->colVal.value.type != TSDB_DATA_TYPE_DECIMAL) { p->colVal = pColVal->colVal; } else { if (COL_VAL_IS_VALUE(&pColVal->colVal)) { diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c index 3b187323b5..6c1e016e24 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c @@ -1069,7 +1069,7 @@ static int32_t tsdbDataFileDoWriteBlockData(SDataFileWriter *writer, SBlockData if ((colData->cflag & COL_SMA_ON) == 0 || ((colData->flag & HAS_VALUE) == 0)) continue; SColumnDataAgg sma[1] = {{.colId = colData->cid}}; - tColDataCalcSMA[colData->type](colData, &sma->sum, &sma->max, &sma->min, &sma->numOfNull); + tColDataCalcSMA[colData->type](colData, sma); TAOS_CHECK_GOTO(tPutColumnDataAgg(&buffers[0], sma), &lino, _exit); } diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 3e4208fc54..1c1917609a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -64,7 +64,7 @@ int32_t tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t num static void freeItem(void *pValue) { SValue *p = (SValue *)pValue; - if (IS_VAR_DATA_TYPE(p->type)) { + if (IS_VAR_DATA_TYPE(p->type) || p->type == TSDB_DATA_TYPE_DECIMAL) { taosMemoryFree(p->pData); } } @@ -359,7 +359,7 @@ static int32_t extractSttBlockInfo(SLDataIter *pIter, const TSttBlkArray *pArray } static int32_t tValueDupPayload(SValue *pVal) { - if (IS_VAR_DATA_TYPE(pVal->type)) { + if (IS_VAR_DATA_TYPE(pVal->type) || pVal->type == TSDB_DATA_TYPE_DECIMAL) { char *p = (char *)pVal->pData; char *pBuf = taosMemoryMalloc(pVal->nData); if (pBuf == NULL) { diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index ca3b82239b..412e87ed27 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -130,7 +130,7 @@ static int32_t tColRowGetPriamyKeyDeepCopy(SBlockData* pBlock, int32_t irow, int pKey->pks[0].type = cv.value.type; if (IS_NUMERIC_TYPE(cv.value.type)) { - pKey->pks[0].val = cv.value.val; + valueCloneDatum(pKey->pks, &cv.value, cv.value.type); } else { pKey->pks[0].nData = cv.value.nData; TAOS_MEMCPY(pKey->pks[0].pData, cv.value.pData, cv.value.nData); @@ -182,7 +182,7 @@ static int32_t tRowGetPrimaryKeyDeepCopy(SRow* pRow, SRowKey* pKey) { tdata += tGetU32v(tdata, &pKey->pks[i].nData); TAOS_MEMCPY(pKey->pks[i].pData, tdata, pKey->pks[i].nData); } else { - TAOS_MEMCPY(&pKey->pks[i].val, data + indices[i].offset, tDataTypes[pKey->pks[i].type].bytes); + valueSetDatum(pKey->pks + i, indices[i].type, data + indices[i].offset, tDataTypes[pKey->pks[i].type].bytes); } } @@ -1084,7 +1084,7 @@ static int32_t updateLastKeyInfo(SRowKey* pKey, SFileDataBlockInfo* pBlockInfo, TSDB_CHECK_NULL(pBlockInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); if (IS_NUMERIC_TYPE(pKey->pks[0].type)) { - pKey->pks[0].val = asc ? pBlockInfo->lastPk.val : pBlockInfo->firstPk.val; + VALUE_SET_TRIVIAL_DATUM(pKey->pks, asc ? pBlockInfo->lastPk.val : pBlockInfo->firstPk.val); } else { uint8_t* p = asc ? pBlockInfo->lastPk.pData : pBlockInfo->firstPk.pData; pKey->pks[0].nData = asc ? varDataLen(pBlockInfo->lastPk.pData) : varDataLen(pBlockInfo->firstPk.pData); @@ -1127,7 +1127,8 @@ static int32_t doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int TSDB_CHECK_CODE(code, lino, _end); } } else { - code = colDataSetVal(pColInfoData, rowIndex, (const char*)&pColVal->value.val, !COL_VAL_IS_VALUE(pColVal)); + code = colDataSetVal(pColInfoData, rowIndex, VALUE_GET_DATUM(&pColVal->value, pColVal->value.type), + !COL_VAL_IS_VALUE(pColVal)); TSDB_CHECK_CODE(code, lino, _end); } @@ -1353,6 +1354,7 @@ static int32_t copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpI case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_DOUBLE: case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_DECIMAL64: case TSDB_DATA_TYPE_UBIGINT: { int32_t mid = dumpedRows >> 1u; int64_t* pts = (int64_t*)pColData->pData; @@ -1401,6 +1403,20 @@ static int32_t copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpI } break; } + case TSDB_DATA_TYPE_DECIMAL: { + int32_t mid = dumpedRows >> 1u; + DecimalWord* pDec = (DecimalWord*)pColData->pData; + DecimalWord tmp[2] = {0}; + for (int32_t j = 0; j < mid; ++j) { + tmp[0] = pDec[2 * j]; + tmp[1] = pDec[2 * j + 1]; + pDec[2 * j] = pDec[2 * (dumpedRows - j - 1)]; + pDec[2 * j + 1] = pDec[2 * (dumpedRows - j - 1) + 1]; + pDec[2 * (dumpedRows - j - 1)] = tmp[0]; + pDec[2 * (dumpedRows - j - 1) + 1] = tmp[1]; + } + break; + } } } @@ -1443,8 +1459,8 @@ static void blockInfoToRecord(SBrinRecord* record, SFileDataBlockInfo* pBlockInf pLast->pData = (uint8_t*)varDataVal(pBlockInfo->lastPk.pData); pLast->nData = varDataLen(pBlockInfo->lastPk.pData); } else { - pFirst->val = pBlockInfo->firstPk.val; - pLast->val = pBlockInfo->lastPk.val; + VALUE_SET_TRIVIAL_DATUM(pFirst, pBlockInfo->firstPk.val); + VALUE_SET_TRIVIAL_DATUM(pLast, pBlockInfo->lastPk.val); } } @@ -1899,7 +1915,7 @@ static bool overlapWithNeighborBlock2(SFileDataBlockInfo* pBlock, SBrinRecord* p if (IS_VAR_DATA_TYPE(pkType)) { v1.pData = (uint8_t*)varDataVal(pBlock->lastPk.pData), v1.nData = varDataLen(pBlock->lastPk.pData); } else { - v1.val = pBlock->lastPk.val; + VALUE_SET_TRIVIAL_DATUM(&v1, pBlock->lastPk.val); } return (tValueCompare(&v1, &pRec->firstKey.key.pks[0]) == 0); } else { // no pk @@ -1915,7 +1931,7 @@ static bool overlapWithNeighborBlock2(SFileDataBlockInfo* pBlock, SBrinRecord* p if (IS_VAR_DATA_TYPE(pkType)) { v1.pData = (uint8_t*)varDataVal(pBlock->firstPk.pData), v1.nData = varDataLen(pBlock->firstPk.pData); } else { - v1.val = pBlock->firstPk.val; + VALUE_SET_TRIVIAL_DATUM(&v1, pBlock->firstPk.val); } return (tValueCompare(&v1, &pRec->lastKey.key.pks[0]) == 0); } else { // no pk @@ -2192,7 +2208,7 @@ static int32_t nextRowFromSttBlocks(SSttBlockReader* pSttBlockReader, STableBloc pNextProc->ts += step; if (pSttBlockReader->numOfPks > 0) { if (IS_NUMERIC_TYPE(pNextProc->pks[0].type)) { - pNextProc->pks[0].val = INT64_MIN; + VALUE_SET_TRIVIAL_DATUM(pNextProc->pks, INT64_MAX); } else { memset(pNextProc->pks[0].pData, 0, pNextProc->pks[0].nData); } @@ -3759,8 +3775,8 @@ static int32_t buildCleanBlockFromDataFiles(STsdbReader* pReader, STableBlockSca if (pReader->suppInfo.numOfPks > 0) { if (IS_NUMERIC_TYPE(pReader->suppInfo.pk.type)) { - pInfo->pks[0].val = pBlockInfo->firstPk.val; - pInfo->pks[1].val = pBlockInfo->lastPk.val; + VALUE_SET_TRIVIAL_DATUM(pInfo->pks, pBlockInfo->firstPk.val); + VALUE_SET_TRIVIAL_DATUM(pInfo->pks + 1, pBlockInfo->lastPk.val); } else { (void)memcpy(pInfo->pks[0].pData, varDataVal(pBlockInfo->firstPk.pData), varDataLen(pBlockInfo->firstPk.pData)); (void)memcpy(pInfo->pks[1].pData, varDataVal(pBlockInfo->lastPk.pData), varDataLen(pBlockInfo->lastPk.pData)); diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c index df2279d3fe..184693b5c1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -221,29 +221,29 @@ int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, in if (asc) { switch (type) { case TSDB_DATA_TYPE_BIGINT: { - pKey->pks[0].val = INT64_MIN; + VALUE_SET_TRIVIAL_DATUM(pKey->pks, INT64_MIN); break; } case TSDB_DATA_TYPE_INT: { int32_t min = INT32_MIN; - (void)memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes); + valueSetDatum(pKey->pks, type, &min, tDataTypes[type].bytes); break; } case TSDB_DATA_TYPE_SMALLINT: { int16_t min = INT16_MIN; - (void)memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes); + valueSetDatum(pKey->pks, type, &min, tDataTypes[type].bytes); break; } case TSDB_DATA_TYPE_TINYINT: { int8_t min = INT8_MIN; - (void)memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes); + valueSetDatum(pKey->pks, type, &min, tDataTypes[type].bytes); break; } case TSDB_DATA_TYPE_UTINYINT: case TSDB_DATA_TYPE_USMALLINT: case TSDB_DATA_TYPE_UINT: case TSDB_DATA_TYPE_UBIGINT: { - pKey->pks[0].val = 0; + VALUE_SET_TRIVIAL_DATUM(pKey->pks, 0); break; } default: @@ -253,28 +253,28 @@ int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, in } else { switch (type) { case TSDB_DATA_TYPE_BIGINT: - pKey->pks[0].val = INT64_MAX; + VALUE_SET_TRIVIAL_DATUM(pKey->pks, INT64_MAX); break; case TSDB_DATA_TYPE_INT: - pKey->pks[0].val = INT32_MAX; + VALUE_SET_TRIVIAL_DATUM(pKey->pks, INT32_MAX); break; case TSDB_DATA_TYPE_SMALLINT: - pKey->pks[0].val = INT16_MAX; + VALUE_SET_TRIVIAL_DATUM(pKey->pks, INT16_MAX); break; case TSDB_DATA_TYPE_TINYINT: - pKey->pks[0].val = INT8_MAX; + VALUE_SET_TRIVIAL_DATUM(pKey->pks, INT8_MAX); break; case TSDB_DATA_TYPE_UBIGINT: - pKey->pks[0].val = UINT64_MAX; + VALUE_SET_TRIVIAL_DATUM(pKey->pks, UINT64_MAX); break; case TSDB_DATA_TYPE_UINT: - pKey->pks[0].val = UINT32_MAX; + VALUE_SET_TRIVIAL_DATUM(pKey->pks, UINT32_MAX); break; case TSDB_DATA_TYPE_USMALLINT: - pKey->pks[0].val = UINT16_MAX; + VALUE_SET_TRIVIAL_DATUM(pKey->pks, UINT16_MAX); break; case TSDB_DATA_TYPE_UTINYINT: - pKey->pks[0].val = UINT8_MAX; + VALUE_SET_TRIVIAL_DATUM(pKey->pks, UINT8_MAX); break; default: code = TSDB_CODE_INVALID_PARA; @@ -722,8 +722,8 @@ int32_t recordToBlockInfo(SFileDataBlockInfo* pBlockInfo, SBrinRecord* record) { TSDB_CHECK_CONDITION((pFirstKey->numOfPKs == pLastKey->numOfPKs), code, lino, _end, TSDB_CODE_INVALID_PARA); if (pFirstKey->numOfPKs > 0) { if (IS_NUMERIC_TYPE(pFirstKey->pks[0].type)) { - pBlockInfo->firstPk.val = pFirstKey->pks[0].val; - pBlockInfo->lastPk.val = pLastKey->pks[0].val; + pBlockInfo->firstPk.val = VALUE_GET_TRIVIAL_DATUM(pFirstKey->pks); + pBlockInfo->lastPk.val = VALUE_GET_TRIVIAL_DATUM(pLastKey->pks); } else { int32_t keyLen = pFirstKey->pks[0].nData; char* p = taosMemoryMalloc(keyLen + VARSTR_HEADER_SIZE); diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 5372456e2b..276f871743 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -615,9 +615,9 @@ void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal * } } else if (pRow->type == TSDBROW_COL_FMT) { if (iCol == 0) { - *pColVal = - COL_VAL_VALUE(PRIMARYKEY_TIMESTAMP_COL_ID, - ((SValue){.type = TSDB_DATA_TYPE_TIMESTAMP, .val = pRow->pBlockData->aTSKEY[pRow->iRow]})); + SValue val = {.type = TSDB_DATA_TYPE_TIMESTAMP}; + VALUE_SET_TRIVIAL_DATUM(&val, pRow->pBlockData->aTSKEY[pRow->iRow]); + *pColVal = COL_VAL_VALUE(PRIMARYKEY_TIMESTAMP_COL_ID, val); } else { SColData *pColData = tBlockDataGetColData(pRow->pBlockData, pTColumn->colId); @@ -715,9 +715,9 @@ SColVal *tsdbRowIterNext(STSDBRowIter *pIter) { return tRowIterNext(pIter->pIter); } else if (pIter->pRow->type == TSDBROW_COL_FMT) { if (pIter->iColData == 0) { - pIter->cv = COL_VAL_VALUE( - PRIMARYKEY_TIMESTAMP_COL_ID, - ((SValue){.type = TSDB_DATA_TYPE_TIMESTAMP, .val = pIter->pRow->pBlockData->aTSKEY[pIter->pRow->iRow]})); + SValue val = {.type = TSDB_DATA_TYPE_TIMESTAMP}; + VALUE_SET_TRIVIAL_DATUM(&val, pIter->pRow->pBlockData->aTSKEY[pIter->pRow->iRow]); + pIter->cv = COL_VAL_VALUE(PRIMARYKEY_TIMESTAMP_COL_ID, val); ++pIter->iColData; return &pIter->cv; } @@ -754,8 +754,9 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) // ts jCol = 0; pTColumn = &pTSchema->columns[jCol++]; - - *pColVal = COL_VAL_VALUE(pTColumn->colId, ((SValue){.type = pTColumn->type, .val = key.ts})); + SValue val = {.type = pTColumn->type}; + VALUE_SET_TRIVIAL_DATUM(&val, key.ts); + *pColVal = COL_VAL_VALUE(pTColumn->colId, val); if (taosArrayPush(pMerger->pArray, pColVal) == NULL) { code = terrno; return code; @@ -776,7 +777,8 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) } tsdbRowGetColVal(pRow, pTSchema, jCol++, pColVal); - if ((!COL_VAL_IS_NONE(pColVal)) && (!COL_VAL_IS_NULL(pColVal)) && IS_VAR_DATA_TYPE(pColVal->value.type)) { + bool usepData = IS_VAR_DATA_TYPE(pColVal->value.type) || pColVal->value.type == TSDB_DATA_TYPE_DECIMAL; + if ((!COL_VAL_IS_NONE(pColVal)) && (!COL_VAL_IS_NULL(pColVal)) && usepData) { uint8_t *pVal = pColVal->value.pData; pColVal->value.pData = NULL; @@ -819,7 +821,7 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) if (key.version > pMerger->version) { if (!COL_VAL_IS_NONE(pColVal)) { - if (IS_VAR_DATA_TYPE(pColVal->value.type)) { + if (IS_VAR_DATA_TYPE(pColVal->value.type) || pColVal->value.type == TSDB_DATA_TYPE_DECIMAL) { SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol); if (!pTColVal) return terrno; if (!COL_VAL_IS_NULL(pColVal)) { @@ -842,7 +844,8 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) } else if (key.version < pMerger->version) { SColVal *tColVal = (SColVal *)taosArrayGet(pMerger->pArray, iCol); if (COL_VAL_IS_NONE(tColVal) && !COL_VAL_IS_NONE(pColVal)) { - if ((!COL_VAL_IS_NULL(pColVal)) && IS_VAR_DATA_TYPE(pColVal->value.type)) { + bool usepData = IS_VAR_DATA_TYPE(pColVal->value.type) || pColVal->value.type == TSDB_DATA_TYPE_DECIMAL; + if ((!COL_VAL_IS_NULL(pColVal)) && usepData) { code = tRealloc(&tColVal->value.pData, pColVal->value.nData); if (code) return code; @@ -878,7 +881,7 @@ int32_t tsdbRowMergerInit(SRowMerger *pMerger, STSchema *pSchema) { void tsdbRowMergerClear(SRowMerger *pMerger) { for (int32_t iCol = 1; iCol < pMerger->pTSchema->numOfCols; iCol++) { SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol); - if (IS_VAR_DATA_TYPE(pTColVal->value.type)) { + if (IS_VAR_DATA_TYPE(pTColVal->value.type) || pTColVal->value.type == TSDB_DATA_TYPE_DECIMAL) { tFree(pTColVal->value.pData); } } @@ -890,7 +893,7 @@ void tsdbRowMergerCleanup(SRowMerger *pMerger) { int32_t numOfCols = taosArrayGetSize(pMerger->pArray); for (int32_t iCol = 1; iCol < numOfCols; iCol++) { SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol); - if (IS_VAR_DATA_TYPE(pTColVal->value.type)) { + if (IS_VAR_DATA_TYPE(pTColVal->value.type) || pTColVal->value.type == TSDB_DATA_TYPE_DECIMAL) { tFree(pTColVal->value.pData); } } @@ -1573,11 +1576,23 @@ int32_t tGetDiskDataHdr(SBufferReader *br, SDiskDataHdr *pHdr) { int32_t tPutColumnDataAgg(SBuffer *buffer, SColumnDataAgg *pColAgg) { int32_t code; - if ((code = tBufferPutI16v(buffer, pColAgg->colId))) return code; - if ((code = tBufferPutI16v(buffer, pColAgg->numOfNull))) return code; - if ((code = tBufferPutI64(buffer, pColAgg->sum))) return code; - if ((code = tBufferPutI64(buffer, pColAgg->max))) return code; - if ((code = tBufferPutI64(buffer, pColAgg->min))) return code; + if (pColAgg->colId & DECIMAL_AGG_FLAG) { + if ((code = tBufferPutI32v(buffer, pColAgg->colId))) return code; + if ((code = tBufferPutI16v(buffer, pColAgg->numOfNull))) return code; + if ((code = tBufferPutU64(buffer, pColAgg->decimal128Sum[0]))) return code; + if ((code = tBufferPutU64(buffer, pColAgg->decimal128Sum[1]))) return code; + if ((code = tBufferPutU64(buffer, pColAgg->decimal128Max[0]))) return code; + if ((code = tBufferPutU64(buffer, pColAgg->decimal128Max[1]))) return code; + if ((code = tBufferPutU64(buffer, pColAgg->decimal128Min[0]))) return code; + if ((code = tBufferPutU64(buffer, pColAgg->decimal128Min[1]))) return code; + if ((code = tBufferPutU8(buffer, pColAgg->overflow))) return code; + } else { + if ((code = tBufferPutI32v(buffer, pColAgg->colId))) return code; + if ((code = tBufferPutI16v(buffer, pColAgg->numOfNull))) return code; + if ((code = tBufferPutI64(buffer, pColAgg->sum))) return code; + if ((code = tBufferPutI64(buffer, pColAgg->max))) return code; + if ((code = tBufferPutI64(buffer, pColAgg->min))) return code; + } return 0; } @@ -1585,11 +1600,22 @@ int32_t tPutColumnDataAgg(SBuffer *buffer, SColumnDataAgg *pColAgg) { int32_t tGetColumnDataAgg(SBufferReader *br, SColumnDataAgg *pColAgg) { int32_t code; - if ((code = tBufferGetI16v(br, &pColAgg->colId))) return code; + if ((code = tBufferGetI32v(br, &pColAgg->colId))) return code; if ((code = tBufferGetI16v(br, &pColAgg->numOfNull))) return code; - if ((code = tBufferGetI64(br, &pColAgg->sum))) return code; - if ((code = tBufferGetI64(br, &pColAgg->max))) return code; - if ((code = tBufferGetI64(br, &pColAgg->min))) return code; + if (pColAgg->colId & DECIMAL_AGG_FLAG) { + pColAgg->colId &= 0xFFFF; + if ((code = tBufferGetU64(br, &pColAgg->decimal128Sum[0]))) return code; + if ((code = tBufferGetU64(br, &pColAgg->decimal128Sum[1]))) return code; + if ((code = tBufferGetU64(br, &pColAgg->decimal128Max[0]))) return code; + if ((code = tBufferGetU64(br, &pColAgg->decimal128Max[1]))) return code; + if ((code = tBufferGetU64(br, &pColAgg->decimal128Min[0]))) return code; + if ((code = tBufferGetU64(br, &pColAgg->decimal128Min[1]))) return code; + if ((code = tBufferGetU8(br, &pColAgg->overflow))) return code; + } else { + if ((code = tBufferGetI64(br, &pColAgg->sum))) return code; + if ((code = tBufferGetI64(br, &pColAgg->max))) return code; + if ((code = tBufferGetI64(br, &pColAgg->min))) return code; + } return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 6567d60956..f041da5ccd 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -34,7 +34,7 @@ void vnodeQueryClose(SVnode *pVnode) { qWorkerDestroy((void **)&pVnode->pQuery); int32_t fillTableColCmpr(SMetaReader *reader, SSchemaExt *pExt, int32_t numOfCol) { int8_t tblType = reader->me.type; - if (useCompress(tblType)) { + if (withExtSchema(tblType)) { SColCmprWrapper *p = &(reader->me.colCmpr); if (numOfCol != p->nCols) { vError("fillTableColCmpr table type:%d, col num:%d, col cmpr num:%d mismatch", tblType, numOfCol, p->nCols); @@ -180,6 +180,9 @@ int32_t vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { if (code < 0) { goto _exit; } + for (int32_t i = 0; i < metaRsp.numOfColumns && pReader->me.pExtSchemas; i++) { + metaRsp.pSchemaExt[i].typeMod = pReader->me.pExtSchemas[i].typeMod; + } } else { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; @@ -348,6 +351,10 @@ int32_t vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { SSchemaExt *pSchExt = cfgRsp.pSchemaExt + i; pSchExt->colId = pCmpr->id; pSchExt->compress = pCmpr->alg; + if (pReader->me.pExtSchemas) + pSchExt->typeMod = pReader->me.pExtSchemas[i].typeMod; + else + pSchExt->typeMod = 0; } //} @@ -753,7 +760,7 @@ int32_t vnodeGetCtbNum(SVnode *pVnode, int64_t suid, int64_t *num) { } int32_t vnodeGetStbColumnNum(SVnode *pVnode, tb_uid_t suid, int *num) { - SSchemaWrapper *pSW = metaGetTableSchema(pVnode->pMeta, suid, -1, 0); + SSchemaWrapper *pSW = metaGetTableSchema(pVnode->pMeta, suid, -1, 0, NULL); if (pSW) { *num = pSW->nCols; tDeleteSchemaWrapper(pSW); diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index df6e377261..982a4c94fa 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -1800,11 +1800,11 @@ static int32_t vnodeCellValConvertToColVal(STColumn *pCol, SCellVal *pCellVal, S pColVal->value.pData = (uint8_t *)varDataVal(pCellVal->val); } else if (TSDB_DATA_TYPE_FLOAT == pCol->type) { float f = GET_FLOAT_VAL(pCellVal->val); - memcpy(&pColVal->value.val, &f, sizeof(f)); + valueSetDatum(&pColVal->value, pCol->type, &f, sizeof(f)); } else if (TSDB_DATA_TYPE_DOUBLE == pCol->type) { taosSetPInt64Aligned(&pColVal->value.val, (int64_t *)pCellVal->val); } else { - GET_TYPED_DATA(pColVal->value.val, int64_t, pCol->type, pCellVal->val); + valueSetDatum(&pColVal->value, pCol->type, pCellVal->val, tDataTypes[pCol->type].bytes); } pColVal->flag = CV_FLAG_VALUE; diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index 033582f2c0..8ee417ac57 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -25,3 +25,4 @@ add_subdirectory(geometry) add_subdirectory(command) add_subdirectory(azure) add_subdirectory(tcs) +add_subdirectory(decimal) diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 4712a942e9..a31e82ccb0 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -598,13 +598,22 @@ int32_t ctgCopyTbMeta(SCatalog *pCtg, SCtgTbMetaCtx *ctx, SCtgDBCache **pDb, SCt } metaSize = CTG_META_SIZE(stbMeta); - *pTableMeta = taosMemoryRealloc(*pTableMeta, metaSize); + int32_t schemaExtSize = 0; + if (stbMeta->schemaExt) { + schemaExtSize = stbMeta->tableInfo.numOfColumns * sizeof(SSchemaExt); + } + *pTableMeta = taosMemoryRealloc(*pTableMeta, metaSize + schemaExtSize); if (NULL == *pTableMeta) { CTG_ERR_RET(terrno); } TAOS_MEMCPY(&(*pTableMeta)->sversion, &stbMeta->sversion, metaSize - sizeof(SCTableMeta)); - (*pTableMeta)->schemaExt = NULL; + if (stbMeta->schemaExt) { + (*pTableMeta)->schemaExt = (SSchemaExt*)((char*)*pTableMeta + metaSize); + TAOS_MEMCPY((*pTableMeta)->schemaExt, stbMeta->schemaExt, schemaExtSize); + } else { + (*pTableMeta)->schemaExt = NULL; + } return TSDB_CODE_SUCCESS; } diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 33fe39a992..cc98721b3e 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -1684,7 +1684,7 @@ int32_t ctgCloneMetaOutput(STableMetaOutput* output, STableMetaOutput** pOutput) if (output->tbMeta) { int32_t metaSize = CTG_META_SIZE(output->tbMeta); int32_t schemaExtSize = 0; - if (useCompress(output->tbMeta->tableType) && (*pOutput)->tbMeta->schemaExt) { + if (withExtSchema(output->tbMeta->tableType) && (*pOutput)->tbMeta->schemaExt) { schemaExtSize = output->tbMeta->tableInfo.numOfColumns * sizeof(SSchemaExt); } @@ -1697,7 +1697,7 @@ int32_t ctgCloneMetaOutput(STableMetaOutput* output, STableMetaOutput** pOutput) } TAOS_MEMCPY((*pOutput)->tbMeta, output->tbMeta, metaSize); - if (useCompress(output->tbMeta->tableType) && (*pOutput)->tbMeta->schemaExt) { + if (withExtSchema(output->tbMeta->tableType) && (*pOutput)->tbMeta->schemaExt) { (*pOutput)->tbMeta->schemaExt = (SSchemaExt*)((char*)(*pOutput)->tbMeta + metaSize); TAOS_MEMCPY((*pOutput)->tbMeta->schemaExt, output->tbMeta->schemaExt, schemaExtSize); } else { diff --git a/source/libs/command/CMakeLists.txt b/source/libs/command/CMakeLists.txt index 308f652861..533c144bb7 100644 --- a/source/libs/command/CMakeLists.txt +++ b/source/libs/command/CMakeLists.txt @@ -8,7 +8,7 @@ target_include_directories( target_link_libraries( command - PRIVATE os util nodes catalog function transport qcom scheduler + PRIVATE os util nodes catalog function transport qcom scheduler decimal ) if(${BUILD_TEST}) diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 1458164e28..e3002766da 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -16,6 +16,7 @@ #include "catalog.h" #include "command.h" #include "commandInt.h" +#include "decimal.h" #include "scheduler.h" #include "systable.h" #include "taosdef.h" @@ -150,7 +151,7 @@ static int32_t setDescResultIntoDataBlock(bool sysInfoUser, SSDataBlock* pBlock, SColumnInfoData* pCol6 = NULL; // level SColumnInfoData* pCol7 = NULL; - if (useCompress(pMeta->tableType)) { + if (withExtSchema(pMeta->tableType)) { pCol5 = taosArrayGet(pBlock->pDataBlock, 4); pCol6 = taosArrayGet(pBlock->pDataBlock, 5); pCol7 = taosArrayGet(pBlock->pDataBlock, 6); @@ -165,7 +166,15 @@ static int32_t setDescResultIntoDataBlock(bool sysInfoUser, SSDataBlock* pBlock, STR_TO_VARSTR(buf, pMeta->schema[i].name); COL_DATA_SET_VAL_AND_CHECK(pCol1, pBlock->info.rows, buf, false); - STR_TO_VARSTR(buf, tDataTypes[pMeta->schema[i].type].name); + if (IS_DECIMAL_TYPE(pMeta->schema[i].type) && withExtSchema(pMeta->tableType)) { + uint8_t prec = 0, scale = 0; + decimalFromTypeMod(pMeta->schemaExt[i].typeMod, &prec, &scale); + size_t len = snprintf(buf + VARSTR_HEADER_SIZE, DESCRIBE_RESULT_FIELD_LEN - VARSTR_HEADER_SIZE, "%s(%hhu, %hhu)", + tDataTypes[pMeta->schema[i].type].name, prec, scale); + varDataSetLen(buf, len); + } else { + STR_TO_VARSTR(buf, tDataTypes[pMeta->schema[i].type].name); + } COL_DATA_SET_VAL_AND_CHECK(pCol2, pBlock->info.rows, buf, false); int32_t bytes = getSchemaBytes(pMeta->schema + i); COL_DATA_SET_VAL_AND_CHECK(pCol3, pBlock->info.rows, (const char*)&bytes, false); @@ -182,7 +191,7 @@ static int32_t setDescResultIntoDataBlock(bool sysInfoUser, SSDataBlock* pBlock, STR_TO_VARSTR(buf, "VIEW COL"); } COL_DATA_SET_VAL_AND_CHECK(pCol4, pBlock->info.rows, buf, false); - if (useCompress(pMeta->tableType) && pMeta->schemaExt) { + if (withExtSchema(pMeta->tableType) && pMeta->schemaExt) { if (i < pMeta->tableInfo.numOfColumns) { STR_TO_VARSTR(buf, columnEncodeStr(COMPRESS_L1_TYPE_U32(pMeta->schemaExt[i].compress))); COL_DATA_SET_VAL_AND_CHECK(pCol5, pBlock->info.rows, buf, false); @@ -235,7 +244,7 @@ static int32_t execDescribe(bool sysInfoUser, SNode* pStmt, SRetrieveTableRsp** code = setDescResultIntoDataBlock(sysInfoUser, pBlock, numOfRows, pDesc->pMeta, biMode); } if (TSDB_CODE_SUCCESS == code) { - if (pDesc->pMeta && useCompress(pDesc->pMeta->tableType) && pDesc->pMeta->schemaExt) { + if (pDesc->pMeta && withExtSchema(pDesc->pMeta->tableType) && pDesc->pMeta->schemaExt) { code = buildRetrieveTableRsp(pBlock, DESCRIBE_RESULT_COLS_COMPRESS, pRsp); } else { code = buildRetrieveTableRsp(pBlock, DESCRIBE_RESULT_COLS, pRsp); @@ -537,9 +546,13 @@ static void appendColumnFields(char* buf, int32_t* len, STableCfg* pCfg) { } else if (TSDB_DATA_TYPE_NCHAR == pSchema->type) { typeLen += tsnprintf(type + typeLen, LTYPE_LEN - typeLen, "(%d)", (int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); + } else if (IS_DECIMAL_TYPE(pSchema->type)) { + uint8_t precision, scale; + decimalFromTypeMod(pCfg->pSchemaExt[i].typeMod, &precision, &scale); + typeLen += tsnprintf(type + typeLen, LTYPE_LEN - typeLen, "(%d,%d)", precision, scale); } - if (useCompress(pCfg->tableType) && pCfg->pSchemaExt) { + if (withExtSchema(pCfg->tableType) && pCfg->pSchemaExt) { typeLen += tsnprintf(type + typeLen, LTYPE_LEN - typeLen, " ENCODE \'%s\'", columnEncodeStr(COMPRESS_L1_TYPE_U32(pCfg->pSchemaExt[i].compress))); typeLen += tsnprintf(type + typeLen, LTYPE_LEN - typeLen, " COMPRESS \'%s\'", @@ -1027,6 +1040,8 @@ static int32_t createSelectResultDataBlock(SNodeList* pProjects, SSDataBlock** p } else { infoData.info.type = pExpr->resType.type; infoData.info.bytes = pExpr->resType.bytes; + infoData.info.precision = pExpr->resType.precision; + infoData.info.scale = pExpr->resType.scale; } QRY_ERR_RET(blockDataAppendColInfo(pBlock, &infoData)); } diff --git a/source/libs/decimal/CMakeLists.txt b/source/libs/decimal/CMakeLists.txt new file mode 100644 index 0000000000..440953216b --- /dev/null +++ b/source/libs/decimal/CMakeLists.txt @@ -0,0 +1,18 @@ +aux_source_directory(src DECIMAL_SRC) + +add_library(decimal STATIC ${DECIMAL_SRC}) + +target_include_directories( + decimal + PUBLIC "${TD_SOURCE_DIR}/include/libs/decimal" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) +target_link_libraries( + decimal + PRIVATE os common wideInteger +) + +if(${BUILD_TEST}) + ADD_SUBDIRECTORY(test) +endif(${BUILD_TEST}) +ADD_SUBDIRECTORY(src/detail) diff --git a/source/libs/decimal/inc/wideInteger.h b/source/libs/decimal/inc/wideInteger.h new file mode 100644 index 0000000000..9d88d2c539 --- /dev/null +++ b/source/libs/decimal/inc/wideInteger.h @@ -0,0 +1,144 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_WIDE_INTEGER_H_ +#define _TD_WIDE_INTEGER_H_ + +#include +#include +#ifdef __cplusplus +extern "C" { +#endif + +struct uint128 { + uint64_t low; + uint64_t high; +}; + +struct int128 { + uint64_t low; + int64_t high; +}; + +struct uint256 { + struct uint128 low; + struct uint128 high; +}; + +struct int256 { + struct uint128 low; + struct int128 high; +}; + +#define UInt128 struct uint128 +#define UInt256 struct uint256 +#define Int128 struct int128 +#define Int256 struct int256 + +#define SAFE_SIGNED_OP(a, b, SIGNED_TYPE, UNSIGNED_TYPE, OP) (SIGNED_TYPE)((UNSIGNED_TYPE)(a)OP(UNSIGNED_TYPE)(b)) +#define SAFE_INT64_ADD(a, b) SAFE_SIGNED_OP(a, b, int64_t, uint64_t, +) +#define SAFE_INT64_SUBTRACT(a, b) SAFE_SIGNED_OP(a, b, int64_t, uint64_t, -) + +void makeUInt128(UInt128* pInt, uint64_t hi, uint64_t lo); +uint64_t uInt128Hi(const UInt128* pInt); +uint64_t uInt128Lo(const UInt128* pInt); +void uInt128Add(UInt128* pLeft, const UInt128* pRight); +void uInt128Subtract(UInt128* pLeft, const UInt128* pRight); +void uInt128Multiply(UInt128* pLeft, const UInt128* pRight); +void uInt128Divide(UInt128* pLeft, const UInt128* pRight); +void uInt128Mod(UInt128* pLeft, const UInt128* pRight); +bool uInt128Lt(const UInt128* pLeft, const UInt128* pRight); +bool uInt128Gt(const UInt128* pLeft, const UInt128* pRight); +bool uInt128Eq(const UInt128* pLeft, const UInt128* pRight); + +extern const UInt128 uInt128_1e18; +extern const UInt128 uInt128Zero; +extern const uint64_t k1e18; +extern const UInt128 uInt128One; +extern const UInt128 uInt128Two; + +Int128 makeInt128(int64_t high, uint64_t low); +int64_t int128Hi(const Int128* pUint128); +uint64_t int128Lo(const Int128* pUint128); +Int128 int128Abs(const Int128* pInt128); +Int128 int128Negate(const Int128* pInt128); +Int128 int128Add(const Int128* pLeft, const Int128* pRight); +Int128 int128Subtract(const Int128* pLeft, const Int128* pRight); +Int128 int128Multiply(const Int128* pLeft, const Int128* pRight); +Int128 int128Divide(const Int128* pLeft, const Int128* pRight); +Int128 int128Mod(const Int128* pLeft, const Int128* pRight); +bool int128Lt(const Int128* pLeft, const Int128* pRight); +bool int128Gt(const Int128* pLeft, const Int128* pRight); +bool int128Eq(const Int128* pLeft, const Int128* pRight); +Int128 int128RightShift(const Int128* pLeft, int32_t shift); + +extern const Int128 int128Zero; +extern const Int128 int128One; + +UInt256 makeUint256(UInt128 high, UInt128 low); +UInt128 uInt256Hi(const UInt256* pUint256); +UInt128 uInt256Lo(const UInt256* pUint256); +UInt256 uInt256Add(const UInt256* pLeft, const UInt256* pRight); +UInt256 uInt256Subtract(const UInt256* pLeft, const UInt256* pRight); +UInt256 uInt256Multiply(const UInt256* pLeft, const UInt256* pRight); +UInt256 uInt256Divide(const UInt256* pLeft, const UInt256* pRight); +UInt256 uInt256Mod(const UInt256* pLeft, const UInt256* pRight); +bool uInt256Lt(const UInt256* pLeft, const UInt256* pRight); +bool uInt256Gt(const UInt256* pLeft, const UInt256* pRight); +bool uInt256Eq(const UInt256* pLeft, const UInt256* pRight); +UInt256 uInt256RightShift(const UInt256* pLeft, int32_t shift); + +extern const UInt256 uInt256Zero; +extern const UInt256 uInt256One; + +Int256 makeInt256(Int128 high, UInt128 low); +Int128 int256Hi(const Int256* pUint256); +UInt128 int256Lo(const Int256* pUint256); +Int256 int256Abs(const Int256* pInt256); +Int256 int256Negate(const Int256* pInt256); +Int256 int256Add(const Int256* pLeft, const Int256* pRight); +Int256 int256Subtract(const Int256* pLeft, const Int256* pRight); +Int256 int256Multiply(const Int256* pLeft, const Int256* pRight); +Int256 int256Divide(const Int256* pLeft, const Int256* pRight); +Int256 int256Mod(const Int256* pLeft, const Int256* pRight); +bool int256Lt(const Int256* pLeft, const Int256* pRight); +bool int256Gt(const Int256* pLeft, const Int256* pRight); +bool int256Eq(const Int256* pLeft, const Int256* pRight); +Int256 int256RightShift(const Int256* pLeft, int32_t shift); + +extern const Int256 int256Zero; +extern const Int256 int256One; +extern const Int256 int256Two; + +#ifdef __cplusplus +} +#endif + +static inline int32_t countLeadingZeros(uint64_t v) { +#if defined(__clang__) || defined(__GNUC__) + if (v == 0) return 64; + return __builtin_clzll(v); +#else + int32_t bitpos = 0; + while (v != 0) { + v >>= 1; + ++bitpos; + } + return 64 - bitpos; +#endif +} + + +#endif /* _TD_WIDE_INTEGER_H_ */ diff --git a/source/libs/decimal/src/decimal.c b/source/libs/decimal/src/decimal.c new file mode 100644 index 0000000000..cc59f4c719 --- /dev/null +++ b/source/libs/decimal/src/decimal.c @@ -0,0 +1,2024 @@ +/* + * + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "decimal.h" +#include "tdataformat.h" +#include "wideInteger.h" + +typedef enum DecimalInternalType { + DECIMAL_64 = 0, + DECIMAL_128 = 1, +} DecimalInternalType; + +typedef enum DecimalRoundType { + ROUND_TYPE_CEIL, + ROUND_TYPE_FLOOR, + ROUND_TYPE_TRUNC, + ROUND_TYPE_HALF_ROUND_UP, +} DecimalRoundType; + +#define DECIMAL_GET_INTERNAL_TYPE(dataType) ((dataType) == TSDB_DATA_TYPE_DECIMAL ? DECIMAL_128 : DECIMAL_64) +#define DECIMAL_GET_WORD_NUM(decimalInternalType) \ + ((decimalInternalType) == DECIMAL_64 ? DECIMAL_WORD_NUM(Decimal64) : DECIMAL_WORD_NUM(Decimal128)) +static SDecimalOps* getDecimalOpsImp(DecimalInternalType t); + +#define DECIMAL_MIN_ADJUSTED_SCALE 6 + +static Decimal64 SCALE_MULTIPLIER_64[TSDB_DECIMAL64_MAX_PRECISION + 1] = {1LL, + 10LL, + 100LL, + 1000LL, + 10000LL, + 100000LL, + 1000000LL, + 10000000LL, + 100000000LL, + 1000000000LL, + 10000000000LL, + 100000000000LL, + 1000000000000LL, + 10000000000000LL, + 100000000000000LL, + 1000000000000000LL, + 10000000000000000LL, + 100000000000000000LL, + 1000000000000000000LL}; + +typedef struct DecimalVar { + DecimalInternalType type; + int8_t precision; + int8_t scale; + int32_t exponent; + int8_t sign; + DecimalType* pDec; + int32_t weight; +} DecimalVar; + +static uint8_t maxPrecision(DecimalInternalType type) { + switch (type) { + case DECIMAL_64: + return TSDB_DECIMAL64_MAX_PRECISION; + case DECIMAL_128: + return TSDB_DECIMAL128_MAX_PRECISION; + default: + return 0; + } +} + +static const uint8_t typeConvertDecimalPrec[] = { + 0, 1, 3, 5, 10, 19, TSDB_DECIMAL128_MAX_PRECISION, TSDB_DECIMAL_MAX_PRECISION, 0, 19, 10, 3, 5, 10, 20, 0, + 0, 0, 0, 0, 0, 0}; + +int32_t decimalGetRetType(const SDataType* pLeftT, const SDataType* pRightT, EOperatorType opType, + SDataType* pOutType) { + if (pLeftT->type == TSDB_DATA_TYPE_JSON || pRightT->type == TSDB_DATA_TYPE_JSON || + pLeftT->type == TSDB_DATA_TYPE_VARBINARY || pRightT->type == TSDB_DATA_TYPE_VARBINARY) + return TSDB_CODE_TSC_INVALID_OPERATION; + if ((pLeftT->type >= TSDB_DATA_TYPE_BLOB && pLeftT->type <= TSDB_DATA_TYPE_GEOMETRY) || + (pRightT->type >= TSDB_DATA_TYPE_BLOB && pRightT->type <= TSDB_DATA_TYPE_GEOMETRY)) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + if (IS_FLOAT_TYPE(pLeftT->type) || IS_FLOAT_TYPE(pRightT->type) || IS_VAR_DATA_TYPE(pLeftT->type) || + IS_VAR_DATA_TYPE(pRightT->type)) { + pOutType->type = TSDB_DATA_TYPE_DOUBLE; + pOutType->bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; + return 0; + } + + if (IS_NULL_TYPE(pLeftT->type) || IS_NULL_TYPE(pRightT->type)) { + pOutType->type = TSDB_DATA_TYPE_NULL; + pOutType->bytes = tDataTypes[TSDB_DATA_TYPE_NULL].bytes; + return 0; + } + uint8_t p1 = pLeftT->precision, s1 = pLeftT->scale, p2 = pRightT->precision, s2 = pRightT->scale; + + if (!IS_DECIMAL_TYPE(pLeftT->type)) { + p1 = typeConvertDecimalPrec[pLeftT->type]; + s1 = 0; + } + if (!IS_DECIMAL_TYPE(pRightT->type)) { + p2 = typeConvertDecimalPrec[pRightT->type]; + s2 = 0; + } + + switch (opType) { + case OP_TYPE_ADD: + case OP_TYPE_SUB: + pOutType->scale = TMAX(s1, s2); + pOutType->precision = TMAX(p1 - s1, p2 - s2) + pOutType->scale + 1; + break; + case OP_TYPE_MULTI: + pOutType->scale = s1 + s2; + pOutType->precision = p1 + p2 + 1; + break; + case OP_TYPE_DIV: + pOutType->scale = TMAX(s1 + p2 + 1, DECIMAL_MIN_ADJUSTED_SCALE); + pOutType->precision = p1 - s1 + s2 + pOutType->scale; + break; + case OP_TYPE_REM: + pOutType->scale = TMAX(s1, s2); + pOutType->precision = TMIN(p1 - s1, p2 - s2) + pOutType->scale; + break; + default: + return TSDB_CODE_TSC_INVALID_OPERATION; + } + if (pOutType->precision > TSDB_DECIMAL_MAX_PRECISION) { + int8_t minScale = TMIN(DECIMAL_MIN_ADJUSTED_SCALE, pOutType->scale); + int8_t delta = pOutType->precision - TSDB_DECIMAL_MAX_PRECISION; + pOutType->precision = TSDB_DECIMAL_MAX_PRECISION; + pOutType->scale = TMAX(minScale, (int8_t)(pOutType->scale) - delta); + } + pOutType->type = TSDB_DATA_TYPE_DECIMAL; + pOutType->bytes = tDataTypes[pOutType->type].bytes; + return 0; +} + +int32_t calcCurPrec(int32_t prec, int32_t places, int32_t exp, int32_t weight, int32_t firstValidScale) { + if (exp == 0) return prec + places; + if (exp < 0) { + if (weight + exp >= 0) return prec + places; + return prec + places - exp - weight; + } + if (weight > 0) return prec + places; + return prec + places - TMIN(firstValidScale - 1, exp); +} + +int32_t calcActualWeight(int32_t prec, int32_t scale, int32_t exp, int32_t weight, int32_t firstValidScale) { + if (exp == 0) return prec - scale; + if (exp < 0) { + if (weight + exp >= 0) return weight + exp; + return 0; + } + if (weight > 0) return weight + exp; + if (firstValidScale == 0) return 0; + return TMAX(0, exp - firstValidScale); +} + +static int32_t decimalVarFromStr(const char* str, int32_t len, DecimalVar* result) { + int32_t code = 0, pos = 0; + int32_t expectPrecision = result->precision; + int32_t expectScale = result->scale; + result->precision = 0; + result->scale = 0; + result->exponent = 0; + bool leadingZeroes = true, afterPoint = false, rounded = false, stop = false; + uint32_t places = 0; + result->sign = 1; + int32_t weight = 0; + int32_t firstValidScale = 0; + + if (len == 0) return TSDB_CODE_INVALID_DATA_FMT; + SDecimalOps* pOps = getDecimalOpsImp(result->type); + + // sign + switch (str[pos]) { + case '-': + result->sign = -1; + case '+': + pos++; + default: + break; + } + int32_t pos2 = pos; + while(pos2 < len) { + if (isdigit(str[pos2] || str[pos] == '.')) continue; + if (str[pos2] == 'e' || str[pos2] == 'E') { + result->exponent = atoi(str + pos2 + 1); + break; + } + pos2++; + } + + for (; pos < len && !stop; ++pos) { + switch (str[pos]) { + case '.': + weight = result->precision; + afterPoint = true; + leadingZeroes = false; + break; + case '0': + if (leadingZeroes) break; + if (afterPoint) { + places++; + break; + } + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': { + leadingZeroes = false; + ++places; + if (firstValidScale == 0 && afterPoint) firstValidScale = places; + + int32_t curPrec = calcCurPrec(result->precision, places, result->exponent, weight, firstValidScale); + int32_t scaleUp = 0; + Decimal64 delta = {0}; + if (curPrec > maxPrecision(result->type)) { + if (!afterPoint) return TSDB_CODE_DECIMAL_OVERFLOW; + int32_t curScale = result->scale - result->exponent + places; + if (rounded || curScale > expectScale + 1 /*scale already overflowed, no need do rounding*/ || + curPrec - 1 != maxPrecision(result->type) /* not the maxPrecision + 1 digit, no need do rounding*/ || + str[pos] < '5') + break; + + // Do rounding for the maxPrecision + 1 digit. + // Here we cannot directly add this digit into the results, because it may cause overflow. + DECIMAL64_SET_VALUE(&delta, 1); + scaleUp = places - 1; + rounded = true; + } else { + scaleUp = places; + DECIMAL64_SET_VALUE(&delta, str[pos] - '0'); + } + + result->precision += scaleUp; + if (afterPoint) result->scale += scaleUp; + while (scaleUp != 0) { + int32_t curScale = TMIN(17, scaleUp); + pOps->multiply(result->pDec, &SCALE_MULTIPLIER_64[curScale], DECIMAL_WORD_NUM(Decimal64)); + scaleUp -= curScale; + } + pOps->add(result->pDec, &delta, DECIMAL_WORD_NUM(Decimal64)); + places = 0; + } break; + case 'e': + case 'E': { + stop = true; + } break; + default: + stop = true; + break; + } + } + result->weight = calcActualWeight(result->precision, result->scale, result->exponent, weight, firstValidScale); + if (result->precision + result->scale > 0) result->scale -= result->exponent; + if (result->sign < 0) { + pOps->negate(result->pDec); + } + return code; +} + +int32_t decimal64ToDataVal(const Decimal64* dec, SValue* pVal) { + VALUE_SET_TRIVIAL_DATUM(pVal, DECIMAL64_GET_VALUE(dec)); + return TSDB_CODE_SUCCESS; +} + +int32_t decimal128ToDataVal(Decimal128* dec, SValue* pVal) { + void* pV = taosMemCalloc(1, sizeof(Decimal128)); + if (!pV) return terrno; + memcpy(pV, dec, DECIMAL_WORD_NUM(Decimal128) * sizeof(DecimalWord)); + valueSetDatum(pVal, TSDB_DATA_TYPE_DECIMAL, pV, DECIMAL_WORD_NUM(Decimal128) * sizeof(DecimalWord)); + return TSDB_CODE_SUCCESS; +} + +#define DECIMAL64_ONE SCALE_MULTIPLIER_64[0] + +#define DECIMAL64_GET_MAX(precision, pMax) \ + do { \ + *(pMax) = SCALE_MULTIPLIER_64[precision]; \ + decimal64Subtract(pMax, &DECIMAL64_ONE, DECIMAL_WORD_NUM(Decimal64)); \ + } while (0) + +#define DECIMAL64_SIGN(pDec) (1 | (DECIMAL64_GET_VALUE(pDec) >> 63)) + +static void decimal64GetWhole(const DecimalType* pDec, int8_t scale, DecimalType* pWhole) { + const SDecimalOps* pOps = getDecimalOps(TSDB_DATA_TYPE_DECIMAL64); + DECIMAL64_CLONE(pWhole, pDec); + Decimal64 scaleMul = SCALE_MULTIPLIER_64[scale]; + pOps->divide(pWhole, &scaleMul, 1, NULL); + pOps->abs(pWhole); +} + +static void decimal64GetFrac(const DecimalType* pDec, int8_t scale, DecimalType* pFrac) { + const SDecimalOps* pOps = getDecimalOpsImp(DECIMAL_64); + DECIMAL64_CLONE(pFrac, pDec); + Decimal64 scaleMul = SCALE_MULTIPLIER_64[scale]; + pOps->mod(pFrac, &scaleMul, 1); + pOps->abs(pFrac); +} + +static void decimal64Negate(DecimalType* pInt); +static void decimal64Abs(DecimalType* pInt); +static void decimal64Add(DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum); +static void decimal64Subtract(DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum); +static void decimal64Multiply(DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum); +static void decimal64divide(DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum, + DecimalType* pRemainder); +static void decimal64Mod(DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum); +static bool decimal64Lt(const DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum); +static bool decimal64Gt(const DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum); +static bool decimal64Eq(const DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum); +static int32_t decimal64ToStr(const DecimalType* pInt, uint8_t scale, char* pBuf, int32_t bufLen); +static void decimal64ScaleDown(Decimal64* pDec, uint8_t scaleDown, bool round); +static void decimal64ScaleUp(Decimal64* pDec, uint8_t scaleUp); +static void decimal64ScaleTo(Decimal64* pDec, uint8_t oldScale, uint8_t newScale); + +static void decimal64RoundWithPositiveScale(Decimal64* pDec, uint8_t prec, int8_t scale, uint8_t toPrec, + uint8_t toScale, DecimalRoundType roundType, bool* overflow); + +static void decimal128Negate(DecimalType* pInt); +static void decimal128Abs(DecimalType* pWord); +static void decimal128Add(DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum); +static void decimal128Subtract(DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum); +static void decimal128Multiply(DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum); +static void decimal128Divide(DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum, + DecimalType* pRemainder); +static void decimal128Mod(DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum); +static bool decimal128Lt(const DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum); +static bool decimal128Gt(const DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum); +static bool decimal128Eq(const DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum); +static int32_t decimal128ToStr(const DecimalType* pInt, uint8_t scale, char* pBuf, int32_t bufLen); +static void decimal128ScaleTo(Decimal128* pDec, uint8_t oldScale, uint8_t newScale); +static void decimal128ScaleDown(Decimal128* pDec, uint8_t scaleDown, bool round); +static void decimal128ScaleUp(Decimal128* pDec, uint8_t scaleUp); +static int32_t decimal128CountLeadingBinaryZeros(const Decimal128* pDec); +static int32_t decimal128FromInt64(DecimalType* pDec, uint8_t prec, uint8_t scale, int64_t val); +static int32_t decimal128FromUint64(DecimalType* pDec, uint8_t prec, uint8_t scale, uint64_t val); +// +// rounding functions +static void decimal128RoundWithPositiveScale(Decimal128* pDec, uint8_t prec, uint8_t scale, uint8_t toPrec, + uint8_t toScale, DecimalRoundType roundType, bool* overflow); +static void decimal128RoundWithNegativeScale(Decimal128* pDec, uint8_t prec, uint8_t scale, int8_t toScale, + DecimalRoundType roundType, bool* overflow); +static void decimal128ModifyScaleAndPrecision(Decimal128* pDec, uint8_t scale, uint8_t toPrec, int8_t toScale, + bool* overflow); +static int32_t decimal128CountRoundingDelta(const Decimal128* pDec, int8_t scale, int8_t toScale, + DecimalRoundType roundType); + +SDecimalOps decimal64Ops = {decimal64Negate, decimal64Abs, decimal64Add, decimal64Subtract, + decimal64Multiply, decimal64divide, decimal64Mod, decimal64Lt, + decimal64Gt, decimal64Eq, decimal64ToStr}; +SDecimalOps decimal128Ops = {decimal128Negate, decimal128Abs, decimal128Add, decimal128Subtract, + decimal128Multiply, decimal128Divide, decimal128Mod, decimal128Lt, + decimal128Gt, decimal128Eq, decimal128ToStr}; + +static SDecimalOps* getDecimalOpsImp(DecimalInternalType t) { + switch (t) { + case DECIMAL_128: + return &decimal128Ops; + case DECIMAL_64: + return &decimal64Ops; + default: + return NULL; + } +} +const SDecimalOps* getDecimalOps(int8_t dataType) { return getDecimalOpsImp(DECIMAL_GET_INTERNAL_TYPE(dataType)); } + +void makeDecimal64(Decimal64* pDec64, int64_t w) { DECIMAL64_SET_VALUE(pDec64, w); } + +void decimal64Negate(DecimalType* pInt) { + Decimal64* pDec = pInt; + DECIMAL64_SET_VALUE(pDec, -DECIMAL64_GET_VALUE(pDec)); +} +void decimal64Abs(DecimalType* pInt) { + Decimal64* pDec = pInt; + DECIMAL64_SET_VALUE(pDec, TABS(DECIMAL64_GET_VALUE(pDec))); +} +void decimal64Add(DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum) { + Decimal64* pDecL = pLeft; + const Decimal64* pDecR = pRight; + DECIMAL64_SET_VALUE(pDecL, SAFE_INT64_ADD(DECIMAL64_GET_VALUE(pDecL), DECIMAL64_GET_VALUE(pDecR))); +} +void decimal64Subtract(DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum) { + Decimal64* pDecL = pLeft; + const Decimal64* pDecR = pRight; + DECIMAL64_SET_VALUE(pDecL, SAFE_INT64_SUBTRACT(DECIMAL64_GET_VALUE(pDecL), DECIMAL64_GET_VALUE(pDecR))); +} +void decimal64Multiply(DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum) { + Decimal64* pDecL = pLeft; + Decimal64 decR = *((Decimal64*)pRight); + bool sign = DECIMAL64_SIGN(pDecL) != DECIMAL64_SIGN(&decR); + decimal64Abs(pLeft); + decimal64Abs(&decR); + uint64_t x = DECIMAL64_GET_VALUE(pDecL), y = DECIMAL64_GET_VALUE(&decR); + x *= y; + DECIMAL64_SET_VALUE(pDecL, x); + if (sign) decimal64Negate(pDecL); +} +void decimal64divide(DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum, DecimalType* pRemainder) { + Decimal64* pDecL = pLeft; + Decimal64 decR = *((Decimal64*)pRight); + Decimal64* pDecRemainder = pRemainder; + bool sign = DECIMAL64_SIGN(pDecL) != DECIMAL64_SIGN(&decR); + decimal64Abs(pDecL); + decimal64Abs(&decR); + uint64_t x = DECIMAL64_GET_VALUE(pDecL), y = DECIMAL64_GET_VALUE(&decR); + uint64_t z = x; + x /= y; + DECIMAL64_SET_VALUE(pDecL, x); + if (sign) decimal64Negate(pDecL); + if (pDecRemainder) { + z %= y; + DECIMAL64_SET_VALUE(pDecRemainder, z); + } +} +void decimal64Mod(DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum) { + Decimal64 remainder = {0}; + Decimal64* pDec = pLeft; + decimal64divide(pLeft, pRight, rightWordNum, &remainder); + DECIMAL64_SET_VALUE(pDec, DECIMAL64_GET_VALUE(&remainder)); +} + +bool decimal64Lt(const DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum) { + const Decimal64 *pDecL = pLeft, *pDecR = pRight; + return DECIMAL64_GET_VALUE(pDecL) < DECIMAL64_GET_VALUE(pDecR); +} +bool decimal64Gt(const DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum) { + return DECIMAL64_GET_VALUE((Decimal64*)pLeft) > DECIMAL64_GET_VALUE((Decimal64*)pRight); +} +bool decimal64Eq(const DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum) { + return DECIMAL64_GET_VALUE((Decimal64*)pLeft) == DECIMAL64_GET_VALUE(((Decimal64*)pRight)); +} +int32_t decimal64ToStr(const DecimalType* pInt, uint8_t scale, char* pBuf, int32_t bufLen) { + if (!pBuf) return TSDB_CODE_INVALID_PARA; + Decimal whole = {0}, frac = {0}; + int32_t pos = 0; + char buf[64] = {0}; + + if (DECIMAL64_SIGN((Decimal64*)pInt) == -1) { + pos = sprintf(buf, "-"); + } + decimal64GetWhole(pInt, scale, &whole); + pos += snprintf(buf + pos, bufLen - pos, "%" PRId64, DECIMAL64_GET_VALUE(&whole)); + if (scale > 0) { + decimal64GetFrac(pInt, scale, &frac); + if (DECIMAL64_GET_VALUE(&frac) != 0 || DECIMAL64_GET_VALUE(&whole) != 0) { + TAOS_STRCAT(buf + pos, "."); + pos += 1; + char format[16] = "\%0"; + snprintf(format + 2, 14, "%" PRIu8 PRIu64, scale); + snprintf(buf + pos, bufLen - pos, format, DECIMAL64_GET_VALUE(&frac)); + } + } + TAOS_STRNCPY(pBuf, buf, bufLen); + return 0; +} + +// return 1 if positive or zero, else return -1 +#define DECIMAL128_SIGN(pDec) (1 | (DECIMAL128_HIGH_WORD(pDec) >> 63)) + +static const Decimal128 SCALE_MULTIPLIER_128[TSDB_DECIMAL128_MAX_PRECISION + 1] = { + DEFINE_DECIMAL128(1LL, 0), + DEFINE_DECIMAL128(10LL, 0), + DEFINE_DECIMAL128(100LL, 0), + DEFINE_DECIMAL128(1000LL, 0), + DEFINE_DECIMAL128(10000LL, 0), + DEFINE_DECIMAL128(100000LL, 0), + DEFINE_DECIMAL128(1000000LL, 0), + DEFINE_DECIMAL128(10000000LL, 0), + DEFINE_DECIMAL128(100000000LL, 0), + DEFINE_DECIMAL128(1000000000LL, 0), + DEFINE_DECIMAL128(10000000000LL, 0), + DEFINE_DECIMAL128(100000000000LL, 0), + DEFINE_DECIMAL128(1000000000000LL, 0), + DEFINE_DECIMAL128(10000000000000LL, 0), + DEFINE_DECIMAL128(100000000000000LL, 0), + DEFINE_DECIMAL128(1000000000000000LL, 0), + DEFINE_DECIMAL128(10000000000000000LL, 0), + DEFINE_DECIMAL128(100000000000000000LL, 0), + DEFINE_DECIMAL128(1000000000000000000LL, 0), + DEFINE_DECIMAL128(10000000000000000000ULL, 0LL), + DEFINE_DECIMAL128(7766279631452241920ULL, 5LL), + DEFINE_DECIMAL128(3875820019684212736ULL, 54LL), + DEFINE_DECIMAL128(1864712049423024128ULL, 542LL), + DEFINE_DECIMAL128(200376420520689664ULL, 5421LL), + DEFINE_DECIMAL128(2003764205206896640ULL, 54210LL), + DEFINE_DECIMAL128(1590897978359414784ULL, 542101LL), + DEFINE_DECIMAL128(15908979783594147840ULL, 5421010LL), + DEFINE_DECIMAL128(11515845246265065472ULL, 54210108LL), + DEFINE_DECIMAL128(4477988020393345024ULL, 542101086LL), + DEFINE_DECIMAL128(7886392056514347008ULL, 5421010862LL), + DEFINE_DECIMAL128(5076944270305263616ULL, 54210108624LL), + DEFINE_DECIMAL128(13875954555633532928ULL, 542101086242LL), + DEFINE_DECIMAL128(9632337040368467968ULL, 5421010862427LL), + DEFINE_DECIMAL128(4089650035136921600ULL, 54210108624275LL), + DEFINE_DECIMAL128(4003012203950112768ULL, 542101086242752LL), + DEFINE_DECIMAL128(3136633892082024448ULL, 5421010862427522LL), + DEFINE_DECIMAL128(12919594847110692864ULL, 54210108624275221LL), + DEFINE_DECIMAL128(68739955140067328ULL, 542101086242752217LL), + DEFINE_DECIMAL128(687399551400673280ULL, 5421010862427522170LL), +}; + +static double getDoubleScaleMultiplier(uint8_t scale) { + static double SCALE_MULTIPLIER_DOUBLE[TSDB_DECIMAL_MAX_PRECISION + 1] = {0}; + static bool initialized = false; + if (!initialized) { + SCALE_MULTIPLIER_DOUBLE[0] = 1.0; + for (int32_t idx = 1; idx <= TSDB_DECIMAL_MAX_PRECISION; ++idx) { + SCALE_MULTIPLIER_DOUBLE[idx] = SCALE_MULTIPLIER_DOUBLE[idx - 1] * 10; + } + initialized = true; + } + return SCALE_MULTIPLIER_DOUBLE[scale]; +}; + +#define DECIMAL128_ONE SCALE_MULTIPLIER_128[0] +#define DECIMAL128_TEN SCALE_MULTIPLIER_128[1] + +// To calculate how many bits for integer X. +// eg. 999(3 digits) -> 1111100111(10 bits) -> bitsForNumDigits[3] = 10 +static const int32_t bitsForNumDigits[] = {0, 4, 7, 10, 14, 17, 20, 24, 27, 30, 34, 37, 40, + 44, 47, 50, 54, 57, 60, 64, 67, 70, 74, 77, 80, 84, + 87, 90, 94, 97, 100, 103, 107, 110, 113, 117, 120, 123, 127}; + +#define DECIMAL128_GET_MAX(precision, pMax) \ + do { \ + *(pMax) = SCALE_MULTIPLIER_128[precision]; \ + decimal128Subtract(pMax, &DECIMAL128_ONE, DECIMAL_WORD_NUM(Decimal128)); \ + } while (0) + +void makeDecimal128(Decimal128* pDec128, int64_t hi, uint64_t low) { + DECIMAL128_SET_HIGH_WORD(pDec128, hi); + DECIMAL128_SET_LOW_WORD(pDec128, low); +} + +static void makeDecimal128FromDecimal64(Decimal128* pTarget, Decimal64 decimal64) { + bool negative = false; + if (DECIMAL64_SIGN(&decimal64) == -1) { + decimal64Negate(&decimal64); + negative = true; + } + makeDecimal128(pTarget, 0, DECIMAL64_GET_VALUE(&decimal64)); + if (negative) decimal128Negate(pTarget); +} + +static void decimal128Negate(DecimalType* pWord) { + Decimal128* pDec = (Decimal128*)pWord; + uint64_t lo = ~DECIMAL128_LOW_WORD(pDec) + 1; + int64_t hi = ~DECIMAL128_HIGH_WORD(pDec); + if (lo == 0) hi = SAFE_INT64_ADD(hi, 1); + makeDecimal128(pDec, hi, lo); +} + +static void decimal128Abs(DecimalType* pWord) { + if (DECIMAL128_SIGN((Decimal128*)pWord) == -1) { + decimal128Negate(pWord); + } +} + +#define DECIMAL128_CHECK_RIGHT_WORD_NUM(rightWordNum, pTarget, rightDec, pWord) \ + if (rightWordNum != DECIMAL_WORD_NUM(Decimal128)) { \ + Decimal64 d64 = {0}; \ + makeDecimal64(&d64, *(int64_t*)pWord); \ + makeDecimal128FromDecimal64(&rightDec, d64); \ + pTarget = &rightDec; \ + } + +static void decimal128Add(DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum) { + Decimal128 *pLeftDec = (Decimal128*)pLeft, *pRightDec = (Decimal128*)pRight; + Decimal128 right = {0}; + DECIMAL128_CHECK_RIGHT_WORD_NUM(rightWordNum, pRightDec, right, pRight); + + int64_t hi = SAFE_INT64_ADD(DECIMAL128_HIGH_WORD(pLeftDec), DECIMAL128_HIGH_WORD(pRightDec)); + uint64_t lo = DECIMAL128_LOW_WORD(pLeftDec) + DECIMAL128_LOW_WORD(pRightDec); + hi = SAFE_INT64_ADD(hi, lo < DECIMAL128_LOW_WORD(pLeftDec)); + makeDecimal128(pLeftDec, hi, lo); +} + +static void decimal128Subtract(DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum) { + Decimal128 *pLeftDec = (Decimal128*)pLeft, *pRightDec = (Decimal128*)pRight; + Decimal128 right = {0}; + DECIMAL128_CHECK_RIGHT_WORD_NUM(rightWordNum, pRightDec, right, pRight); + + int64_t hi = SAFE_INT64_SUBTRACT(DECIMAL128_HIGH_WORD(pLeftDec), DECIMAL128_HIGH_WORD(pRightDec)); + uint64_t lo = DECIMAL128_LOW_WORD(pLeftDec) - DECIMAL128_LOW_WORD(pRightDec); + hi = SAFE_INT64_SUBTRACT(hi, lo > DECIMAL128_LOW_WORD(pLeftDec)); + makeDecimal128(pLeftDec, hi, lo); +} + +static void decimal128Multiply(DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum) { + Decimal128 *pLeftDec = (Decimal128*)pLeft, *pRightDec = (Decimal128*)pRight; + Decimal128 right = {0}; + DECIMAL128_CHECK_RIGHT_WORD_NUM(rightWordNum, pRightDec, right, pRight); + + bool negate = DECIMAL128_SIGN(pLeftDec) != DECIMAL128_SIGN(pRightDec); + Decimal128 x = *pLeftDec, y = *pRightDec; + decimal128Abs(&x); + decimal128Abs(&y); + + UInt128 res = {0}, tmp = {0}; + makeUInt128(&res, DECIMAL128_HIGH_WORD(&x), DECIMAL128_LOW_WORD(&x)); + makeUInt128(&tmp, DECIMAL128_HIGH_WORD(&y), DECIMAL128_LOW_WORD(&y)); + uInt128Multiply(&res, &tmp); + makeDecimal128(pLeftDec, uInt128Hi(&res), uInt128Lo(&res)); + if (negate) decimal128Negate(pLeftDec); +} + +static bool decimal128Lt(const DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum) { + Decimal128 *pLeftDec = (Decimal128*)pLeft, *pRightDec = (Decimal128*)pRight; + Decimal128 right = {0}; + DECIMAL128_CHECK_RIGHT_WORD_NUM(rightWordNum, pRightDec, right, pRight); + + return DECIMAL128_HIGH_WORD(pLeftDec) < DECIMAL128_HIGH_WORD(pRightDec) || + (DECIMAL128_HIGH_WORD(pLeftDec) == DECIMAL128_HIGH_WORD(pRightDec) && + DECIMAL128_LOW_WORD(pLeftDec) < DECIMAL128_LOW_WORD(pRightDec)); +} + +static void decimal128Divide(DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum, + DecimalType* pRemainder) { + Decimal128 *pLeftDec = (Decimal128*)pLeft, *pRightDec = (Decimal128*)pRight, *pRemainderDec = (Decimal128*)pRemainder; + Decimal128 right = {0}; + DECIMAL128_CHECK_RIGHT_WORD_NUM(rightWordNum, pRightDec, right, pRight); + + bool leftNegate = DECIMAL128_SIGN(pLeftDec) == -1, rightNegate = DECIMAL128_SIGN(pRightDec) == -1; + UInt128 a = {0}, b = {0}, c = {0}, d = {0}; + Decimal128 x = *pLeftDec, y = *pRightDec; + decimal128Abs(&x); + decimal128Abs(&y); + makeUInt128(&a, DECIMAL128_HIGH_WORD(&x), DECIMAL128_LOW_WORD(&x)); + makeUInt128(&d, DECIMAL128_HIGH_WORD(&x), DECIMAL128_LOW_WORD(&x)); + makeUInt128(&b, DECIMAL128_HIGH_WORD(&y), DECIMAL128_LOW_WORD(&y)); + uInt128Divide(&a, &b); + uInt128Mod(&d, &b); + makeDecimal128(pLeftDec, uInt128Hi(&a), uInt128Lo(&a)); + if (pRemainder) makeDecimal128(pRemainderDec, uInt128Hi(&d), uInt128Lo(&d)); + if (leftNegate != rightNegate) decimal128Negate(pLeftDec); + if (leftNegate && pRemainder) decimal128Negate(pRemainderDec); +} + +static void decimal128Mod(DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum) { + Decimal128 pLeftDec = *(Decimal128*)pLeft, *pRightDec = (Decimal128*)pRight, right = {0}; + DECIMAL128_CHECK_RIGHT_WORD_NUM(rightWordNum, pRightDec, right, pRight); + + decimal128Divide(&pLeftDec, pRightDec, DECIMAL_WORD_NUM(Decimal128), + pLeft); +} + +static bool decimal128Gt(const DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum) { + Decimal128 *pLeftDec = (Decimal128*)pLeft, *pRightDec = (Decimal128*)pRight; + Decimal128 right = {0}; + DECIMAL128_CHECK_RIGHT_WORD_NUM(rightWordNum, pRightDec, right, pRight); + + return decimal128Lt(pRightDec, pLeftDec, DECIMAL_WORD_NUM(Decimal128)); +} + +static bool decimal128Eq(const DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum) { + Decimal128 *pLeftDec = (Decimal128*)pLeft, *pRightDec = (Decimal128*)pRight; + Decimal128 right = {0}; + DECIMAL128_CHECK_RIGHT_WORD_NUM(rightWordNum, pRightDec, right, pRight); + + return DECIMAL128_HIGH_WORD(pLeftDec) == DECIMAL128_HIGH_WORD(pRightDec) && + DECIMAL128_LOW_WORD(pLeftDec) == DECIMAL128_LOW_WORD(pRightDec); +} + +#define DIGIT_NUM_ONCE 18 +static void extractDecimal128Digits(const Decimal128* pDec, uint64_t* digits, int32_t* digitNum) { + UInt128 a = {0}; + UInt128 b = {0}; + *digitNum = 0; + makeUInt128(&a, DECIMAL128_HIGH_WORD(pDec), DECIMAL128_LOW_WORD(pDec)); + while (!uInt128Eq(&a, &uInt128Zero)) { + uint64_t hi = uInt128Hi(&a); + uint64_t lo = uInt128Lo(&a); + + uint64_t hiQuotient = hi / k1e18; + uint64_t hiRemainder = hi % k1e18; + makeUInt128(&b, hiRemainder, lo); + uInt128Divide(&b, &uInt128_1e18); + uint64_t loQuotient = uInt128Lo(&b); + makeUInt128(&b, hiRemainder, lo); + uInt128Mod(&b, &uInt128_1e18); + uint64_t loRemainder = uInt128Lo(&b); + makeUInt128(&a, hiQuotient, loQuotient); + digits[(*digitNum)++] = loRemainder; + } +} + +static int32_t decimal128ToStr(const DecimalType* pInt, uint8_t scale, char* pBuf, int32_t bufLen) { + if (!pBuf) return TSDB_CODE_INVALID_PARA; + const Decimal128* pDec = (const Decimal128*)pInt; + bool negative = DECIMAL128_SIGN(pDec) == -1; + uint64_t segments[3] = {0}; + int32_t digitNum = 0; + char buf[64] = {0}, buf2[64] = {0}; + int32_t len = 0; + if (negative) { + Decimal128 copy = {0}; + makeDecimal128(©, DECIMAL128_HIGH_WORD(pDec), DECIMAL128_LOW_WORD(pDec)); + decimal128Abs(©); + extractDecimal128Digits(©, segments, &digitNum); + TAOS_STRNCAT(buf2, "-", 2); + } else { + extractDecimal128Digits(pDec, segments, &digitNum); + } + if (digitNum == 0) { + TAOS_STRNCPY(pBuf, "0", bufLen); + return 0; + } + for (int32_t i = digitNum - 1; i >= 0; --i) { + len += snprintf(buf + len, 64 - len, i == digitNum - 1 ? "%" PRIu64 : "%018" PRIu64, segments[i]); + } + int32_t wholeLen = len - scale; + if (wholeLen > 0) { + TAOS_STRNCAT(buf2, buf, wholeLen); + } else { + TAOS_STRNCAT(buf2, "0", 2); + } + if (scale > 0) { + static const char *format = "0000000000000000000000000000000000000000"; + TAOS_STRNCAT(buf2, ".", 2); + if (wholeLen < 0) TAOS_STRNCAT(buf2, format, TABS(wholeLen)); + TAOS_STRNCAT(buf2, buf + TMAX(0, wholeLen), scale); + } + TAOS_STRNCPY(pBuf, buf2, bufLen); + return 0; +} +int32_t decimalToStr(const DecimalType* pDec, int8_t dataType, int8_t precision, int8_t scale, char* pBuf, + int32_t bufLen) { + DecimalInternalType iType = DECIMAL_GET_INTERNAL_TYPE(dataType); + switch (iType) { + case DECIMAL_64: + return decimal64ToStr(pDec, scale, pBuf, bufLen); + case DECIMAL_128: + return decimal128ToStr(pDec, scale, pBuf, bufLen); + default: + break; + } + return TSDB_CODE_INVALID_PARA; +} + +static int32_t decimalAddLargePositive(Decimal* pX, const SDataType* pXT, const Decimal* pY, const SDataType* pYT, + const SDataType* pOT) { + Decimal wholeX = *pX, wholeY = *pY, fracX = {0}, fracY = {0}; + decimal128Divide(&wholeX, &SCALE_MULTIPLIER_128[pXT->scale], DECIMAL_WORD_NUM(Decimal), &fracX); + decimal128Divide(&wholeY, &SCALE_MULTIPLIER_128[pYT->scale], DECIMAL_WORD_NUM(Decimal), &fracY); + + uint8_t maxScale = TMAX(pXT->scale, pYT->scale); + decimal128ScaleUp(&fracX, maxScale - pXT->scale); + decimal128ScaleUp(&fracY, maxScale - pYT->scale); + + Decimal pMultiplier = SCALE_MULTIPLIER_128[maxScale]; + Decimal right = fracX; + Decimal carry = {0}; + decimal128Subtract(&pMultiplier, &fracY, DECIMAL_WORD_NUM(Decimal)); + if (!decimal128Gt(&pMultiplier, &fracX, DECIMAL_WORD_NUM(Decimal))) { + decimal128Subtract(&right, &pMultiplier, DECIMAL_WORD_NUM(Decimal)); + makeDecimal128(&carry, 0, 1); + } else { + decimal128Add(&right, &fracY, DECIMAL_WORD_NUM(Decimal)); + } + + decimal128ScaleDown(&right, maxScale - pOT->scale, true); + if (decimal128AddCheckOverflow(&wholeX, &wholeY, DECIMAL_WORD_NUM(Decimal))) { + return TSDB_CODE_DECIMAL_OVERFLOW; + } + decimal128Add(&wholeX, &wholeY, DECIMAL_WORD_NUM(Decimal)); + if (decimal128AddCheckOverflow(&wholeX, &carry, DECIMAL_WORD_NUM(Decimal))) { + return TSDB_CODE_DECIMAL_OVERFLOW; + } + decimal128Add(&wholeX, &carry, DECIMAL_WORD_NUM(Decimal)); + decimal128Multiply(&wholeX, &SCALE_MULTIPLIER_128[pOT->scale], DECIMAL_WORD_NUM(Decimal)); + decimal128Add(&wholeX, &right, DECIMAL_WORD_NUM(Decimal)); + *pX = wholeX; + return 0; +} + +static void decimalAddLargeNegative(Decimal* pX, const SDataType* pXT, const Decimal* pY, const SDataType* pYT, + const SDataType* pOT) { + Decimal wholeX = *pX, wholeY = *pY, fracX = {0}, fracY = {0}; + decimal128Divide(&wholeX, &SCALE_MULTIPLIER_128[pXT->scale], DECIMAL_WORD_NUM(Decimal), &fracX); + decimal128Divide(&wholeY, &SCALE_MULTIPLIER_128[pYT->scale], DECIMAL_WORD_NUM(Decimal), &fracY); + + uint8_t maxScale = TMAX(pXT->scale, pYT->scale); + decimal128ScaleUp(&fracX, maxScale - pXT->scale); + decimal128ScaleUp(&fracY, maxScale - pYT->scale); + + decimal128Add(&wholeX, &wholeY, DECIMAL_WORD_NUM(Decimal)); + decimal128Add(&fracX, &fracY, DECIMAL_WORD_NUM(Decimal)); + + if (DECIMAL128_SIGN(&wholeX) == -1 && decimal128Gt(&fracX, &DECIMAL128_ZERO, DECIMAL_WORD_NUM(Decimal128))) { + decimal128Add(&wholeX, &DECIMAL128_ONE, DECIMAL_WORD_NUM(Decimal)); + decimal128Subtract(&fracX, &SCALE_MULTIPLIER_128[maxScale], DECIMAL_WORD_NUM(Decimal)); + } else if (decimal128Gt(&wholeX, &DECIMAL128_ZERO, DECIMAL_WORD_NUM(Decimal128)) && DECIMAL128_SIGN(&fracX) == -1) { + decimal128Subtract(&wholeX, &DECIMAL128_ONE, DECIMAL_WORD_NUM(Decimal)); + decimal128Add(&fracX, &SCALE_MULTIPLIER_128[maxScale], DECIMAL_WORD_NUM(Decimal)); + } + + decimal128ScaleDown(&fracX, maxScale - pOT->scale, true); + decimal128Multiply(&wholeX, &SCALE_MULTIPLIER_128[pOT->scale], DECIMAL_WORD_NUM(Decimal)); + decimal128Add(&wholeX, &fracX, DECIMAL_WORD_NUM(Decimal)); + *pX = wholeX; +} + +static int32_t decimalAdd(Decimal* pX, const SDataType* pXT, const Decimal* pY, const SDataType* pYT, + const SDataType* pOT) { + int32_t code = 0; + if (pOT->precision < TSDB_DECIMAL_MAX_PRECISION) { + uint8_t maxScale = TMAX(pXT->scale, pYT->scale); + Decimal tmpY = *pY; + decimal128ScaleTo(pX, pXT->scale, maxScale); + decimal128ScaleTo(&tmpY, pYT->scale, maxScale); + decimal128Add(pX, &tmpY, DECIMAL_WORD_NUM(Decimal)); + } else { + int8_t signX = DECIMAL128_SIGN(pX), signY = DECIMAL128_SIGN(pY); + if (signX == 1 && signY == 1) { + code = decimalAddLargePositive(pX, pXT, pY, pYT, pOT); + } else if (signX == -1 && signY == -1) { + decimal128Negate(pX); + Decimal y = *pY; + decimal128Negate(&y); + code = decimalAddLargePositive(pX, pXT, &y, pYT, pOT); + decimal128Negate(pX); + } else { + decimalAddLargeNegative(pX, pXT, pY, pYT, pOT); + } + } + return code; +} + +static void makeInt256FromDecimal128(Int256* pTarget, const Decimal128* pDec) { + bool negative = DECIMAL128_SIGN(pDec) == -1; + Decimal128 abs = *pDec; + decimal128Abs(&abs); + UInt128 tmp = {DECIMAL128_LOW_WORD(&abs), DECIMAL128_HIGH_WORD(&abs)}; + *pTarget = makeInt256(int128Zero, tmp); + if (negative) { + *pTarget = int256Negate(pTarget); + } +} + +static Int256 int256ScaleBy(const Int256* pX, int32_t scale) { + Int256 result = *pX; + if (scale > 0) { + Int256 multiplier = {0}; + makeInt256FromDecimal128(&multiplier, &SCALE_MULTIPLIER_128[scale]); + result = int256Multiply(pX, &multiplier); + } else if (scale < 0) { + Int256 divisor = {0}; + makeInt256FromDecimal128(&divisor, &SCALE_MULTIPLIER_128[-scale]); + result = int256Divide(pX, &divisor); + Int256 remainder = int256Mod(pX, &divisor); + Int256 afterShift = int256RightShift(&divisor, 1); + remainder = int256Abs(&remainder); + if (!int256Gt(&afterShift, &remainder)) { + if (int256Gt(pX, &int256Zero)) { + result = int256Add(&result, &int256One); + } else { + result = int256Subtract(&result, &int256One); + } + } + } + return result; +} + +static bool convertInt256ToDecimal128(const Int256* pX, Decimal128* pDec) { + bool overflow = false; + Int256 abs = int256Abs(pX); + bool isNegative = int256Lt(pX, &int256Zero); + UInt128 low = int256Lo(&abs); + uint64_t lowLow= uInt128Lo(&low); + uint64_t lowHigh = uInt128Hi(&low); + Int256 afterShift = int256RightShift(&abs, 128); + + if (int256Gt(&afterShift, &int256Zero)) { + overflow = true; + } else if (lowHigh > INT64_MAX) { + overflow = true; + } else { + makeDecimal128(pDec, lowHigh, lowLow); + if (decimal128Gt(pDec, &decimal128Max, DECIMAL_WORD_NUM(Decimal128))) { + overflow = true; + } + } + if (isNegative) { + decimal128Negate(pDec); + } + return overflow; +} + +static int32_t decimalMultiply(Decimal* pX, const SDataType* pXT, const Decimal* pY, const SDataType* pYT, + const SDataType* pOT) { + if (pOT->precision < TSDB_DECIMAL_MAX_PRECISION) { + decimal128Multiply(pX, pY, DECIMAL_WORD_NUM(Decimal)); + } else if (decimal128Eq(pX, &DECIMAL128_ZERO, DECIMAL_WORD_NUM(Decimal)) || + decimal128Eq(pY, &DECIMAL128_ZERO, DECIMAL_WORD_NUM(Decimal))) { + makeDecimal128(pX, 0, 0); + } else { + int8_t deltaScale = pXT->scale + pYT->scale - pOT->scale; + Decimal xAbs = *pX, yAbs = *pY; + decimal128Abs(&xAbs); + decimal128Abs(&yAbs); + if (deltaScale == 0) { + // no need to trim scale + Decimal max = DECIMAL128_MAX; + + decimal128Divide(&max, &yAbs, DECIMAL_WORD_NUM(Decimal), NULL); + if (decimal128Gt(&xAbs, &max, DECIMAL_WORD_NUM(Decimal))) { + return TSDB_CODE_DECIMAL_OVERFLOW; + } else { + decimal128Multiply(pX, pY, DECIMAL_WORD_NUM(Decimal)); + } + } else { + int32_t leadingZeros = decimal128CountLeadingBinaryZeros(&xAbs) + decimal128CountLeadingBinaryZeros(&yAbs); + if (leadingZeros <= 128) { + // need to trim scale + Int256 x256 = {0}, y256 = {0}; + makeInt256FromDecimal128(&x256, pX); + makeInt256FromDecimal128(&y256, pY); + Int256 res = int256Multiply(&x256, &y256); + if (deltaScale != 0) { + res = int256ScaleBy(&res, -deltaScale); + } + bool overflow = convertInt256ToDecimal128(&res, pX); + if (overflow) return TSDB_CODE_DECIMAL_OVERFLOW; + } else { + // no need to trim scale + if (deltaScale <= 38) { + decimal128Multiply(pX, pY, DECIMAL_WORD_NUM(Decimal)); + decimal128ScaleDown(pX, deltaScale, true); + } else { + makeDecimal128(pX, 0, 0); + } + } + } + } + return 0; +} + +static int32_t decimalDivide(Decimal* pX, const SDataType* pXT, const Decimal* pY, const SDataType* pYT, + const SDataType* pOT) { + if (decimal128Eq(pY, &DECIMAL128_ZERO, DECIMAL_WORD_NUM(Decimal))) { + return TSDB_CODE_DIVISION_BY_ZERO; + } + + int8_t deltaScale = pOT->scale + pYT->scale - pXT->scale; + + Decimal xTmp = *pX; + decimal128Abs(&xTmp); + int32_t bitsOccupied = 128 - decimal128CountLeadingBinaryZeros(&xTmp); + if (bitsOccupied + bitsForNumDigits[deltaScale] <= 127) { + xTmp = *pX; + decimal128ScaleUp(&xTmp, deltaScale); + Decimal remainder = {0}; + decimal128Divide(&xTmp, pY, DECIMAL_WORD_NUM(Decimal), &remainder); + + Decimal tmpY = *pY; + decimal128Abs(&tmpY); + decimal128Multiply(&remainder, &decimal128Two, DECIMAL_WORD_NUM(Decimal)); + decimal128Abs(&remainder); + if (!decimal128Lt(&remainder, &tmpY, DECIMAL_WORD_NUM(Decimal))) { + Decimal64 extra = {(DECIMAL128_SIGN(pX) ^ DECIMAL128_SIGN(pY)) + 1}; + decimal128Add(&xTmp, &extra, DECIMAL_WORD_NUM(Decimal64)); + } + *pX = xTmp; + } else { + Int256 x256 = {0}, y256 = {0}; + makeInt256FromDecimal128(&x256, pX); + Int256 xScaledUp = int256ScaleBy(&x256, deltaScale); + makeInt256FromDecimal128(&y256, pY); + Int256 res = int256Divide(&xScaledUp, &y256); + Int256 remainder = int256Mod(&xScaledUp, &y256); + + remainder = int256Multiply(&remainder, &int256Two); + remainder = int256Abs(&remainder); + y256 = int256Abs(&y256); + if (!int256Lt(&remainder, &y256)) { + if ((DECIMAL128_SIGN(pX) ^ DECIMAL128_SIGN(pY)) == 0) { + res = int256Add(&res, &int256One); + } else { + res = int256Subtract(&res, &int256One); + } + } + bool overflow = convertInt256ToDecimal128(&res, pX); + if (overflow) return TSDB_CODE_DECIMAL_OVERFLOW; + } + return 0; +} + +static int32_t decimalMod(Decimal* pX, const SDataType* pXT, const Decimal* pY, const SDataType* pYT, + const SDataType* pOT) { + if (decimal128Eq(pY, &DECIMAL128_ZERO, DECIMAL_WORD_NUM(Decimal))) { + return TSDB_CODE_DIVISION_BY_ZERO; + } + Decimal xAbs = *pX, yAbs = *pY; + decimal128Abs(&xAbs); + decimal128Abs(&yAbs); + int32_t xlz = decimal128CountLeadingBinaryZeros(&xAbs), ylz = decimal128CountLeadingBinaryZeros(&yAbs); + if (pXT->scale < pYT->scale) { + // x scale up + xlz = xlz - bitsForNumDigits[pYT->scale - pXT->scale]; + } else if (pXT->scale > pYT->scale) { + // y scale up + ylz = ylz - bitsForNumDigits[pXT->scale - pYT->scale]; + } + int32_t lz = TMIN(xlz, ylz); + if (lz >= 2) { + // it's safe to scale up + yAbs = *pY; + decimal128ScaleTo(pX, pXT->scale, TMAX(pXT->scale, pYT->scale)); + decimal128ScaleTo(&yAbs, pYT->scale, TMAX(pXT->scale, pYT->scale)); + decimal128Mod(pX, &yAbs, DECIMAL_WORD_NUM(Decimal)); + } else { + Int256 x256 = {0}, y256 = {0}; + makeInt256FromDecimal128(&x256, pX); + makeInt256FromDecimal128(&y256, pY); + if (pXT->scale < pYT->scale) { + x256 = int256ScaleBy(&x256, pYT->scale - pXT->scale); + } else if (pXT->scale > pYT->scale) { + y256 = int256ScaleBy(&y256, pXT->scale - pYT->scale); + } + Int256 res = int256Mod(&x256, &y256); + if (convertInt256ToDecimal128(&res, pX)) { + return TSDB_CODE_DECIMAL_OVERFLOW; + } + } + return 0; +} + +int32_t decimalOp(EOperatorType op, const SDataType* pLeftT, const SDataType* pRightT, const SDataType* pOutT, + const void* pLeftData, const void* pRightData, void* pOutputData) { + int32_t code = 0; + + Decimal left = {0}, right = {0}; + SDataType lt = {.type = TSDB_DATA_TYPE_DECIMAL, + .precision = TSDB_DECIMAL_MAX_PRECISION, + .bytes = tDataTypes[TSDB_DATA_TYPE_DECIMAL].bytes, + .scale = pLeftT->scale}; + SDataType rt = {.type = TSDB_DATA_TYPE_DECIMAL, + .precision = TSDB_DECIMAL_MAX_PRECISION, + .bytes = tDataTypes[TSDB_DATA_TYPE_DECIMAL].bytes, + .scale = 0}; + if (pRightT) rt.scale = pRightT->scale; + if (TSDB_DATA_TYPE_DECIMAL != pLeftT->type) { + code = convertToDecimal(pLeftData, pLeftT, &left, <); + if (TSDB_CODE_SUCCESS != code) return code; + } else { + left = *(Decimal*)pLeftData; + } + if (pRightT && TSDB_DATA_TYPE_DECIMAL != pRightT->type) { + code = convertToDecimal(pRightData, pRightT, &right, &rt); + if (TSDB_CODE_SUCCESS != code) return code; + pRightData = &right; + } else if (pRightData){ + right = *(Decimal*)pRightData; + } +#ifdef DEBUG + char left_var[64] = {0}, right_var[64] = {0}; + decimal128ToStr(&left, lt.scale, left_var, 64); + decimal128ToStr(&right, rt.scale, right_var, 64); +#endif + + switch (op) { + case OP_TYPE_ADD: + code = decimalAdd(&left, <, &right, &rt, pOutT); + break; + case OP_TYPE_SUB: + decimal128Negate(&right); + code = decimalAdd(&left, <, &right, &rt, pOutT); + break; + case OP_TYPE_MULTI: + code = decimalMultiply(&left, <, &right, &rt, pOutT); + break; + case OP_TYPE_DIV: + code = decimalDivide(&left, <, &right, &rt, pOutT); + break; + case OP_TYPE_REM: + code = decimalMod(&left, <, &right, &rt, pOutT); + break; + case OP_TYPE_MINUS: + decimal128Negate(&left); + break; + default: + code = TSDB_CODE_TSC_INVALID_OPERATION; + break; + } + if (0 == code && pOutT->type != TSDB_DATA_TYPE_DECIMAL) { + lt = *pOutT; + lt.type = TSDB_DATA_TYPE_DECIMAL; + code = convertToDecimal(&left, <, pOutputData, pOutT); + } else { + *(Decimal*)pOutputData = left; + } + return code; +} + +bool doCompareDecimal128(EOperatorType op, const Decimal128* pLeftDec, const Decimal128* pRightDec) { + switch (op) { + case OP_TYPE_GREATER_THAN: + return decimal128Gt(pLeftDec, pRightDec, DECIMAL_WORD_NUM(Decimal)); + case OP_TYPE_GREATER_EQUAL: + return !decimal128Lt(pLeftDec, pRightDec, DECIMAL_WORD_NUM(Decimal)); + case OP_TYPE_LOWER_THAN: + return decimal128Lt(pLeftDec, pRightDec, DECIMAL_WORD_NUM(Decimal)); + case OP_TYPE_LOWER_EQUAL: + return !decimal128Gt(pLeftDec, pRightDec, DECIMAL_WORD_NUM(Decimal)); + case OP_TYPE_EQUAL: + return decimal128Eq(pLeftDec, pRightDec, DECIMAL_WORD_NUM(Decimal)); + case OP_TYPE_NOT_EQUAL: + return !decimal128Eq(pLeftDec, pRightDec, DECIMAL_WORD_NUM(Decimal)); + default: + break; + } + return false; +} + +bool decimal64Compare(EOperatorType op, const SDecimalCompareCtx* pLeft, const SDecimalCompareCtx* pRight) { + bool ret = false; + uint8_t leftPrec = 0, leftScale = 0, rightPrec = 0, rightScale = 0; + decimalFromTypeMod(pLeft->typeMod, &leftPrec, &leftScale); + decimalFromTypeMod(pRight->typeMod, &rightPrec, &rightScale); + int32_t deltaScale = leftScale - rightScale; + + Decimal64 leftDec = *(Decimal64*)pLeft->pData, rightDec = *(Decimal64*)pRight->pData; + + if (deltaScale != 0) { + bool needInt128 = (deltaScale < 0 && leftPrec - deltaScale > TSDB_DECIMAL64_MAX_PRECISION) || + (rightPrec + deltaScale > TSDB_DECIMAL64_MAX_PRECISION); + if (needInt128) { + Decimal128 dec128L = {0}, dec128R = {0}; + makeDecimal128FromDecimal64(&dec128L, leftDec); + makeDecimal128FromDecimal64(&dec128R, rightDec); + return doCompareDecimal128(op, &dec128L, &dec128R); + } else { + if (deltaScale < 0) { + decimal64ScaleUp(&leftDec, -deltaScale); + } else { + decimal64ScaleUp(&rightDec, deltaScale); + } + } + } + + switch (op) { + case OP_TYPE_GREATER_THAN: + return decimal64Gt(&leftDec, &rightDec, DECIMAL_WORD_NUM(Decimal64)); + case OP_TYPE_GREATER_EQUAL: + return !decimal64Lt(&leftDec, &rightDec, DECIMAL_WORD_NUM(Decimal64)); + case OP_TYPE_LOWER_THAN: + return decimal64Lt(&leftDec, &rightDec, DECIMAL_WORD_NUM(Decimal64)); + case OP_TYPE_LOWER_EQUAL: + return !decimal64Gt(&leftDec, &rightDec, DECIMAL_WORD_NUM(Decimal64)); + case OP_TYPE_EQUAL: + return decimal64Eq(&leftDec, &rightDec, DECIMAL_WORD_NUM(Decimal64)); + case OP_TYPE_NOT_EQUAL: + return !decimal64Eq(&leftDec, &rightDec, DECIMAL_WORD_NUM(Decimal64)); + default: + break; + } + return ret; +} + +bool decimalCompare(EOperatorType op, const SDecimalCompareCtx* pLeft, const SDecimalCompareCtx* pRight) { + if (pLeft->type == TSDB_DATA_TYPE_DECIMAL64 && pRight->type == TSDB_DATA_TYPE_DECIMAL64) { + return decimal64Compare(op, pLeft, pRight); + } + bool ret = false; + uint8_t leftPrec = 0, leftScale = 0, rightPrec = 0, rightScale = 0; + decimalFromTypeMod(pLeft->typeMod, &leftPrec, &leftScale); + decimalFromTypeMod(pRight->typeMod, &rightPrec, &rightScale); + + if (pLeft->type == TSDB_DATA_TYPE_DECIMAL64) { + Decimal128 dec128 = {0}; + makeDecimal128FromDecimal64(&dec128, *(Decimal64*)pLeft->pData); + SDecimalCompareCtx leftCtx = {.pData = &dec128, + .type = TSDB_DATA_TYPE_DECIMAL, + .typeMod = decimalCalcTypeMod(TSDB_DECIMAL128_MAX_PRECISION, leftScale)}; + return decimalCompare(op, &leftCtx, pRight); + } else if (pRight->type == TSDB_DATA_TYPE_DECIMAL64) { + Decimal128 dec128 = {0}; + makeDecimal128FromDecimal64(&dec128, *(Decimal64*)pRight->pData); + SDecimalCompareCtx rightCtx = {.pData = &dec128, + .type = TSDB_DATA_TYPE_DECIMAL, + .typeMod = decimalCalcTypeMod(TSDB_DECIMAL128_MAX_PRECISION, rightScale)}; + return decimalCompare(op, pLeft, &rightCtx); + } + int32_t deltaScale = leftScale - rightScale; + Decimal pLeftDec = *(Decimal*)pLeft->pData, pRightDec = *(Decimal*)pRight->pData; + + if (deltaScale != 0) { + bool needInt256 = (deltaScale < 0 && leftPrec - deltaScale > TSDB_DECIMAL_MAX_PRECISION) || + (rightPrec + deltaScale > TSDB_DECIMAL_MAX_PRECISION); + if (needInt256) { + Int256 x = {0}, y = {0}; + makeInt256FromDecimal128(&x, &pLeftDec); + makeInt256FromDecimal128(&y, &pRightDec); + if (leftScale < rightScale) { + x = int256ScaleBy(&x, rightScale - leftScale); + } else { + y = int256ScaleBy(&y, leftScale - rightScale); + } + switch (op) { + case OP_TYPE_GREATER_THAN: + return int256Gt(&x, &y); + case OP_TYPE_GREATER_EQUAL: + return !int256Lt(&x, &y); + case OP_TYPE_LOWER_THAN: + return int256Lt(&x, &y); + case OP_TYPE_LOWER_EQUAL: + return !int256Gt(&x, &y); + case OP_TYPE_EQUAL: + return int256Eq(&x, &y); + case OP_TYPE_NOT_EQUAL: + return !int256Eq(&x, &y); + default: + break; + } + return false; + } else { + if (deltaScale < 0) { + decimal128ScaleUp(&pLeftDec, -deltaScale); + } else { + decimal128ScaleUp(&pRightDec, deltaScale); + } + } + } + return doCompareDecimal128(op, &pLeftDec, &pRightDec); +} + +#define ABS_INT64(v) (v) == INT64_MIN ? (uint64_t)INT64_MAX + 1 : (uint64_t)llabs(v) +#define ABS_UINT64(v) (v) + +static int64_t int64FromDecimal64(const DecimalType* pDec, uint8_t prec, uint8_t scale) { + Decimal64 rounded = *(Decimal64*)pDec; + bool overflow = false; + decimal64RoundWithPositiveScale(&rounded, prec, scale, prec, 0, ROUND_TYPE_HALF_ROUND_UP, &overflow); + if (overflow) return 0; + + return DECIMAL64_GET_VALUE(&rounded); +} + +static uint64_t uint64FromDecimal64(const DecimalType* pDec, uint8_t prec, uint8_t scale) { + Decimal64 rounded = *(Decimal64*)pDec; + bool overflow = false; + decimal64RoundWithPositiveScale(&rounded, prec, scale, prec, 0, ROUND_TYPE_HALF_ROUND_UP, &overflow); + if (overflow) return 0; + + return DECIMAL64_GET_VALUE(&rounded); +} + +static int32_t decimal64FromInt64(DecimalType* pDec, uint8_t prec, uint8_t scale, int64_t val) { + Decimal64 max = {0}; + DECIMAL64_GET_MAX(prec - scale, &max); + if (DECIMAL64_GET_VALUE(&max) < val || -DECIMAL64_GET_VALUE(&max) > val) { + return TSDB_CODE_DECIMAL_OVERFLOW; + } + DECIMAL64_SET_VALUE((Decimal64*)pDec, val); + decimal64ScaleUp(pDec, scale); + return 0; +} + +static int32_t decimal64FromUint64(DecimalType* pDec, uint8_t prec, uint8_t scale, uint64_t val) { + Decimal64 max = {0}; + DECIMAL64_GET_MAX(prec - scale, &max); + if ((uint64_t)DECIMAL64_GET_VALUE(&max) < val) return TSDB_CODE_DECIMAL_OVERFLOW; + DECIMAL64_SET_VALUE((Decimal64*)pDec, val); + decimal64ScaleUp(pDec, scale); + return 0; +} + +static int32_t decimal64FromDouble(DecimalType* pDec, uint8_t prec, uint8_t scale, double val) { + double unscaled = val * getDoubleScaleMultiplier(scale); + if (isnan(unscaled)) { + goto __OVERFLOW__; + } + unscaled = round(unscaled); + + bool negative = unscaled < 0 ? true : false; + double abs = TABS(unscaled); + if (abs > ldexp(1.0, 63) - 1) { + goto __OVERFLOW__; + } + + uint64_t result = (uint64_t)abs; + makeDecimal64(pDec, result); + Decimal64 max = {0}; + DECIMAL64_GET_MAX(prec, &max); + if (decimal64Gt(pDec, &max, DECIMAL_WORD_NUM(Decimal64))) goto __OVERFLOW__; + if (negative) decimal64Negate(pDec); + return 0; + +__OVERFLOW__: + makeDecimal64(pDec, 0); + return TSDB_CODE_DECIMAL_OVERFLOW; +} + +static int32_t decimal64FromDecimal128(DecimalType* pDec, uint8_t prec, uint8_t scale, const DecimalType* pVal, + uint8_t valPrec, uint8_t valScale) { + Decimal128 dec128 = *(Decimal128*)pVal, tmpDec128 = {0}; + bool negative = DECIMAL128_SIGN(&dec128) == -1; + if (negative) decimal128Negate(&dec128); + tmpDec128 = dec128; + + Decimal64 max = {0}; + DECIMAL64_GET_MAX(prec - scale, &max); + decimal128ScaleDown(&dec128, valScale, false); + if (decimal128Gt(&dec128, &max, DECIMAL_WORD_NUM(Decimal64))) { + return TSDB_CODE_DECIMAL_OVERFLOW; + } + decimal128ScaleTo(&tmpDec128, valScale, scale); + DECIMAL64_SET_VALUE((Decimal64*)pDec, DECIMAL128_LOW_WORD(&tmpDec128)); + DECIMAL64_GET_MAX(prec, &max); + if (decimal64Lt(&max, pDec, DECIMAL_WORD_NUM(Decimal64))) { + return TSDB_CODE_DECIMAL_OVERFLOW; + } + if (negative) decimal64Negate(pDec); + return 0; +} + +static int32_t decimal64FromDecimal64(DecimalType* pDec, uint8_t prec, uint8_t scale, const DecimalType* pVal, + uint8_t valPrec, uint8_t valScale) { + Decimal64 dec64 = *(Decimal64*)pVal, max = {0}; + bool negative = DECIMAL64_SIGN(&dec64) == -1; + if (negative) decimal64Negate(&dec64); + *(Decimal64*)pDec = dec64; + + DECIMAL64_GET_MAX(prec - scale, &max); + decimal64ScaleDown(&dec64, valScale, false); + if (decimal64Lt(&max, &dec64, DECIMAL_WORD_NUM(Decimal64))) { + return TSDB_CODE_DECIMAL_OVERFLOW; + } + decimal64ScaleTo(pDec, valScale, scale); + DECIMAL64_GET_MAX(prec, &max); + if (decimal64Lt(&max, pDec, DECIMAL_WORD_NUM(Decimal64))) { + return TSDB_CODE_DECIMAL_OVERFLOW; + } + if (negative) decimal64Negate(pDec); + return 0; +} + +static int64_t int64FromDecimal128(const DecimalType* pDec, uint8_t prec, uint8_t scale) { + Decimal128 rounded = *(Decimal128*)pDec; + bool overflow = false; + decimal128RoundWithPositiveScale(&rounded, prec, scale, prec, 0, ROUND_TYPE_HALF_ROUND_UP, &overflow); + if (overflow) { + return 0; + } + Decimal128 max = {0}, min = {0}; + (void)decimal128FromInt64(&max, TSDB_DECIMAL128_MAX_PRECISION, 0, INT64_MAX); + (void)decimal128FromInt64(&min, TSDB_DECIMAL128_MAX_PRECISION, 0, INT64_MIN); + if (decimal128Gt(&rounded, &max, DECIMAL_WORD_NUM(Decimal128)) || decimal128Lt(&rounded, &min, DECIMAL_WORD_NUM(Decimal128))) { + overflow = true; + return (int64_t)DECIMAL128_LOW_WORD(&rounded); + } + + return (int64_t)DECIMAL128_LOW_WORD(&rounded); +} + +static uint64_t uint64FromDecimal128(const DecimalType* pDec, uint8_t prec, uint8_t scale) { + Decimal128 rounded = *(Decimal128*)pDec; + bool overflow = false; + decimal128RoundWithPositiveScale(&rounded, prec, scale, prec, 0, ROUND_TYPE_HALF_ROUND_UP, &overflow); + if (overflow) return 0; + + Decimal128 max = {0}; + (void)decimal128FromUint64(&max, TSDB_DECIMAL128_MAX_PRECISION, 0, UINT64_MAX); + if (decimal128Gt(&rounded, &max, DECIMAL_WORD_NUM(Decimal128)) || + decimal128Lt(&rounded, &decimal128Zero, DECIMAL_WORD_NUM(Decimal128))) { + overflow = true; + return DECIMAL128_LOW_WORD(&rounded); + } + return DECIMAL128_LOW_WORD(&rounded); +} + +static int32_t decimal128FromInt64(DecimalType* pDec, uint8_t prec, uint8_t scale, int64_t val) { + if (prec - scale <= 18) { + Decimal64 max = {0}; + DECIMAL64_GET_MAX(prec - scale, &max); + if (DECIMAL64_GET_VALUE(&max) < val || -DECIMAL64_GET_VALUE(&max) > val) return TSDB_CODE_DECIMAL_OVERFLOW; + } + uint64_t valAbs = ABS_INT64(val); + makeDecimal128(pDec, 0, valAbs); + if (val < 0) decimal128Negate(pDec); + decimal128ScaleUp(pDec, scale); + return 0; +} + +static int32_t decimal128FromUint64(DecimalType* pDec, uint8_t prec, uint8_t scale, uint64_t val) { + if (prec - scale <= 19) { + Decimal128 max = {0}, decVal = {0}; + DECIMAL128_GET_MAX(prec - scale, &max); + makeDecimal128(&decVal, 0, val); + if (decimal128Gt(&decVal, &max, DECIMAL_WORD_NUM(Decimal128))) { + return TSDB_CODE_DECIMAL_OVERFLOW; + } + } + makeDecimal128(pDec, 0, val); + decimal128ScaleUp(pDec, scale); + return 0; +} + +static int32_t decimal128FromDouble(DecimalType* pDec, uint8_t prec, uint8_t scale, double val) { + double unscaled = val * getDoubleScaleMultiplier(scale); + if (isnan(unscaled)) { + goto __OVERFLOW__; + } + unscaled = round(unscaled); + + bool negative = unscaled < 0 ? true : false; + double abs = TABS(unscaled); + if (abs > ldexp(1.0, 127) - 1) { + goto __OVERFLOW__; + } + + uint64_t hi = (uint64_t)ldexp(abs, -64), lo = (uint64_t)(abs - ldexp((double)hi, 64)); + makeDecimal128(pDec, hi, lo); + Decimal128 max = {0}; + DECIMAL128_GET_MAX(prec, &max); + if (decimal128Gt(pDec, &max, DECIMAL_WORD_NUM(Decimal128))) goto __OVERFLOW__; + if (negative) decimal128Negate(pDec); + return 0; + +__OVERFLOW__: + *(Decimal128*)pDec = decimal128Zero; + return TSDB_CODE_DECIMAL_OVERFLOW; +} + +static int32_t decimal128FromDecimal64(DecimalType* pDec, uint8_t prec, uint8_t scale, const DecimalType* pVal, + uint8_t valPrec, uint8_t valScale) { + Decimal64 dec64 = *(Decimal64*)pVal; + bool negative = DECIMAL64_SIGN(&dec64) == -1; + if (negative) decimal64Negate(&dec64); + + makeDecimal128(pDec, 0, DECIMAL64_GET_VALUE(&dec64)); + Decimal128 max = {0}; + DECIMAL128_GET_MAX(prec - scale, &max); + decimal64ScaleDown(&dec64, valScale, false); + if (decimal128Lt(&max, &dec64, DECIMAL_WORD_NUM(Decimal64))) { + return TSDB_CODE_DECIMAL_OVERFLOW; + } + decimal128ScaleTo(pDec, valScale, scale); + DECIMAL128_GET_MAX(prec, &max); + if (decimal128Lt(&max, pDec, DECIMAL_WORD_NUM(Decimal128))) { + return TSDB_CODE_DECIMAL_OVERFLOW; + } + if (negative) decimal128Negate(pDec); + return 0; +} + +static int32_t decimal128FromDecimal128(DecimalType* pDec, uint8_t prec, uint8_t scale, const DecimalType* pVal, + uint8_t valPrec, uint8_t valScale) { + bool negative = DECIMAL128_SIGN((Decimal128*)pVal) == -1; + Decimal128 tmpDec = *(Decimal128*)pVal; + if (negative) decimal128Negate(&tmpDec); + *(Decimal128*)pDec = tmpDec; + + Decimal128 max = {0}; + DECIMAL128_GET_MAX(prec - scale, &max); + decimal128ScaleDown(&tmpDec, valScale, false); + if (decimal128Lt(&max, &tmpDec, DECIMAL_WORD_NUM(Decimal128))) { + return TSDB_CODE_DECIMAL_OVERFLOW; + } + decimal128ScaleTo(pDec, valScale, scale); + DECIMAL128_GET_MAX(prec, &max); + if (decimal128Lt(&max, pDec, DECIMAL_WORD_NUM(Decimal128))) { + return TSDB_CODE_DECIMAL_OVERFLOW; + } + if (negative) decimal128Negate(pDec); + return 0; +} + +#define CONVERT_TO_DECIMAL(pData, pInputType, pOut, pOutType, decimal) \ + do { \ + int64_t val = 0; \ + uint64_t uval = 0; \ + double dval = 0; \ + switch (pInputType->type) { \ + case TSDB_DATA_TYPE_NULL: \ + break; \ + case TSDB_DATA_TYPE_BOOL: \ + uval = *(const bool*)pData; \ + code = decimal##FromUint64(pOut, pOutType->precision, pOutType->scale, uval); \ + break; \ + case TSDB_DATA_TYPE_TINYINT: \ + val = *(const int8_t*)pData; \ + code = decimal##FromInt64(pOut, pOutType->precision, pOutType->scale, val); \ + break; \ + case TSDB_DATA_TYPE_SMALLINT: \ + val = *(const int16_t*)pData; \ + code = decimal##FromInt64(pOut, pOutType->precision, pOutType->scale, val); \ + break; \ + case TSDB_DATA_TYPE_INT: \ + val = *(const int32_t*)pData; \ + code = decimal##FromInt64(pOut, pOutType->precision, pOutType->scale, val); \ + break; \ + case TSDB_DATA_TYPE_TIMESTAMP: \ + case TSDB_DATA_TYPE_BIGINT: \ + val = *(const int64_t*)pData; \ + code = decimal##FromInt64(pOut, pOutType->precision, pOutType->scale, val); \ + break; \ + case TSDB_DATA_TYPE_UTINYINT: \ + uval = *(const uint8_t*)pData; \ + code = decimal##FromUint64(pOut, pOutType->precision, pOutType->scale, uval); \ + break; \ + case TSDB_DATA_TYPE_USMALLINT: \ + uval = *(const uint16_t*)pData; \ + code = decimal##FromUint64(pOut, pOutType->precision, pOutType->scale, uval); \ + break; \ + case TSDB_DATA_TYPE_UINT: \ + uval = *(const uint32_t*)pData; \ + code = decimal##FromUint64(pOut, pOutType->precision, pOutType->scale, uval); \ + break; \ + case TSDB_DATA_TYPE_UBIGINT: \ + uval = *(const uint64_t*)pData; \ + code = decimal##FromUint64(pOut, pOutType->precision, pOutType->scale, uval); \ + break; \ + case TSDB_DATA_TYPE_FLOAT: { \ + dval = *(const float*)pData; \ + code = decimal##FromDouble(pOut, pOutType->precision, pOutType->scale, dval); \ + } break; \ + case TSDB_DATA_TYPE_DOUBLE: { \ + dval = *(const double*)pData; \ + code = decimal##FromDouble(pOut, pOutType->precision, pOutType->scale, dval); \ + } break; \ + case TSDB_DATA_TYPE_DECIMAL64: { \ + code = decimal##FromDecimal64(pOut, pOutType->precision, pOutType->scale, pData, pInputType->precision, \ + pInputType->scale); \ + } break; \ + case TSDB_DATA_TYPE_DECIMAL: { \ + code = decimal##FromDecimal128(pOut, pOutType->precision, pOutType->scale, pData, pInputType->precision, \ + pInputType->scale); \ + } break; \ + case TSDB_DATA_TYPE_VARCHAR: \ + case TSDB_DATA_TYPE_VARBINARY: \ + case TSDB_DATA_TYPE_NCHAR: { \ + code = decimal##FromStr(pData, pInputType->bytes - VARSTR_HEADER_SIZE, pOutType->precision, pOutType->scale, \ + pOut); \ + } break; \ + default: \ + code = TSDB_CODE_OPS_NOT_SUPPORT; \ + break; \ + } \ + } while (0) + +int32_t convertToDecimal(const void* pData, const SDataType* pInputType, void* pOut, const SDataType* pOutType) { + int32_t code = 0; + + switch (pOutType->type) { + case TSDB_DATA_TYPE_DECIMAL64: { + CONVERT_TO_DECIMAL(pData, pInputType, pOut, pOutType, decimal64); + } break; + case TSDB_DATA_TYPE_DECIMAL: { + CONVERT_TO_DECIMAL(pData, pInputType, pOut, pOutType, decimal128); + } break; + default: + code = TSDB_CODE_INTERNAL_ERROR; + break; + } + return code; +} + +void decimal64ScaleDown(Decimal64* pDec, uint8_t scaleDown, bool round) { + if (scaleDown > 0) { + Decimal64 divisor = SCALE_MULTIPLIER_64[scaleDown], remainder = {0}; + decimal64divide(pDec, &divisor, DECIMAL_WORD_NUM(Decimal64), &remainder); + if (round) { + decimal64Abs(&remainder); + Decimal64 half = SCALE_MULTIPLIER_64[scaleDown]; + decimal64divide(&half, &decimal64Two, DECIMAL_WORD_NUM(Decimal64), NULL); + if (!decimal64Lt(&remainder, &half, DECIMAL_WORD_NUM(Decimal64))) { + Decimal64 delta = {DECIMAL64_SIGN(pDec)}; + decimal64Add(pDec, &delta, DECIMAL_WORD_NUM(Decimal64)); + } + } + } +} + +void decimal64ScaleUp(Decimal64* pDec, uint8_t scaleUp) { + if (scaleUp > 0) { + Decimal64 multiplier = SCALE_MULTIPLIER_64[scaleUp]; + decimal64Multiply(pDec, &multiplier, DECIMAL_WORD_NUM(Decimal64)); + } +} + +static void decimal64ScaleTo(Decimal64* pDec, uint8_t oldScale, uint8_t newScale) { + if (newScale > oldScale) + decimal64ScaleUp(pDec, newScale - oldScale); + else if (newScale < oldScale) + decimal64ScaleDown(pDec, oldScale - newScale, true); +} + +static void decimal64ScaleAndCheckOverflow(Decimal64* pDec, int8_t scale, uint8_t toPrec, uint8_t toScale, + bool* overflow) { + int8_t deltaScale = toScale - scale; + if (deltaScale >= 0) { + Decimal64 max = {0}; + DECIMAL64_GET_MAX(toPrec - deltaScale, &max); + Decimal64 abs = *pDec; + decimal64Abs(&abs); + if (decimal64Gt(&abs, &max, DECIMAL_WORD_NUM(Decimal64))) { + if (overflow) *overflow = true; + } else { + decimal64ScaleUp(pDec, deltaScale); + } + } else if (deltaScale < 0) { + Decimal64 res = *pDec, max = {0}; + decimal64ScaleDown(&res, -deltaScale, false); + DECIMAL64_GET_MAX(toPrec, &max); + Decimal64 abs = res; + decimal64Abs(&abs); + if (decimal64Gt(&abs, &max, DECIMAL_WORD_NUM(Decimal64))) { + if (overflow) *overflow = true; + } else { + *pDec = res; + } + } +} + +static int32_t decimal64CountRoundingDelta(const Decimal64* pDec, int8_t scale, int8_t toScale, + DecimalRoundType roundType) { + if (roundType == ROUND_TYPE_TRUNC || toScale >= scale) return 0; + + Decimal64 dec = *pDec; + int32_t res = 0; + switch (roundType) { + case ROUND_TYPE_HALF_ROUND_UP: { + Decimal64 trailing = dec; + decimal64Mod(&trailing, &SCALE_MULTIPLIER_64[scale - toScale], DECIMAL_WORD_NUM(Decimal64)); + if (decimal64Eq(&trailing, &decimal64Zero, DECIMAL_WORD_NUM(Decimal64))) { + res = 0; + break; + } + Decimal64 trailingAbs = trailing, baseDiv2 = SCALE_MULTIPLIER_64[scale - toScale]; + decimal64Abs(&trailingAbs); + decimal64divide(&baseDiv2, &decimal64Two, DECIMAL_WORD_NUM(Decimal64), NULL); + if (decimal64Lt(&trailingAbs, &baseDiv2, DECIMAL_WORD_NUM(Decimal64))) { + res = 0; + break; + } + res = DECIMAL64_SIGN(pDec) == 1 ? 1 : -1; + } break; + default: + break; + } + return res; +} + +static void decimal64RoundWithPositiveScale(Decimal64* pDec, uint8_t prec, int8_t scale, uint8_t toPrec, + uint8_t toScale, DecimalRoundType roundType, bool* overflow) { + Decimal64 scaled = *pDec; + bool overflowLocal = false; + // scale up or down to toScale + decimal64ScaleAndCheckOverflow(&scaled, scale, toPrec, toScale, &overflowLocal); + if (overflowLocal) { + if (overflow) *overflow = true; + *pDec = decimal64Zero; + return; + } + + // calc rounding delta + int32_t delta = decimal64CountRoundingDelta(pDec, scale, toScale, roundType); + if (delta == 0) { + *pDec = scaled; + return; + } + + Decimal64 deltaDec = {delta}; + // add the delta + decimal64Add(&scaled, &deltaDec, DECIMAL_WORD_NUM(Decimal64)); + + // check overflow again + if (toPrec < prec) { + Decimal64 max = {0}; + DECIMAL64_GET_MAX(toPrec, &max); + Decimal64 scaledAbs = scaled; + decimal64Abs(&scaledAbs); + if (decimal64Gt(&scaledAbs, &max, DECIMAL_WORD_NUM(Decimal64))) { + if (overflow) *overflow = true; + *pDec = decimal64Zero; + return; + } + } + *pDec = scaled; +} + +int32_t decimal64FromStr(const char* str, int32_t len, uint8_t expectPrecision, uint8_t expectScale, Decimal64* pRes) { + int32_t code = 0; + DecimalVar var = {.type = DECIMAL_64, .pDec = pRes->words, .precision = expectPrecision, .scale = expectScale}; + DECIMAL64_SET_VALUE(pRes, 0); + code = decimalVarFromStr(str, len, &var); + if (TSDB_CODE_SUCCESS != code) return code; + if (var.weight > (int32_t)expectPrecision - expectScale) { + return TSDB_CODE_DECIMAL_OVERFLOW; + } + bool overflow = false; + decimal64RoundWithPositiveScale(pRes, var.precision, var.scale, expectPrecision, expectScale, + ROUND_TYPE_HALF_ROUND_UP, &overflow); + if (overflow) { + return TSDB_CODE_DECIMAL_OVERFLOW; + } + return code; +} + +static void decimal128ScaleDown(Decimal128* pDec, uint8_t scaleDown, bool round) { + if (scaleDown > 0) { + Decimal128 divisor = SCALE_MULTIPLIER_128[scaleDown], remainder = {0}; + decimal128Divide(pDec, &divisor, 2, &remainder); + if (round) { + decimal128Abs(&remainder); + Decimal128 half = SCALE_MULTIPLIER_128[scaleDown]; + decimal128Divide(&half, &decimal128Two, 2, NULL); + if (!decimal128Lt(&remainder, &half, DECIMAL_WORD_NUM(Decimal128))) { + Decimal64 delta = {DECIMAL128_SIGN(pDec)}; + decimal128Add(pDec, &delta, DECIMAL_WORD_NUM(Decimal64)); + } + } + } +} + +static void decimal128ScaleUp(Decimal128* pDec, uint8_t scaleUp) { + if (scaleUp > 0) { + Decimal128 multiplier = SCALE_MULTIPLIER_128[scaleUp]; + decimal128Multiply(pDec, &multiplier, DECIMAL_WORD_NUM(Decimal128)); + } +} + +static void decimal128ScaleTo(Decimal128* pDec, uint8_t oldScale, uint8_t newScale) { + if (newScale > oldScale) + decimal128ScaleUp(pDec, newScale - oldScale); + else if (newScale < oldScale) + decimal128ScaleDown(pDec, oldScale - newScale, true); +} + +int32_t decimal128FromStr(const char* str, int32_t len, uint8_t expectPrecision, uint8_t expectScale, + Decimal128* pRes) { + int32_t code = 0; + DecimalVar var = {.type = DECIMAL_128, .pDec = pRes->words, .precision = expectPrecision, .scale = expectScale}; + DECIMAL128_SET_HIGH_WORD(pRes, 0); + DECIMAL128_SET_LOW_WORD(pRes, 0); + code = decimalVarFromStr(str, len, &var); + if (TSDB_CODE_SUCCESS != code) return code; + if (var.weight > (int32_t)expectPrecision - expectScale) { + return TSDB_CODE_DECIMAL_OVERFLOW; + } + bool overflow = false; + decimal128RoundWithPositiveScale(pRes, var.precision, var.scale, expectPrecision, expectScale, + ROUND_TYPE_HALF_ROUND_UP, &overflow); + if (overflow) { + return TSDB_CODE_DECIMAL_OVERFLOW; + } + return code; +} + +#if 0 +__int128 decimal128ToInt128(const Decimal128* pDec) { + __int128 ret = 0; + ret = DECIMAL128_HIGH_WORD(pDec); + ret <<= 64; + ret |= DECIMAL128_LOW_WORD(pDec); + return ret; +} +#endif + +static int32_t decimal128CountLeadingBinaryZeros(const Decimal128* pDec) { + if (DECIMAL128_HIGH_WORD(pDec) == 0) { + return 64 + countLeadingZeros(DECIMAL128_LOW_WORD(pDec)); + } else { + return countLeadingZeros((uint64_t)DECIMAL128_HIGH_WORD(pDec)); + } +} + +#define IMPL_INTEGER_TYPE_FROM_DECIMAL_TYPE(oType, decimalType, sign) \ + oType oType##From##decimalType(const DecimalType* pDec, uint8_t prec, uint8_t scale) { \ + return (oType)sign##int64##From##decimalType(pDec, prec, scale); \ + } +#define IMP_SIGNED_INTEGER_TYPE_FROM_DECIMAL_TYPE(oType, decimalType) \ + IMPL_INTEGER_TYPE_FROM_DECIMAL_TYPE(oType, decimalType, ) +#define IMP_UNSIGNED_INTEGER_TYPE_FROM_DECIMAL_TYPE(oType, decimalType) \ + IMPL_INTEGER_TYPE_FROM_DECIMAL_TYPE(oType, decimalType, u) + +IMP_SIGNED_INTEGER_TYPE_FROM_DECIMAL_TYPE(int8_t, Decimal64) +IMP_SIGNED_INTEGER_TYPE_FROM_DECIMAL_TYPE(int16_t, Decimal64) +IMP_SIGNED_INTEGER_TYPE_FROM_DECIMAL_TYPE(int32_t, Decimal64) +IMP_SIGNED_INTEGER_TYPE_FROM_DECIMAL_TYPE(int64_t, Decimal64) + +IMP_UNSIGNED_INTEGER_TYPE_FROM_DECIMAL_TYPE(uint8_t, Decimal64) +IMP_UNSIGNED_INTEGER_TYPE_FROM_DECIMAL_TYPE(uint16_t, Decimal64) +IMP_UNSIGNED_INTEGER_TYPE_FROM_DECIMAL_TYPE(uint32_t, Decimal64) +IMP_UNSIGNED_INTEGER_TYPE_FROM_DECIMAL_TYPE(uint64_t, Decimal64) + +double doubleFromDecimal64(const void* pDec, uint8_t prec, uint8_t scale) { + int32_t sign = DECIMAL64_SIGN((Decimal64*)pDec); + Decimal64 abs = *(Decimal64*)pDec; + decimal64Abs(&abs); + return (double)DECIMAL64_GET_VALUE(&abs) * sign / getDoubleScaleMultiplier(scale); +} + +bool boolFromDecimal64(const void* pDec, uint8_t prec, uint8_t scale) { + return !decimal64Eq(pDec, &decimal64Zero, DECIMAL_WORD_NUM(Decimal64)); +} + +#define IMPL_REAL_TYPE_FROM_DECIMAL_TYPE(oType, decimalType) \ + oType oType##From##decimalType(const DecimalType* pDec, uint8_t prec, uint8_t scale) { \ + return (oType) double##From##decimalType(pDec, prec, scale); \ + } + +IMPL_REAL_TYPE_FROM_DECIMAL_TYPE(float, Decimal64); + +IMP_SIGNED_INTEGER_TYPE_FROM_DECIMAL_TYPE(int8_t, Decimal128) +IMP_SIGNED_INTEGER_TYPE_FROM_DECIMAL_TYPE(int16_t, Decimal128) +IMP_SIGNED_INTEGER_TYPE_FROM_DECIMAL_TYPE(int32_t, Decimal128) +IMP_SIGNED_INTEGER_TYPE_FROM_DECIMAL_TYPE(int64_t, Decimal128) + +IMP_UNSIGNED_INTEGER_TYPE_FROM_DECIMAL_TYPE(uint8_t, Decimal128) +IMP_UNSIGNED_INTEGER_TYPE_FROM_DECIMAL_TYPE(uint16_t, Decimal128) +IMP_UNSIGNED_INTEGER_TYPE_FROM_DECIMAL_TYPE(uint32_t, Decimal128) +IMP_UNSIGNED_INTEGER_TYPE_FROM_DECIMAL_TYPE(uint64_t, Decimal128) + +bool boolFromDecimal128(const void* pDec, uint8_t prec, uint8_t scale) { + return !decimal128Eq(pDec, &decimal128Zero, DECIMAL_WORD_NUM(Decimal128)); +} + +double doubleFromDecimal128(const void* pDec, uint8_t prec, uint8_t scale) { + int32_t sign = DECIMAL128_SIGN((Decimal128*)pDec); + Decimal128 abs = *(Decimal128*)pDec; + decimal128Abs(&abs); + double unscaled = DECIMAL128_LOW_WORD(&abs); + unscaled += ldexp((double)DECIMAL128_HIGH_WORD(&abs), 64); + return (unscaled * sign) / getDoubleScaleMultiplier(scale); +} + +IMPL_REAL_TYPE_FROM_DECIMAL_TYPE(float, Decimal128); + +static void decimal128RoundWithPositiveScale(Decimal128* pDec, uint8_t prec, uint8_t scale, uint8_t toPrec, + uint8_t toScale, DecimalRoundType roundType, bool* overflow) { + Decimal128 scaled = *pDec; + bool overflowLocal = false; + // scale up or down to toScale + decimal128ModifyScaleAndPrecision(&scaled, scale, toPrec, toScale, &overflowLocal); + if (overflowLocal) { + if (overflow) *overflow = true; + *pDec = decimal128Zero; + return; + } + + // calc rounding delta, 1 or -1 + int32_t delta = decimal128CountRoundingDelta(pDec, scale, toScale, roundType); + if (delta == 0) { + *pDec = scaled; + return; + } + + Decimal64 deltaDec = {delta}; + // add the delta + decimal128Add(&scaled, &deltaDec, DECIMAL_WORD_NUM(Decimal64)); + + // check overflow again + if (toPrec < prec) { + Decimal128 max = {0}; + DECIMAL128_GET_MAX(toPrec, &max); + Decimal128 scaledAbs = scaled; + decimal128Abs(&scaledAbs); + if (decimal128Gt(&scaledAbs, &max, DECIMAL_WORD_NUM(Decimal128))) { + if (overflow) *overflow = true; + *(Decimal128*)pDec = decimal128Zero; + return; + } + } + *(Decimal128*)pDec = scaled; +} + +static void decimal128ModifyScaleAndPrecision(Decimal128* pDec, uint8_t scale, uint8_t toPrec, int8_t toScale, + bool* overflow) { + int8_t deltaScale = toScale - scale; + if (deltaScale >= 0) { + Decimal128 max = {0}; + DECIMAL128_GET_MAX(toPrec - deltaScale, &max); + Decimal128 abs = *pDec; + decimal128Abs(&abs); + if (decimal128Gt(&abs, &max, DECIMAL_WORD_NUM(Decimal128))) { + if (overflow) *overflow = true; + } else { + decimal128ScaleUp(pDec, deltaScale); + } + } else { + Decimal128 res = *pDec, max = {0}; + decimal128ScaleDown(&res, -deltaScale, false); + DECIMAL128_GET_MAX(toPrec, &max); + if (decimal128Gt(&res, &max, DECIMAL_WORD_NUM(Decimal128))) { + if (overflow) *overflow = true; + } else { + *(Decimal128*)pDec = res; + } + } +} + +static int32_t decimal128CountRoundingDelta(const Decimal128* pDec, int8_t scale, int8_t toScale, + DecimalRoundType roundType) { + if (roundType == ROUND_TYPE_TRUNC || toScale >= scale) return 0; + Decimal128 dec128 = *pDec; + int32_t res = 0; + switch (roundType) { + case ROUND_TYPE_HALF_ROUND_UP: { + Decimal128 trailing = dec128; + decimal128Mod(&trailing, &SCALE_MULTIPLIER_128[scale - toScale], DECIMAL_WORD_NUM(Decimal128)); + if (decimal128Eq(&trailing, &decimal128Zero, DECIMAL_WORD_NUM(Decimal128))) { + res = 0; + break; + } + Decimal128 tailingAbs = trailing, baseDiv2 = SCALE_MULTIPLIER_128[scale - toScale]; + decimal128Abs(&tailingAbs); + decimal128Divide(&baseDiv2, &decimal128Two, DECIMAL_WORD_NUM(Decimal128), NULL); + if (decimal128Lt(&tailingAbs, &baseDiv2, DECIMAL_WORD_NUM(Decimal128))) { + res = 0; + break; + } + res = DECIMAL128_SIGN(pDec) == -1 ? -1 : 1; + } break; + case ROUND_TYPE_TRUNC: + default: + break; + } + return res; +} + +bool decimal128AddCheckOverflow(const Decimal128* pLeft, const DecimalType* pRight, uint8_t rightWordNum) { + if (DECIMAL128_SIGN(pLeft) == 0) { + Decimal128 max = decimal128Max; + decimal128Subtract(&max, pLeft, DECIMAL_WORD_NUM(Decimal128)); + return decimal128Lt(&max, pRight, rightWordNum); + } else { + Decimal128 min = decimal128Min; + decimal128Subtract(&min, pLeft, DECIMAL_WORD_NUM(Decimal128)); + return decimal128Gt(&min, pRight, rightWordNum); + } +} + +int32_t TEST_decimal64From_int64_t(Decimal64* pDec, uint8_t prec, uint8_t scale, int64_t v) { + return decimal64FromInt64(pDec, prec, scale, v); +} +int32_t TEST_decimal64From_uint64_t(Decimal64* pDec, uint8_t prec, uint8_t scale, uint64_t v) { + return decimal64FromUint64(pDec, prec, scale, v); +} +int32_t TEST_decimal64From_double(Decimal64* pDec, uint8_t prec, uint8_t scale, double v) { + return decimal64FromDouble(pDec, prec, scale, v); +} +double TEST_decimal64ToDouble(Decimal64* pDec, uint8_t prec, uint8_t scale) { + return doubleFromDecimal64(pDec, prec, scale); +} + +int32_t TEST_decimal128From_int64_t(Decimal128* pDec, uint8_t prec, uint8_t scale, int64_t v) { + return decimal128FromInt64(pDec, prec, scale, v); +} +int32_t TEST_decimal128From_uint64_t(Decimal128* pDec, uint8_t prec, uint8_t scale, uint64_t v) { + return decimal128FromUint64(pDec, prec, scale, v); +} +int32_t TEST_decimal128From_double(Decimal128* pDec, uint8_t prec, uint8_t scale, double v) { + return decimal128FromDouble(pDec, prec, scale, v); +} +double TEST_decimal128ToDouble(Decimal128* pDec, uint8_t prec, uint8_t scale) { + return doubleFromDecimal128(pDec, prec, scale); +} + +int32_t TEST_decimal64FromDecimal64(const Decimal64* pInput, uint8_t inputPrec, uint8_t inputScale, Decimal64* pOutput, + uint8_t outputPrec, uint8_t outputScale) { + return decimal64FromDecimal64(pOutput, outputPrec, outputScale, pInput, inputPrec, inputScale); +} + +int32_t TEST_decimal64FromDecimal128(const Decimal128* pInput, uint8_t prec, uint8_t inputScale, Decimal64* pOutput, + uint8_t outputPrec, uint8_t outputScale) { + return decimal64FromDecimal128(pOutput, outputPrec, outputScale, pInput, prec, inputScale); +} + +int32_t TEST_decimal128FromDecimal64(const Decimal64* pInput, uint8_t inputPrec, uint8_t inputScale, + Decimal128* pOutput, uint8_t outputPrec, uint8_t outputScale) { + return decimal128FromDecimal64(pOutput, outputPrec, outputScale, pInput, inputPrec, inputScale); +} +int32_t TEST_decimal128FromDecimal128(const Decimal128* pDec, uint8_t prec, uint8_t scale, Decimal128* pOutput, + uint8_t outputPrec, uint8_t outputScale) { + return decimal128FromDecimal128(pOutput, outputPrec, outputScale, pDec, prec, scale); +} + +void encodeDecimal(const DecimalType* pDec, int8_t type, void* pBuf) { + switch (type) { + case TSDB_DATA_TYPE_DECIMAL64: + *(DecimalWord*)pBuf = htobe64((DecimalWord)DECIMAL64_GET_VALUE((Decimal64*)pDec)); + break; + case TSDB_DATA_TYPE_DECIMAL: + ((Decimal128*)pBuf)->words[0] = htobe64(DECIMAL128_LOW_WORD((Decimal128*)pDec)); + ((Decimal128*)pBuf)->words[1] = htobe64(DECIMAL128_HIGH_WORD((Decimal128*)pDec)); + break; + default: + break; + } +} + +void decodeDecimal(const void* pBuf, int8_t type, DecimalType* pDec) { + switch (type) { + case TSDB_DATA_TYPE_DECIMAL64: + DECIMAL64_SET_VALUE((Decimal64*)pDec, be64toh(*(DecimalWord*)pBuf)); + break; + case TSDB_DATA_TYPE_DECIMAL: + DECIMAL128_SET_LOW_WORD((Decimal128*)pDec, be64toh(((Decimal128*)pBuf)->words[0])); + DECIMAL128_SET_HIGH_WORD((Decimal128*)pDec, be64toh(((Decimal128*)pBuf)->words[1])); + break; + default: + break; + } +} diff --git a/source/libs/decimal/src/detail/CMakeLists.txt b/source/libs/decimal/src/detail/CMakeLists.txt new file mode 100644 index 0000000000..51245312db --- /dev/null +++ b/source/libs/decimal/src/detail/CMakeLists.txt @@ -0,0 +1,15 @@ +MESSAGE(STATUS "Building decimal/src/detail") +aux_source_directory(. WIDE_INTEGER_SRC) + +SET(CMAKE_CXX_STANDARD 14) +add_library(wideInteger STATIC ${WIDE_INTEGER_SRC}) + +target_include_directories( + wideInteger + PUBLIC "${TD_SOURCE_DIR}/source/libs/decimal/inc/" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/intx/" +) +target_link_libraries( + wideInteger + PUBLIC +) diff --git a/source/libs/decimal/src/detail/intx/int128.hpp b/source/libs/decimal/src/detail/intx/int128.hpp new file mode 100644 index 0000000000..b351c1e4cf --- /dev/null +++ b/source/libs/decimal/src/detail/intx/int128.hpp @@ -0,0 +1,885 @@ +// intx: extended precision integer library. +// Copyright 2019-2020 Pawel Bylica. +// Licensed under the Apache License, Version 2.0. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _MSC_VER + #include +#endif + +#ifdef _MSC_VER + #define INTX_UNREACHABLE __assume(0) +#else + #define INTX_UNREACHABLE __builtin_unreachable() +#endif + +#ifdef _MSC_VER + #define INTX_UNLIKELY(EXPR) (bool{EXPR}) +#else + #define INTX_UNLIKELY(EXPR) __builtin_expect(bool{EXPR}, false) +#endif + +#ifdef NDEBUG + #define INTX_REQUIRE(X) (X) ? (void)0 : INTX_UNREACHABLE +#else + #include + #define INTX_REQUIRE assert +#endif + +namespace intx +{ +template +struct uint; + +/// The 128-bit unsigned integer. +/// +/// This type is defined as a specialization of uint<> to easier integration with full intx package, +/// however, uint128 may be used independently. +template <> +struct uint<128> +{ + static constexpr unsigned num_bits = 128; + + uint64_t lo = 0; + uint64_t hi = 0; + + constexpr uint() noexcept = default; + + constexpr uint(uint64_t high, uint64_t low) noexcept : lo{low}, hi{high} {} + + template ::value>> + constexpr uint(T x) noexcept : lo(static_cast(x)) // NOLINT + {} + +#ifdef __SIZEOF_INT128__ + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wpedantic" + constexpr uint(unsigned __int128 x) noexcept // NOLINT + : lo{uint64_t(x)}, hi{uint64_t(x >> 64)} + {} + + constexpr explicit operator unsigned __int128() const noexcept + { + return (static_cast(hi) << 64) | lo; + } + #pragma GCC diagnostic pop +#endif + + constexpr explicit operator bool() const noexcept { return hi | lo; } + + /// Explicit converting operator for all builtin integral types. + template ::value>::type> + constexpr explicit operator Int() const noexcept + { + return static_cast(lo); + } +}; + +using uint128 = uint<128>; + + +/// Contains result of add/sub/etc with a carry flag. +template +struct result_with_carry +{ + T value; + bool carry; + + /// Conversion to tuple of references, to allow usage with std::tie(). + constexpr operator std::tuple() noexcept { return {value, carry}; } +}; + + +/// Linear arithmetic operators. +/// @{ + +constexpr inline result_with_carry add_with_carry( + uint64_t x, uint64_t y, bool carry = false) noexcept +{ + const auto s = x + y; + const auto carry1 = s < x; + const auto t = s + carry; + const auto carry2 = t < s; + return {t, carry1 || carry2}; +} + +template +constexpr result_with_carry> add_with_carry( + const uint& a, const uint& b, bool carry = false) noexcept +{ + const auto lo = add_with_carry(a.lo, b.lo, carry); + const auto hi = add_with_carry(a.hi, b.hi, lo.carry); + return {{hi.value, lo.value}, hi.carry}; +} + +constexpr inline uint128 operator+(uint128 x, uint128 y) noexcept +{ + return add_with_carry(x, y).value; +} + +constexpr inline uint128 operator+(uint128 x) noexcept +{ + return x; +} + +constexpr inline result_with_carry sub_with_carry( + uint64_t x, uint64_t y, bool carry = false) noexcept +{ + const auto d = x - y; + const auto carry1 = d > x; + const auto e = d - carry; + const auto carry2 = e > d; + return {e, carry1 || carry2}; +} + +/// Performs subtraction of two unsigned numbers and returns the difference +/// and the carry bit (aka borrow, overflow). +template +constexpr inline result_with_carry> sub_with_carry( + const uint& a, const uint& b, bool carry = false) noexcept +{ + const auto lo = sub_with_carry(a.lo, b.lo, carry); + const auto hi = sub_with_carry(a.hi, b.hi, lo.carry); + return {{hi.value, lo.value}, hi.carry}; +} + +constexpr inline uint128 operator-(uint128 x, uint128 y) noexcept +{ + return sub_with_carry(x, y).value; +} + +constexpr inline uint128 operator-(uint128 x) noexcept +{ + // Implementing as subtraction is better than ~x + 1. + // Clang9: Perfect. + // GCC8: Does something weird. + return 0 - x; +} + +inline uint128& operator++(uint128& x) noexcept +{ + return x = x + 1; +} + +inline uint128& operator--(uint128& x) noexcept +{ + return x = x - 1; +} + +inline uint128 operator++(uint128& x, int) noexcept +{ + auto ret = x; + ++x; + return ret; +} + +inline uint128 operator--(uint128& x, int) noexcept +{ + auto ret = x; + --x; + return ret; +} + +/// Optimized addition. +/// +/// This keeps the multiprecision addition until CodeGen so the pattern is not +/// broken during other optimizations. +constexpr uint128 fast_add(uint128 x, uint128 y) noexcept +{ +#ifdef __SIZEOF_INT128__ + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wpedantic" + using uint128_native = unsigned __int128; + return uint128_native{x} + uint128_native{y}; + #pragma GCC diagnostic pop +#else + // Fallback to regular addition. + return x + y; +#endif +} + +/// @} + + +/// Comparison operators. +/// +/// In all implementations bitwise operators are used instead of logical ones +/// to avoid branching. +/// +/// @{ + +constexpr bool operator==(uint128 x, uint128 y) noexcept +{ + // Clang7: generates perfect xor based code, + // much better than __int128 where it uses vector instructions. + // GCC8: generates a bit worse cmp based code + // although it generates the xor based one for __int128. + return (x.lo == y.lo) & (x.hi == y.hi); +} + +constexpr bool operator!=(uint128 x, uint128 y) noexcept +{ + // Analogous to ==, but == not used directly, because that confuses GCC 8-9. + return (x.lo != y.lo) | (x.hi != y.hi); +} + +constexpr bool operator<(uint128 x, uint128 y) noexcept +{ + // OPT: This should be implemented by checking the borrow of x - y, + // but compilers (GCC8, Clang7) + // have problem with properly optimizing subtraction. + return (x.hi < y.hi) | ((x.hi == y.hi) & (x.lo < y.lo)); +} + +constexpr bool operator<=(uint128 x, uint128 y) noexcept +{ + return !(y < x); +} + +constexpr bool operator>(uint128 x, uint128 y) noexcept +{ + return y < x; +} + +constexpr bool operator>=(uint128 x, uint128 y) noexcept +{ + return !(x < y); +} + +/// @} + + +/// Bitwise operators. +/// @{ + +constexpr uint128 operator~(uint128 x) noexcept +{ + return {~x.hi, ~x.lo}; +} + +constexpr uint128 operator|(uint128 x, uint128 y) noexcept +{ + // Clang7: perfect. + // GCC8: stupidly uses a vector instruction in all bitwise operators. + return {x.hi | y.hi, x.lo | y.lo}; +} + +constexpr uint128 operator&(uint128 x, uint128 y) noexcept +{ + return {x.hi & y.hi, x.lo & y.lo}; +} + +constexpr uint128 operator^(uint128 x, uint128 y) noexcept +{ + return {x.hi ^ y.hi, x.lo ^ y.lo}; +} + +constexpr uint128 operator<<(uint128 x, unsigned shift) noexcept +{ + return (shift < 64) ? + // Find the part moved from lo to hi. + // For shift == 0 right shift by (64 - shift) is invalid so + // split it into 2 shifts by 1 and (63 - shift). + uint128{(x.hi << shift) | ((x.lo >> 1) >> (63 - shift)), x.lo << shift} : + + // Guarantee "defined" behavior for shifts larger than 128. + (shift < 128) ? uint128{x.lo << (shift - 64), 0} : 0; +} + +constexpr uint128 operator<<(uint128 x, uint128 shift) noexcept +{ + if (shift < 128) + return x << unsigned(shift); + return 0; +} + +constexpr uint128 operator>>(uint128 x, unsigned shift) noexcept +{ + return (shift < 64) ? + // Find the part moved from lo to hi. + // For shift == 0 left shift by (64 - shift) is invalid so + // split it into 2 shifts by 1 and (63 - shift). + uint128{x.hi >> shift, (x.lo >> shift) | ((x.hi << 1) << (63 - shift))} : + + // Guarantee "defined" behavior for shifts larger than 128. + (shift < 128) ? uint128{0, x.hi >> (shift - 64)} : 0; +} + +constexpr uint128 operator>>(uint128 x, uint128 shift) noexcept +{ + if (shift < 128) + return x >> unsigned(shift); + return 0; +} + + +/// @} + + +/// Multiplication +/// @{ + +/// Portable full unsigned multiplication 64 x 64 -> 128. +constexpr uint128 constexpr_umul(uint64_t x, uint64_t y) noexcept +{ + uint64_t xl = x & 0xffffffff; + uint64_t xh = x >> 32; + uint64_t yl = y & 0xffffffff; + uint64_t yh = y >> 32; + + uint64_t t0 = xl * yl; + uint64_t t1 = xh * yl; + uint64_t t2 = xl * yh; + uint64_t t3 = xh * yh; + + uint64_t u1 = t1 + (t0 >> 32); + uint64_t u2 = t2 + (u1 & 0xffffffff); + + uint64_t lo = (u2 << 32) | (t0 & 0xffffffff); + uint64_t hi = t3 + (u2 >> 32) + (u1 >> 32); + return {hi, lo}; +} + +/// Full unsigned multiplication 64 x 64 -> 128. +inline uint128 umul(uint64_t x, uint64_t y) noexcept +{ +#if defined(__SIZEOF_INT128__) + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wpedantic" + const auto p = static_cast(x) * y; + return {uint64_t(p >> 64), uint64_t(p)}; + #pragma GCC diagnostic pop +#elif defined(_MSC_VER) + unsigned __int64 hi; + const auto lo = _umul128(x, y, &hi); + return {hi, lo}; +#else + return constexpr_umul(x, y); +#endif +} + +inline uint128 operator*(uint128 x, uint128 y) noexcept +{ + auto p = umul(x.lo, y.lo); + p.hi += (x.lo * y.hi) + (x.hi * y.lo); + return {p.hi, p.lo}; +} + +constexpr uint128 constexpr_mul(uint128 x, uint128 y) noexcept +{ + auto p = constexpr_umul(x.lo, y.lo); + p.hi += (x.lo * y.hi) + (x.hi * y.lo); + return {p.hi, p.lo}; +} + +/// @} + + +/// Assignment operators. +/// @{ + +constexpr uint128& operator+=(uint128& x, uint128 y) noexcept +{ + return x = x + y; +} + +constexpr uint128& operator-=(uint128& x, uint128 y) noexcept +{ + return x = x - y; +} + +inline uint128& operator*=(uint128& x, uint128 y) noexcept +{ + return x = x * y; +} + +constexpr uint128& operator|=(uint128& x, uint128 y) noexcept +{ + return x = x | y; +} + +constexpr uint128& operator&=(uint128& x, uint128 y) noexcept +{ + return x = x & y; +} + +constexpr uint128& operator^=(uint128& x, uint128 y) noexcept +{ + return x = x ^ y; +} + +constexpr uint128& operator<<=(uint128& x, unsigned shift) noexcept +{ + return x = x << shift; +} + +constexpr uint128& operator>>=(uint128& x, unsigned shift) noexcept +{ + return x = x >> shift; +} + +/// @} + + +constexpr unsigned clz_generic(uint32_t x) noexcept +{ + unsigned n = 32; + for (int i = 4; i >= 0; --i) + { + const auto s = unsigned{1} << i; + const auto hi = x >> s; + if (hi != 0) + { + n -= s; + x = hi; + } + } + return n - x; +} + +constexpr unsigned clz_generic(uint64_t x) noexcept +{ + unsigned n = 64; + for (int i = 5; i >= 0; --i) + { + const auto s = unsigned{1} << i; + const auto hi = x >> s; + if (hi != 0) + { + n -= s; + x = hi; + } + } + return n - static_cast(x); +} + +constexpr inline unsigned clz(uint32_t x) noexcept +{ +#ifdef _MSC_VER + return clz_generic(x); +#else + return x != 0 ? unsigned(__builtin_clz(x)) : 32; +#endif +} + +constexpr inline unsigned clz(uint64_t x) noexcept +{ +#ifdef _MSC_VER + return clz_generic(x); +#else + return x != 0 ? unsigned(__builtin_clzll(x)) : 64; +#endif +} + +constexpr inline unsigned clz(uint128 x) noexcept +{ + // In this order `h == 0` we get less instructions than in case of `h != 0`. + return x.hi == 0 ? clz(x.lo) + 64 : clz(x.hi); +} + + +inline uint64_t bswap(uint64_t x) noexcept +{ +#ifdef _MSC_VER + return _byteswap_uint64(x); +#else + return __builtin_bswap64(x); +#endif +} + +inline uint128 bswap(uint128 x) noexcept +{ + return {bswap(x.lo), bswap(x.hi)}; +} + + +/// Division. +/// @{ + +template +struct div_result +{ + QuotT quot; + RemT rem; + + /// Conversion to tuple of references, to allow usage with std::tie(). + constexpr operator std::tuple() noexcept { return {quot, rem}; } +}; + +namespace internal +{ +constexpr uint16_t reciprocal_table_item(uint8_t d9) noexcept +{ + return uint16_t(0x7fd00 / (0x100 | d9)); +} + +#define REPEAT4(x) \ + reciprocal_table_item((x) + 0), reciprocal_table_item((x) + 1), \ + reciprocal_table_item((x) + 2), reciprocal_table_item((x) + 3) + +#define REPEAT32(x) \ + REPEAT4((x) + 4 * 0), REPEAT4((x) + 4 * 1), REPEAT4((x) + 4 * 2), REPEAT4((x) + 4 * 3), \ + REPEAT4((x) + 4 * 4), REPEAT4((x) + 4 * 5), REPEAT4((x) + 4 * 6), REPEAT4((x) + 4 * 7) + +#define REPEAT256() \ + REPEAT32(32 * 0), REPEAT32(32 * 1), REPEAT32(32 * 2), REPEAT32(32 * 3), REPEAT32(32 * 4), \ + REPEAT32(32 * 5), REPEAT32(32 * 6), REPEAT32(32 * 7) + +/// Reciprocal lookup table. +constexpr uint16_t reciprocal_table[] = {REPEAT256()}; + +#undef REPEAT4 +#undef REPEAT32 +#undef REPEAT256 +} // namespace internal + +/// Computes the reciprocal (2^128 - 1) / d - 2^64 for normalized d. +/// +/// Based on Algorithm 2 from "Improved division by invariant integers". +inline uint64_t reciprocal_2by1(uint64_t d) noexcept +{ + INTX_REQUIRE(d & 0x8000000000000000); // Must be normalized. + + const uint64_t d9 = d >> 55; + const uint32_t v0 = internal::reciprocal_table[d9 - 256]; + + const uint64_t d40 = (d >> 24) + 1; + const uint64_t v1 = (v0 << 11) - uint32_t(v0 * v0 * d40 >> 40) - 1; + + const uint64_t v2 = (v1 << 13) + (v1 * (0x1000000000000000 - v1 * d40) >> 47); + + const uint64_t d0 = d & 1; + const uint64_t d63 = (d >> 1) + d0; // ceil(d/2) + const uint64_t e = ((v2 >> 1) & (0 - d0)) - v2 * d63; + const uint64_t v3 = (umul(v2, e).hi >> 1) + (v2 << 31); + + const uint64_t v4 = v3 - (umul(v3, d) + d).hi - d; + return v4; +} + +inline uint64_t reciprocal_3by2(uint128 d) noexcept +{ + auto v = reciprocal_2by1(d.hi); + auto p = d.hi * v; + p += d.lo; + if (p < d.lo) + { + --v; + if (p >= d.hi) + { + --v; + p -= d.hi; + } + p -= d.hi; + } + + const auto t = umul(v, d.lo); + + p += t.hi; + if (p < t.hi) + { + --v; + if (p >= d.hi) + { + if (p > d.hi || t.lo >= d.lo) + --v; + } + } + return v; +} + +inline div_result udivrem_2by1(uint128 u, uint64_t d, uint64_t v) noexcept +{ + auto q = umul(v, u.hi); + q = fast_add(q, u); + + ++q.hi; + + auto r = u.lo - q.hi * d; + + if (r > q.lo) + { + --q.hi; + r += d; + } + + if (r >= d) + { + ++q.hi; + r -= d; + } + + return {q.hi, r}; +} + +inline div_result udivrem_3by2( + uint64_t u2, uint64_t u1, uint64_t u0, uint128 d, uint64_t v) noexcept +{ + auto q = umul(v, u2); + q = fast_add(q, {u2, u1}); + + auto r1 = u1 - q.hi * d.hi; + + auto t = umul(d.lo, q.hi); + + auto r = uint128{r1, u0} - t - d; + r1 = r.hi; + + ++q.hi; + + if (r1 >= q.lo) + { + --q.hi; + r += d; + } + + if (r >= d) + { + ++q.hi; + r -= d; + } + + return {q.hi, r}; +} + +inline div_result udivrem(uint128 x, uint128 y) noexcept +{ + if (y.hi == 0) + { + INTX_REQUIRE(y.lo != 0); // Division by 0. + + const auto lsh = clz(y.lo); + const auto rsh = (64 - lsh) % 64; + const auto rsh_mask = uint64_t{lsh == 0} - 1; + + const auto yn = y.lo << lsh; + const auto xn_lo = x.lo << lsh; + const auto xn_hi = (x.hi << lsh) | ((x.lo >> rsh) & rsh_mask); + const auto xn_ex = (x.hi >> rsh) & rsh_mask; + + const auto v = reciprocal_2by1(yn); + const auto res1 = udivrem_2by1({xn_ex, xn_hi}, yn, v); + const auto res2 = udivrem_2by1({res1.rem, xn_lo}, yn, v); + return {{res1.quot, res2.quot}, res2.rem >> lsh}; + } + + if (y.hi > x.hi) + return {0, x}; + + const auto lsh = clz(y.hi); + if (lsh == 0) + { + const auto q = unsigned{y.hi < x.hi} | unsigned{y.lo <= x.lo}; + return {q, x - (q ? y : 0)}; + } + + const auto rsh = 64 - lsh; + + const auto yn_lo = y.lo << lsh; + const auto yn_hi = (y.hi << lsh) | (y.lo >> rsh); + const auto xn_lo = x.lo << lsh; + const auto xn_hi = (x.hi << lsh) | (x.lo >> rsh); + const auto xn_ex = x.hi >> rsh; + + const auto v = reciprocal_3by2({yn_hi, yn_lo}); + const auto res = udivrem_3by2(xn_ex, xn_hi, xn_lo, {yn_hi, yn_lo}, v); + + return {res.quot, res.rem >> lsh}; +} + +inline div_result sdivrem(uint128 x, uint128 y) noexcept +{ + constexpr auto sign_mask = uint128{1} << 127; + const auto x_is_neg = (x & sign_mask) != 0; + const auto y_is_neg = (y & sign_mask) != 0; + + const auto x_abs = x_is_neg ? -x : x; + const auto y_abs = y_is_neg ? -y : y; + + const auto q_is_neg = x_is_neg ^ y_is_neg; + + const auto res = udivrem(x_abs, y_abs); + + return {q_is_neg ? -res.quot : res.quot, x_is_neg ? -res.rem : res.rem}; +} + +inline uint128 operator/(uint128 x, uint128 y) noexcept +{ + return udivrem(x, y).quot; +} + +inline uint128 operator%(uint128 x, uint128 y) noexcept +{ + return udivrem(x, y).rem; +} + +inline uint128& operator/=(uint128& x, uint128 y) noexcept +{ + return x = x / y; +} + +inline uint128& operator%=(uint128& x, uint128 y) noexcept +{ + return x = x % y; +} + +/// @} + +} // namespace intx + + +namespace std +{ +template +struct numeric_limits> +{ + using type = intx::uint; + + static constexpr bool is_specialized = true; + static constexpr bool is_integer = true; + static constexpr bool is_signed = false; + static constexpr bool is_exact = true; + static constexpr bool has_infinity = false; + static constexpr bool has_quiet_NaN = false; + static constexpr bool has_signaling_NaN = false; + static constexpr float_denorm_style has_denorm = denorm_absent; + static constexpr bool has_denorm_loss = false; + static constexpr float_round_style round_style = round_toward_zero; + static constexpr bool is_iec559 = false; + static constexpr bool is_bounded = true; + static constexpr bool is_modulo = true; + static constexpr int digits = CHAR_BIT * sizeof(type); + static constexpr int digits10 = int(0.3010299956639812 * digits); + static constexpr int max_digits10 = 0; + static constexpr int radix = 2; + static constexpr int min_exponent = 0; + static constexpr int min_exponent10 = 0; + static constexpr int max_exponent = 0; + static constexpr int max_exponent10 = 0; + static constexpr bool traps = std::numeric_limits::traps; + static constexpr bool tinyness_before = false; + + static constexpr type min() noexcept { return 0; } + static constexpr type lowest() noexcept { return min(); } + static constexpr type max() noexcept { return ~type{0}; } + static constexpr type epsilon() noexcept { return 0; } + static constexpr type round_error() noexcept { return 0; } + static constexpr type infinity() noexcept { return 0; } + static constexpr type quiet_NaN() noexcept { return 0; } + static constexpr type signaling_NaN() noexcept { return 0; } + static constexpr type denorm_min() noexcept { return 0; } +}; +} // namespace std + +namespace intx +{ +template +[[noreturn]] inline void throw_(const char* what) +{ +#if __cpp_exceptions + throw T{what}; +#else + std::fputs(what, stderr); + std::abort(); +#endif +} + +constexpr inline int from_dec_digit(char c) +{ + if (c < '0' || c > '9') + throw_("invalid digit"); + return c - '0'; +} + +constexpr inline int from_hex_digit(char c) +{ + if (c >= 'a' && c <= 'f') + return c - ('a' - 10); + if (c >= 'A' && c <= 'F') + return c - ('A' - 10); + return from_dec_digit(c); +} + +template +constexpr Int from_string(const char* str) +{ + auto s = str; + auto x = Int{}; + int num_digits = 0; + + if (s[0] == '0' && s[1] == 'x') + { + s += 2; + while (const auto c = *s++) + { + if (++num_digits > int{sizeof(x) * 2}) + throw_(str); + x = (x << 4) | from_hex_digit(c); + } + return x; + } + + while (const auto c = *s++) + { + if (num_digits++ > std::numeric_limits::digits10) + throw_(str); + + const auto d = from_dec_digit(c); + x = constexpr_mul(x, Int{10}) + d; + if (x < d) + throw_(str); + } + return x; +} + +template +constexpr Int from_string(const std::string& s) +{ + return from_string(s.c_str()); +} + +constexpr uint128 operator""_u128(const char* s) +{ + return from_string(s); +} + +template +inline std::string to_string(uint x, int base = 10) +{ + if (base < 2 || base > 36) + base = 10; + + if (x == 0) + return "0"; + + auto s = std::string{}; + while (x != 0) + { + // TODO: Use constexpr udivrem_1? + const auto res = udivrem(x, uint{base}); + const auto d = int(res.rem); + const auto c = d < 10 ? '0' + d : 'a' + d - 10; + s.push_back(char(c)); + x = res.quot; + } + std::reverse(s.begin(), s.end()); + return s; +} + +template +inline std::string hex(uint x) +{ + return to_string(x, 16); +} +} // namespace intx diff --git a/source/libs/decimal/src/detail/intx/intx.hpp b/source/libs/decimal/src/detail/intx/intx.hpp new file mode 100644 index 0000000000..e91809d647 --- /dev/null +++ b/source/libs/decimal/src/detail/intx/intx.hpp @@ -0,0 +1,1221 @@ +// intx: extended precision integer library. +// Copyright 2019-2020 Pawel Bylica. +// Licensed under the Apache License, Version 2.0. + +#pragma once + +#include "int128.hpp" +#include +#include +#include +#include +#include +#include + +namespace intx +{ +template +struct uint +{ + static_assert((N & (N - 1)) == 0, "Number of bits must be power of 2"); + static_assert(N >= 256, "Number of bits must be at lest 256"); + + using word_type = uint64_t; + + /// The 2x smaller type. + using half_type = uint; + + static constexpr auto num_bits = N; + static constexpr auto num_words = N / 8 / sizeof(word_type); + + half_type lo = 0; + half_type hi = 0; + + constexpr uint() noexcept = default; + + constexpr uint(half_type high, half_type low) noexcept : lo(low), hi(high) {} + + /// Implicit converting constructor for the half type. + constexpr uint(half_type x) noexcept : lo(x) {} // NOLINT + + /// Implicit converting constructor for types convertible to the half type. + template ::value>::type> + constexpr uint(T x) noexcept : lo(x) // NOLINT + {} + + constexpr explicit operator bool() const noexcept + { + return static_cast(lo) | static_cast(hi); + } + + /// Explicit converting operator for all builtin integral types. + template ::value>::type> + explicit operator Int() const noexcept + { + return static_cast(lo); + } +}; + +using uint256 = uint<256>; +using uint512 = uint<512>; + +constexpr uint8_t lo_half(uint16_t x) +{ + return static_cast(x); +} + +constexpr uint16_t lo_half(uint32_t x) +{ + return static_cast(x); +} + +constexpr uint32_t lo_half(uint64_t x) +{ + return static_cast(x); +} + +constexpr uint8_t hi_half(uint16_t x) +{ + return static_cast(x >> 8); +} + +constexpr uint16_t hi_half(uint32_t x) +{ + return static_cast(x >> 16); +} + +constexpr uint32_t hi_half(uint64_t x) +{ + return static_cast(x >> 32); +} + +template +inline constexpr auto lo_half(const uint& x) noexcept +{ + return x.lo; +} + +template +inline constexpr auto hi_half(const uint& x) noexcept +{ + return x.hi; +} + +template +constexpr unsigned num_bits(const T&) noexcept +{ + return sizeof(T) * 8; +} + +template +constexpr bool operator==(const uint& a, const uint& b) noexcept +{ + return (a.lo == b.lo) & (a.hi == b.hi); +} + +template >::value>::type> +constexpr bool operator==(const uint& x, const T& y) noexcept +{ + return x == uint(y); +} + +template >::value>::type> +constexpr bool operator==(const T& x, const uint& y) noexcept +{ + return uint(y) == x; +} + + +template +constexpr bool operator!=(const uint& a, const uint& b) noexcept +{ + return !(a == b); +} + +template >::value>::type> +constexpr bool operator!=(const uint& x, const T& y) noexcept +{ + return x != uint(y); +} + +template >::value>::type> +constexpr bool operator!=(const T& x, const uint& y) noexcept +{ + return uint(x) != y; +} + + +template +constexpr bool operator<(const uint& a, const uint& b) noexcept +{ + // Bitwise operators are used to implement logic here to avoid branching. + // It also should make the function smaller, but no proper benchmark has + // been done. + return (a.hi < b.hi) | ((a.hi == b.hi) & (a.lo < b.lo)); +} + +template >::value>::type> +constexpr bool operator<(const uint& x, const T& y) noexcept +{ + return x < uint(y); +} + +template >::value>::type> +constexpr bool operator<(const T& x, const uint& y) noexcept +{ + return uint(x) < y; +} + + +template +constexpr bool operator>(const uint& a, const uint& b) noexcept +{ + return b < a; +} + +template >::value>::type> +constexpr bool operator>(const uint& x, const T& y) noexcept +{ + return x > uint(y); +} + +template >::value>::type> +constexpr bool operator>(const T& x, const uint& y) noexcept +{ + return uint(x) > y; +} + + +template +constexpr bool operator>=(const uint& a, const uint& b) noexcept +{ + return !(a < b); +} + +template >::value>::type> +constexpr bool operator>=(const uint& x, const T& y) noexcept +{ + return x >= uint(y); +} + +template >::value>::type> +constexpr bool operator>=(const T& x, const uint& y) noexcept +{ + return uint(x) >= y; +} + + +template +constexpr bool operator<=(const uint& a, const uint& b) noexcept +{ + return !(b < a); +} + +template >::value>::type> +constexpr bool operator<=(const uint& x, const T& y) noexcept +{ + return x <= uint(y); +} + +template >::value>::type> +constexpr bool operator<=(const T& x, const uint& y) noexcept +{ + return uint(x) <= y; +} + +template +constexpr uint operator|(const uint& x, const uint& y) noexcept +{ + return {x.hi | y.hi, x.lo | y.lo}; +} + +template +constexpr uint operator&(const uint& x, const uint& y) noexcept +{ + return {x.hi & y.hi, x.lo & y.lo}; +} + +template +constexpr uint operator^(const uint& x, const uint& y) noexcept +{ + return {x.hi ^ y.hi, x.lo ^ y.lo}; +} + +template +constexpr uint operator~(const uint& x) noexcept +{ + return {~x.hi, ~x.lo}; +} + +template +constexpr uint operator<<(const uint& x, unsigned shift) noexcept +{ + constexpr auto num_bits = N; + constexpr auto half_bits = num_bits / 2; + + if (shift < half_bits) + { + const auto lo = x.lo << shift; + + // Find the part moved from lo to hi. + // The shift right here can be invalid: + // for shift == 0 => lshift == half_bits. + // Split it into 2 valid shifts by (rshift - 1) and 1. + const auto rshift = half_bits - shift; + const auto lo_overflow = (x.lo >> (rshift - 1)) >> 1; + const auto hi = (x.hi << shift) | lo_overflow; + return {hi, lo}; + } + + // This check is only needed if we want "defined" behavior for shifts + // larger than size of the Int. + if (shift < num_bits) + return {x.lo << (shift - half_bits), 0}; + + return 0; +} + +template +inline Target narrow_cast(uint64_t x) noexcept +{ + return static_cast(x); +} + +template +inline Target narrow_cast(const Int& x) noexcept +{ + return narrow_cast(x.lo); +} + +template +constexpr uint operator>>(const uint& x, unsigned shift) noexcept +{ + constexpr auto half_bits = N / 2; + + if (shift < half_bits) + { + auto hi = x.hi >> shift; + + // Find the part moved from hi to lo. + // To avoid invalid shift left, + // split them into 2 valid shifts by (lshift - 1) and 1. + unsigned lshift = half_bits - shift; + auto hi_overflow = (x.hi << (lshift - 1)) << 1; + auto lo_part = x.lo >> shift; + auto lo = lo_part | hi_overflow; + return {hi, lo}; + } + + if (shift < num_bits(x)) + return {0, x.hi >> (shift - half_bits)}; + + return 0; +} + + +template >::value>::type> +constexpr uint operator<<(const uint& x, const T& shift) noexcept +{ + if (shift < T{sizeof(x) * 8}) + return x << static_cast(shift); + return 0; +} + +template >::value>::type> +constexpr uint operator>>(const uint& x, const T& shift) noexcept +{ + if (shift < T{sizeof(x) * 8}) + return x >> static_cast(shift); + return 0; +} + +template +inline uint& operator>>=(uint& x, unsigned shift) noexcept +{ + return x = x >> shift; +} + + +constexpr uint64_t* as_words(uint128& x) noexcept +{ + return &x.lo; +} + +constexpr const uint64_t* as_words(const uint128& x) noexcept +{ + return &x.lo; +} + +template +constexpr uint64_t* as_words(uint& x) noexcept +{ + return as_words(x.lo); +} + +template +constexpr const uint64_t* as_words(const uint& x) noexcept +{ + return as_words(x.lo); +} + +template +inline uint8_t* as_bytes(uint& x) noexcept +{ + return reinterpret_cast(as_words(x)); +} + +template +inline const uint8_t* as_bytes(const uint& x) noexcept +{ + return reinterpret_cast(as_words(x)); +} + +/// Implementation of shift left as a loop. +/// This one is slower than the one using "split" strategy. +template +inline uint shl_loop(const uint& x, unsigned shift) +{ + auto r = uint{}; + constexpr unsigned word_bits = sizeof(uint64_t) * 8; + constexpr size_t num_words = sizeof(uint) / sizeof(uint64_t); + auto rw = as_words(r); + auto words = as_words(x); + unsigned s = shift % word_bits; + unsigned skip = shift / word_bits; + + uint64_t carry = 0; + for (size_t i = 0; i < (num_words - skip); ++i) + { + auto w = words[i]; + auto v = (w << s) | carry; + carry = (w >> (word_bits - s - 1)) >> 1; + rw[i + skip] = v; + } + return r; +} + +template +inline uint add_loop(const uint& a, const uint& b) noexcept +{ + static constexpr auto num_words = sizeof(a) / sizeof(uint64_t); + + auto x = as_words(a); + auto y = as_words(b); + + uint s; + auto z = as_words(s); + + bool k = false; + for (size_t i = 0; i < num_words; ++i) + { + z[i] = x[i] + y[i]; + auto k1 = z[i] < x[i]; + z[i] += k; + k = (z[i] < k) || k1; + } + + return s; +} + +template +constexpr uint operator+(const uint& x, const uint& y) noexcept +{ + return add_with_carry(x, y).value; +} + +template +constexpr uint operator-(const uint& x) noexcept +{ + return ~x + uint{1}; +} + +template +constexpr uint operator-(const uint& x, const uint& y) noexcept +{ + return sub_with_carry(x, y).value; +} + +template >::value>::type> +constexpr uint& operator+=(uint& x, const T& y) noexcept +{ + return x = x + y; +} + +template >::value>::type> +constexpr uint& operator-=(uint& x, const T& y) noexcept +{ + return x = x - y; +} + + +template +inline uint<2 * N> umul(const uint& x, const uint& y) noexcept +{ + const auto t0 = umul(x.lo, y.lo); + const auto t1 = umul(x.hi, y.lo); + const auto t2 = umul(x.lo, y.hi); + const auto t3 = umul(x.hi, y.hi); + + const auto u1 = t1 + t0.hi; + const auto u2 = t2 + u1.lo; + + const auto lo = (u2 << (num_bits(x) / 2)) | t0.lo; + const auto hi = t3 + u2.hi + u1.hi; + + return {hi, lo}; +} + +template +constexpr uint<2 * N> constexpr_umul(const uint& x, const uint& y) noexcept +{ + auto t0 = constexpr_umul(x.lo, y.lo); + auto t1 = constexpr_umul(x.hi, y.lo); + auto t2 = constexpr_umul(x.lo, y.hi); + auto t3 = constexpr_umul(x.hi, y.hi); + + auto u1 = t1 + t0.hi; + auto u2 = t2 + u1.lo; + + auto lo = (u2 << (num_bits(x) / 2)) | t0.lo; + auto hi = t3 + u2.hi + u1.hi; + + return {hi, lo}; +} + +template +inline uint mul(const uint& a, const uint& b) noexcept +{ + // Requires 1 full mul, 2 muls and 2 adds. + // Clang & GCC implements 128-bit multiplication this way. + + const auto t = umul(a.lo, b.lo); + const auto hi = (a.lo * b.hi) + (a.hi * b.lo) + t.hi; + + return {hi, t.lo}; +} + +template +inline uint sqr(const uint& a) noexcept +{ + // Based on mul() implementation. + + const auto t = umul(a.lo, a.lo); + const auto hi = 2 * (a.lo * a.hi) + t.hi; + + return {hi, t.lo}; +} + + +template +constexpr uint constexpr_mul(const uint& a, const uint& b) noexcept +{ + auto t = constexpr_umul(a.lo, b.lo); + auto hi = constexpr_mul(a.lo, b.hi) + constexpr_mul(a.hi, b.lo) + t.hi; + return {hi, t.lo}; +} + + +template +inline uint<2 * N> umul_loop(const uint& x, const uint& y) noexcept +{ + constexpr int num_words = sizeof(uint) / sizeof(uint64_t); + + uint<2 * N> p; + auto pw = as_words(p); + auto uw = as_words(x); + auto vw = as_words(y); + + for (int j = 0; j < num_words; ++j) + { + uint64_t k = 0; + for (int i = 0; i < num_words; ++i) + { + auto t = umul(uw[i], vw[j]) + pw[i + j] + k; + pw[i + j] = t.lo; + k = t.hi; + } + pw[j + num_words] = k; + } + return p; +} + +template +inline uint mul_loop_opt(const uint& u, const uint& v) noexcept +{ + constexpr int num_words = sizeof(uint) / sizeof(uint64_t); + + uint p; + auto pw = as_words(p); + auto uw = as_words(u); + auto vw = as_words(v); + + for (int j = 0; j < num_words; j++) + { + uint64_t k = 0; + for (int i = 0; i < (num_words - j - 1); i++) + { + auto t = umul(uw[i], vw[j]) + pw[i + j] + k; + pw[i + j] = t.lo; + k = t.hi; + } + pw[num_words - 1] += uw[num_words - j - 1] * vw[j] + k; + } + return p; +} + +inline uint256 operator*(const uint256& x, const uint256& y) noexcept +{ + return mul(x, y); +} + +template +inline uint operator*(const uint& x, const uint& y) noexcept +{ + return mul_loop_opt(x, y); +} + + +template >::value>::type> +constexpr uint& operator*=(uint& x, const T& y) noexcept +{ + return x = x * y; +} + +template +constexpr uint exp(uint base, uint exponent) noexcept +{ + auto result = uint{1}; + if (base == 2) + return result << exponent; + + while (exponent != 0) + { + if ((exponent & 1) != 0) + result *= base; + base = sqr(base); + exponent >>= 1; + } + return result; +} + +template +constexpr unsigned clz(const uint& x) noexcept +{ + const auto half_bits = num_bits(x) / 2; + + // TODO: Try: + // bool take_hi = h != 0; + // bool take_lo = !take_hi; + // unsigned clz_hi = take_hi * clz(h); + // unsigned clz_lo = take_lo * (clz(l) | half_bits); + // return clz_hi | clz_lo; + + // In this order `h == 0` we get less instructions than in case of `h != 0`. + return x.hi == 0 ? clz(x.lo) + half_bits : clz(x.hi); +} + +template +std::array to_words(Int x) noexcept +{ + std::array words; + std::memcpy(&words, &x, sizeof(x)); + return words; +} + +template +unsigned count_significant_words_loop(uint256 x) noexcept +{ + auto words = to_words(x); + for (size_t i = words.size(); i > 0; --i) + { + if (words[i - 1] != 0) + return static_cast(i); + } + return 0; +} + +template +inline typename std::enable_if::type count_significant_words( + const Int& x) noexcept +{ + return x != 0 ? 1 : 0; +} + +template +inline typename std::enable_if::type count_significant_words( + const Int& x) noexcept +{ + constexpr auto num_words = static_cast(sizeof(x) / sizeof(Word)); + auto h = count_significant_words(hi_half(x)); + auto l = count_significant_words(lo_half(x)); + return h != 0 ? h + (num_words / 2) : l; +} + + +namespace internal +{ +template +struct normalized_div_args +{ + uint divisor; + uint numerator; + typename uint::word_type numerator_ex; + int num_divisor_words; + int num_numerator_words; + unsigned shift; +}; + +template +[[gnu::always_inline]] inline normalized_div_args normalize( + const IntT& numerator, const IntT& denominator) noexcept +{ + // FIXME: Make the implementation type independent + static constexpr auto num_words = IntT::num_words; + + auto* u = as_words(numerator); + auto* v = as_words(denominator); + + normalized_div_args na; + auto* un = as_words(na.numerator); + auto* vn = as_words(na.divisor); + + auto& m = na.num_numerator_words; + for (m = num_words; m > 0 && u[m - 1] == 0; --m) + ; + + auto& n = na.num_divisor_words; + for (n = num_words; n > 0 && v[n - 1] == 0; --n) + ; + + na.shift = clz(v[n - 1]); + if (na.shift) + { + for (int i = num_words - 1; i > 0; --i) + vn[i] = (v[i] << na.shift) | (v[i - 1] >> (64 - na.shift)); + vn[0] = v[0] << na.shift; + + un[num_words] = u[num_words - 1] >> (64 - na.shift); + for (int i = num_words - 1; i > 0; --i) + un[i] = (u[i] << na.shift) | (u[i - 1] >> (64 - na.shift)); + un[0] = u[0] << na.shift; + } + else + { + na.numerator_ex = 0; + na.numerator = numerator; + na.divisor = denominator; + } + + // Skip the highest word of numerator if not significant. + if (un[m] != 0 || un[m - 1] >= vn[n - 1]) + ++m; + + return na; +} + +/// Divides arbitrary long unsigned integer by 64-bit unsigned integer (1 word). +/// @param u The array of a normalized numerator words. It will contain +/// the quotient after execution. +/// @param len The number of numerator words. +/// @param d The normalized divisor. +/// @return The remainder. +inline uint64_t udivrem_by1(uint64_t u[], int len, uint64_t d) noexcept +{ + INTX_REQUIRE(len >= 2); + + const auto reciprocal = reciprocal_2by1(d); + + auto rem = u[len - 1]; // Set the top word as remainder. + u[len - 1] = 0; // Reset the word being a part of the result quotient. + + auto it = &u[len - 2]; + do + { + std::tie(*it, rem) = udivrem_2by1({rem, *it}, d, reciprocal); + } while (it-- != &u[0]); + + return rem; +} + +/// Divides arbitrary long unsigned integer by 128-bit unsigned integer (2 words). +/// @param u The array of a normalized numerator words. It will contain the +/// quotient after execution. +/// @param len The number of numerator words. +/// @param d The normalized divisor. +/// @return The remainder. +inline uint128 udivrem_by2(uint64_t u[], int len, uint128 d) noexcept +{ + INTX_REQUIRE(len >= 3); + + const auto reciprocal = reciprocal_3by2(d); + + auto rem = uint128{u[len - 1], u[len - 2]}; // Set the 2 top words as remainder. + u[len - 1] = u[len - 2] = 0; // Reset these words being a part of the result quotient. + + auto it = &u[len - 3]; + do + { + std::tie(*it, rem) = udivrem_3by2(rem.hi, rem.lo, *it, d, reciprocal); + } while (it-- != &u[0]); + + return rem; +} + +/// s = x + y. +inline bool add(uint64_t s[], const uint64_t x[], const uint64_t y[], int len) noexcept +{ + // OPT: Add MinLen template parameter and unroll first loop iterations. + INTX_REQUIRE(len >= 2); + + bool carry = false; + for (int i = 0; i < len; ++i) + std::tie(s[i], carry) = add_with_carry(x[i], y[i], carry); + return carry; +} + +/// r = x - multiplier * y. +inline uint64_t submul( + uint64_t r[], const uint64_t x[], const uint64_t y[], int len, uint64_t multiplier) noexcept +{ + // OPT: Add MinLen template parameter and unroll first loop iterations. + INTX_REQUIRE(len >= 1); + + uint64_t borrow = 0; + for (int i = 0; i < len; ++i) + { + const auto s = sub_with_carry(x[i], borrow); + const auto p = umul(y[i], multiplier); + const auto t = sub_with_carry(s.value, p.lo); + r[i] = t.value; + borrow = p.hi + s.carry + t.carry; + } + return borrow; +} + +inline void udivrem_knuth( + uint64_t q[], uint64_t u[], int ulen, const uint64_t d[], int dlen) noexcept +{ + INTX_REQUIRE(dlen >= 3); + INTX_REQUIRE(ulen >= dlen); + + const auto divisor = uint128{d[dlen - 1], d[dlen - 2]}; + const auto reciprocal = reciprocal_3by2(divisor); + for (int j = ulen - dlen - 1; j >= 0; --j) + { + const auto u2 = u[j + dlen]; + const auto u1 = u[j + dlen - 1]; + const auto u0 = u[j + dlen - 2]; + + uint64_t qhat; + if (INTX_UNLIKELY(uint128(u2, u1) == divisor)) // Division overflows. + { + qhat = ~uint64_t{0}; + + u[j + dlen] = u2 - submul(&u[j], &u[j], d, dlen, qhat); + } + else + { + uint128 rhat; + std::tie(qhat, rhat) = udivrem_3by2(u2, u1, u0, divisor, reciprocal); + + bool carry; + const auto overflow = submul(&u[j], &u[j], d, dlen - 2, qhat); + std::tie(u[j + dlen - 2], carry) = sub_with_carry(rhat.lo, overflow); + std::tie(u[j + dlen - 1], carry) = sub_with_carry(rhat.hi, carry); + + if (INTX_UNLIKELY(carry)) + { + --qhat; + u[j + dlen - 1] += divisor.hi + add(&u[j], &u[j], d, dlen - 1); + } + } + + q[j] = qhat; // Store quotient digit. + } +} + +} // namespace internal + +template +div_result> udivrem(const uint& u, const uint& v) noexcept +{ + auto na = internal::normalize(u, v); + + if (na.num_numerator_words <= na.num_divisor_words) + return {0, u}; + + if (na.num_divisor_words == 1) + { + const auto r = internal::udivrem_by1( + as_words(na.numerator), na.num_numerator_words, as_words(na.divisor)[0]); + return {na.numerator, r >> na.shift}; + } + + if (na.num_divisor_words == 2) + { + const auto d = as_words(na.divisor); + const auto r = + internal::udivrem_by2(as_words(na.numerator), na.num_numerator_words, {d[1], d[0]}); + return {na.numerator, r >> na.shift}; + } + + auto un = as_words(na.numerator); // Will be modified. + + uint q; + internal::udivrem_knuth( + as_words(q), &un[0], na.num_numerator_words, as_words(na.divisor), na.num_divisor_words); + + uint r; + auto rw = as_words(r); + for (int i = 0; i < na.num_divisor_words - 1; ++i) + rw[i] = na.shift ? (un[i] >> na.shift) | (un[i + 1] << (64 - na.shift)) : un[i]; + rw[na.num_divisor_words - 1] = un[na.num_divisor_words - 1] >> na.shift; + + return {q, r}; +} + +template +constexpr div_result> sdivrem(const uint& u, const uint& v) noexcept +{ + const auto sign_mask = uint{1} << (sizeof(u) * 8 - 1); + auto u_is_neg = (u & sign_mask) != 0; + auto v_is_neg = (v & sign_mask) != 0; + + auto u_abs = u_is_neg ? -u : u; + auto v_abs = v_is_neg ? -v : v; + + auto q_is_neg = u_is_neg ^ v_is_neg; + + auto res = udivrem(u_abs, v_abs); + + return {q_is_neg ? -res.quot : res.quot, u_is_neg ? -res.rem : res.rem}; +} + +template +constexpr uint operator/(const uint& x, const uint& y) noexcept +{ + return udivrem(x, y).quot; +} + +template +constexpr uint operator%(const uint& x, const uint& y) noexcept +{ + return udivrem(x, y).rem; +} + +template >::value>::type> +constexpr uint& operator/=(uint& x, const T& y) noexcept +{ + return x = x / y; +} + +template >::value>::type> +constexpr uint& operator%=(uint& x, const T& y) noexcept +{ + return x = x % y; +} + +template +inline uint bswap(const uint& x) noexcept +{ + return {bswap(x.lo), bswap(x.hi)}; +} + + +// Support for type conversions for binary operators. + +template >::value>::type> +constexpr uint operator+(const uint& x, const T& y) noexcept +{ + return x + uint(y); +} + +template >::value>::type> +constexpr uint operator+(const T& x, const uint& y) noexcept +{ + return uint(x) + y; +} + +template >::value>::type> +constexpr uint operator-(const uint& x, const T& y) noexcept +{ + return x - uint(y); +} + +template >::value>::type> +constexpr uint operator-(const T& x, const uint& y) noexcept +{ + return uint(x) - y; +} + +template >::value>::type> +constexpr uint operator*(const uint& x, const T& y) noexcept +{ + return x * uint(y); +} + +template >::value>::type> +constexpr uint operator*(const T& x, const uint& y) noexcept +{ + return uint(x) * y; +} + +template >::value>::type> +constexpr uint operator/(const uint& x, const T& y) noexcept +{ + return x / uint(y); +} + +template >::value>::type> +constexpr uint operator/(const T& x, const uint& y) noexcept +{ + return uint(x) / y; +} + +template >::value>::type> +constexpr uint operator%(const uint& x, const T& y) noexcept +{ + return x % uint(y); +} + +template >::value>::type> +constexpr uint operator%(const T& x, const uint& y) noexcept +{ + return uint(x) % y; +} + +template >::value>::type> +constexpr uint operator|(const uint& x, const T& y) noexcept +{ + return x | uint(y); +} + +template >::value>::type> +constexpr uint operator|(const T& x, const uint& y) noexcept +{ + return uint(x) | y; +} + +template >::value>::type> +constexpr uint operator&(const uint& x, const T& y) noexcept +{ + return x & uint(y); +} + +template >::value>::type> +constexpr uint operator&(const T& x, const uint& y) noexcept +{ + return uint(x) & y; +} + +template >::value>::type> +constexpr uint operator^(const uint& x, const T& y) noexcept +{ + return x ^ uint(y); +} + +template >::value>::type> +constexpr uint operator^(const T& x, const uint& y) noexcept +{ + return uint(x) ^ y; +} + +template >::value>::type> +constexpr uint& operator|=(uint& x, const T& y) noexcept +{ + return x = x | y; +} + +template >::value>::type> +constexpr uint& operator&=(uint& x, const T& y) noexcept +{ + return x = x & y; +} + +template >::value>::type> +constexpr uint& operator^=(uint& x, const T& y) noexcept +{ + return x = x ^ y; +} + +template >::value>::type> +constexpr uint& operator<<=(uint& x, const T& y) noexcept +{ + return x = x << y; +} + +template >::value>::type> +constexpr uint& operator>>=(uint& x, const T& y) noexcept +{ + return x = x >> y; +} + + +inline uint256 addmod(const uint256& x, const uint256& y, const uint256& mod) noexcept +{ + const auto s = add_with_carry(x, y); + return (uint512{s.carry, s.value} % mod).lo; +} + +inline uint256 mulmod(const uint256& x, const uint256& y, const uint256& mod) noexcept +{ + return (umul(x, y) % mod).lo; +} + + +constexpr uint256 operator"" _u256(const char* s) noexcept +{ + return from_string(s); +} + +constexpr uint512 operator"" _u512(const char* s) noexcept +{ + return from_string(s); +} + +namespace le // Conversions to/from LE bytes. +{ +template +inline IntT load(const uint8_t (&bytes)[M]) noexcept +{ + static_assert(M == IntT::num_bits / 8, + "the size of source bytes must match the size of the destination uint"); + auto x = IntT{}; + std::memcpy(&x, bytes, sizeof(x)); + return x; +} + +template +inline void store(uint8_t (&dst)[N / 8], const intx::uint& x) noexcept +{ + std::memcpy(dst, &x, sizeof(x)); +} + +} // namespace le + + +namespace be // Conversions to/from BE bytes. +{ +/// Loads an uint value from bytes of big-endian order. +/// If the size of bytes is smaller than the result uint, the value is zero-extended. +template +inline IntT load(const uint8_t (&bytes)[M]) noexcept +{ + static_assert(M <= IntT::num_bits / 8, + "the size of source bytes must not exceed the size of the destination uint"); + auto x = IntT{}; + std::memcpy(&as_bytes(x)[IntT::num_bits / 8 - M], bytes, M); + return bswap(x); +} + +template +inline IntT load(const T& t) noexcept +{ + return load(t.bytes); +} + +/// Stores an uint value in a bytes array in big-endian order. +template +inline void store(uint8_t (&dst)[N / 8], const intx::uint& x) noexcept +{ + const auto d = bswap(x); + std::memcpy(dst, &d, sizeof(d)); +} + +/// Stores an uint value in .bytes field of type T. The .bytes must be an array of uint8_t +/// of the size matching the size of uint. +template +inline T store(const intx::uint& x) noexcept +{ + T r{}; + store(r.bytes, x); + return r; +} + +/// Stores the truncated value of an uint in a bytes array. +/// Only the least significant bytes from big-endian representation of the uint +/// are stored in the result bytes array up to array's size. +template +inline void trunc(uint8_t (&dst)[M], const intx::uint& x) noexcept +{ + static_assert(M < N / 8, "destination must be smaller than the source value"); + const auto d = bswap(x); + const auto b = as_bytes(d); + std::memcpy(dst, &b[sizeof(d) - M], M); +} + +/// Stores the truncated value of an uint in the .bytes field of an object of type T. +template +inline T trunc(const intx::uint& x) noexcept +{ + T r{}; + trunc(r.bytes, x); + return r; +} + +namespace unsafe +{ +/// Loads an uint value from a buffer. The user must make sure +/// that the provided buffer is big enough. Therefore marked "unsafe". +template +inline IntT load(const uint8_t* bytes) noexcept +{ + auto x = IntT{}; + std::memcpy(&x, bytes, sizeof(x)); + return bswap(x); +} + +/// Stores an uint value at the provided pointer in big-endian order. The user must make sure +/// that the provided buffer is big enough to fit the value. Therefore marked "unsafe". +template +inline void store(uint8_t* dst, const intx::uint& x) noexcept +{ + const auto d = bswap(x); + std::memcpy(dst, &d, sizeof(d)); +} +} // namespace unsafe + +} // namespace be + +} // namespace intx diff --git a/source/libs/decimal/src/detail/wideInteger.cpp b/source/libs/decimal/src/detail/wideInteger.cpp new file mode 100644 index 0000000000..41fdbd7f65 --- /dev/null +++ b/source/libs/decimal/src/detail/wideInteger.cpp @@ -0,0 +1,262 @@ +#include "wideInteger.h" +#include "intx/int128.hpp" +#include "intx/intx.hpp" + + +const UInt128 uInt128Zero = {0, 0}; +const uint64_t k1e18 = 1000000000000000000LL; +const UInt128 uInt128_1e18 = {k1e18, 0}; +const UInt128 uInt128One = {1, 0}; +const UInt128 uInt128Two = {2, 0}; + +void makeUInt128(uint128* pUint128, uint64_t high, uint64_t low) { + intx::uint128* pIntxUint = (intx::uint128*)pUint128; + pIntxUint->hi = high; + pIntxUint->lo = low; +} + +uint64_t uInt128Hi(const UInt128* pInt) { + intx::uint128 *pIntUint = (intx::uint128*)pInt; + return pIntUint->hi; +} + +uint64_t uInt128Lo(const UInt128* pInt) { + intx::uint128 *pIntUint = (intx::uint128*)pInt; + return pIntUint->lo; +} + +void uInt128Add(UInt128* pLeft, const UInt128* pRight) { + intx::uint128 *pX = (intx::uint128*)pLeft; + const intx::uint128 *pY = (const intx::uint128*)pRight; + *pX += *pY; +} +void uInt128Subtract(UInt128* pLeft, const UInt128* pRight) { + intx::uint128 *pX = (intx::uint128*)pLeft; + const intx::uint128 *pY = (const intx::uint128*)pRight; + *pX -= *pY; +} +void uInt128Multiply(UInt128* pLeft, const UInt128* pRight) { + intx::uint128 *pX = (intx::uint128*)pLeft; + const intx::uint128 *pY = (const intx::uint128*)pRight; + *pX *= *pY; + /* __uint128_t *px = (__uint128_t*)pLeft; + const __uint128_t *py = (__uint128_t*)pRight; + *px = *px * *py; */ +} +void uInt128Divide(UInt128* pLeft, const UInt128* pRight) { + intx::uint128 *pX = (intx::uint128*)pLeft; + const intx::uint128 *pY = (const intx::uint128*)pRight; + *pX /= *pY; + /* __uint128_t *px = (__uint128_t*)pLeft; + const __uint128_t *py = (__uint128_t*)pRight; + *px = *px / *py; */ +} +void uInt128Mod(UInt128* pLeft, const UInt128* pRight) { + intx::uint128 *pX = (intx::uint128*)pLeft; + const intx::uint128 *pY = (const intx::uint128*)pRight; + *pX %= *pY; + /* __uint128_t *px = (__uint128_t*)pLeft; + const __uint128_t *py = (__uint128_t*)pRight; + *px = *px % *py; */ +} +bool uInt128Lt(const UInt128* pLeft, const UInt128* pRight) { + const intx::uint128 *pX = (const intx::uint128*)pLeft; + const intx::uint128 *pY = (const intx::uint128*)pRight; + return *pX < *pY; +} +bool uInt128Gt(const UInt128* pLeft, const UInt128* pRight) { + const intx::uint128 *pX = (const intx::uint128*)pLeft; + const intx::uint128 *pY = (const intx::uint128*)pRight; + return *pX > *pY; +} +bool uInt128Eq(const UInt128* pLeft, const UInt128* pRight) { + const intx::uint128 *pX = (const intx::uint128*)pLeft; + const intx::uint128 *pY = (const intx::uint128*)pRight; + return *pX == *pY; +} + +Int128 makeInt128(int64_t high, uint64_t low) { + Int128 int128 = {low, high}; + return int128; +} +int64_t int128Hi(const Int128* pUint128) { + return pUint128->high; +} +uint64_t int128Lo(const Int128* pUint128) { + return pUint128->low; +} +Int128 int128Abs(const Int128* pInt128) { + if (int128Lt(pInt128, &int128Zero)) { + return int128Negate(pInt128); + } + return *pInt128; +} +Int128 int128Negate(const Int128* pInt128) { + uint64_t low = ~pInt128->low + 1; + int64_t high = ~pInt128->high; + if (low == 0) high += 1; + return makeInt128(high, low); +} +Int128 int128Add(const Int128* pLeft, const Int128* pRight) { + intx::uint128 result = *(intx::uint128*)pLeft + *(intx::uint128*)pRight; + return *(Int128*)&result; +} +Int128 int128Subtract(const Int128* pLeft, const Int128* pRight) { + intx::uint128 result = *(intx::uint128*)pLeft - *(intx::uint128*)pRight; + return *(Int128*)&result; +} +Int128 int128Multiply(const Int128* pLeft, const Int128* pRight) { + intx::uint128 result = *(intx::uint128*)pLeft * *(intx::uint128*)pRight; + return *(Int128*)&result; +} +Int128 int128Divide(const Int128* pLeft, const Int128* pRight) { + intx::uint128 result = *(intx::uint128*)pLeft / *(intx::uint128*)pRight; + return *(Int128*)&result; +} +Int128 int128Mod(const Int128* pLeft, const Int128* pRight) { + intx::uint128 result = *(intx::uint128*)pLeft % *(intx::uint128*)pRight; + return *(Int128*)&result; +} +bool int128Lt(const Int128* pLeft, const Int128* pRight) { + return pLeft->high < pRight->high || (pLeft->high == pRight->high && pLeft->low < pRight->low); +} +bool int128Gt(const Int128* pLeft, const Int128* pRight) { + return int128Lt(pRight, pLeft); +} +bool int128Eq(const Int128* pLeft, const Int128* pRight) { + return pLeft->high == pRight->high && pLeft->low == pRight->low; +} +Int128 int128RightShift(const Int128* pLeft, int32_t shift) { + intx::uint128 result = *(intx::uint128*)pLeft >> shift; + return *(Int128*)&result; +} + +const Int128 int128Zero = {0, 0}; +const Int128 int128One = {1, 0}; + +UInt256 makeUint256(UInt128 high, UInt128 low) { + UInt256 uint256 = {high, low}; + return uint256; +} +uint128 uInt256Hi(const UInt256* pUint256) { + return pUint256->high; +} +uint128 uInt256Lo(const UInt256* pUint256) { + return pUint256->low; +} +UInt256 uInt256Add(const UInt256* pLeft, const UInt256* pRight) { + intx::uint256 result = *(intx::uint256*)pLeft + *(intx::uint256*)pRight; + return *(UInt256*)&result; +} +UInt256 uInt256Subtract(const UInt256* pLeft, const UInt256* pRight) { + intx::uint256 result = *(intx::uint256*)pLeft - *(intx::uint256*)pRight; + return *(UInt256*)&result; +} +UInt256 uInt256Multiply(const UInt256* pLeft, const UInt256* pRight) { + intx::uint256 result = *(intx::uint256*)pLeft * *(intx::uint256*)pRight; + return *(UInt256*)&result; +} +UInt256 uInt256Divide(const UInt256* pLeft, const UInt256* pRight) { + intx::uint256 result = *(intx::uint256*)pLeft / *(intx::uint256*)pRight; + return *(UInt256*)&result; +} +UInt256 uInt256Mod(const UInt256* pLeft, const UInt256* pRight) { + intx::uint256 result = *(intx::uint256*)pLeft % *(intx::uint256*)pRight; + return *(UInt256*)&result; +} +bool uInt256Lt(const UInt256* pLeft, const UInt256* pRight) { + return *(intx::uint256*)pLeft < *(intx::uint256*)pRight; +} +bool uInt256Gt(const UInt256* pLeft, const UInt256* pRight) { + return *(intx::uint256*)pLeft > *(intx::uint256*)pRight; +} +bool uInt256Eq(const UInt256* pLeft, const UInt256* pRight) { + return *(intx::uint256*)pLeft == *(intx::uint256*)pRight; +} +UInt256 uInt256RightShift(const UInt256* pLeft, int32_t shift) { + intx::uint256 result = *(intx::uint256*)pLeft >> shift; + return *(UInt256*)&result; +} + +Int256 makeInt256(Int128 high, UInt128 low) { + Int256 int256 = {low, high}; + return int256; +} +Int128 int256Hi(const Int256* pUint256) { + return pUint256->high; +} +UInt128 int256Lo(const Int256* pUint256) { + return pUint256->low; +} +Int256 int256Abs(const Int256* pInt256) { + if (int256Lt(pInt256, &int256Zero)) { + return int256Negate(pInt256); + } + return *pInt256; +} + +Int256 int256Negate(const Int256* pInt256) { + return int256Subtract(&int256Zero, pInt256); +} +Int256 int256Add(const Int256* pLeft, const Int256* pRight) { + intx::uint256 result = *(intx::uint256*)pLeft + *(intx::uint256*)pRight; + return *(Int256*)&result; +} +Int256 int256Subtract(const Int256* pLeft, const Int256* pRight) { + intx::uint256 result = *(intx::uint256*)pLeft - *(intx::uint256*)pRight; + return *(Int256*)&result; +} +Int256 int256Multiply(const Int256* pLeft, const Int256* pRight) { + intx::uint256 result = *(intx::uint256*)pLeft * *(intx::uint256*)pRight; + return *(Int256*)&result; +} +Int256 int256Divide(const Int256* pLeft, const Int256* pRight) { + Int256 l = *pLeft, r = *pRight; + bool leftNegative = int256Lt(pLeft, &int256Zero), rightNegative = int256Lt(pRight, &int256Zero); + if (leftNegative) { + l = int256Abs(pLeft); + } + if (rightNegative) { + r = int256Abs(pRight); + } + intx::uint256 result = *(intx::uint256*)&l / *(intx::uint256*)&r; + Int256 res = *(Int256*)&result; + if (leftNegative != rightNegative) + res = int256Negate(&res); + return res; +} + +Int256 int256Mod(const Int256* pLeft, const Int256* pRight) { + Int256 left = *pLeft, right = *pRight; + bool leftNegative = int256Lt(pLeft, &int256Zero); + if (leftNegative) { + left = int256Abs(&left); + } + bool rightNegate = int256Lt(pRight, &int256Zero); + if (rightNegate) right = int256Abs(pRight); + intx::uint256 result = *(intx::uint256*)&left % *(intx::uint256*)&right; + Int256 res = *(Int256*)&result; + if (leftNegative) res = int256Negate(&res); + return res; +} +bool int256Lt(const Int256* pLeft, const Int256* pRight) { + Int128 hiLeft = int256Hi(pLeft), hiRight = int256Hi(pRight); + UInt128 lowLeft = int256Lo(pLeft), lowRight = int256Lo(pRight); + return int128Lt(&hiLeft, &hiRight) || (int128Eq(&hiLeft, &hiRight) && uInt128Lt(&lowLeft, &lowRight)); +} +bool int256Gt(const Int256* pLeft, const Int256* pRight) { + return int256Lt(pRight, pLeft); +} +bool int256Eq(const Int256* pLeft, const Int256* pRight) { + Int128 hiLeft = int256Hi(pLeft), hiRight = int256Hi(pRight); + UInt128 lowLeft = int256Lo(pLeft), lowRight = int256Lo(pRight); + return int128Eq(&hiLeft, &hiRight) && uInt128Eq(&lowLeft, &lowRight); +} +Int256 int256RightShift(const Int256* pLeft, int32_t shift) { + intx::uint256 result = *(intx::uint256*)pLeft >> shift; + return *(Int256*)&result; +} + +const Int256 int256One = {uInt128One, int128Zero}; +const Int256 int256Zero = {uInt128Zero, int128Zero}; +const Int256 int256Two = {uInt128Two, int128Zero}; diff --git a/source/libs/decimal/test/CMakeLists.txt b/source/libs/decimal/test/CMakeLists.txt new file mode 100644 index 0000000000..237a8dd220 --- /dev/null +++ b/source/libs/decimal/test/CMakeLists.txt @@ -0,0 +1,23 @@ +MESSAGE(STATUS "build decimal unit test") + +# GoogleTest requires at least C++11 +SET(CMAKE_CXX_STANDARD 11) +AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) + +ADD_EXECUTABLE(decimalTest ${SOURCE_LIST}) + +TARGET_INCLUDE_DIRECTORIES( + decimalTest + PUBLIC "${TD_SOURCE_DIR}/include/libs/decimal/" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) + +TARGET_LINK_LIBRARIES( + decimalTest + PUBLIC gtest taos os common +) + +add_test( + NAME decimalTest + COMMAND decimalTest +) diff --git a/source/libs/decimal/test/decimalTest.cpp b/source/libs/decimal/test/decimalTest.cpp new file mode 100644 index 0000000000..d1ca3ff153 --- /dev/null +++ b/source/libs/decimal/test/decimalTest.cpp @@ -0,0 +1,1545 @@ +#include +#include +#include +#include +#include +#define ALLOW_FORBID_FUNC + +#include "decimal.h" +#include "tdatablock.h" +using namespace std; + +template +void printArray(const std::array& arr) { + auto it = arr.rbegin(); + for (; it != arr.rend(); ++it) { + cout << *it; + } + cout << endl; +} + +#if 0 +template +void extractWideInteger(__int128 a) { + uint64_t k = 10; + std::array segments{}; + int seg_num = 0; + for (int i = 0; i < DIGIT_NUM; ++i) { + k *= 10; + } + + while (a != 0) { + uint64_t hi = a >> 64; + uint64_t lo = a; + cout << "hi: " << hi << " lo: " << lo << endl; + uint64_t hi_quotient = hi / k; + uint64_t hi_remainder = hi % k; + // cout << "hi % 1e9: " << hi_remainder << endl; + __int128 tmp = ((__int128)hi_remainder << 64) | (__int128)lo; + uint64_t lo_remainder = tmp % k; + uint64_t lo_quotient = tmp / k; + a = (__int128)hi_quotient << 64 | (__int128)lo_quotient; + segments[seg_num++] = lo_remainder; + } + printArray<38 / DIGIT_NUM>(segments); +} + +void printDecimal(const DecimalType* pDec, uint8_t type, uint8_t prec, uint8_t scale) { + char buf[64] = {0}; + int32_t code = decimalToStr(pDec, type, prec, scale, buf, 64); + ASSERT_EQ(code, 0); + cout << buf; +} + +__int128 generate_big_int128(uint32_t digitNum) { + __int128 a = 0; + for (int i = 0; i < digitNum + 1; ++i) { + a *= 10; + a += (i % 10); + } + return a; +} +#endif + +void checkDecimal(const DecimalType* pDec, uint8_t t, uint8_t prec, uint8_t scale, const char* valExpect) { + ASSERT_TRUE(t == prec > 18 ? TSDB_DATA_TYPE_DECIMAL : TSDB_DATA_TYPE_DECIMAL64); + ASSERT_TRUE(scale <= prec); + char buf[64] = {0}; + int32_t code = decimalToStr(pDec, t, prec, scale, buf, 64); + ASSERT_EQ(code, 0); + ASSERT_STREQ(buf, valExpect); + cout << "decimal" << (prec > 18 ? 128 : 64) << " " << (int32_t)prec << ":" << (int32_t)scale << " -> " << buf << endl; +} + +class Numeric128; +class Numeric64 { + Decimal64 dec_; + static constexpr uint8_t DECIMAL_WORD_NUM = DECIMAL_WORD_NUM(Decimal64); + + public: + friend class Numeric128; + Numeric64() { dec_ = {0}; } + int32_t fromStr(const string& str, uint8_t prec, uint8_t scale) { + return decimal64FromStr(str.c_str(), str.size(), prec, scale, &dec_); + } + Numeric64& operator+=(const Numeric64& r) { + getOps()->add(&dec_, &r.dec_, DECIMAL_WORD_NUM); + return *this; + } + // Numeric64& operator+=(const Numeric128& r); + + bool operator==(const Numeric64& r) const { return getOps()->eq(&dec_, &r.dec_, DECIMAL_WORD_NUM); } + Numeric64& operator=(const Numeric64& r); + Numeric64& operator=(const Numeric128& r); + + static const SDecimalOps* getOps() { return getDecimalOps(TSDB_DATA_TYPE_DECIMAL64); } +}; + +class Numeric128 { + Decimal128 dec_; + static constexpr uint8_t DECIMAL_WORD_NUM = DECIMAL_WORD_NUM(Decimal128); + + public: + friend Numeric64; + Numeric128() { dec_ = {0}; } + Numeric128(const Numeric128& r) = default; + int32_t fromStr(const string& str, uint8_t prec, uint8_t scale) { + return decimal128FromStr(str.c_str(), str.size(), prec, scale, &dec_); + } + Numeric128& operator+=(const Numeric128& r) { return *this; } + Numeric128& operator+=(const Numeric64& r) { + getOps()->add(&dec_, &r.dec_, Numeric64::DECIMAL_WORD_NUM); + return *this; + } + + static const SDecimalOps* getOps() { return getDecimalOps(TSDB_DATA_TYPE_DECIMAL); } +}; + +Numeric64& Numeric64::operator=(const Numeric64& r) { + dec_ = r.dec_; + return *this; +} + +template +struct NumericType {}; + +template <> +struct NumericType<64> { + using Type = Numeric64; + static constexpr int8_t dataType = TSDB_DATA_TYPE_DECIMAL64; + static constexpr int8_t maxPrec = TSDB_DECIMAL64_MAX_PRECISION; + static constexpr int8_t bytes = DECIMAL64_BYTES; +}; + +template <> +struct NumericType<128> { + using Type = Numeric128; + static constexpr int8_t dataType = TSDB_DATA_TYPE_DECIMAL; + static constexpr int8_t maxPrec = TSDB_DECIMAL_MAX_PRECISION; + static constexpr int8_t bytes = DECIMAL128_BYTES; +}; + +template +struct TrivialTypeInfo { + using TrivialType = T; +}; + +#define DEFINE_TRIVIAL_TYPE_HELPER(type, tsdb_type) \ + template <> \ + struct TrivialTypeInfo { \ + static constexpr int8_t dataType = tsdb_type; \ + static constexpr int32_t bytes = sizeof(type); \ + } + +DEFINE_TRIVIAL_TYPE_HELPER(int8_t, TSDB_DATA_TYPE_TINYINT); +DEFINE_TRIVIAL_TYPE_HELPER(uint8_t, TSDB_DATA_TYPE_UTINYINT); +DEFINE_TRIVIAL_TYPE_HELPER(int16_t, TSDB_DATA_TYPE_SMALLINT); +DEFINE_TRIVIAL_TYPE_HELPER(uint16_t, TSDB_DATA_TYPE_USMALLINT); +DEFINE_TRIVIAL_TYPE_HELPER(int32_t, TSDB_DATA_TYPE_INT); +DEFINE_TRIVIAL_TYPE_HELPER(uint32_t, TSDB_DATA_TYPE_UINT); +DEFINE_TRIVIAL_TYPE_HELPER(int64_t, TSDB_DATA_TYPE_BIGINT); +DEFINE_TRIVIAL_TYPE_HELPER(uint64_t, TSDB_DATA_TYPE_UBIGINT); +DEFINE_TRIVIAL_TYPE_HELPER(float, TSDB_DATA_TYPE_FLOAT); +DEFINE_TRIVIAL_TYPE_HELPER(double, TSDB_DATA_TYPE_DOUBLE); +DEFINE_TRIVIAL_TYPE_HELPER(bool, TSDB_DATA_TYPE_BOOL); + +template +class Numeric { + static_assert(BitNum == 64 || BitNum == 128, "only support Numeric64 and Numeric128"); + using Type = typename NumericType::Type; + Type dec_; + uint8_t prec_; + uint8_t scale_; + + public: + Numeric(uint8_t prec, uint8_t scale, const std::string& str) : prec_(prec), scale_(scale) { + if (prec > NumericType::maxPrec) throw std::string("prec too big") + std::to_string(prec); + int32_t code = dec_.fromStr(str, prec, scale) != 0; + if (code != 0) { + cout << "failed to init decimal(" << (int32_t)prec << "," << (int32_t)scale << ") from str: " << str << "\t"; + throw std::overflow_error(tstrerror(code)); + } + } + Numeric() = default; + Numeric(const Numeric& o) = default; + ~Numeric() = default; + Numeric& operator=(const Numeric& o) = default; + + static SDataType getRetType(EOperatorType op, const SDataType& lt, const SDataType& rt) { + SDataType ot = {0}; + int32_t code = decimalGetRetType(<, &rt, op, &ot); + if (code != 0) throw std::runtime_error(tstrerror(code)); + return ot; + } + SDataType type() const { return {NumericType::dataType, prec(), scale(), NumericType::bytes}; } + uint8_t prec() const { return prec_; } + uint8_t scale() const { return scale_; } + const Type& dec() const { return dec_; } + STypeMod get_type_mod() const { return decimalCalcTypeMod(prec(), scale()); } + + template + Numeric& binaryOp(const Numeric& r, EOperatorType op) { + auto out = binaryOp(r, op); + return *this = out; + } + + template + Numeric binaryOp(const Numeric& r, EOperatorType op) { + SDataType lt{NumericType::dataType, prec_, scale_, NumericType::bytes}; + SDataType rt{NumericType::dataType, r.prec(), r.scale(), NumericType::bytes}; + SDataType ot = getRetType(op, lt, rt); + Numeric out{ot.precision, ot.scale, "0"}; + int32_t code = decimalOp(op, <, &rt, &ot, &dec_, &r.dec(), &out); + if (code != 0) throw std::overflow_error(tstrerror(code)); + return out; + } + + template + Numeric binaryOp(const T& r, EOperatorType op) { + using TypeInfo = TrivialTypeInfo; + SDataType lt{NumericType::dataType, prec_, scale_, NumericType::bytes}; + SDataType rt{TypeInfo::dataType, 0, 0, TypeInfo::bytes}; + SDataType ot = getRetType(op, lt, rt); + Numeric out{ot.precision, ot.scale, "0"}; + int32_t code = decimalOp(op, <, &rt, &ot, &dec_, &r, &out); + if (code == TSDB_CODE_DECIMAL_OVERFLOW) throw std::overflow_error(tstrerror(code)); + if (code != 0) throw std::runtime_error(tstrerror(code)); + return out; + } +#define DEFINE_OPERATOR(op, op_type) \ + template \ + Numeric operator op(const Numeric& r) { \ + cout << *this << " " #op " " << r << " = "; \ + auto res = binaryOp(r, op_type); \ + cout << res << endl; \ + return res; \ + } + + DEFINE_OPERATOR(+, OP_TYPE_ADD); + DEFINE_OPERATOR(-, OP_TYPE_SUB); + DEFINE_OPERATOR(*, OP_TYPE_MULTI); + DEFINE_OPERATOR(/, OP_TYPE_DIV); + DEFINE_OPERATOR(%, OP_TYPE_REM); + +#define DEFINE_TYPE_OP(op, op_type) \ + template \ + Numeric operator op(const T & r) { \ + cout << *this << " " #op " " << r << "(" << typeid(T).name() << ")" << " = "; \ + Numeric res = {}; \ + try { \ + res = binaryOp(r, op_type); \ + } catch (...) { \ + throw; \ + } \ + cout << res << endl; \ + return res; \ + } + DEFINE_TYPE_OP(+, OP_TYPE_ADD); + DEFINE_TYPE_OP(-, OP_TYPE_SUB); + DEFINE_TYPE_OP(*, OP_TYPE_MULTI); + DEFINE_TYPE_OP(/, OP_TYPE_DIV); + DEFINE_TYPE_OP(%, OP_TYPE_REM); + +#define DEFINE_REAL_OP(op) \ + double operator op(double v) { \ + if (BitNum == 128) \ + return TEST_decimal128ToDouble((Decimal128*)&dec_, prec(), scale()) / v; \ + else if (BitNum == 64) \ + return TEST_decimal64ToDouble((Decimal64*)&dec_, prec(), scale()) / v; \ + return 0; \ + } + DEFINE_REAL_OP(+); + DEFINE_REAL_OP(-); + DEFINE_REAL_OP(*); + DEFINE_REAL_OP(/); + + template + Numeric& operator+=(const Numeric& r) { + return binaryOp(r, OP_TYPE_ADD); + } + + std::string toString() const { + char buf[64] = {0}; + int32_t code = decimalToStr(&dec_, NumericType::dataType, prec(), scale(), buf, 64); + if (code != 0) throw std::string(tstrerror(code)); + return {buf}; + } + + std::string toStringTrimTailingZeros() const { + auto ret = toString(); + int32_t sizeToRemove = 0; + auto it = ret.rbegin(); + for (; it != ret.rend(); ++it) { + if (*it == '0') { + ++sizeToRemove; + continue; + } + break; + } + if (ret.size() - sizeToRemove > 0) ret.resize(ret.size() - sizeToRemove); + return ret; + } +#define DEFINE_OPERATOR_T(type) \ + operator type() { \ + if (BitNum == 64) { \ + return type##FromDecimal64(&dec_, prec(), scale()); \ + } else if (BitNum == 128) { \ + return type##FromDecimal128(&dec_, prec(), scale()); \ + } \ + return 0; \ + } + DEFINE_OPERATOR_T(bool); + DEFINE_OPERATOR_T(int8_t); + DEFINE_OPERATOR_T(uint8_t); + DEFINE_OPERATOR_T(int16_t); + DEFINE_OPERATOR_T(uint16_t); + DEFINE_OPERATOR_T(int32_t); + DEFINE_OPERATOR_T(uint32_t); + DEFINE_OPERATOR_T(int64_t); + DEFINE_OPERATOR_T(uint64_t); + DEFINE_OPERATOR_T(float); + DEFINE_OPERATOR_T(double); + + Numeric& operator=(const char* str) { + std::string s = str; + int32_t code = 0; + if (BitNum == 64) { + code = decimal64FromStr(s.c_str(), s.size(), prec(), scale(), (Decimal64*)&dec_); + } else if (BitNum == 128) { + code = decimal128FromStr(s.c_str(), s.size(), prec(), scale(), (Decimal128*)&dec_); + } + if (TSDB_CODE_SUCCESS != code) { + throw std::string("failed to convert str to decimal64: ") + s + " " + tstrerror(code); + } + return *this; + } + +#define DEFINE_OPERATOR_FROM_FOR_BITNUM(type, BitNum) \ + if (std::is_floating_point::value) { \ + code = TEST_decimal##BitNum##From_double((Decimal##BitNum*)&dec_, prec(), scale(), v); \ + } else if (std::is_signed::value) { \ + code = TEST_decimal##BitNum##From_int64_t((Decimal##BitNum*)&dec_, prec(), scale(), v); \ + } else if (std::is_unsigned::value) { \ + code = TEST_decimal##BitNum##From_uint64_t((Decimal##BitNum*)&dec_, prec(), scale(), v); \ + } + +#define DEFINE_OPERATOR_EQ_T(type) \ + Numeric& operator=(type v) { \ + int32_t code = 0; \ + if (BitNum == 64) { \ + DEFINE_OPERATOR_FROM_FOR_BITNUM(type, 64); \ + } else if (BitNum == 128) { \ + DEFINE_OPERATOR_FROM_FOR_BITNUM(type, 128); \ + } \ + return *this; \ + } + DEFINE_OPERATOR_EQ_T(int64_t); + DEFINE_OPERATOR_EQ_T(int32_t); + DEFINE_OPERATOR_EQ_T(int16_t); + DEFINE_OPERATOR_EQ_T(int8_t); + + DEFINE_OPERATOR_EQ_T(uint64_t); + DEFINE_OPERATOR_EQ_T(uint32_t); + DEFINE_OPERATOR_EQ_T(uint16_t); + DEFINE_OPERATOR_EQ_T(uint8_t); + + DEFINE_OPERATOR_EQ_T(bool); + DEFINE_OPERATOR_EQ_T(double); + DEFINE_OPERATOR_EQ_T(float); + + Numeric& operator=(const Decimal128& d) { + SDataType inputDt = {TSDB_DATA_TYPE_DECIMAL, prec(), scale(), DECIMAL128_BYTES}; + SDataType outputDt = {NumericType::dataType, prec(), scale(), NumericType::bytes}; + int32_t code = convertToDecimal(&d, &inputDt, &dec_, &outputDt); + if (code == TSDB_CODE_DECIMAL_OVERFLOW) throw std::overflow_error(tstrerror(code)); + if (code != 0) throw std::runtime_error(tstrerror(code)); + return *this; + } + Numeric& operator=(const Decimal64& d) { + SDataType inputDt = {TSDB_DATA_TYPE_DECIMAL64, prec(), scale(), DECIMAL64_BYTES}; + SDataType outputDt = {NumericType::dataType, prec_, scale_, NumericType::bytes}; + int32_t code = convertToDecimal(&d, &inputDt, &dec_, &outputDt); + if (code == TSDB_CODE_DECIMAL_OVERFLOW) throw std::overflow_error(tstrerror(code)); + if (code != 0) throw std::runtime_error(tstrerror(code)); + return *this; + } + + template + Numeric(const Numeric& num2) { + Numeric(); + *this = num2; + } + + template + Numeric& operator=(const Numeric& num2) { + static_assert(BitNum2 == 64 || BitNum2 == 128, "Only support decimal128/decimal64"); + SDataType inputDt = {num2.type().type, num2.prec(), num2.scale(), num2.type().bytes}; + SDataType outputDt = {NumericType::dataType, NumericType::maxPrec, num2.scale(), + NumericType::bytes}; + int32_t code = convertToDecimal(&num2.dec(), &inputDt, &dec_, &outputDt); + if (code == TSDB_CODE_DECIMAL_OVERFLOW) throw std::overflow_error(tstrerror(code)); + if (code != 0) throw std::runtime_error(tstrerror(code)); + prec_ = outputDt.precision; + scale_ = outputDt.scale; + return *this; + } +#define DEFINE_COMPARE_OP(op, op_type) \ + template \ + bool operator op(const T& t) { \ + Numeric<128> lDec = *this, rDec = *this; \ + rDec = t; \ + SDecimalCompareCtx l = {(void*)&lDec.dec(), TSDB_DATA_TYPE_DECIMAL, lDec.get_type_mod()}, \ + r = {(void*)&rDec.dec(), TSDB_DATA_TYPE_DECIMAL, rDec.get_type_mod()}; \ + return decimalCompare(op_type, &l, &r); \ + } + + DEFINE_COMPARE_OP(>, OP_TYPE_GREATER_THAN); + DEFINE_COMPARE_OP(>=, OP_TYPE_GREATER_EQUAL); + DEFINE_COMPARE_OP(<, OP_TYPE_LOWER_THAN); + DEFINE_COMPARE_OP(<=, OP_TYPE_LOWER_EQUAL); + DEFINE_COMPARE_OP(==, OP_TYPE_EQUAL); + DEFINE_COMPARE_OP(!=, OP_TYPE_NOT_EQUAL); +}; + +template +ostream& operator<<(ostream& os, const Numeric& n) { + os << n.toString() << "(" << (int32_t)n.prec() << ":" << (int32_t)n.scale() << ")"; + return os; +} + +TEST(decimal, numeric) { + Numeric<64> dec{10, 4, "123.456"}; + Numeric<64> dec2{18, 10, "123456.123123"}; + auto o = dec + dec2; + ASSERT_EQ(o.toString(), "123579.5791230000"); + + Numeric<128> dec128{37, 10, "123456789012300.09876543"}; + o = dec + dec128; + ASSERT_EQ(o.toStringTrimTailingZeros(), "123456789012423.55476543"); + ASSERT_EQ(o.toString(), "123456789012423.5547654300"); + + auto os = o - dec; + ASSERT_EQ(os.toStringTrimTailingZeros(), dec128.toStringTrimTailingZeros()); + + auto os2 = o - dec128; + ASSERT_EQ(os2.toStringTrimTailingZeros(), dec.toStringTrimTailingZeros()); + + os = dec * dec2; + ASSERT_EQ(os.toStringTrimTailingZeros(), "15241399.136273088"); + ASSERT_EQ(os.toString(), "15241399.13627308800000"); + + os = dec * dec128; + ASSERT_EQ(os.toStringTrimTailingZeros(), "15241481344302520.993185"); + ASSERT_EQ(os.toString(), "15241481344302520.993185"); + + os2 = os / dec128; + ASSERT_EQ(os2.toStringTrimTailingZeros(), "123.456"); + ASSERT_EQ(os2.toString(), "123.456000"); + + os = dec2 / dec; + ASSERT_EQ(os.toString(), "1000.000997302682737169518"); + + int32_t a = 123; + os = dec + a; + ASSERT_EQ(os.toString(), "246.4560"); + + os = dec * a; + ASSERT_EQ(os.toString(), "15185.0880"); + + os = dec / 2; + ASSERT_EQ(os.toStringTrimTailingZeros(), "61.728"); + + os = dec2 / 2; + ASSERT_EQ(os.toStringTrimTailingZeros(), "61728.0615615"); + + os = dec128 / 2; + ASSERT_EQ(os.toStringTrimTailingZeros(), "61728394506150.049382715"); + + auto dec3 = Numeric<64>(10, 2, "171154.38"); + os = dec3 / 2; + ASSERT_EQ(os.toStringTrimTailingZeros(), "85577.19"); + + auto dec4 = Numeric<64>(10, 5, "1.23456"); + os = dec4 / 2; + ASSERT_EQ(os.toStringTrimTailingZeros(), "0.61728"); + + os = dec4 / 123123123; + ASSERT_EQ(os.toStringTrimTailingZeros(), "0.0000000100270361"); + + os = dec4 / (int64_t)123123123; + ASSERT_EQ(os.toStringTrimTailingZeros(), "0.0000000100270361075880117"); + + double dv = dec4 / 123123.123; + + Numeric<128> max{38, 0, "99999999999999999999999999999999999999.000"}; + ASSERT_EQ(max.toString(), "99999999999999999999999999999999999999"); + Numeric<128> zero{38, 0, "0"}; + auto min = zero - max; + ASSERT_EQ(min.toString(), "-99999999999999999999999999999999999999"); + + dec = 123.456; + ASSERT_EQ(dec.toString(), "123.4560"); + + dec = 47563.36; + dec128 = 0; + o = dec128 + dec; // (37, 10) + (10, 4) = (38, 10) + ASSERT_EQ(o.toString(), "47563.3600000000"); + dec = 3749.00; + o = o + dec; // (38, 10) + (10, 4) = (38, 9) + ASSERT_EQ(o.toString(), "51312.360000000"); +} + +TEST(decimal, decimalFromType) { + Numeric<128> dec1{20, 4, "0"}; + dec1 = 123.123; + ASSERT_EQ(dec1.toString(), "123.1230"); + dec1 = (float)123.123; + ASSERT_EQ(dec1.toString(), "123.1230"); + dec1 = (int64_t)-9999999; + ASSERT_EQ(dec1.toString(), "-9999999.0000"); + dec1 = "99.99999"; + ASSERT_EQ(dec1.toString(), "100.0000"); +} + +TEST(decimal, typeFromDecimal) { + Numeric<128> dec1{18, 4, "1234"}; + Numeric<64> dec2{18, 4, "1234"}; + int64_t intv = dec1; + uint64_t uintv = dec1; + double doublev = dec1; + ASSERT_EQ(intv, 1234); + ASSERT_EQ(uintv, 1234); + ASSERT_EQ(doublev, 1234); + doublev = dec2; + ASSERT_EQ(doublev, 1234); + intv = dec1 = "123.43"; + uintv = dec1; + doublev = dec1; + ASSERT_EQ(intv, 123); + ASSERT_EQ(uintv, 123); + ASSERT_EQ(doublev, 123.43); + doublev = dec2 = "123.54"; + ASSERT_EQ(doublev, 123.54); + intv = dec1 = "123.66"; + uintv = dec1; + doublev = dec1; + ASSERT_EQ(intv, 124); + ASSERT_EQ(uintv, 124); + ASSERT_EQ(doublev, 123.66); + + intv = dec1 = "-123.44"; + uintv = dec1; + doublev = dec1; + ASSERT_EQ(intv, -123); + ASSERT_EQ(uintv, 18446744073709551493ULL); + ASSERT_EQ(doublev, -123.44); + intv = dec1 = "-123.99"; + uintv = dec1; + doublev = dec1; + ASSERT_EQ(intv, -124); + ASSERT_EQ(uintv, 18446744073709551492ULL); + ASSERT_EQ(doublev, -123.99); + + bool boolv = false; + boolv = dec1; + ASSERT_TRUE(boolv); + boolv = dec1 = "0"; + ASSERT_FALSE(boolv); +} + +#if 0 +TEST(decimal128, to_string) { + __int128 i = generate_big_int128(37); + int64_t hi = i >> 64; + uint64_t lo = i; + Decimal128 d; + makeDecimal128(&d, hi, lo); + char buf[64] = {0}; + decimalToStr(d.words, TSDB_DATA_TYPE_DECIMAL, 38, 10, buf, 64); + ASSERT_STREQ(buf, "123456789012345678901234567.8901234567"); + + buf[0] = '\0'; + decimalToStr(d.words, TSDB_DATA_TYPE_DECIMAL, 38, 9, buf, 64); + ASSERT_STREQ(buf, "1234567890123456789012345678.901234567"); +} + +TEST(decimal, divide) { + __int128 i = generate_big_int128(15); + int64_t hi = i >> 64; + uint64_t lo = i; + Decimal128 d; + makeDecimal128(&d, hi, lo); + + Decimal128 d2 = {0}; + makeDecimal128(&d2, 0, 12345678); + + auto ops = getDecimalOps(TSDB_DATA_TYPE_DECIMAL); + Decimal128 remainder = {0}; + int8_t precision1 = 38, scale1 = 5, precision2 = 10, scale2 = 2; + int8_t out_scale = 25; + int8_t out_precision = std::min(precision1 - scale1 + scale2 + out_scale, 38); + int8_t delta_scale = out_scale + scale2 - scale1; + printDecimal(&d, TSDB_DATA_TYPE_DECIMAL, precision1, scale1); + __int128 a = 1; + while (delta_scale-- > 0) a *= 10; + Decimal128 multiplier = {0}; + makeDecimal128(&multiplier, a >> 64, a); + ops->multiply(d.words, multiplier.words, 2); + cout << " / "; + printDecimal(&d2, TSDB_DATA_TYPE_DECIMAL, precision2, scale2); + cout << " = "; + ops->divide(d.words, d2.words, 2, remainder.words); + printDecimal(&d, TSDB_DATA_TYPE_DECIMAL, out_precision, out_scale); + ASSERT_TRUE(1); +} +#endif + +TEST(decimal, conversion) { + // convert uint8 to decimal + char buf[64] = {0}; + int8_t i8 = 22; + SDataType inputType = {TSDB_DATA_TYPE_TINYINT, 1}; + uint8_t prec = 10, scale = 2; + SDataType decType = {TSDB_DATA_TYPE_DECIMAL64, prec, scale, 8}; + Decimal64 dec64 = {0}; + int32_t code = convertToDecimal(&i8, &inputType, &dec64, &decType); + ASSERT_TRUE(code == 0); + cout << "convert uint8: " << (int32_t)i8 << " to "; + checkDecimal(&dec64, TSDB_DATA_TYPE_DECIMAL64, prec, scale, "22.00"); + + Decimal128 dec128 = {0}; + decType.type = TSDB_DATA_TYPE_DECIMAL; + decType.precision = 38; + decType.scale = 10; + decType.bytes = 16; + code = convertToDecimal(&i8, &inputType, &dec128, &decType); + ASSERT_TRUE(code == 0); + cout << "convert uint8: " << (int32_t)i8 << " to "; + checkDecimal(&dec128, TSDB_DATA_TYPE_DECIMAL, decType.precision, decType.scale, "22.0000000000"); + + char inputBuf[64] = "123.000000000000000000000000000000001"; + code = decimal128FromStr(inputBuf, strlen(inputBuf), 38, 35, &dec128); + ASSERT_EQ(code, 0); + checkDecimal(&dec128, TSDB_DATA_TYPE_DECIMAL, 38, 35, "123.00000000000000000000000000000000100"); + + inputType.type = TSDB_DATA_TYPE_DECIMAL64; + inputType.precision = prec; + inputType.scale = scale; + code = convertToDecimal(&dec64, &inputType, &dec128, &decType); + ASSERT_EQ(code, 0); + checkDecimal(&dec128, TSDB_DATA_TYPE_DECIMAL, 38, 10, "22.0000000000"); +} + +static constexpr uint64_t k1E16 = 10000000000000000LL; + +#if 0 +TEST(decimal, decimalFromStr) { + char inputBuf[64] = "123.000000000000000000000000000000001"; + Decimal128 dec128 = {0}; + int32_t code = decimal128FromStr(inputBuf, strlen(inputBuf), 38, 35, &dec128); + ASSERT_EQ(code, 0); + __int128 res = decimal128ToInt128(&dec128); + __int128 resExpect = 123; + resExpect *= k1E16; + resExpect *= k1E16; + resExpect *= 10; + resExpect += 1; + resExpect *= 100; + ASSERT_EQ(res, resExpect); + + char buf[64] = "999.999"; + Decimal64 dec64 = {0}; + code = decimal64FromStr(buf, strlen(buf), 6, 3, &dec64); + ASSERT_EQ(code, 0); + ASSERT_EQ(999999, DECIMAL64_GET_VALUE(&dec64)); + +} +#endif + +TEST(decimal, toStr) { + Decimal64 dec = {0}; + char buf[64] = {0}; + int32_t code = decimalToStr(&dec, TSDB_DATA_TYPE_DECIMAL64, 10, 2, buf, 64); + ASSERT_EQ(code, 0); + ASSERT_STREQ(buf, "0"); + + Decimal128 dec128 = {0}; + code = decimalToStr(&dec128, TSDB_DATA_TYPE_DECIMAL, 38, 10, buf, 64); + ASSERT_EQ(code, 0); + ASSERT_STREQ(buf, "0"); + + char buf2[1]; + code = decimalToStr(&dec128, TSDB_DATA_TYPE_DECIMAL, 38, 10, buf2, 1); + ASSERT_TRUE(buf2[0] == '0'); + + code = decimalToStr(&dec128, TSDB_DATA_TYPE_DECIMAL, 38, 10, NULL, 100); + ASSERT_EQ(code, TSDB_CODE_INVALID_PARA); + + makeDecimal128(&dec128, 999999999999, 999999999999); + ASSERT_EQ(TSDB_CODE_SUCCESS, decimalToStr(&dec128, TSDB_DATA_TYPE_DECIMAL, 38, 10, buf, 64)); + code = decimalToStr(&dec128, TSDB_DATA_TYPE_DECIMAL, 38, 10, buf2, 1); + ASSERT_EQ(buf2[0], buf[0]); +} + +SDataType getDecimalType(uint8_t prec, uint8_t scale) { + if (prec > TSDB_DECIMAL_MAX_PRECISION) throw std::string("invalid prec: ") + std::to_string(prec); + uint8_t type = decimalTypeFromPrecision(prec); + return {type, prec, scale, tDataTypes[type].bytes}; +} + +bool operator==(const SDataType& lt, const SDataType& rt) { + return lt.type == rt.type && lt.precision == rt.precision && lt.scale == rt.scale && lt.bytes == rt.bytes; +} + +TEST(decimal, decimalOpRetType) { + EOperatorType op = OP_TYPE_ADD; + auto ta = getDecimalType(10, 2); + auto tb = getDecimalType(10, 2); + SDataType tc{}, tExpect = {TSDB_DATA_TYPE_DECIMAL, 11, 2, sizeof(Decimal)}; + int32_t code = decimalGetRetType(&ta, &tb, op, &tc); + ASSERT_EQ(code, 0); + ASSERT_EQ(tExpect, tc); + + ta.bytes = 8; + ta.type = TSDB_DATA_TYPE_TIMESTAMP; + + code = decimalGetRetType(&ta, &tb, op, &tc); + ASSERT_EQ(code, 0); + tExpect.type = TSDB_DATA_TYPE_DECIMAL; + tExpect.precision = 22; + tExpect.scale = 2; + tExpect.bytes = sizeof(Decimal); + ASSERT_EQ(tExpect, tc); + + ta.bytes = 8; + ta.type = TSDB_DATA_TYPE_DOUBLE; + tc = {0}; + code = decimalGetRetType(&ta, &tb, op, &tc); + ASSERT_EQ(code, 0); + tExpect.type = TSDB_DATA_TYPE_DOUBLE; + tExpect.precision = 0; + tExpect.scale = 0; + tExpect.bytes = 8; + ASSERT_EQ(tExpect, tc); + + op = OP_TYPE_DIV; + ta = getDecimalType(10, 2); + tb = getDecimalType(10, 2); + tExpect.type = TSDB_DATA_TYPE_DECIMAL; + tExpect.precision = 23; + tExpect.scale = 13; + tExpect.bytes = sizeof(Decimal); + code = decimalGetRetType(&ta, &tb, op, &tc); + + Numeric<64> aNum = {10, 2, "123.99"}; + int64_t bInt64 = 317759474393305778; + auto res = aNum / bInt64; + ASSERT_EQ(res.scale(), 22); +} + +TEST(decimal, op) { + const char * stra = "123.99", *strb = "456.12"; + EOperatorType op = OP_TYPE_ADD; + auto ta = getDecimalType(10, 2); + Decimal64 a = {0}; + int32_t code = decimal64FromStr(stra, strlen(stra), ta.precision, ta.scale, &a); + ASSERT_EQ(code, 0); + + auto tb = getDecimalType(10, 2); + Decimal64 b{0}; + code = decimal64FromStr(strb, strlen(strb), tb.precision, tb.scale, &b); + ASSERT_EQ(code, 0); + + SDataType tc{}, tExpect{TSDB_DATA_TYPE_DECIMAL, 11, 2, sizeof(Decimal)}; + code = decimalGetRetType(&ta, &tb, op, &tc); + ASSERT_EQ(code, 0); + ASSERT_EQ(tc, tExpect); + Decimal res{}; + code = decimalOp(op, &ta, &tb, &tc, &a, &b, &res); + ASSERT_EQ(code, 0); + + checkDecimal(&res, TSDB_DATA_TYPE_DECIMAL, tc.precision, tc.scale, "580.11"); + + a = {1234567890}; + b = {9876543210}; + ta = getDecimalType(18, 5); + tb = getDecimalType(15, 3); + code = decimalGetRetType(&ta, &tb, op, &tc); + ASSERT_EQ(code, 0); + tExpect.precision = 19; + tExpect.scale = 5; + tExpect.type = TSDB_DATA_TYPE_DECIMAL; + tExpect.bytes = sizeof(Decimal128); + ASSERT_EQ(tExpect, tc); + Decimal128 res128 = {0}; + code = decimalOp(op, &ta, &tb, &tc, &a, &b, &res128); + ASSERT_EQ(code, 0); + checkDecimal(&res128, 0, tExpect.precision, tExpect.scale, "9888888.88890"); +} + +struct DecimalStringRandomGeneratorConfig { + uint8_t prec = 38; + uint8_t scale = 10; + bool enableWeightOverflow = false; + float weightOverflowRatio = 0.001; + bool enableScaleOverflow = true; + float scaleOverFlowRatio = 0.1; + bool enablePositiveSign = false; + bool withCornerCase = true; + float cornerCaseRatio = 0.1; + float positiveRatio = 0.7; +}; + +class DecimalStringRandomGenerator { + std::random_device rd_; + std::mt19937 gen_; + std::uniform_int_distribution dis_; + static const std::array cornerCases; + static const unsigned int ratio_base = 1000000; + + public: + DecimalStringRandomGenerator() : gen_(rd_()), dis_(0, ratio_base) {} + std::string generate(const DecimalStringRandomGeneratorConfig& config) { + std::string ret; + auto sign = generateSign(config.positiveRatio); + if (config.enablePositiveSign || sign != '+') ret.push_back(sign); + if (config.withCornerCase && currentShouldGenerateCornerCase(config.cornerCaseRatio)) { + ret += generateCornerCase(config); + } else { + uint8_t prec = randomInt(config.prec - config.scale), scale = randomInt(config.scale); + for (int i = 0; i < prec; ++i) { + ret.push_back(generateDigit()); + } + if (config.enableWeightOverflow && possible(config.weightOverflowRatio)) { + int extra_weight = config.prec - prec + 1 + randomInt(TSDB_DECIMAL_MAX_PRECISION); + while (extra_weight--) { + ret.push_back(generateDigit()); + } + } + ret.push_back('.'); + for (int i = 0; i < scale; ++i) { + ret.push_back(generateDigit()); + } + if (config.enableScaleOverflow && possible(config.scaleOverFlowRatio)) { + int extra_scale = config.scale - scale + 1 + randomInt(TSDB_DECIMAL_MAX_SCALE); + while (extra_scale--) { + ret.push_back(generateDigit()); + } + } + } + return ret; + } + + private: + int randomInt(int modulus) { return dis_(gen_) % modulus; } + char generateSign(float positive_ratio) { return possible(positive_ratio) ? '+' : '-'; } + char generateDigit() { return randomInt(10) + '0'; } + bool currentShouldGenerateCornerCase(float corner_case_ratio) { return possible(corner_case_ratio); } + string generateCornerCase(const DecimalStringRandomGeneratorConfig& config) { + string res{}; + if (possible(0.8)) { + res = cornerCases[randomInt(cornerCases.size())]; + } else { + res = std::string(config.prec - config.scale, generateDigit()); + if (possible(0.8)) { + res.push_back('.'); + if (possible(0.8)) { + res += std::string(config.scale, generateDigit()); + } + } + } + return res; + } + + bool possible(float ratio) { return randomInt(ratio_base) <= ratio * ratio_base; } +}; + +const std::array DecimalStringRandomGenerator::cornerCases = {"0", "NULL", "0.", ".0", "00000.000000"}; + +TEST(decimal, randomGenerator) { + GTEST_SKIP(); + DecimalStringRandomGeneratorConfig config; + DecimalStringRandomGenerator generator; + for (int i = 0; i < 1000; ++i) { + auto str = generator.generate(config); + cout << str << endl; + } +} + +#define ASSERT_OVERFLOW(op) \ + do { \ + try { \ + auto res = op; \ + } catch (std::overflow_error & e) { \ + cout << " overflow" << endl; \ + break; \ + } catch (std::exception & e) { \ + FAIL(); \ + } \ + FAIL(); \ + } while (0) + +#define ASSERT_RUNTIME_ERROR(op) \ + do { \ + try { \ + auto res = op; \ + } catch (std::overflow_error & e) { \ + FAIL(); \ + } catch (std::runtime_error & e) { \ + cout << " runtime error" << endl; \ + break; \ + } catch (std::exception & e) { \ + FAIL(); \ + } \ + FAIL(); \ + } while (0) + +template +struct DecimalFromStrTestUnit { + uint8_t precision; + uint8_t scale; + std::string input; + std::string expect; + bool overflow; +}; + +template +void testDecimalFromStr(std::vector>& units) { + for (auto& unit : units) { + if (unit.overflow) { + auto ff = [&]() { + Numeric dec = {unit.precision, unit.scale, unit.input}; + return dec; + }; + ASSERT_OVERFLOW(ff()); + continue; + } + cout << unit.input << " convert to decimal: (" << (int32_t)unit.precision << "," << (int32_t)unit.scale + << "): " << unit.expect << endl; + Numeric dec = {unit.precision, unit.scale, unit.input}; + ASSERT_EQ(dec.toString(), unit.expect); + } +} + +TEST(decimal, decimalFromStr_all) { + std::vector> units = { + {10, 5, "1e+2.1", "100.00000", false}, + {10, 5, "0e+1000", "0", false}, + {10, 5, "0e-1000", "0", false}, + {10, 5, "0e-100", "0", false}, + {10, 5, "0e100", "0", false}, + {10, 5, "0e10", "0", false}, + {10, 5, "1.634e-5", "0.00002", false}, + {18, 16, "-0.0009900000000000000009e5", "-99.0000000000000001", false}, + {18, 10, "1.23456e7", "12345600.0000000000", false}, + {10, 5, "0e-10", "0", false}, + {10, 8, "1.000000000000000009e2", "", true}, + {10, 8, "1.2345e2", "", true}, + {18, 18, "-0.0000000000000100000010000000900000009e10", "-0.000100000010000001", false}, + {18, 18, "-0.1999999999999999999e-1", "-0.020000000000000000", false}, + {18, 18, "-0.9999999999999999999e-1", "-0.100000000000000000", false}, + {18, 18, "-9.999999999999999999e-1", "", true}, + {10, 10, "-9.9999999e-2", "-0.0999999990", false}, + {10, 6, "9.99999e4", "", true}, + {18, 4, "9.999999e1", "100.0000", false}, + {18, 18, "0.0000000000000000000000000010000000000000000199999e26", "0.100000000000000002", false}, + {18, 18, "0.000000000000000000000000001000000000000000009e26", "0.100000000000000001", false}, + {10, 10, "0.000000000010000000009e10", "0.1000000001", false}, + {10, 10, "0.000000000000000000009e10", "0.0000000001", false}, + {10, 10, "0.00000000001e10", "0.1000000000", false}, + {10, 7, "-1234567890e-8", "-12.3456789", false}, + {10, 4, "1e5", "100000.0000", false}, + {10, 3, "123.000E4", "1230000.000", false}, + {10, 3, "123.456E2", "12345.600", false}, + {18, 2, "1.2345e8", "123450000.00", false}, + {10, 2, "1.2345e8", "123450000.00", true}, + {18, 4, "9.99999", "10.0000", false}, + {10, 2, "123.45", "123.45", false}, + {10, 2, "123.456", "123.46", false}, + {10, 2, "123.454", "123.45"}, + {18, 2, "1234567890123456.456", "1234567890123456.46", false}, + {18, 2, "9999999999999999.995", "", true}, + {18, 2, "9999999999999999.994", "9999999999999999.99", false}, + {18, 2, "-9999999999999999.995", "", true}, + {18, 2, "-9999999999999999.994", "-9999999999999999.99", false}, + {18, 2, "-9999999999999999.9999999", "", true}, + {10, 2, "12345678.456", "12345678.46", false}, + {10, 2, "12345678.454", "12345678.45", false}, + {10, 2, "99999999.999", "", true}, + {10, 2, "-99999999.992", "-99999999.99", false}, + {10, 2, "-99999999.999", "", true}, + {10, 2, "-99999989.998", "-99999990.00", false}, + {10, 2, "-99999998.997", "-99999999.00", false}, + {10, 2, "-99999999.009", "-99999999.01", false}, + {18, 17, "-9.99999999999999999999", "", true}, + {18, 16, "-99.999999999999999899999", "-99.9999999999999999", false}, + {18, 16, "-99.999999999999990099999", "-99.9999999999999901", false}, + {18, 18, "0.0000000000000000099", "0.000000000000000010", false}, + {18, 18, "0.0000000000000000001", "0", false}, + {18, 18, "0.0000000000000000005", "0.000000000000000001", false}, + {18, 18, "-0.0000000000000000001", "0", false}, + {18, 18, "-0.00000000000000000019999", "0", false}, + {18, 18, "-0.0000000000000000005", "-0.000000000000000001", false}, + {18, 18, "-0.00000000000000000000000000123123123", "0", false}, + {18, 18, "0.10000000000000000000000000123123123", "0.100000000000000000", false}, + {18, 18, "0.000000000000000000000000000000000000006", "0", false}, + {18, 17, "1.00000000000000000999", "1.00000000000000001", false}, + {18, 17, "1.00000000000000000199", "1.00000000000000000", false}, + {15, 1, "-00000.", "0", false}, + {14, 12, "-.000", "0", false}, + {14, 12, "-.000000000000", "0", false}, + {14, 12, "-.", "0", false}, + {14, 10, "12345.12345", "", true}, + {14, 0, "123456789012345.123123", "", true}, + {18, 0, "1234567890123456789.123123", "", true}, + {18, 0, "1.23456e18", "", true}, + {18, 18, "1.23456e0", "", true}, + {18, 18, "1.23456e-1", "0.123456000000000000", false}, + }; + testDecimalFromStr(units); + + std::vector> dec128Units = { + {38, 38, "1.23456789121312312312312312355e-10", "0.00000000012345678912131231231231231236", false}, + {38, 10, "610854818164978322886511028.733672028246706062283745797933332437780013", + "610854818164978322886511028.7336720282", false}, + {38, 10, "0e-10", "0", false}, + {38, 10, "0e10", "0", false}, + {38, 10, "e-100000", "0", false}, + {38, 10, "e-10", "0", false}, + {38, 10, "e10", "0", false}, + {38, 10, "-1.23456789012300000000000000000099000009e20", "-123456789012300000000.0000000001", false}, + {38, 10, "-1.234567890123e20", "-123456789012300000000.0000000000", false}, + {20, 15, "1234567890.9999999999e-6", "1234.567891000000000", false}, + {20, 15, "1234567890.9999999999999999999e-5", "12345.678910000000000", false}, + {20, 15, "1234567890.99999999999e-5", "12345.678910000000000", false}, + {20, 10, "12345667788.12312e-10", "1.2345667788", false}, + {20, 20, "1.234567123123123e-20", "0.00000000000000000001", false}, + {38, 38, "1.23456e-10", "0.00000000012345600000000000000000000000", false}, + {38, 10, "123456789012345678901234567.89012345679", "123456789012345678901234567.8901234568", false}, + {38, 10, "123456789012345678901234567.89012345670", "123456789012345678901234567.8901234567", false}, + {38, 10, "-123456789012345678901234567.89012345671", "-123456789012345678901234567.8901234567", false}, + {38, 10, "-123456789012345678901234567.89012345679", "-123456789012345678901234567.8901234568", false}, + {38, 10, "-9999999999999999999999999999.99999999995", "", true}, + {38, 10, "-9999999999999999999999999999.99999999994", "-9999999999999999999999999999.9999999999", false}, + {38, 10, "9999999999999999999999999999.99999999996", "", true}, + {38, 10, "9999999999999999999999999999.99999999994", "9999999999999999999999999999.9999999999", false}, + {36, 35, "9.99999999999999999999999999999999999", "9.99999999999999999999999999999999999", false}, + {36, 35, "9.999999999999999999999999999999999999111231231", "", true}, + {38, 38, "0.000000000000000000000000000000000000001", "0", false}, + {38, 38, "0.000000000000000000000000000000000000006", "0.00000000000000000000000000000000000001", false}, + {38, 35, "123.000000000000000000000000000000001", "123.00000000000000000000000000000000100", false}, + {38, 5, "123.", "123.00000", false}, + {20, 4, "-.12345", "-0.1235", false}, + {20, 4, "-.", "0", false}, + {30, 10, "1.2345e+20", "", true}, + {38, 38, "1.23456e0", "", true}, + {38, 38, "1.23456e-1", "0.12345600000000000000000000000000000000", false}, + }; + testDecimalFromStr(dec128Units); +} + +TEST(decimal, op_overflow) { + // divide 0 error + Numeric<128> dec{38, 2, string(36, '9') + ".99"}; + ASSERT_RUNTIME_ERROR(dec / 0); + + // test decimal128Max + Numeric<128> max{38, 10, "0"}; + max = decimal128Max; + ASSERT_EQ(max.toString(), "9999999999999999999999999999.9999999999"); + + { + // multiply no overflow, trim scale + auto res = max * 10; // scale will be trimed to 6, and round up + ASSERT_EQ(res.scale(), 6); + ASSERT_EQ(res.toString(), "100000000000000000000000000000.000000"); + + // multiply not overflow, no trim scale + Numeric<64> dec64{18, 10, "99999999.9999999999"}; + Numeric<128> dec128{19, 10, "999999999.9999999999"}; + + auto rett = Numeric<64>::getRetType(OP_TYPE_MULTI, dec64.type(), dec128.type()); + ASSERT_EQ(rett.precision, 38); + ASSERT_EQ(rett.type, TSDB_DATA_TYPE_DECIMAL); + ASSERT_EQ(rett.scale, dec64.scale() + dec128.scale()); + + res = dec64 * dec128; + ASSERT_EQ(res.toString(), "99999999999999999.89000000000000000001"); + + // multiply not overflow, trim scale from 20 - 19 + Numeric<128> dec128_2{20, 10, "9999999999.9999999999"}; + rett = Numeric<128>::getRetType(OP_TYPE_MULTI, dec64.type(), dec128_2.type()); + ASSERT_EQ(rett.scale, 19); + res = dec64 * dec128_2; + ASSERT_EQ(res.toString(), "999999999999999998.9900000000000000000"); + + // trim scale from 20 - 18 + dec128_2 = {21, 10, "99999999999.9999999999"}; + rett = Numeric<128>::getRetType(OP_TYPE_MULTI, dec64.type(), dec128_2.type()); + ASSERT_EQ(rett.scale, 18); + res = dec64 * dec128_2; + ASSERT_EQ(res.toString(), "9999999999999999989.990000000000000000"); + + // trim scale from 20 -> 17 + dec128_2 = {22, 10, "999999999999.9999999999"}; + rett = Numeric<128>::getRetType(OP_TYPE_MULTI, dec64.type(), dec128_2.type()); + ASSERT_EQ(rett.scale, 17); + res = dec64 * dec128_2; + ASSERT_EQ(res.toString(), "99999999999999999899.99000000000000000"); + + // trim scale from 20 -> 6 + dec128_2 = {33, 10, "99999999999999999999999.9999999999"}; + rett = Numeric<128>::getRetType(OP_TYPE_MULTI, dec64.type(), dec128_2.type()); + ASSERT_EQ(rett.scale, 6); + res = dec64 * dec128_2; + ASSERT_EQ(res.toString(), "9999999999999999989999999999999.990000"); + + dec128_2 = {34, 10, "999999999999999999999999.9999999999"}; + rett = Numeric<128>::getRetType(OP_TYPE_MULTI, dec64.type(), dec128_2.type()); + ASSERT_EQ(rett.scale, 6); + res = dec64 * dec128_2; + ASSERT_EQ(res.toString(), "99999999999999999899999999999999.990000"); + + dec128_2 = {35, 10, "9999999999999999999999999.9999999999"}; + rett = Numeric<128>::getRetType(OP_TYPE_MULTI, dec64.type(), dec128_2.type()); + ASSERT_EQ(rett.scale, 6); + ASSERT_OVERFLOW(dec64 * dec128_2); + } + { + // divide not overflow but trim scale + Numeric<128> dec128{19, 10, "999999999.9999999999"}; + Numeric<64> dec64{10, 10, "0.10000000"}; + auto res = dec128 / dec64; + ASSERT_EQ(res.scale(), 19); + ASSERT_EQ(res.toString(), "9999999999.9999999990000000000"); + + dec64 = {10, 10, "0.1111111111"}; + res = dec128 / dec64; + ASSERT_EQ(res.scale(), 19); + ASSERT_EQ(res.toString(), "9000000000.8999999991899999999"); + + dec64 = {10, 2, "0.01"}; + res = dec128 / dec64; + ASSERT_EQ(res.scale(), 21); + ASSERT_EQ(res.prec(), 32); + ASSERT_EQ(res.toString(), "99999999999.999999990000000000000"); + + dec64 = {10, 2, "7.77"}; + int32_t a = 2; + res = dec64 % a; + ASSERT_EQ(res.toString(), "1.77"); + + dec128 = {38, 10, "999999999999999999999999999.9999999999"}; + res = dec128 % dec64; + ASSERT_EQ(res.toString(), "5.4399999999"); + + dec64 = {18, 10, "99999999.9999999999"}; + res = dec128 % dec64; + ASSERT_EQ(res.toString(), "0.0000000009"); + + Numeric<128> dec128_2 = {38, 10, "9988888888888888888888888.1111111111"}; + res = dec128 % dec128_2; + ASSERT_EQ(res.toString(), "1111111111111111111111188.8888888899"); + + dec128 = {38, 10, "9999999999999999999999999999.9999999999"}; + dec128_2 = {38, 2, "999999999999999999999999999988123123.88"}; + res = dec128 % dec128_2; + ASSERT_EQ(res.toString(), "9999999999999999999999999999.9999999999"); + } +} + +EOperatorType get_op_type(char op) { + switch (op) { + case '+': + return OP_TYPE_ADD; + case '-': + return OP_TYPE_SUB; + case '*': + return OP_TYPE_MULTI; + case '/': + return OP_TYPE_DIV; + case '%': + return OP_TYPE_REM; + default: + return OP_TYPE_IS_UNKNOWN; + } +} + +std::string get_op_str(EOperatorType op) { + switch (op) { + case OP_TYPE_ADD: + return "+"; + case OP_TYPE_SUB: + return "-"; + case OP_TYPE_MULTI: + return "*"; + case OP_TYPE_DIV: + return "/"; + case OP_TYPE_REM: + return "%"; + default: + return "unknown"; + } +} + +struct DecimalRetTypeCheckConfig { + bool check_res_type = true; + bool check_bytes = true; + bool log = true; +}; +struct DecimalRetTypeCheckContent { + DecimalRetTypeCheckContent(const SDataType& a, const SDataType& b, const SDataType& out, EOperatorType op) + : type_a(a), type_b(b), type_out(out), op_type(op) {} + SDataType type_a; + SDataType type_b; + SDataType type_out; + EOperatorType op_type; + // (1, 0) / (1, 1) = (8, 6) + // (1, 0) / (1, 1) = (8, 6) + DecimalRetTypeCheckContent(const std::string& s) { + char op = '\0'; + sscanf(s.c_str(), "(%hhu, %hhu) %c (%hhu, %hhu) = (%hhu, %hhu)", &type_a.precision, &type_a.scale, &op, + &type_b.precision, &type_b.scale, &type_out.precision, &type_out.scale); + type_a = getDecimalType(type_a.precision, type_a.scale); + type_b = getDecimalType(type_b.precision, type_b.scale); + type_out = getDecimalType(type_out.precision, type_out.scale); + op_type = get_op_type(op); + } + + void check(const DecimalRetTypeCheckConfig& config = DecimalRetTypeCheckConfig()) { + SDataType ret = {0}; + try { + if (config.log) + cout << "check ret type for type: (" << (int)type_a.type << " " << (int)type_a.precision << " " + << (int)type_a.scale << ") " << get_op_str(op_type) << " (" << (int)type_b.type << " " + << (int)type_b.precision << " " << (int)type_b.scale << ") = \n"; + ret = Numeric<64>::getRetType(op_type, type_a, type_b); + } catch (std::runtime_error& e) { + ASSERT_EQ(type_out.type, TSDB_DATA_TYPE_MAX); + if (config.log) cout << "not support!" << endl; + return; + } + if (config.log) + cout << "(" << (int)ret.type << " " << (int)ret.precision << " " << (int)ret.scale << ") expect:" << endl + << "(" << (int)type_out.type << " " << (int)type_out.precision << " " << (int)type_out.scale << ")" << endl; + if (config.check_res_type) ASSERT_EQ(ret.type, type_out.type); + ASSERT_EQ(ret.precision, type_out.precision); + ASSERT_EQ(ret.scale, type_out.scale); + if (config.check_bytes) ASSERT_EQ(ret.bytes, type_out.bytes); + } +}; + +TEST(decimal_all, ret_type_load_from_file) { + GTEST_SKIP(); + std::string fname = "/tmp/ret_type.txt"; + std::ifstream ifs(fname, std::ios_base::in); + if (!ifs.is_open()) { + std::cerr << "open file " << fname << " failed" << std::endl; + FAIL(); + } + char buf[64]; + int32_t total_lines = 0; + while (ifs.getline(buf, 64, '\n')) { + DecimalRetTypeCheckContent dcc(buf); + DecimalRetTypeCheckConfig config; + config.check_res_type = false; + config.check_bytes = false; + config.log = false; + dcc.check(config); + ++total_lines; + } + ASSERT_EQ(total_lines, 3034205); +} + +TEST(decimal_all, test_decimal_compare) { + Numeric<64> dec64 = {10, 2, "123.23"}; + Numeric<64> dec64_2 = {11, 10, "1.23"}; + ASSERT_FALSE(dec64_2 > dec64); + dec64 = "10123456.23"; + ASSERT_FALSE(dec64_2 > dec64); + ASSERT_TRUE(dec64 > dec64_2); + ASSERT_TRUE(dec64_2 < 100); + Numeric<128> dec128 = {38, 10, "1.23"}; + ASSERT_TRUE(dec128 == dec64_2); +} + +TEST(decimal_all, ret_type_for_non_decimal_types) { + std::vector non_decimal_types; + SDataType decimal_type = {TSDB_DATA_TYPE_DECIMAL64, 10, 2, 8}; + EOperatorType op = OP_TYPE_DIV; + std::vector out_types; + auto count_digits = [](uint64_t v) { return std::floor(std::log10(v) + 1); }; + std::vector equivalent_decimal_types; + // #define TSDB_DATA_TYPE_NULL 0 // 1 bytes + equivalent_decimal_types.push_back({TSDB_DATA_TYPE_NULL, 0, 0, tDataTypes[TSDB_DATA_TYPE_NULL].bytes}); + // #define TSDB_DATA_TYPE_BOOL 1 // 1 bytes + equivalent_decimal_types.push_back(getDecimalType(1, 0)); + // #define TSDB_DATA_TYPE_TINYINT 2 // 1 byte + equivalent_decimal_types.push_back(getDecimalType(count_digits(INT8_MAX), 0)); + // #define TSDB_DATA_TYPE_SMALLINT 3 // 2 bytes + equivalent_decimal_types.push_back(getDecimalType(count_digits(INT16_MAX), 0)); + // #define TSDB_DATA_TYPE_INT 4 // 4 bytes + equivalent_decimal_types.push_back(getDecimalType(count_digits(INT32_MAX), 0)); + // #define TSDB_DATA_TYPE_BIGINT 5 // 8 bytes + equivalent_decimal_types.push_back(getDecimalType(count_digits(INT64_MAX), 0)); + // #define TSDB_DATA_TYPE_FLOAT 6 // 4 bytes + equivalent_decimal_types.push_back({TSDB_DATA_TYPE_DOUBLE, 0, 0, tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes}); + // #define TSDB_DATA_TYPE_DOUBLE 7 // 8 bytes + equivalent_decimal_types.push_back({TSDB_DATA_TYPE_DOUBLE, 0, 0, tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes}); + // #define TSDB_DATA_TYPE_VARCHAR 8 // string, alias for varchar + equivalent_decimal_types.push_back({TSDB_DATA_TYPE_DOUBLE, 0, 0, tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes}); + // #define TSDB_DATA_TYPE_TIMESTAMP 9 // 8 bytes + equivalent_decimal_types.push_back(getDecimalType(count_digits(INT64_MAX), 0)); + // #define TSDB_DATA_TYPE_NCHAR 10 // unicode string + equivalent_decimal_types.push_back({TSDB_DATA_TYPE_DOUBLE, 0, 0, tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes}); + // #define TSDB_DATA_TYPE_UTINYINT 11 // 1 byte + equivalent_decimal_types.push_back(getDecimalType(count_digits(UINT8_MAX), 0)); + // #define TSDB_DATA_TYPE_USMALLINT 12 // 2 bytes + equivalent_decimal_types.push_back(getDecimalType(count_digits(UINT16_MAX), 0)); + // #define TSDB_DATA_TYPE_UINT 13 // 4 bytes + equivalent_decimal_types.push_back(getDecimalType(count_digits(UINT32_MAX), 0)); + // #define TSDB_DATA_TYPE_UBIGINT 14 // 8 bytes + equivalent_decimal_types.push_back(getDecimalType(count_digits(UINT64_MAX), 0)); + // #define TSDB_DATA_TYPE_JSON 15 // json string + equivalent_decimal_types.push_back({TSDB_DATA_TYPE_MAX, 0, 0, 0}); + // #define TSDB_DATA_TYPE_VARBINARY 16 // binary + equivalent_decimal_types.push_back({TSDB_DATA_TYPE_MAX, 0, 0, 0}); + // #define TSDB_DATA_TYPE_DECIMAL 17 // decimal + equivalent_decimal_types.push_back({TSDB_DATA_TYPE_MAX, 0, 0, 0}); + // #define TSDB_DATA_TYPE_BLOB 18 // binary + equivalent_decimal_types.push_back({TSDB_DATA_TYPE_MAX, 0, 0, 0}); + // #define TSDB_DATA_TYPE_MEDIUMBLOB 19 + equivalent_decimal_types.push_back({TSDB_DATA_TYPE_MAX, 0, 0, 0}); + // #define TSDB_DATA_TYPE_GEOMETRY 20 // geometry + equivalent_decimal_types.push_back({TSDB_DATA_TYPE_MAX, 0, 0, 0}); + // #define TSDB_DATA_TYPE_DECIMAL64 21 // decimal64 + equivalent_decimal_types.push_back({TSDB_DATA_TYPE_MAX, 0, 0, 0}); + + for (uint8_t i = 0; i < TSDB_DATA_TYPE_MAX; ++i) { + if (IS_DECIMAL_TYPE(i)) continue; + SDataType equivalent_out_type = equivalent_decimal_types[i]; + if (equivalent_out_type.type != TSDB_DATA_TYPE_MAX) + equivalent_out_type = Numeric<128>::getRetType(op, decimal_type, equivalent_decimal_types[i]); + DecimalRetTypeCheckContent dcc{decimal_type, {i, 0, 0, tDataTypes[i].bytes}, equivalent_out_type, op}; + dcc.check(); + + if (equivalent_out_type.type != TSDB_DATA_TYPE_MAX) { + equivalent_out_type = Numeric<128>::getRetType(op, equivalent_decimal_types[i], decimal_type); + } + DecimalRetTypeCheckContent dcc2{{i, 0, 0, tDataTypes[i].bytes}, decimal_type, equivalent_out_type, op}; + dcc2.check(); + } +} + +class DecimalTest : public ::testing::Test { + TAOS* get_connection() { + auto conn = taos_connect(host, user, passwd, db, 0); + if (!conn) { + cout << "taos connect failed: " << host << " " << taos_errstr(NULL); + } + return conn; + } + TAOS* default_conn_ = NULL; + static constexpr const char* host = "127.0.0.1"; + static constexpr const char* user = "root"; + static constexpr const char* passwd = "taosdata"; + static constexpr const char* db = "test"; + DecimalStringRandomGenerator generator_; + DecimalStringRandomGeneratorConfig generator_config_; + + public: + void SetUp() override { + default_conn_ = get_connection(); + if (!default_conn_) { + FAIL(); + } + } + void TearDown() override { + if (default_conn_) { + taos_close(default_conn_); + } + } + + std::string generate_decimal_str() { return generator_.generate(generator_config_); } +}; + +TEST(decimal, fillDecimalInfoInBytes) { + auto d = getDecimalType(10, 2); + int32_t bytes = 0; + fillBytesForDecimalType(&bytes, d.type, d.precision, d.scale); + + uint8_t prec = 0, scale = 0; + extractDecimalTypeInfoFromBytes(&bytes, &prec, &scale); + ASSERT_EQ(bytes, tDataTypes[d.type].bytes); + ASSERT_EQ(prec, d.precision); + ASSERT_EQ(scale, d.scale); +} + +#if 0 +TEST_F(DecimalTest, api_taos_fetch_rows) { + GTEST_SKIP_(""); + const char* host = "127.0.0.1"; + const char* user = "root"; + const char* passwd = "taosdata"; + const char* db = "test_api"; + const char* create_tb = "create table if not exists test_api.nt(ts timestamp, c1 decimal(10, 2), c2 decimal(38, 10), c3 varchar(255))"; + const char* sql = "select c1, c2,c3 from test_api.nt"; + const char* sql_insert = "insert into test_api.nt values(now, 123456.123, 98472981092.1209111)"; + + TAOS* pTaos = taos_connect(host, user, passwd, NULL, 0); + if (!pTaos) { + cout << "taos connect failed: " << host << " " << taos_errstr(NULL); + FAIL(); + } + + auto* res = taos_query(pTaos, (std::string("create database if not exists ") + db).c_str()); + taos_free_result(res); + res = taos_query(pTaos, create_tb); + taos_free_result(res); + res = taos_query(pTaos, sql_insert); + taos_free_result(res); + + res = taos_query(pTaos, sql); + int32_t code = taos_errno(res); + if (code != 0) { + cout << "taos_query with sql: " << sql << " failed: " << taos_errstr(res); + FAIL(); + } + + char buf[1024] = {0}; + auto* fields = taos_fetch_fields(res); + auto fieldNum = taos_field_count(res); + while (auto row = taos_fetch_row(res)) { + taos_print_row(buf, row, fields, fieldNum); + cout << buf << endl; + } + auto* fields_e = taos_fetch_fields_e(res); + ASSERT_EQ(fields_e[0].type, TSDB_DATA_TYPE_DECIMAL64); + ASSERT_EQ(fields_e[1].type, TSDB_DATA_TYPE_DECIMAL); + ASSERT_EQ(fields_e[0].precision, 10); + ASSERT_EQ(fields_e[0].scale, 2); + ASSERT_EQ(fields_e[1].precision, 38); + ASSERT_EQ(fields_e[1].scale, 10); + ASSERT_EQ(fields_e[2].type, TSDB_DATA_TYPE_VARCHAR); + ASSERT_EQ(fields_e[2].bytes, 255); + + taos_free_result(res); + + res = taos_query(pTaos, sql); + code = taos_errno(res); + if (code != 0) { + cout << "taos_query with sql: " << sql << " failed: " << taos_errstr(res); + taos_free_result(res); + FAIL(); + } + + void* pData = NULL; + int32_t numOfRows = 0; + code = taos_fetch_raw_block(res, &numOfRows, &pData); + if (code != 0) { + cout << "taos_query with sql: " << sql << " failed: " << taos_errstr(res); + FAIL(); + } + if (numOfRows > 0) { + int32_t version = *(int32_t*)pData; + ASSERT_EQ(version, BLOCK_VERSION_1); + int32_t rows = *(int32_t*)((char*)pData + 4 + 4); + int32_t colNum = *(int32_t*)((char*)pData + 4 + 4 + 4); + int32_t bytes_skip = 4 + 4 + 4 + 4 + 4 + 8; + char* p = (char*)pData + bytes_skip; + // col1 + int8_t t = *(int8_t*)p; + int32_t type_mod = *(int32_t*)(p + 1); + + ASSERT_EQ(t, TSDB_DATA_TYPE_DECIMAL64); + auto check_type_mod = [](char* pStart, uint8_t prec, uint8_t scale, int32_t bytes) { + int32_t d = *(int32_t*)pStart; + ASSERT_EQ(d >> 24, bytes); + ASSERT_EQ((d & 0xFF00) >> 8, prec); + ASSERT_EQ(d & 0xFF, scale); + }; + check_type_mod(p + 1, 10, 2, 8); + + // col2 + p += 5; + t = *(int8_t*)p; + type_mod = *(int32_t*)(p + 1); + check_type_mod(p + 1, 38, 10, 16); + + p = p + 5 + BitmapLen(numOfRows) + colNum * 4; + int64_t row1Val = *(int64_t*)p; + ASSERT_EQ(row1Val, 12345612); + } + taos_free_result(res); + + taos_close(pTaos); +} +#endif + +#if 0 +TEST_F(DecimalTest, decimalFromStr) { + Numeric<64> numeric64 = {10, 2, "0"}; + + numeric64 = {18, 0, "0"}; + + numeric64 = {18, 18, "0"}; + + numeric64 = {18, 2, "0"}; + Numeric<128> numeric128 = {38, 10, "0"}; +} +#endif + +TEST(decimal, test_add_check_overflow) { + Numeric<128> dec128 = {38, 10, "9999999999999999999999999999.9999999999"}; + Numeric<64> dec64 = {18, 2, "123.12"}; + bool overflow = decimal128AddCheckOverflow((Decimal128*)&dec128.dec(), &dec64.dec(), DECIMAL_WORD_NUM(Decimal64)); + ASSERT_TRUE(overflow); + auto ret = dec128 + dec64; + dec128 = {38, 10, "-9999999999999999999999999999.9999999999"}; + ASSERT_FALSE(decimal128AddCheckOverflow((Decimal128*)&dec128.dec(), &dec64.dec(), DECIMAL_WORD_NUM(Decimal64))); + dec64 = {18, 2, "-123.1"}; + ASSERT_TRUE(decimal128AddCheckOverflow((Decimal128*)&dec128.dec(), &dec64.dec(), DECIMAL_WORD_NUM(Decimal64))); + + dec128 = {38, 0, "99999999999999999999999999999999999999"}; + dec64= {18, 0, "123"}; + Numeric<128> dec128_2 = {38, 2, "999999999999999999999999999999999999.99"}; + ASSERT_OVERFLOW(dec128 + dec128_2); + ASSERT_OVERFLOW(dec128 + dec64); + ASSERT_RUNTIME_ERROR(dec128 / 0); + + dec64 = {10, 2, "99999999.99"}; + Decimal64 tmp = {0}; + ASSERT_EQ(TSDB_CODE_DECIMAL_OVERFLOW, TEST_decimal64FromDecimal64((Decimal64*)&dec64.dec(), 10, 2, &tmp, 9, 1)); + dec128 = {20, 12, "99999999.999999999999"}; + ASSERT_EQ(TSDB_CODE_DECIMAL_OVERFLOW, TEST_decimal64FromDecimal128((Decimal128*)&dec128.dec(), 20, 12, &tmp, 9, 1)); + + dec64 = {18, 10, "99999999.9999999999"}; + Decimal128 tmp2 = {0}; + ASSERT_EQ(TSDB_CODE_DECIMAL_OVERFLOW, TEST_decimal128FromDecimal64((Decimal64*)&dec64.dec(), 18, 10, &tmp2, 20, 20)); + ASSERT_EQ(TSDB_CODE_DECIMAL_OVERFLOW, TEST_decimal128FromDecimal128((Decimal128*)&dec128.dec(), 20, 12, &tmp2, 19, 11)); +} + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index 3bd6f4e64a..87e9a2fb97 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -264,7 +264,6 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp } break; } - case TSDB_DATA_TYPE_DECIMAL: case TSDB_DATA_TYPE_BLOB: case TSDB_DATA_TYPE_JSON: case TSDB_DATA_TYPE_MEDIUMBLOB: @@ -295,7 +294,7 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp } SValue sv = {.type = pCol->type}; - TAOS_MEMCPY(&sv.val, var, tDataTypes[pCol->type].bytes); + valueSetDatum(&sv, sv.type, var, tDataTypes[pCol->type].bytes); SColVal cv = COL_VAL_VALUE(pCol->colId, sv); if (NULL == taosArrayPush(pVals, &cv)) { goto _end; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 61547f5fb3..bf835acc4e 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1536,6 +1536,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT SMetaTableInfo mtInfo = {0}; code = pTaskInfo->storageAPI.snapshotFn.getMetaTableInfoFromSnapshot(sContext, &mtInfo); if (code != 0) { + destroyMetaTableInfo(&mtInfo); return code; } pTaskInfo->storageAPI.tsdReader.tsdReaderClose(pInfo->dataReader); @@ -1545,7 +1546,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT tableListClear(pTableListInfo); if (mtInfo.uid == 0) { - tDeleteSchemaWrapper(mtInfo.schema); + destroyMetaTableInfo(&mtInfo); goto end; // no data } @@ -1553,7 +1554,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT code = initQueryTableDataCondForTmq(&pTaskInfo->streamInfo.tableCond, sContext, &mtInfo); if (code != TSDB_CODE_SUCCESS) { qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); - tDeleteSchemaWrapper(mtInfo.schema); + destroyMetaTableInfo(&mtInfo); return code; } if (pAPI->snapshotFn.taosXGetTablePrimaryKey(sContext)) { @@ -1564,7 +1565,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT code = tableListAddTableInfo(pTableListInfo, mtInfo.uid, 0); if (code != TSDB_CODE_SUCCESS) { - tDeleteSchemaWrapper(mtInfo.schema); + destroyMetaTableInfo(&mtInfo); qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); return code; } @@ -1572,14 +1573,14 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT STableKeyInfo* pList = tableListGetInfo(pTableListInfo, 0); if (!pList) { qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); - tDeleteSchemaWrapper(mtInfo.schema); + destroyMetaTableInfo(&mtInfo); return code; } int32_t size = 0; code = tableListGetSize(pTableListInfo, &size); if (code != TSDB_CODE_SUCCESS) { qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); - tDeleteSchemaWrapper(mtInfo.schema); + destroyMetaTableInfo(&mtInfo); return code; } @@ -1587,7 +1588,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT NULL, (void**)&pInfo->dataReader, NULL, NULL); if (code != TSDB_CODE_SUCCESS) { qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); - tDeleteSchemaWrapper(mtInfo.schema); + destroyMetaTableInfo(&mtInfo); return code; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 4ce04ed3da..1957a45b83 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2868,7 +2868,8 @@ static int32_t doBlockDataPrimaryKeyFilter(SSDataBlock* pBlock, STqOffsetVal* of SColumnInfoData* pColTs = taosArrayGet(pBlock->pDataBlock, 0); SColumnInfoData* pColPk = taosArrayGet(pBlock->pDataBlock, 1); - qDebug("doBlockDataWindowFilter primary key, ts:%" PRId64 " %" PRId64, offset->ts, offset->primaryKey.val); + qDebug("doBlockDataWindowFilter primary key, ts:%" PRId64 " %" PRId64, offset->ts, + VALUE_GET_TRIVIAL_DATUM(&offset->primaryKey)); QUERY_CHECK_CONDITION((pColPk->info.type == offset->primaryKey.type), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); @@ -2885,7 +2886,7 @@ static int32_t doBlockDataPrimaryKeyFilter(SSDataBlock* pBlock, STqOffsetVal* of p[i] = (*ts > offset->ts) || (func(data, tmq) > 0); taosMemoryFree(tmq); } else { - p[i] = (*ts > offset->ts) || (func(data, &offset->primaryKey.val) > 0); + p[i] = (*ts > offset->ts) || (func(data, VALUE_GET_DATUM(&offset->primaryKey, pColPk->info.type)) > 0); } if (!p[i]) { @@ -3124,7 +3125,7 @@ static int32_t processPrimaryKey(SSDataBlock* pBlock, bool hasPrimaryKey, STqOff val.nData = varDataLen(tmp); memcpy(val.pData, varDataVal(tmp), varDataLen(tmp)); } else { - memcpy(&val.val, tmp, pColPk->info.bytes); + valueSetDatum(&val, pColPk->info.type, tmp, pColPk->info.bytes); } } tqOffsetResetToData(offset, pBlock->info.id.uid, pBlock->info.window.ekey, val); @@ -4149,7 +4150,7 @@ static int32_t doRawScanNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) { code = pAPI->snapshotFn.getMetaTableInfoFromSnapshot(pInfo->sContext, &mtInfo); QUERY_CHECK_CODE(code, lino, _end); if (code != 0) { - tDeleteSchemaWrapper(mtInfo.schema); + destroyMetaTableInfo(&mtInfo); QUERY_CHECK_CODE(code, lino, _end); } STqOffsetVal offset = {0}; @@ -4161,7 +4162,7 @@ static int32_t doRawScanNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) { tqOffsetResetToData(&offset, mtInfo.uid, INT64_MIN, val); qDebug("tmqsnap change get data uid:%" PRId64 "", mtInfo.uid); } - tDeleteSchemaWrapper(mtInfo.schema); + destroyMetaTableInfo(&mtInfo); code = qStreamPrepareScan(pTaskInfo, &offset, pInfo->sContext->subType); QUERY_CHECK_CODE(code, lino, _end); (*ppRes) = NULL; @@ -5303,7 +5304,6 @@ int32_t createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* p nodesRewriteExprPostOrder(&pTagCond, tagScanRewriteTagColumn, (void*)&pInfo->filterCtx); } } - // TODO wjm check pInfo->filterCtx.code __optr_fn_t tagScanNextFn = (pTagScanNode->onlyMetaCtbIdx) ? doTagScanFromCtbIdxNext : doTagScanFromMetaEntryNext; pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, tagScanNextFn, NULL, destroyTagScanOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); diff --git a/source/libs/executor/src/streamfilloperator.c b/source/libs/executor/src/streamfilloperator.c index ffaa62721e..0d14343988 100644 --- a/source/libs/executor/src/streamfilloperator.c +++ b/source/libs/executor/src/streamfilloperator.c @@ -603,7 +603,7 @@ static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo* cur.key = pFillInfo->current; cur.val = taosMemoryCalloc(1, pCell->bytes); QUERY_CHECK_NULL(cur.val, code, lino, _end, terrno); - taosGetLinearInterpolationVal(&cur, pCell->type, &start, pEnd, pCell->type); + taosGetLinearInterpolationVal(&cur, pCell->type, &start, pEnd, pCell->type, typeGetTypeModFromColInfo(&pColData->info)); code = colDataSetVal(pColData, index, (const char*)cur.val, false); QUERY_CHECK_CODE(code, lino, _end); destroySPoint(&cur); @@ -1712,15 +1712,15 @@ static void setValueForFillInfo(SStreamFillSupporter* pFillSup, SStreamFillInfo* SVariant* pVar = &(pFillCol->fillVal); if (pCell->type == TSDB_DATA_TYPE_FLOAT) { float v = 0; - GET_TYPED_DATA(v, float, pVar->nType, &pVar->i); + GET_TYPED_DATA(v, float, pVar->nType, &pVar->i, 0); SET_TYPED_DATA(pCell->pData, pCell->type, v); } else if (IS_FLOAT_TYPE(pCell->type)) { double v = 0; - GET_TYPED_DATA(v, double, pVar->nType, &pVar->i); + GET_TYPED_DATA(v, double, pVar->nType, &pVar->i, 0); SET_TYPED_DATA(pCell->pData, pCell->type, v); } else if (IS_INTEGER_TYPE(pCell->type)) { int64_t v = 0; - GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i); + GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i, 0); SET_TYPED_DATA(pCell->pData, pCell->type, v); } else { pCell->isNull = true; diff --git a/source/libs/executor/src/streamintervalsliceoperator.c b/source/libs/executor/src/streamintervalsliceoperator.c index cc06f5b693..80e59445f5 100644 --- a/source/libs/executor/src/streamintervalsliceoperator.c +++ b/source/libs/executor/src/streamintervalsliceoperator.c @@ -217,15 +217,15 @@ void doStreamSliceInterpolation(SSliceRowData* pPrevWinVal, TSKEY winKey, TSKEY double prevVal = 0, curVal = 0, winVal = 0; SResultCellData* pCell = getSliceResultCell((SResultCellData*)pPrevWinVal->pRowVal, pParam->pCol->slotId, pOffsetInfo); - GET_TYPED_DATA(prevVal, double, pCell->type, pCell->pData); - GET_TYPED_DATA(curVal, double, pColInfo->info.type, colDataGetData(pColInfo, curRowIndex)); + GET_TYPED_DATA(prevVal, double, pCell->type, pCell->pData, typeGetTypeModFromColInfo(&pColInfo->info)); + GET_TYPED_DATA(curVal, double, pColInfo->info.type, colDataGetData(pColInfo, curRowIndex), typeGetTypeModFromColInfo(&pColInfo->info)); SPoint point1 = (SPoint){.key = pPrevWinVal->key, .val = &prevVal}; SPoint point2 = (SPoint){.key = curTs, .val = &curVal}; SPoint point = (SPoint){.key = winKey, .val = &winVal}; if (!fmIsElapsedFunc(pCtx[k].functionId)) { - taosGetLinearInterpolationVal(&point, TSDB_DATA_TYPE_DOUBLE, &point1, &point2, TSDB_DATA_TYPE_DOUBLE); + taosGetLinearInterpolationVal(&point, TSDB_DATA_TYPE_DOUBLE, &point1, &point2, TSDB_DATA_TYPE_DOUBLE, 0); } if (type == INTERVAL_SLICE_START) { diff --git a/source/libs/executor/src/streamtimesliceoperator.c b/source/libs/executor/src/streamtimesliceoperator.c index f511c5ab4f..7401de3ee6 100644 --- a/source/libs/executor/src/streamtimesliceoperator.c +++ b/source/libs/executor/src/streamtimesliceoperator.c @@ -515,7 +515,7 @@ static void fillLinearRange(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFi cur.val = taosMemoryCalloc(1, pCell->bytes); QUERY_CHECK_NULL(cur.val, code, lino, _end, terrno); - taosGetLinearInterpolationVal(&cur, pCell->type, &start, pEnd, pCell->type); + taosGetLinearInterpolationVal(&cur, pCell->type, &start, pEnd, pCell->type, typeGetTypeModFromColInfo(&pDstCol->info)); code = colDataSetVal(pDstCol, index, (const char*)cur.val, false); QUERY_CHECK_CODE(code, lino, _end); @@ -1980,15 +1980,15 @@ static void copyFillValueInfo(SStreamFillSupporter* pFillSup, SStreamFillInfo* p SVariant* pVar = &(pValueCol->fillVal); if (pCell->type == TSDB_DATA_TYPE_FLOAT) { float v = 0; - GET_TYPED_DATA(v, float, pVar->nType, &pVar->i); + GET_TYPED_DATA(v, float, pVar->nType, &pVar->i, 0); SET_TYPED_DATA(pCell->pData, pCell->type, v); } else if (IS_FLOAT_TYPE(pCell->type)) { double v = 0; - GET_TYPED_DATA(v, double, pVar->nType, &pVar->i); + GET_TYPED_DATA(v, double, pVar->nType, &pVar->i, 0); SET_TYPED_DATA(pCell->pData, pCell->type, v); } else if (IS_INTEGER_TYPE(pCell->type)) { int64_t v = 0; - GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i); + GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i, 0); SET_TYPED_DATA(pCell->pData, pCell->type, v); } else { pCell->isNull = true; diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index ecbb389f8a..104b731e77 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -85,33 +85,37 @@ static int32_t doSetUserSpecifiedValue(SColumnInfoData* pDst, SVariant* pVar, in bool isNull = (TSDB_DATA_TYPE_NULL == pVar->nType) ? true : false; if (pDst->info.type == TSDB_DATA_TYPE_FLOAT) { float v = 0; - GET_TYPED_DATA(v, float, pVar->nType, &pVar->f); + GET_TYPED_DATA(v, float, pVar->nType, &pVar->f, typeGetTypeModFromColInfo(&pDst->info)); code = colDataSetVal(pDst, rowIndex, (char*)&v, isNull); QUERY_CHECK_CODE(code, lino, _end); } else if (pDst->info.type == TSDB_DATA_TYPE_DOUBLE) { double v = 0; - GET_TYPED_DATA(v, double, pVar->nType, &pVar->d); + GET_TYPED_DATA(v, double, pVar->nType, &pVar->d, typeGetTypeModFromColInfo(&pDst->info)); code = colDataSetVal(pDst, rowIndex, (char*)&v, isNull); QUERY_CHECK_CODE(code, lino, _end); } else if (IS_SIGNED_NUMERIC_TYPE(pDst->info.type) || pDst->info.type == TSDB_DATA_TYPE_BOOL) { int64_t v = 0; - GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i); + GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i, typeGetTypeModFromColInfo(&pDst->info)); code = colDataSetVal(pDst, rowIndex, (char*)&v, isNull); QUERY_CHECK_CODE(code, lino, _end); } else if (IS_UNSIGNED_NUMERIC_TYPE(pDst->info.type)) { uint64_t v = 0; - GET_TYPED_DATA(v, uint64_t, pVar->nType, &pVar->u); + GET_TYPED_DATA(v, uint64_t, pVar->nType, &pVar->u, typeGetTypeModFromColInfo(&pDst->info)); code = colDataSetVal(pDst, rowIndex, (char*)&v, isNull); QUERY_CHECK_CODE(code, lino, _end); } else if (pDst->info.type == TSDB_DATA_TYPE_TIMESTAMP) { int64_t v = 0; - GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->u); + GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->u, typeGetTypeModFromColInfo(&pDst->info)); code = colDataSetVal(pDst, rowIndex, (const char*)&v, isNull); QUERY_CHECK_CODE(code, lino, _end); } else if (pDst->info.type == TSDB_DATA_TYPE_NCHAR || pDst->info.type == TSDB_DATA_TYPE_VARCHAR || pDst->info.type == TSDB_DATA_TYPE_VARBINARY) { code = colDataSetVal(pDst, rowIndex, pVar->pz, isNull); QUERY_CHECK_CODE(code, lino, _end); + } else if (pDst->info.type == TSDB_DATA_TYPE_DECIMAL64) { + code = colDataSetVal(pDst, rowIndex, (char*)&pVar->i, isNull); + } else if (pDst->info.type == TSDB_DATA_TYPE_DECIMAL) { + code = colDataSetVal(pDst, rowIndex, (char*)pVar->pz, isNull); } else { // others data colDataSetNULL(pDst, rowIndex); } @@ -232,7 +236,7 @@ static void doFillOneRow(SFillInfo* pFillInfo, SSDataBlock* pBlock, SSDataBlock* int64_t out = 0; point = (SPoint){.key = pFillInfo->currentKey, .val = &out}; - taosGetLinearInterpolationVal(&point, type, &point1, &point2, type); + taosGetLinearInterpolationVal(&point, type, &point1, &point2, type, typeGetTypeModFromColInfo(&pDstCol->info)); code = colDataSetVal(pDstCol, index, (const char*)&out, false); QUERY_CHECK_CODE(code, lino, _end); @@ -721,10 +725,10 @@ int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, TSKEY ekey, int32_t ma } void taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* point1, SPoint* point2, - int32_t inputType) { + int32_t inputType, STypeMod inputTypeMod) { double v1 = -1, v2 = -1; - GET_TYPED_DATA(v1, double, inputType, point1->val); - GET_TYPED_DATA(v2, double, inputType, point2->val); + GET_TYPED_DATA(v1, double, inputType, point1->val, inputTypeMod); + GET_TYPED_DATA(v2, double, inputType, point2->val, inputTypeMod); double r = 0; if (!IS_BOOLEAN_TYPE(inputType)) { diff --git a/source/libs/executor/src/timesliceoperator.c b/source/libs/executor/src/timesliceoperator.c index 96ad51149c..7e43fbd6b7 100644 --- a/source/libs/executor/src/timesliceoperator.c +++ b/source/libs/executor/src/timesliceoperator.c @@ -193,7 +193,7 @@ static void tRowGetKeyFromColData(int64_t ts, SColumnInfoData* pPkCol, int32_t r pKey->pks[0].type = t; if (IS_NUMERIC_TYPE(t)) { - GET_TYPED_DATA(pKey->pks[0].val, int64_t, t, colDataGetNumData(pPkCol, rowIndex)); + valueSetDatum(pKey->pks, t, colDataGetData(pPkCol, rowIndex), tDataTypes[t].bytes); } else { char* p = colDataGetVarData(pPkCol, rowIndex); pKey->pks[0].pData = (uint8_t*)varDataVal(p); @@ -215,7 +215,7 @@ static bool checkDuplicateTimestamps(STimeSliceOperatorInfo* pSliceInfo, SColumn if (IS_VAR_DATA_TYPE(pPkCol->info.type)) { cur.pks[0].pData = (uint8_t*)colDataGetVarData(pPkCol, curIndex); } else { - memcpy(&cur.pks[0].val, colDataGetData(pPkCol, curIndex), pPkCol->info.bytes); + valueSetDatum(cur.pks, pPkCol->info.type, colDataGetData(pPkCol, curIndex), pPkCol->info.bytes); } } @@ -460,7 +460,7 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp if (pDst->info.type == TSDB_DATA_TYPE_FLOAT) { float v = 0; if (!IS_VAR_DATA_TYPE(pVar->nType)) { - GET_TYPED_DATA(v, float, pVar->nType, &pVar->f); + GET_TYPED_DATA(v, float, pVar->nType, &pVar->f, 0); } else { v = taosStr2Float(varDataVal(pVar->pz), NULL); } @@ -469,7 +469,7 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp } else if (pDst->info.type == TSDB_DATA_TYPE_DOUBLE) { double v = 0; if (!IS_VAR_DATA_TYPE(pVar->nType)) { - GET_TYPED_DATA(v, double, pVar->nType, &pVar->d); + GET_TYPED_DATA(v, double, pVar->nType, &pVar->d, 0); } else { v = taosStr2Double(varDataVal(pVar->pz), NULL); } @@ -478,7 +478,7 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp } else if (IS_SIGNED_NUMERIC_TYPE(pDst->info.type)) { int64_t v = 0; if (!IS_VAR_DATA_TYPE(pVar->nType)) { - GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i); + GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i, 0); } else { v = taosStr2Int64(varDataVal(pVar->pz), NULL, 10); } @@ -487,7 +487,7 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp } else if (IS_UNSIGNED_NUMERIC_TYPE(pDst->info.type)) { uint64_t v = 0; if (!IS_VAR_DATA_TYPE(pVar->nType)) { - GET_TYPED_DATA(v, uint64_t, pVar->nType, &pVar->u); + GET_TYPED_DATA(v, uint64_t, pVar->nType, &pVar->u, 0); } else { v = taosStr2UInt64(varDataVal(pVar->pz), NULL, 10); } @@ -496,7 +496,7 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp } else if (IS_BOOLEAN_TYPE(pDst->info.type)) { bool v = false; if (!IS_VAR_DATA_TYPE(pVar->nType)) { - GET_TYPED_DATA(v, bool, pVar->nType, &pVar->i); + GET_TYPED_DATA(v, bool, pVar->nType, &pVar->i, 0); } else { v = taosStr2Int8(varDataVal(pVar->pz), NULL, 10); } @@ -537,7 +537,7 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp current.val = taosMemoryCalloc(pLinearInfo->bytes, 1); QUERY_CHECK_NULL(current.val, code, lino, _end, terrno); - taosGetLinearInterpolationVal(¤t, pLinearInfo->type, &start, &end, pLinearInfo->type); + taosGetLinearInterpolationVal(¤t, pLinearInfo->type, &start, &end, pLinearInfo->type, typeGetTypeModFromColInfo(&pDst->info)); code = colDataSetVal(pDst, rows, (char*)current.val, false); QUERY_CHECK_CODE(code, lino, _end); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 956598eb6b..d3185f898e 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -236,12 +236,12 @@ void doTimeWindowInterpolation(SArray* pPrevValues, SArray* pDataBlock, TSKEY pr double v1 = 0, v2 = 0, v = 0; if (prevRowIndex == -1) { SGroupKeys* p = taosArrayGet(pPrevValues, index); - GET_TYPED_DATA(v1, double, pColInfo->info.type, p->pData); + GET_TYPED_DATA(v1, double, pColInfo->info.type, p->pData, typeGetTypeModFromColInfo(&pColInfo->info)); } else { - GET_TYPED_DATA(v1, double, pColInfo->info.type, colDataGetData(pColInfo, prevRowIndex)); + GET_TYPED_DATA(v1, double, pColInfo->info.type, colDataGetData(pColInfo, prevRowIndex), typeGetTypeModFromColInfo(&pColInfo->info)); } - GET_TYPED_DATA(v2, double, pColInfo->info.type, colDataGetData(pColInfo, curRowIndex)); + GET_TYPED_DATA(v2, double, pColInfo->info.type, colDataGetData(pColInfo, curRowIndex), typeGetTypeModFromColInfo(&pColInfo->info)); #if 0 if (functionId == FUNCTION_INTERP) { @@ -271,7 +271,7 @@ void doTimeWindowInterpolation(SArray* pPrevValues, SArray* pDataBlock, TSKEY pr SPoint point = (SPoint){.key = windowKey, .val = &v}; if (!fmIsElapsedFunc(pCtx[k].functionId)) { - taosGetLinearInterpolationVal(&point, TSDB_DATA_TYPE_DOUBLE, &point1, &point2, TSDB_DATA_TYPE_DOUBLE); + taosGetLinearInterpolationVal(&point, TSDB_DATA_TYPE_DOUBLE, &point1, &point2, TSDB_DATA_TYPE_DOUBLE, 0); } if (type == RESULT_ROW_START_INTERP) { diff --git a/source/libs/function/CMakeLists.txt b/source/libs/function/CMakeLists.txt index f31b70325c..6c3542aaef 100644 --- a/source/libs/function/CMakeLists.txt +++ b/source/libs/function/CMakeLists.txt @@ -29,6 +29,7 @@ target_link_libraries( PRIVATE qcom PRIVATE scalar PRIVATE geometry + PRIVATE decimal PRIVATE transport PUBLIC uv_a ) diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index d548ae956d..9d11d78c33 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -78,7 +78,7 @@ int32_t avgInvertFunction(SqlFunctionCtx* pCtx); #endif int32_t avgCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); -int32_t getAvgInfoSize(); +int32_t getAvgInfoSize(SFunctionNode* pFunc); bool getStdFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); int32_t stdFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); diff --git a/source/libs/function/inc/functionMgtInt.h b/source/libs/function/inc/functionMgtInt.h index d676d4b728..029151da09 100644 --- a/source/libs/function/inc/functionMgtInt.h +++ b/source/libs/function/inc/functionMgtInt.h @@ -90,6 +90,7 @@ extern "C" { #define FUNC_PARAM_SUPPORT_INTEGER_TYPE FUNC_MGT_FUNC_PARAM_SUPPORT_TYPE(21) #define FUNC_PARAM_SUPPORT_NULL_TYPE FUNC_MGT_FUNC_PARAM_SUPPORT_TYPE(22) #define FUNC_PARAM_SUPPORT_UNIX_TS_TYPE FUNC_MGT_FUNC_PARAM_SUPPORT_TYPE(23) +#define FUNC_PARAM_SUPPORT_DECIMAL_TYPE FUNC_MGT_FUNC_PARAM_SUPPORT_TYPE(24) diff --git a/source/libs/function/inc/functionResInfoInt.h b/source/libs/function/inc/functionResInfoInt.h index f97d2e8024..763bd4331d 100644 --- a/source/libs/function/inc/functionResInfoInt.h +++ b/source/libs/function/inc/functionResInfoInt.h @@ -25,6 +25,7 @@ extern "C" { #include "tdigest.h" #include "functionResInfo.h" #include "tpercentile.h" +#include "decimal.h" #define USE_ARRAYLIST @@ -46,10 +47,68 @@ typedef struct SSumRes { bool overflow; // if overflow is true, dsum to be used for any type; } SSumRes; +typedef struct SDecimalSumRes { + Decimal128 sum; + int16_t type; + int64_t prevTs; + bool isPrevTsSet; + bool overflow; + uint32_t flag; // currently not used +} SDecimalSumRes; + +#define SUM_RES_GET_RES(pSumRes) ((SSumRes*)pSumRes) +#define SUM_RES_GET_DECIMAL_RES(pSumRes) ((SDecimalSumRes*)pSumRes) + +#define SUM_RES_GET_SIZE(type) IS_DECIMAL_TYPE(type) ? sizeof(SDecimalSumRes) : sizeof(SSumRes) + +#define SUM_RES_SET_TYPE(pSumRes, inputType, _type) \ + do { \ + if (IS_DECIMAL_TYPE(inputType)) \ + SUM_RES_GET_DECIMAL_RES(pSumRes)->type = _type; \ + else \ + SUM_RES_GET_RES(pSumRes)->type = _type; \ + } while (0) + +#define SUM_RES_GET_TYPE(pSumRes, inputType) \ + (IS_DECIMAL_TYPE(inputType) ? SUM_RES_GET_DECIMAL_RES(pSumRes)->type : SUM_RES_GET_RES(pSumRes)->type) +#define SUM_RES_GET_PREV_TS(pSumRes, inputType) \ + (IS_DECIMAL_TYPE(inputType) ? SUM_RES_GET_DECIMAL_RES(pSumRes)->prevTs : SUM_RES_GET_RES(pSumRes)->prevTs) +#define SUM_RES_GET_OVERFLOW(pSumRes, checkInputType, inputType) \ + (checkInputType && IS_DECIMAL_TYPE(inputType) ? SUM_RES_GET_DECIMAL_RES(pSumRes)->overflow \ + : SUM_RES_GET_RES(pSumRes)->overflow) + +#define SUM_RES_GET_ISUM(pSumRes) (((SSumRes*)(pSumRes))->isum) +#define SUM_RES_GET_USUM(pSumRes) (((SSumRes*)(pSumRes))->usum) +#define SUM_RES_GET_DSUM(pSumRes) (((SSumRes*)(pSumRes))->dsum) +#define SUM_RES_INC_ISUM(pSumRes, val) ((SSumRes*)(pSumRes))->isum += val +#define SUM_RES_INC_USUM(pSumRes, val) ((SSumRes*)(pSumRes))->usum += val +#define SUM_RES_INC_DSUM(pSumRes, val) ((SSumRes*)(pSumRes))->dsum += val + +#define SUM_RES_GET_DECIMAL_SUM(pSumRes) ((SDecimalSumRes*)(pSumRes))->sum +#define SUM_RES_INC_DECIMAL_SUM(pSumRes, pVal, type) \ + do { \ + const SDecimalOps* pOps = getDecimalOps(TSDB_DATA_TYPE_DECIMAL); \ + int32_t wordNum = 0; \ + if (type == TSDB_DATA_TYPE_DECIMAL64) { \ + wordNum = DECIMAL_WORD_NUM(Decimal64); \ + overflow = decimal128AddCheckOverflow(&SUM_RES_GET_DECIMAL_SUM(pSumRes), pVal, wordNum); \ + } else { \ + wordNum = DECIMAL_WORD_NUM(Decimal); \ + overflow = decimal128AddCheckOverflow(&SUM_RES_GET_DECIMAL_SUM(pSumRes), pVal, wordNum); \ + } \ + pOps->add(&SUM_RES_GET_DECIMAL_SUM(pSumRes), pVal, wordNum); \ + if (overflow) break; \ + } while (0) + typedef struct SMinmaxResInfo { bool assign; // assign the first value or not - int64_t v; - char *str; + union { + struct { + int64_t v; + char* str; + }; + int64_t dec[2]; // for decimal types + }; STuplePos tuplePos; STuplePos nullTuplePos; @@ -57,6 +116,17 @@ typedef struct SMinmaxResInfo { int16_t type; } SMinmaxResInfo; +typedef struct SOldMinMaxResInfo { + bool assign; // assign the first value or not + int64_t v; + char* str; + STuplePos tuplePos; + + STuplePos nullTuplePos; + bool nullTupleSaved; + int16_t type; +} SOldMinMaxResInfo; + typedef struct SStdRes { double result; int64_t count; @@ -133,6 +203,55 @@ typedef struct SAvgRes { int16_t type; // store the original input type, used in merge function } SAvgRes; +typedef struct SDecimalAvgRes { + Decimal128 avg; + SDecimalSumRes sum; + int64_t count; + int16_t type; // store the original input type and scale, used in merge function + uint8_t scale; +} SDecimalAvgRes; + +#define AVG_RES_GET_RES(pAvgRes) ((SAvgRes*)pAvgRes) +#define AVG_RES_GET_DECIMAL_RES(pAvgRes) ((SDecimalAvgRes*)pAvgRes) +#define AVG_RES_SET_TYPE(pAvgRes, inputType, _type) \ + do { \ + if (IS_DECIMAL_TYPE(inputType)) \ + AVG_RES_GET_DECIMAL_RES(pAvgRes)->type = _type; \ + else \ + AVG_RES_GET_RES(pAvgRes)->type = _type; \ + } while (0) + +#define AVG_RES_SET_INPUT_SCALE(pAvgRes, _scale) \ + do { \ + AVG_RES_GET_DECIMAL_RES(pAvgRes)->scale = _scale; \ + } while (0) + +#define AVG_RES_GET_INPUT_SCALE(pAvgRes) (AVG_RES_GET_DECIMAL_RES(pAvgRes)->scale) + +#define AVG_RES_GET_TYPE(pAvgRes, inputType) \ + (IS_DECIMAL_TYPE(inputType) ? AVG_RES_GET_DECIMAL_RES(pAvgRes)->type : AVG_RES_GET_RES(pAvgRes)->type) + +#define AVG_RES_GET_SIZE(inputType) (IS_DECIMAL_TYPE(inputType) ? sizeof(SDecimalAvgRes) : sizeof(SAvgRes)) +#define AVG_RES_GET_AVG(pAvgRes) (AVG_RES_GET_RES(pAvgRes)->result) +#define AVG_RES_GET_SUM(pAvgRes) (AVG_RES_GET_RES(pAvgRes)->sum) +#define AVG_RES_GET_COUNT(pAvgRes, checkInputType, inputType) \ + (checkInputType && IS_DECIMAL_TYPE(inputType) ? AVG_RES_GET_DECIMAL_RES(pAvgRes)->count \ + : AVG_RES_GET_RES(pAvgRes)->count) +#define AVG_RES_INC_COUNT(pAvgRes, inputType, val) \ + do { \ + if (IS_DECIMAL_TYPE(inputType)) \ + AVG_RES_GET_DECIMAL_RES(pAvgRes)->count += val; \ + else \ + AVG_RES_GET_RES(pAvgRes)->count += val; \ + } while (0) + +#define AVG_RES_GET_DECIMAL_AVG(pAvgRes) (((SDecimalAvgRes*)(pAvgRes))->avg) +#define AVG_RES_GET_DECIMAL_SUM(pAvgRes) (((SDecimalAvgRes*)(pAvgRes))->sum) + +#define AVG_RES_GET_SUM_OVERFLOW(pAvgRes, checkInputType, inputType) \ + checkInputType&& IS_DECIMAL_TYPE(inputType) \ + ? SUM_RES_GET_OVERFLOW(&AVG_RES_GET_DECIMAL_SUM(pAvgRes), true, inputType) \ + : SUM_RES_GET_OVERFLOW(&AVG_RES_GET_SUM(pAvgRes), false, inputType) // structs above are used in stream @@ -195,6 +314,7 @@ typedef struct tMemBucket { int16_t numOfSlots; int16_t type; int32_t bytes; + STypeMod typeMod; int32_t total; int32_t elemPerPage; // number of elements for each object int32_t maxCapacity; // maximum allowed number of elements that can be sort directly to get the result diff --git a/source/libs/function/inc/tpercentile.h b/source/libs/function/inc/tpercentile.h index 4738301cd3..beec7d4307 100644 --- a/source/libs/function/inc/tpercentile.h +++ b/source/libs/function/inc/tpercentile.h @@ -25,7 +25,7 @@ extern "C" { struct tMemBucket; -int32_t tMemBucketCreate(int32_t nElemSize, int16_t dataType, double minval, double maxval, bool hasWindowOrGroup, +int32_t tMemBucketCreate(int32_t nElemSize, int16_t dataType, STypeMod typeMod, double minval, double maxval, bool hasWindowOrGroup, struct tMemBucket **pBucket, int32_t numOfElements); void tMemBucketDestroy(struct tMemBucket **pBucket); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index c2979e85a3..c191630ecd 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -410,6 +410,11 @@ static bool paramSupportGeometry(uint64_t typeFlag) { FUNC_MGT_TEST_MASK(typeFlag, FUNC_PARAM_SUPPORT_VAR_TYPE); } +static bool paramSupportDecimal(uint64_t typeFlag) { + return FUNC_MGT_TEST_MASK(typeFlag, FUNC_PARAM_SUPPORT_DECIMAL_TYPE) || + FUNC_MGT_TEST_MASK(typeFlag, FUNC_PARAM_SUPPORT_ALL_TYPE); +} + static bool paramSupportValueNode(uint64_t typeFlag) { return FUNC_MGT_TEST_MASK(typeFlag, FUNC_PARAM_SUPPORT_VALUE_NODE) || FUNC_MGT_TEST_MASK(typeFlag, FUNC_PARAM_SUPPORT_EXPR_NODE); @@ -502,6 +507,9 @@ static bool paramSupportDataType(SDataType* pDataType, uint64_t typeFlag) { return paramSupportVarBinary(typeFlag); case TSDB_DATA_TYPE_GEOMETRY: return paramSupportGeometry(typeFlag); + case TSDB_DATA_TYPE_DECIMAL64: + case TSDB_DATA_TYPE_DECIMAL: + return paramSupportDecimal(typeFlag); default: return false; } @@ -965,7 +973,35 @@ static int32_t translateMinMax(SFunctionNode* pFunc, char* pErrBuf, int32_t len) SDataType* dataType = getSDataTypeFromNode(nodesListGetNode(pFunc->pParameterList, 0)); uint8_t paraType = IS_NULL_TYPE(dataType->type) ? TSDB_DATA_TYPE_BIGINT : dataType->type; int32_t bytes = IS_STR_DATA_TYPE(paraType) ? dataType->bytes : tDataTypes[paraType].bytes; - pFunc->node.resType = (SDataType){.bytes = bytes, .type = paraType}; + uint8_t prec = IS_DECIMAL_TYPE(paraType) ? dataType->precision : pFunc->node.resType.precision; + pFunc->node.resType = (SDataType){.bytes = bytes, .type = paraType, .precision = prec, .scale = dataType->scale}; + return TSDB_CODE_SUCCESS; +} + +static int32_t translateAvg(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + FUNC_ERR_RET(validateParam(pFunc, pErrBuf, len)); + + uint8_t dt = TSDB_DATA_TYPE_DOUBLE, prec = 0, scale = 0; + + bool isMergeFunc = pFunc->funcType == FUNCTION_TYPE_AVG_MERGE || pFunc->funcType == FUNCTION_TYPE_AVG_STATE_MERGE; + SDataType* pInputDt = getSDataTypeFromNode( + nodesListGetNode(isMergeFunc ? pFunc->pSrcFuncRef->pParameterList : pFunc->pParameterList, 0)); + pFunc->srcFuncInputType = *pInputDt; + if (IS_DECIMAL_TYPE(pInputDt->type)) { + SDataType sumDt = {.type = TSDB_DATA_TYPE_DECIMAL, + .bytes = tDataTypes[TSDB_DATA_TYPE_DECIMAL].bytes, + .precision = TSDB_DECIMAL_MAX_PRECISION, + .scale = pInputDt->scale}; + SDataType countDt = { + .type = TSDB_DATA_TYPE_BIGINT, .bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .precision = 0, .scale = 0}; + SDataType avgDt = {0}; + int32_t code = decimalGetRetType(&sumDt, &countDt, OP_TYPE_DIV, &avgDt); + if (code != 0) return code; + dt = TSDB_DATA_TYPE_DECIMAL; + prec = TSDB_DECIMAL_MAX_PRECISION; + scale = avgDt.scale; + } + pFunc->node.resType = (SDataType){.bytes = tDataTypes[dt].bytes, .type = dt, .precision = prec, .scale = scale}; return TSDB_CODE_SUCCESS; } @@ -1016,15 +1052,20 @@ static int32_t translateSum(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { uint8_t paraType = getSDataTypeFromNode(nodesListGetNode(pFunc->pParameterList, 0))->type; uint8_t resType = 0; + uint8_t prec = 0, scale = 0; if (IS_SIGNED_NUMERIC_TYPE(paraType) || TSDB_DATA_TYPE_BOOL == paraType || IS_NULL_TYPE(paraType)) { resType = TSDB_DATA_TYPE_BIGINT; } else if (IS_UNSIGNED_NUMERIC_TYPE(paraType)) { resType = TSDB_DATA_TYPE_UBIGINT; } else if (IS_FLOAT_TYPE(paraType)) { resType = TSDB_DATA_TYPE_DOUBLE; + } else if (IS_DECIMAL_TYPE(paraType)) { + resType = TSDB_DATA_TYPE_DECIMAL; + prec = TSDB_DECIMAL_MAX_PRECISION; + scale = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.scale; } - pFunc->node.resType = (SDataType){.bytes = tDataTypes[resType].bytes, .type = resType}; + pFunc->node.resType = (SDataType){.bytes = tDataTypes[resType].bytes, .type = resType, .precision = prec, .scale = scale}; return TSDB_CODE_SUCCESS; } @@ -1650,8 +1691,12 @@ static int32_t translateOutVarchar(SFunctionNode* pFunc, char* pErrBuf, int32_t break; case FUNCTION_TYPE_AVG_PARTIAL: case FUNCTION_TYPE_AVG_STATE: + pFunc->srcFuncInputType = ((SExprNode*)pFunc->pParameterList->pHead->pNode)->resType; + bytes = getAvgInfoSize(pFunc) + VARSTR_HEADER_SIZE; + break; case FUNCTION_TYPE_AVG_STATE_MERGE: - bytes = getAvgInfoSize() + VARSTR_HEADER_SIZE; + if (pFunc->pSrcFuncRef) pFunc->srcFuncInputType = pFunc->pSrcFuncRef->srcFuncInputType; + bytes = getAvgInfoSize(pFunc) + VARSTR_HEADER_SIZE; break; case FUNCTION_TYPE_HISTOGRAM_PARTIAL: bytes = getHistogramInfoSize() + VARSTR_HEADER_SIZE; @@ -1846,11 +1891,11 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .inputParaInfo[0][0] = {.isLastParam = true, .startParam = 1, .endParam = 1, - .validDataType = FUNC_PARAM_SUPPORT_NUMERIC_TYPE | FUNC_PARAM_SUPPORT_NULL_TYPE, + .validDataType = FUNC_PARAM_SUPPORT_NUMERIC_TYPE | FUNC_PARAM_SUPPORT_NULL_TYPE | FUNC_PARAM_SUPPORT_DECIMAL_TYPE, .validNodeType = FUNC_PARAM_SUPPORT_EXPR_NODE, .paramAttribute = FUNC_PARAM_NO_SPECIFIC_ATTRIBUTE, .valueRangeFlag = FUNC_PARAM_NO_SPECIFIC_VALUE,}, - .outputParaInfo = {.validDataType = FUNC_PARAM_SUPPORT_BIGINT_TYPE | FUNC_PARAM_SUPPORT_DOUBLE_TYPE | FUNC_PARAM_SUPPORT_UBIGINT_TYPE}}, + .outputParaInfo = {.validDataType = FUNC_PARAM_SUPPORT_BIGINT_TYPE | FUNC_PARAM_SUPPORT_DOUBLE_TYPE | FUNC_PARAM_SUPPORT_UBIGINT_TYPE | FUNC_PARAM_SUPPORT_DECIMAL_TYPE}}, .translateFunc = translateSum, .dataRequiredFunc = statisDataRequired, .getEnvFunc = getSumFuncEnv, @@ -1876,7 +1921,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .inputParaInfo[0][0] = {.isLastParam = true, .startParam = 1, .endParam = 1, - .validDataType = FUNC_PARAM_SUPPORT_NUMERIC_TYPE | FUNC_PARAM_SUPPORT_STRING_TYPE | FUNC_PARAM_SUPPORT_NULL_TYPE, + .validDataType = FUNC_PARAM_SUPPORT_NUMERIC_TYPE | FUNC_PARAM_SUPPORT_STRING_TYPE | FUNC_PARAM_SUPPORT_NULL_TYPE | FUNC_PARAM_SUPPORT_DECIMAL_TYPE, .validNodeType = FUNC_PARAM_SUPPORT_EXPR_NODE, .paramAttribute = FUNC_PARAM_NO_SPECIFIC_ATTRIBUTE, .valueRangeFlag = FUNC_PARAM_NO_SPECIFIC_VALUE,}, @@ -1891,7 +1936,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .combineFunc = minCombine, .pPartialFunc = "min", .pStateFunc = "min", - .pMergeFunc = "min" + .pMergeFunc = "min", }, { .name = "max", @@ -1903,7 +1948,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .inputParaInfo[0][0] = {.isLastParam = true, .startParam = 1, .endParam = 1, - .validDataType = FUNC_PARAM_SUPPORT_NUMERIC_TYPE | FUNC_PARAM_SUPPORT_STRING_TYPE | FUNC_PARAM_SUPPORT_NULL_TYPE, + .validDataType = FUNC_PARAM_SUPPORT_NUMERIC_TYPE | FUNC_PARAM_SUPPORT_STRING_TYPE | FUNC_PARAM_SUPPORT_NULL_TYPE | FUNC_PARAM_SUPPORT_DECIMAL_TYPE, .validNodeType = FUNC_PARAM_SUPPORT_EXPR_NODE, .paramAttribute = FUNC_PARAM_NO_SPECIFIC_ATTRIBUTE, .valueRangeFlag = FUNC_PARAM_NO_SPECIFIC_VALUE,}, @@ -1918,7 +1963,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .combineFunc = maxCombine, .pPartialFunc = "max", .pStateFunc = "max", - .pMergeFunc = "max" + .pMergeFunc = "max", }, { .name = "stddev", @@ -2044,12 +2089,12 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .inputParaInfo[0][0] = {.isLastParam = true, .startParam = 1, .endParam = 1, - .validDataType = FUNC_PARAM_SUPPORT_NUMERIC_TYPE | FUNC_PARAM_SUPPORT_NULL_TYPE, + .validDataType = FUNC_PARAM_SUPPORT_NUMERIC_TYPE | FUNC_PARAM_SUPPORT_NULL_TYPE | FUNC_PARAM_SUPPORT_DECIMAL_TYPE, .validNodeType = FUNC_PARAM_SUPPORT_EXPR_NODE, .paramAttribute = FUNC_PARAM_NO_SPECIFIC_ATTRIBUTE, .valueRangeFlag = FUNC_PARAM_NO_SPECIFIC_VALUE,}, - .outputParaInfo = {.validDataType = FUNC_PARAM_SUPPORT_DOUBLE_TYPE}}, - .translateFunc = translateOutDouble, + .outputParaInfo = {.validDataType = FUNC_PARAM_SUPPORT_DOUBLE_TYPE | FUNC_PARAM_SUPPORT_DECIMAL_TYPE}}, + .translateFunc = translateAvg, .dataRequiredFunc = statisDataRequired, .getEnvFunc = getAvgFuncEnv, .initFunc = avgFunctionSetup, @@ -2075,7 +2120,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .inputParaInfo[0][0] = {.isLastParam = true, .startParam = 1, .endParam = 1, - .validDataType = FUNC_PARAM_SUPPORT_NUMERIC_TYPE | FUNC_PARAM_SUPPORT_NULL_TYPE, + .validDataType = FUNC_PARAM_SUPPORT_NUMERIC_TYPE | FUNC_PARAM_SUPPORT_NULL_TYPE | FUNC_PARAM_SUPPORT_DECIMAL_TYPE, .validNodeType = FUNC_PARAM_SUPPORT_EXPR_NODE, .paramAttribute = FUNC_PARAM_NO_SPECIFIC_ATTRIBUTE, .valueRangeFlag = FUNC_PARAM_NO_SPECIFIC_VALUE,}, @@ -2106,7 +2151,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .paramAttribute = FUNC_PARAM_NO_SPECIFIC_ATTRIBUTE, .valueRangeFlag = FUNC_PARAM_NO_SPECIFIC_VALUE,}, .outputParaInfo = {.validDataType = FUNC_PARAM_SUPPORT_DOUBLE_TYPE}}, - .translateFunc = translateOutDouble, + .translateFunc = translateAvg, .getEnvFunc = getAvgFuncEnv, .initFunc = avgFunctionSetup, .processFunc = avgFunctionMerge, @@ -4779,7 +4824,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .inputParaInfo[0][0] = {.isLastParam = true, .startParam = 1, .endParam = 1, - .validDataType = FUNC_PARAM_SUPPORT_NUMERIC_TYPE | FUNC_PARAM_SUPPORT_NULL_TYPE, + .validDataType = FUNC_PARAM_SUPPORT_NUMERIC_TYPE | FUNC_PARAM_SUPPORT_NULL_TYPE | FUNC_PARAM_SUPPORT_DECIMAL_TYPE, .validNodeType = FUNC_PARAM_SUPPORT_EXPR_NODE, .paramAttribute = FUNC_PARAM_NO_SPECIFIC_ATTRIBUTE, .valueRangeFlag = FUNC_PARAM_NO_SPECIFIC_VALUE,}, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 9c1eb8ba11..2c9d8ca358 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -15,6 +15,7 @@ #include "builtinsimpl.h" #include "cJSON.h" +#include "decimal.h" #include "function.h" #include "functionResInfoInt.h" #include "query.h" @@ -105,6 +106,21 @@ typedef enum { } \ } while (0) +#define LIST_ADD_DECIMAL_N(_res, _col, _start, _rows, _t, numOfElem) \ + do { \ + _t* d = (_t*)(_col->pData); \ + const SDecimalOps* pOps = getDecimalOps(TSDB_DATA_TYPE_DECIMAL); \ + for (int32_t i = (_start); i < (_rows) + (_start); ++i) { \ + if (((_col)->hasNull) && colDataIsNull_f((_col)->nullbitmap, i)) { \ + continue; \ + }; \ + overflow = overflow || decimal128AddCheckOverflow((Decimal*)_res, d + i, DECIMAL_WORD_NUM(_t)); \ + if (overflow) break; \ + pOps->add(_res, d + i, DECIMAL_WORD_NUM(_t)); \ + (numOfElem)++; \ + } \ + } while (0) + #define LIST_SUB_N(_res, _col, _start, _rows, _t, numOfElem) \ do { \ _t* d = (_t*)(_col->pData); \ @@ -619,9 +635,10 @@ int32_t sumFunction(SqlFunctionCtx* pCtx) { SInputColumnInfoData* pInput = &pCtx->input; SColumnDataAgg* pAgg = pInput->pColumnDataAgg[0]; int32_t type = pInput->pData[0]->info.type; + pCtx->inputType = type; - SSumRes* pSumRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - pSumRes->type = type; + void* pSumRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + SUM_RES_SET_TYPE(pSumRes, pCtx->inputType, type); if (IS_NULL_TYPE(type)) { numOfElem = 0; @@ -632,11 +649,19 @@ int32_t sumFunction(SqlFunctionCtx* pCtx) { numOfElem = pInput->numOfRows - pAgg->numOfNull; if (IS_SIGNED_NUMERIC_TYPE(type)) { - pSumRes->isum += pAgg->sum; + SUM_RES_INC_ISUM(pSumRes, pAgg->sum); } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { - pSumRes->usum += pAgg->sum; + SUM_RES_INC_USUM(pSumRes, pAgg->sum); } else if (IS_FLOAT_TYPE(type)) { - pSumRes->dsum += GET_DOUBLE_VAL((const char*)&(pAgg->sum)); + SUM_RES_INC_DSUM(pSumRes, GET_DOUBLE_VAL((const char*)&(pAgg->sum))); + } else if (IS_DECIMAL_TYPE(type)) { + SUM_RES_SET_TYPE(pSumRes, pCtx->inputType, TSDB_DATA_TYPE_DECIMAL); + const SDecimalOps* pOps = getDecimalOps(TSDB_DATA_TYPE_DECIMAL); + if (pAgg->overflow || decimal128AddCheckOverflow((Decimal*)&SUM_RES_GET_DECIMAL_SUM(pSumRes), + &pAgg->decimal128Sum, DECIMAL_WORD_NUM(Decimal))) { + return TSDB_CODE_DECIMAL_OVERFLOW; + } + pOps->add(&SUM_RES_GET_DECIMAL_SUM(pSumRes), &pAgg->decimal128Sum, DECIMAL_WORD_NUM(Decimal)); } } else { // computing based on the true data block SColumnInfoData* pCol = pInput->pData[0]; @@ -646,33 +671,42 @@ int32_t sumFunction(SqlFunctionCtx* pCtx) { if (IS_SIGNED_NUMERIC_TYPE(type) || type == TSDB_DATA_TYPE_BOOL) { if (type == TSDB_DATA_TYPE_TINYINT || type == TSDB_DATA_TYPE_BOOL) { - LIST_ADD_N(pSumRes->isum, pCol, start, numOfRows, int8_t, numOfElem); + LIST_ADD_N(SUM_RES_GET_ISUM(pSumRes), pCol, start, numOfRows, int8_t, numOfElem); } else if (type == TSDB_DATA_TYPE_SMALLINT) { - LIST_ADD_N(pSumRes->isum, pCol, start, numOfRows, int16_t, numOfElem); + LIST_ADD_N(SUM_RES_GET_ISUM(pSumRes), pCol, start, numOfRows, int16_t, numOfElem); } else if (type == TSDB_DATA_TYPE_INT) { - LIST_ADD_N(pSumRes->isum, pCol, start, numOfRows, int32_t, numOfElem); + LIST_ADD_N(SUM_RES_GET_ISUM(pSumRes), pCol, start, numOfRows, int32_t, numOfElem); } else if (type == TSDB_DATA_TYPE_BIGINT) { - LIST_ADD_N(pSumRes->isum, pCol, start, numOfRows, int64_t, numOfElem); + LIST_ADD_N(SUM_RES_GET_ISUM(pSumRes), pCol, start, numOfRows, int64_t, numOfElem); } } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { if (type == TSDB_DATA_TYPE_UTINYINT) { - LIST_ADD_N(pSumRes->usum, pCol, start, numOfRows, uint8_t, numOfElem); + LIST_ADD_N(SUM_RES_GET_USUM(pSumRes), pCol, start, numOfRows, uint8_t, numOfElem); } else if (type == TSDB_DATA_TYPE_USMALLINT) { - LIST_ADD_N(pSumRes->usum, pCol, start, numOfRows, uint16_t, numOfElem); + LIST_ADD_N(SUM_RES_GET_USUM(pSumRes), pCol, start, numOfRows, uint16_t, numOfElem); } else if (type == TSDB_DATA_TYPE_UINT) { - LIST_ADD_N(pSumRes->usum, pCol, start, numOfRows, uint32_t, numOfElem); + LIST_ADD_N(SUM_RES_GET_USUM(pSumRes), pCol, start, numOfRows, uint32_t, numOfElem); } else if (type == TSDB_DATA_TYPE_UBIGINT) { - LIST_ADD_N(pSumRes->usum, pCol, start, numOfRows, uint64_t, numOfElem); + LIST_ADD_N(SUM_RES_GET_USUM(pSumRes), pCol, start, numOfRows, uint64_t, numOfElem); } } else if (type == TSDB_DATA_TYPE_DOUBLE) { - LIST_ADD_N(pSumRes->dsum, pCol, start, numOfRows, double, numOfElem); + LIST_ADD_N(SUM_RES_GET_DSUM(pSumRes), pCol, start, numOfRows, double, numOfElem); } else if (type == TSDB_DATA_TYPE_FLOAT) { - LIST_ADD_N(pSumRes->dsum, pCol, start, numOfRows, float, numOfElem); + LIST_ADD_N(SUM_RES_GET_DSUM(pSumRes), pCol, start, numOfRows, float, numOfElem); + } else if (IS_DECIMAL_TYPE(type)) { + SUM_RES_SET_TYPE(pSumRes, pCtx->inputType, TSDB_DATA_TYPE_DECIMAL); + int32_t overflow = false; + if (TSDB_DATA_TYPE_DECIMAL64 == type) { + LIST_ADD_DECIMAL_N(&SUM_RES_GET_DECIMAL_SUM(pSumRes), pCol, start, numOfRows, Decimal64, numOfElem); + } else if (TSDB_DATA_TYPE_DECIMAL == type) { + LIST_ADD_DECIMAL_N(&SUM_RES_GET_DECIMAL_SUM(pSumRes), pCol, start, numOfRows, Decimal128, numOfElem); + } + if (overflow) return TSDB_CODE_DECIMAL_OVERFLOW; } } // check for overflow - if (IS_FLOAT_TYPE(type) && (isinf(pSumRes->dsum) || isnan(pSumRes->dsum))) { + if (IS_FLOAT_TYPE(type) && (isinf(SUM_RES_GET_DSUM(pSumRes)) || isnan(SUM_RES_GET_DSUM(pSumRes)))) { numOfElem = 0; } @@ -750,26 +784,30 @@ int32_t sumInvertFunction(SqlFunctionCtx* pCtx) { int32_t sumCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx); - SSumRes* pDBuf = GET_ROWCELL_INTERBUF(pDResInfo); + void* pDBuf = GET_ROWCELL_INTERBUF(pDResInfo); + int16_t type = SUM_RES_GET_TYPE(pDBuf, pDestCtx->inputType); SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx); - SSumRes* pSBuf = GET_ROWCELL_INTERBUF(pSResInfo); - int16_t type = pDBuf->type == TSDB_DATA_TYPE_NULL ? pSBuf->type : pDBuf->type; + void* pSBuf = GET_ROWCELL_INTERBUF(pSResInfo); + type = (type == TSDB_DATA_TYPE_NULL) ? SUM_RES_GET_TYPE(pSBuf, pDestCtx->inputType) : type; if (IS_SIGNED_NUMERIC_TYPE(type) || type == TSDB_DATA_TYPE_BOOL) { - pDBuf->isum += pSBuf->isum; + SUM_RES_INC_ISUM(pDBuf, SUM_RES_GET_ISUM(pSBuf)); } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { - pDBuf->usum += pSBuf->usum; + SUM_RES_INC_USUM(pDBuf, SUM_RES_GET_USUM(pSBuf)); + } else if (IS_DECIMAL_TYPE(type)) { + bool overflow = false; + SUM_RES_INC_DECIMAL_SUM(pDBuf, &SUM_RES_GET_DECIMAL_SUM(pSBuf), type); } else if (type == TSDB_DATA_TYPE_DOUBLE || type == TSDB_DATA_TYPE_FLOAT) { - pDBuf->dsum += pSBuf->dsum; + SUM_RES_INC_DSUM(pDBuf, SUM_RES_GET_DSUM(pSBuf)); } pDResInfo->numOfRes = TMAX(pDResInfo->numOfRes, pSResInfo->numOfRes); pDResInfo->isNullRes &= pSResInfo->isNullRes; return TSDB_CODE_SUCCESS; } -bool getSumFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { - pEnv->calcMemSize = sizeof(SSumRes); +bool getSumFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { + pEnv->calcMemSize = SUM_RES_GET_SIZE(pFunc->node.resType.type); return true; } @@ -824,6 +862,7 @@ int32_t minmaxFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultIn } bool getMinmaxFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { + COMPILE_TIME_ASSERT(sizeof(SMinmaxResInfo) == sizeof(SOldMinMaxResInfo)); pEnv->calcMemSize = sizeof(SMinmaxResInfo); return true; } @@ -904,12 +943,18 @@ int32_t minmaxFunctionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { } break; } + case TSDB_DATA_TYPE_DECIMAL64: + code = colDataSetVal(pCol, currentRow, (const char*)&pRes->v, false); + break; + case TSDB_DATA_TYPE_DECIMAL: + code = colDataSetVal(pCol, currentRow, (void*)pRes->dec, false); + break; } } else { colDataSetNULL(pCol, currentRow); } - taosMemoryFreeClear(pRes->str); + if (IS_VAR_DATA_TYPE(pCol->info.type)) taosMemoryFreeClear(pRes->str); if (pCtx->subsidiaries.num > 0) { if (pEntryInfo->numOfRes > 0) { code = setSelectivityValue(pCtx, pBlock, &pRes->tuplePos, currentRow); @@ -1037,7 +1082,6 @@ int32_t minMaxCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int3 int16_t type = pDBuf->type == TSDB_DATA_TYPE_NULL ? pSBuf->type : pDBuf->type; switch (type) { - case TSDB_DATA_TYPE_DOUBLE: case TSDB_DATA_TYPE_UBIGINT: case TSDB_DATA_TYPE_BIGINT: if (pSBuf->assign && (COMPARE_MINMAX_DATA(int64_t) || !pDBuf->assign)) { @@ -1071,6 +1115,7 @@ int32_t minMaxCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int3 pDBuf->assign = true; } break; + case TSDB_DATA_TYPE_DOUBLE: case TSDB_DATA_TYPE_FLOAT: { if (pSBuf->assign && (COMPARE_MINMAX_DATA(double) || !pDBuf->assign)) { pDBuf->v = pSBuf->v; @@ -1079,12 +1124,28 @@ int32_t minMaxCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int3 } break; } - default: - if (pSBuf->assign && (strcmp((char*)&pDBuf->v, (char*)&pSBuf->v) || !pDBuf->assign)) { + case TSDB_DATA_TYPE_DECIMAL64: { + const SDecimalOps* pOps = getDecimalOps(type); + if (pSBuf->assign && ((pOps->lt(&pDBuf->v, &pSBuf->v, DECIMAL_WORD_NUM(Decimal64)) ^ isMinFunc) || !pDBuf->assign)) { pDBuf->v = pSBuf->v; replaceTupleData(&pDBuf->tuplePos, &pSBuf->tuplePos); pDBuf->assign = true; } + } break; + case TSDB_DATA_TYPE_DECIMAL: { + const SDecimalOps* pOps = getDecimalOps(type); + if (pSBuf->assign && (pOps->lt(pDBuf->dec, pSBuf->dec, DECIMAL_WORD_NUM(Decimal)) ^ isMinFunc) || !pDBuf->assign) { + memcpy(pDBuf->dec, pSBuf->dec, DECIMAL128_BYTES); + replaceTupleData(&pDBuf->tuplePos, &pSBuf->tuplePos); + pDBuf->assign = true; + } + } break; + default: + if (pSBuf->assign && (strcmp(pDBuf->str, pSBuf->str) || !pDBuf->assign)) { + memcpy(pDBuf->str, pSBuf->str, varDataLen(pSBuf->str)); + replaceTupleData(&pDBuf->tuplePos, &pSBuf->tuplePos); + pDBuf->assign = true; + } break; } pDResInfo->numOfRes = TMAX(pDResInfo->numOfRes, pSResInfo->numOfRes); @@ -1532,8 +1593,10 @@ int32_t leastSQRFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResult SLeastSQRInfo* pInfo = GET_ROWCELL_INTERBUF(pResultInfo); - GET_TYPED_DATA(pInfo->startVal, double, pCtx->param[1].param.nType, &pCtx->param[1].param.i); - GET_TYPED_DATA(pInfo->stepVal, double, pCtx->param[2].param.nType, &pCtx->param[2].param.i); + GET_TYPED_DATA(pInfo->startVal, double, pCtx->param[1].param.nType, &pCtx->param[1].param.i, + typeGetTypeModFromCol(pCtx->param[1].pCol)); + GET_TYPED_DATA(pInfo->stepVal, double, pCtx->param[2].param.nType, &pCtx->param[2].param.i, + typeGetTypeModFromCol(pCtx->param[2].pCol)); return TSDB_CODE_SUCCESS; } @@ -1828,7 +1891,7 @@ int32_t percentileFunction(SqlFunctionCtx* pCtx) { pResInfo->complete = true; return TSDB_CODE_SUCCESS; } else { - code = tMemBucketCreate(pCol->info.bytes, type, pInfo->minval, pInfo->maxval, pCtx->hasWindowOrGroup, + code = tMemBucketCreate(pCol->info.bytes, type, typeGetTypeModFromColInfo(&pCol->info), pInfo->minval, pInfo->maxval, pCtx->hasWindowOrGroup, &pInfo->pMemBucket, pInfo->numOfElems); if (TSDB_CODE_SUCCESS != code) { return code; @@ -1871,7 +1934,7 @@ int32_t percentileFunction(SqlFunctionCtx* pCtx) { char* data = colDataGetData(pCol, i); double v = 0; - GET_TYPED_DATA(v, double, type, data); + GET_TYPED_DATA(v, double, type, data, typeGetTypeModFromColInfo(&pCol->info)); if (v < GET_DOUBLE_VAL(&pInfo->minval)) { SET_DOUBLE_VAL(&pInfo->minval, v); } @@ -1926,7 +1989,7 @@ int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { for (int32_t i = 1; i < pCtx->numOfParams; ++i) { SVariant* pVal = &pCtx->param[i].param; - GET_TYPED_DATA(v, double, pVal->nType, &pVal->i); + GET_TYPED_DATA(v, double, pVal->nType, &pVal->i, typeGetTypeModFromCol(pCtx->param[i].pCol)); code = getPercentile((*pMemBucket), v, &ppInfo->result); if (code != TSDB_CODE_SUCCESS) { @@ -1958,7 +2021,7 @@ int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { } else { SVariant* pVal = &pCtx->param[1].param; - GET_TYPED_DATA(v, double, pVal->nType, &pVal->i); + GET_TYPED_DATA(v, double, pVal->nType, &pVal->i, typeGetTypeModFromCol(pCtx->param[1].pCol)); code = getPercentile((*pMemBucket), v, &ppInfo->result); if (code != TSDB_CODE_SUCCESS) { @@ -2027,7 +2090,7 @@ int32_t apercentileFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pRes SVariant* pVal = &pCtx->param[1].param; pInfo->percent = 0; - GET_TYPED_DATA(pInfo->percent, double, pVal->nType, &pVal->i); + GET_TYPED_DATA(pInfo->percent, double, pVal->nType, &pVal->i, typeGetTypeModFromCol(pCtx->param[1].pCol)); if (pCtx->numOfParams == 2) { pInfo->algo = APERCT_ALGO_DEFAULT; @@ -2074,7 +2137,7 @@ int32_t apercentileFunction(SqlFunctionCtx* pCtx) { double v = 0; // value int64_t w = 1; // weigth - GET_TYPED_DATA(v, double, type, data); + GET_TYPED_DATA(v, double, type, data, typeGetTypeModFromColInfo(&pCol->info)); int32_t code = tdigestAdd(pInfo->pTDigest, v, w); if (code != TSDB_CODE_SUCCESS) { return code; @@ -2095,7 +2158,7 @@ int32_t apercentileFunction(SqlFunctionCtx* pCtx) { char* data = colDataGetData(pCol, i); double v = 0; - GET_TYPED_DATA(v, double, type, data); + GET_TYPED_DATA(v, double, type, data, typeGetTypeModFromColInfo(&pCol->info)); int32_t code = tHistogramAdd(&pInfo->pHisto, v); if (code != TSDB_CODE_SUCCESS) { return code; @@ -2310,16 +2373,16 @@ static int32_t comparePkDataWithSValue(int8_t pkType, char* pkData, SValue* pVal char numVal[8] = {0}; switch (pkType) { case TSDB_DATA_TYPE_INT: - *(int32_t*)numVal = (int32_t)pVal->val; + *(int32_t*)numVal = (int32_t)VALUE_GET_TRIVIAL_DATUM(pVal); break; case TSDB_DATA_TYPE_UINT: - *(uint32_t*)numVal = (uint32_t)pVal->val; + *(uint32_t*)numVal = (uint32_t)VALUE_GET_TRIVIAL_DATUM(pVal); break; case TSDB_DATA_TYPE_BIGINT: - *(int64_t*)numVal = (int64_t)pVal->val; + *(int64_t*)numVal = (int64_t)VALUE_GET_TRIVIAL_DATUM(pVal); break; case TSDB_DATA_TYPE_UBIGINT: - *(uint64_t*)numVal = (uint64_t)pVal->val; + *(uint64_t*)numVal = (uint64_t)VALUE_GET_TRIVIAL_DATUM(pVal); break; default: break; @@ -4146,7 +4209,7 @@ int32_t spreadFunction(SqlFunctionCtx* pCtx) { char* data = colDataGetData(pCol, i); double v = 0; - GET_TYPED_DATA(v, double, type, data); + GET_TYPED_DATA(v, double, type, data, typeGetTypeModFromColInfo(&pCol->info)); if (v < GET_DOUBLE_VAL(&pInfo->min)) { SET_DOUBLE_VAL(&pInfo->min, v); } @@ -4701,7 +4764,7 @@ static int32_t histogramFunctionImpl(SqlFunctionCtx* pCtx, bool isPartial) { char* data = colDataGetData(pCol, i); double v; - GET_TYPED_DATA(v, double, type, data); + GET_TYPED_DATA(v, double, type, data, typeGetTypeModFromColInfo(&pCol->info)); for (int32_t k = 0; k < pInfo->numOfBins; ++k) { if (v > pInfo->bins[k].lower && v <= pInfo->bins[k].upper) { @@ -5338,7 +5401,7 @@ int32_t csumFunction(SqlFunctionCtx* pCtx) { char* data = colDataGetData(pInputCol, i); if (IS_SIGNED_NUMERIC_TYPE(type)) { int64_t v; - GET_TYPED_DATA(v, int64_t, type, data); + GET_TYPED_DATA(v, int64_t, type, data, typeGetTypeModFromColInfo(&pInputCol->info)); pSumRes->isum += v; code = colDataSetVal(pOutput, pos, (char*)&pSumRes->isum, false); if (TSDB_CODE_SUCCESS != code) { @@ -5346,7 +5409,7 @@ int32_t csumFunction(SqlFunctionCtx* pCtx) { } } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { uint64_t v; - GET_TYPED_DATA(v, uint64_t, type, data); + GET_TYPED_DATA(v, uint64_t, type, data, typeGetTypeModFromColInfo(&pInputCol->info)); pSumRes->usum += v; code = colDataSetVal(pOutput, pos, (char*)&pSumRes->usum, false); if (TSDB_CODE_SUCCESS != code) { @@ -5354,7 +5417,7 @@ int32_t csumFunction(SqlFunctionCtx* pCtx) { } } else if (IS_FLOAT_TYPE(type)) { double v; - GET_TYPED_DATA(v, double, type, data); + GET_TYPED_DATA(v, double, type, data, typeGetTypeModFromColInfo(&pInputCol->info)); pSumRes->dsum += v; // check for overflow if (isinf(pSumRes->dsum) || isnan(pSumRes->dsum)) { @@ -5440,7 +5503,7 @@ int32_t mavgFunction(SqlFunctionCtx* pCtx) { char* data = colDataGetData(pInputCol, i); double v; - GET_TYPED_DATA(v, double, type, data); + GET_TYPED_DATA(v, double, type, data, typeGetTypeModFromColInfo(&pInputCol->info)); if (!pInfo->pointsMeet && (pInfo->pos < pInfo->numOfPoints - 1)) { pInfo->points[pInfo->pos] = v; @@ -6132,7 +6195,7 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { last->key = row.ts; - GET_TYPED_DATA(last->val, double, pInputCol->info.type, row.pData); + GET_TYPED_DATA(last->val, double, pInputCol->info.type, row.pData, typeGetTypeModFromColInfo(&pInputCol->info)); pInfo->dOutput += twa_get_area(pCtx->start, *last); pInfo->win.skey = pCtx->start.key; @@ -6154,7 +6217,7 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { last->key = row.ts; - GET_TYPED_DATA(last->val, double, pInputCol->info.type, row.pData); + GET_TYPED_DATA(last->val, double, pInputCol->info.type, row.pData, typeGetTypeModFromColInfo(&pInputCol->info)); pInfo->win.skey = last->key; pInfo->numOfElems++; @@ -6687,7 +6750,7 @@ int32_t derivativeFunction(SqlFunctionCtx* pCtx) { } char* d = row.pData; - GET_TYPED_DATA(v, double, pInputCol->info.type, d); + GET_TYPED_DATA(v, double, pInputCol->info.type, d, typeGetTypeModFromColInfo(&pInputCol->info)); int32_t pos = pCtx->offset + numOfElems; if (!pDerivInfo->valueSet) { // initial value is not set yet @@ -6743,7 +6806,7 @@ int32_t derivativeFunction(SqlFunctionCtx* pCtx) { } char* d = row.pData; - GET_TYPED_DATA(v, double, pInputCol->info.type, d); + GET_TYPED_DATA(v, double, pInputCol->info.type, d, typeGetTypeModFromColInfo(&pInputCol->info)); int32_t pos = pCtx->offset + numOfElems; if (!pDerivInfo->valueSet) { // initial value is not set yet @@ -6901,7 +6964,7 @@ int32_t irateFunction(SqlFunctionCtx* pCtx) { char* data = row.pData; double v = 0; - GET_TYPED_DATA(v, double, type, data); + GET_TYPED_DATA(v, double, type, data, typeGetTypeModFromColInfo(&pInputCol->info)); if (INT64_MIN == pRateInfo->lastKey) { doSaveRateInfo(pRateInfo, false, row.ts, row.pPk, v); diff --git a/source/libs/function/src/detail/tavgfunction.c b/source/libs/function/src/detail/tavgfunction.c index 7313fc82f7..76ec692f6c 100644 --- a/source/libs/function/src/detail/tavgfunction.c +++ b/source/libs/function/src/detail/tavgfunction.c @@ -42,60 +42,74 @@ } while (0) // define signed number sum with check overflow -#define CHECK_OVERFLOW_SUM_SIGNED(out, val) \ - if (out->sum.overflow) { \ - out->sum.dsum += val; \ - } else if (out->sum.isum > 0 && val > 0 && INT64_MAX - out->sum.isum <= val || \ - out->sum.isum < 0 && val < 0 && INT64_MIN - out->sum.isum >= val) { \ - double dsum = (double)out->sum.isum; \ - out->sum.overflow = true; \ - out->sum.dsum = dsum + val; \ - } else { \ - out->sum.isum += val; \ - } +#define CHECK_OVERFLOW_SUM_SIGNED(pAvgRes, val) \ + do { \ + SAvgRes* out = pAvgRes; \ + if (out->sum.overflow) { \ + out->sum.dsum += val; \ + } else if (out->sum.isum > 0 && val > 0 && INT64_MAX - out->sum.isum <= val || \ + out->sum.isum < 0 && val < 0 && INT64_MIN - out->sum.isum >= val) { \ + double dsum = (double)out->sum.isum; \ + out->sum.overflow = true; \ + out->sum.dsum = dsum + val; \ + } else { \ + out->sum.isum += val; \ + } \ + } while (0) // val is big than INT64_MAX, val come from merge -#define CHECK_OVERFLOW_SUM_SIGNED_BIG(out, val, big) \ - if (out->sum.overflow) { \ - out->sum.dsum += val; \ - } else if (out->sum.isum > 0 && val > 0 && INT64_MAX - out->sum.isum <= val || \ - out->sum.isum < 0 && val < 0 && INT64_MIN - out->sum.isum >= val || \ - big) { \ - double dsum = (double)out->sum.isum; \ - out->sum.overflow = true; \ - out->sum.dsum = dsum + val; \ - } else { \ - out->sum.isum += val; \ - } +#define CHECK_OVERFLOW_SUM_SIGNED_BIG(pAvgRes, val, big) \ + do { \ + SAvgRes* out = pAvgRes; \ + if (out->sum.overflow) { \ + out->sum.dsum += val; \ + } else if (out->sum.isum > 0 && val > 0 && INT64_MAX - out->sum.isum <= val || \ + out->sum.isum < 0 && val < 0 && INT64_MIN - out->sum.isum >= val || big) { \ + double dsum = (double)out->sum.isum; \ + out->sum.overflow = true; \ + out->sum.dsum = dsum + val; \ + } else { \ + SUM_RES_INC_ISUM(&AVG_RES_GET_SUM(out), val); \ + } \ + } while (0) // define unsigned number sum with check overflow -#define CHECK_OVERFLOW_SUM_UNSIGNED(out, val) \ - if (out->sum.overflow) { \ - out->sum.dsum += val; \ - } else if (UINT64_MAX - out->sum.usum <= val) { \ - double dsum = (double)out->sum.usum; \ - out->sum.overflow = true; \ - out->sum.dsum = dsum + val; \ - } else { \ - out->sum.usum += val; \ - } +#define CHECK_OVERFLOW_SUM_UNSIGNED(pAvgRes, val) \ + do { \ + SAvgRes* out = pAvgRes; \ + if (out->sum.overflow) { \ + out->sum.dsum += val; \ + } else if (UINT64_MAX - out->sum.usum <= val) { \ + double dsum = (double)out->sum.usum; \ + out->sum.overflow = true; \ + out->sum.dsum = dsum + val; \ + } else { \ + out->sum.usum += val; \ + } \ + } while (0) // val is big than UINT64_MAX, val come from merge -#define CHECK_OVERFLOW_SUM_UNSIGNED_BIG(out, val, big) \ - if (out->sum.overflow) { \ - out->sum.dsum += val; \ - } else if (UINT64_MAX - out->sum.usum <= val || big) { \ - double dsum = (double)out->sum.usum; \ - out->sum.overflow = true; \ - out->sum.dsum = dsum + val; \ - } else { \ - out->sum.usum += val; \ - } +#define CHECK_OVERFLOW_SUM_UNSIGNED_BIG(pAvgRes, val, big) \ + do { \ + SAvgRes* out = pAvgRes; \ + if (out->sum.overflow) { \ + out->sum.dsum += val; \ + } else if (UINT64_MAX - out->sum.usum <= val || big) { \ + double dsum = (double)out->sum.usum; \ + out->sum.overflow = true; \ + out->sum.dsum = dsum + val; \ + } else { \ + out->sum.usum += val; \ + } \ + } while (0) -int32_t getAvgInfoSize() { return (int32_t)sizeof(SAvgRes); } +int32_t getAvgInfoSize(SFunctionNode* pFunc) { + if (pFunc->pSrcFuncRef) return AVG_RES_GET_SIZE(pFunc->pSrcFuncRef->srcFuncInputType.type); + return AVG_RES_GET_SIZE(pFunc->srcFuncInputType.type); +} -bool getAvgFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { - pEnv->calcMemSize = sizeof(SAvgRes); +bool getAvgFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { + pEnv->calcMemSize =getAvgInfoSize(pFunc); return true; } @@ -107,27 +121,33 @@ int32_t avgFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) return TSDB_CODE_FUNC_SETUP_ERROR; } - SAvgRes* pRes = GET_ROWCELL_INTERBUF(pResultInfo); - (void)memset(pRes, 0, sizeof(SAvgRes)); + void* pRes = GET_ROWCELL_INTERBUF(pResultInfo); + (void)memset(pRes, 0, pCtx->resDataInfo.interBufSize); return TSDB_CODE_SUCCESS; } -static int32_t calculateAvgBySMAInfo(SAvgRes* pRes, int32_t numOfRows, int32_t type, const SColumnDataAgg* pAgg) { +static int32_t calculateAvgBySMAInfo(void* pRes, int32_t numOfRows, int32_t type, const SColumnDataAgg* pAgg, int32_t* pNumOfElem) { int32_t numOfElem = numOfRows - pAgg->numOfNull; - pRes->count += numOfElem; + AVG_RES_INC_COUNT(pRes, type, numOfElem); if (IS_SIGNED_NUMERIC_TYPE(type)) { CHECK_OVERFLOW_SUM_SIGNED(pRes, pAgg->sum); } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { CHECK_OVERFLOW_SUM_UNSIGNED(pRes, pAgg->sum); } else if (IS_FLOAT_TYPE(type)) { - pRes->sum.dsum += GET_DOUBLE_VAL((const char*)&(pAgg->sum)); + SUM_RES_INC_DSUM(&AVG_RES_GET_SUM(pRes), GET_DOUBLE_VAL((const char*)&(pAgg->sum))); + } else if (IS_DECIMAL_TYPE(type)) { + bool overflow = pAgg->overflow; + if (overflow) return TSDB_CODE_DECIMAL_OVERFLOW; + SUM_RES_INC_DECIMAL_SUM(&AVG_RES_GET_DECIMAL_SUM(pRes), &pAgg->decimal128Sum, TSDB_DATA_TYPE_DECIMAL); + if (overflow) return TSDB_CODE_DECIMAL_OVERFLOW; } - return numOfElem; + *pNumOfElem = numOfElem; + return 0; } -static int32_t doAddNumericVector(SColumnInfoData* pCol, int32_t type, SInputColumnInfoData *pInput, SAvgRes* pRes) { +static int32_t doAddNumericVector(SColumnInfoData* pCol, int32_t type, SInputColumnInfoData *pInput, void* pRes, int32_t* pNumOfElem) { int32_t start = pInput->startRowIndex; int32_t numOfRows = pInput->numOfRows; int32_t numOfElems = 0; @@ -141,8 +161,8 @@ static int32_t doAddNumericVector(SColumnInfoData* pCol, int32_t type, SInputCol } numOfElems += 1; - pRes->count += 1; - CHECK_OVERFLOW_SUM_SIGNED(pRes, plist[i]) + AVG_RES_INC_COUNT(pRes, TSDB_DATA_TYPE_TINYINT, 1); + CHECK_OVERFLOW_SUM_SIGNED(pRes, plist[i]); } break; @@ -156,8 +176,8 @@ static int32_t doAddNumericVector(SColumnInfoData* pCol, int32_t type, SInputCol } numOfElems += 1; - pRes->count += 1; - CHECK_OVERFLOW_SUM_SIGNED(pRes, plist[i]) + AVG_RES_INC_COUNT(pRes, TSDB_DATA_TYPE_SMALLINT, 1); + CHECK_OVERFLOW_SUM_SIGNED(pRes, plist[i]); } break; } @@ -170,8 +190,8 @@ static int32_t doAddNumericVector(SColumnInfoData* pCol, int32_t type, SInputCol } numOfElems += 1; - pRes->count += 1; - CHECK_OVERFLOW_SUM_SIGNED(pRes, plist[i]) + AVG_RES_INC_COUNT(pRes, TSDB_DATA_TYPE_INT, 1); + CHECK_OVERFLOW_SUM_SIGNED(pRes, plist[i]); } break; @@ -185,8 +205,8 @@ static int32_t doAddNumericVector(SColumnInfoData* pCol, int32_t type, SInputCol } numOfElems += 1; - pRes->count += 1; - CHECK_OVERFLOW_SUM_SIGNED(pRes, plist[i]) + AVG_RES_INC_COUNT(pRes, TSDB_DATA_TYPE_BIGINT, 1); + CHECK_OVERFLOW_SUM_SIGNED(pRes, plist[i]); } break; } @@ -199,8 +219,8 @@ static int32_t doAddNumericVector(SColumnInfoData* pCol, int32_t type, SInputCol } numOfElems += 1; - pRes->count += 1; - CHECK_OVERFLOW_SUM_UNSIGNED(pRes, plist[i]) + AVG_RES_INC_COUNT(pRes, TSDB_DATA_TYPE_UTINYINT, 1); + CHECK_OVERFLOW_SUM_UNSIGNED(pRes, plist[i]); } break; @@ -214,8 +234,8 @@ static int32_t doAddNumericVector(SColumnInfoData* pCol, int32_t type, SInputCol } numOfElems += 1; - pRes->count += 1; - CHECK_OVERFLOW_SUM_UNSIGNED(pRes, plist[i]) + AVG_RES_INC_COUNT(pRes, TSDB_DATA_TYPE_USMALLINT, 1); + CHECK_OVERFLOW_SUM_UNSIGNED(pRes, plist[i]); } break; } @@ -228,8 +248,8 @@ static int32_t doAddNumericVector(SColumnInfoData* pCol, int32_t type, SInputCol } numOfElems += 1; - pRes->count += 1; - CHECK_OVERFLOW_SUM_UNSIGNED(pRes, plist[i]) + AVG_RES_INC_COUNT(pRes, TSDB_DATA_TYPE_UINT, 1); + CHECK_OVERFLOW_SUM_UNSIGNED(pRes, plist[i]); } break; @@ -243,8 +263,8 @@ static int32_t doAddNumericVector(SColumnInfoData* pCol, int32_t type, SInputCol } numOfElems += 1; - pRes->count += 1; - CHECK_OVERFLOW_SUM_UNSIGNED(pRes, plist[i]) + AVG_RES_INC_COUNT(pRes, TSDB_DATA_TYPE_UBIGINT, 1); + CHECK_OVERFLOW_SUM_UNSIGNED(pRes, plist[i]); } break; @@ -258,8 +278,8 @@ static int32_t doAddNumericVector(SColumnInfoData* pCol, int32_t type, SInputCol } numOfElems += 1; - pRes->count += 1; - pRes->sum.dsum += plist[i]; + AVG_RES_INC_COUNT(pRes, TSDB_DATA_TYPE_FLOAT, 1); + SUM_RES_INC_DSUM(&AVG_RES_GET_SUM(pRes), plist[i]); } break; } @@ -272,17 +292,32 @@ static int32_t doAddNumericVector(SColumnInfoData* pCol, int32_t type, SInputCol } numOfElems += 1; - pRes->count += 1; - pRes->sum.dsum += plist[i]; + AVG_RES_INC_COUNT(pRes, TSDB_DATA_TYPE_DOUBLE, 1); + SUM_RES_INC_DSUM(&AVG_RES_GET_SUM(pRes), plist[i]); } break; } + case TSDB_DATA_TYPE_DECIMAL64: + case TSDB_DATA_TYPE_DECIMAL: { + const char* pDec = pCol->pData; + for (int32_t i = start; i < numOfRows + start; ++i) { + if (colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + numOfElems += 1; + AVG_RES_INC_COUNT(pRes, type, 1); + bool overflow = false; + SUM_RES_INC_DECIMAL_SUM(&AVG_RES_GET_DECIMAL_SUM(pRes), (const void*)(pDec + i * tDataTypes[type].bytes), type); + if (overflow) return TSDB_CODE_DECIMAL_OVERFLOW; + } + } break; default: break; } - return numOfElems; + *pNumOfElem = numOfElems; + return 0; } int32_t avgFunction(SqlFunctionCtx* pCtx) { @@ -292,8 +327,9 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { SInputColumnInfoData* pInput = &pCtx->input; SColumnDataAgg* pAgg = pInput->pColumnDataAgg[0]; int32_t type = pInput->pData[0]->info.type; + pCtx->inputType = type; - SAvgRes* pAvgRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + void* pAvgRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); // computing based on the true data block SColumnInfoData* pCol = pInput->pData[0]; @@ -305,13 +341,15 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { goto _over; } - pAvgRes->type = type; + AVG_RES_SET_TYPE(pAvgRes, pCtx->inputType, type); + if (IS_DECIMAL_TYPE(type)) AVG_RES_SET_INPUT_SCALE(pAvgRes, pInput->pData[0]->info.scale); if (pInput->colDataSMAIsSet) { // try to use SMA if available - numOfElem = calculateAvgBySMAInfo(pAvgRes, numOfRows, type, pAgg); + int32_t code = calculateAvgBySMAInfo(pAvgRes, numOfRows, type, pAgg, &numOfElem); + if (code != 0) return code; } else if (!pCol->hasNull) { // try to employ the simd instructions to speed up the loop numOfElem = pInput->numOfRows; - pAvgRes->count += pInput->numOfRows; + AVG_RES_INC_COUNT(pAvgRes, pCtx->inputType, pInput->numOfRows); switch(type) { case TSDB_DATA_TYPE_UTINYINT: @@ -320,9 +358,9 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { if (type == TSDB_DATA_TYPE_TINYINT) { - CHECK_OVERFLOW_SUM_SIGNED(pAvgRes, plist[i]) + CHECK_OVERFLOW_SUM_SIGNED(pAvgRes, plist[i]); } else { - CHECK_OVERFLOW_SUM_UNSIGNED(pAvgRes, (uint8_t)plist[i]) + CHECK_OVERFLOW_SUM_UNSIGNED(pAvgRes, (uint8_t)plist[i]); } } break; @@ -334,9 +372,9 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { if (type == TSDB_DATA_TYPE_SMALLINT) { - CHECK_OVERFLOW_SUM_SIGNED(pAvgRes, plist[i]) + CHECK_OVERFLOW_SUM_SIGNED(pAvgRes, plist[i]); } else { - CHECK_OVERFLOW_SUM_UNSIGNED(pAvgRes, (uint16_t)plist[i]) + CHECK_OVERFLOW_SUM_UNSIGNED(pAvgRes, (uint16_t)plist[i]); } } break; @@ -348,9 +386,9 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { if (type == TSDB_DATA_TYPE_INT) { - CHECK_OVERFLOW_SUM_SIGNED(pAvgRes, plist[i]) + CHECK_OVERFLOW_SUM_SIGNED(pAvgRes, plist[i]); } else { - CHECK_OVERFLOW_SUM_UNSIGNED(pAvgRes, (uint32_t)plist[i]) + CHECK_OVERFLOW_SUM_UNSIGNED(pAvgRes, (uint32_t)plist[i]); } } break; @@ -362,9 +400,9 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { if (type == TSDB_DATA_TYPE_BIGINT) { - CHECK_OVERFLOW_SUM_SIGNED(pAvgRes, plist[i]) + CHECK_OVERFLOW_SUM_SIGNED(pAvgRes, plist[i]); } else { - CHECK_OVERFLOW_SUM_UNSIGNED(pAvgRes, (uint64_t)plist[i]) + CHECK_OVERFLOW_SUM_UNSIGNED(pAvgRes, (uint64_t)plist[i]); } } break; @@ -374,7 +412,7 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { const float* plist = (const float*) pCol->pData; for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { - pAvgRes->sum.dsum += plist[i]; + SUM_RES_INC_DSUM(&AVG_RES_GET_SUM(pAvgRes), plist[i]); } break; } @@ -382,15 +420,31 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { const double* plist = (const double*)pCol->pData; for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { - pAvgRes->sum.dsum += plist[i]; + SUM_RES_INC_DSUM(&AVG_RES_GET_SUM(pAvgRes), plist[i]); } break; } + case TSDB_DATA_TYPE_DECIMAL: + case TSDB_DATA_TYPE_DECIMAL64: { + const char* pDec = pCol->pData; + for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { + bool overflow = false; + if (type == TSDB_DATA_TYPE_DECIMAL64) { + SUM_RES_INC_DECIMAL_SUM(&AVG_RES_GET_DECIMAL_SUM(pAvgRes), (const void*)(pDec + i * tDataTypes[type].bytes), + TSDB_DATA_TYPE_DECIMAL64); + } else { + SUM_RES_INC_DECIMAL_SUM(&AVG_RES_GET_DECIMAL_SUM(pAvgRes), (const void*)(pDec + i * tDataTypes[type].bytes), + TSDB_DATA_TYPE_DECIMAL); + } + if (overflow) return TSDB_CODE_DECIMAL_OVERFLOW; + } + } break; default: return TSDB_CODE_FUNC_FUNTION_PARA_TYPE; } } else { - numOfElem = doAddNumericVector(pCol, type, pInput, pAvgRes); + int32_t code = doAddNumericVector(pCol, type, pInput, pAvgRes, &numOfElem); + if (code) return code; } _over: @@ -399,23 +453,33 @@ _over: return TSDB_CODE_SUCCESS; } -static void avgTransferInfo(SAvgRes* pInput, SAvgRes* pOutput) { - if (IS_NULL_TYPE(pInput->type)) { - return; +static int32_t avgTransferInfo(SqlFunctionCtx* pCtx, void* pInput, void* pOutput) { + int32_t inputDT = pCtx->pExpr->pExpr->_function.pFunctNode->srcFuncInputType.type; + int32_t type = AVG_RES_GET_TYPE(pInput, inputDT); + pCtx->inputType = type; + if (IS_NULL_TYPE(type)) { + return 0; } - pOutput->type = pInput->type; - if (IS_SIGNED_NUMERIC_TYPE(pOutput->type)) { - bool overflow = pInput->sum.overflow; - CHECK_OVERFLOW_SUM_SIGNED_BIG(pOutput, (overflow ? pInput->sum.dsum : pInput->sum.isum), overflow); - } else if (IS_UNSIGNED_NUMERIC_TYPE(pOutput->type)) { - bool overflow = pInput->sum.overflow; - CHECK_OVERFLOW_SUM_UNSIGNED_BIG(pOutput, (overflow ? pInput->sum.dsum : pInput->sum.usum), overflow); + + AVG_RES_SET_TYPE(pOutput, inputDT, type); + if (IS_SIGNED_NUMERIC_TYPE(type)) { + bool overflow = AVG_RES_GET_SUM_OVERFLOW(pInput, false, 0); + CHECK_OVERFLOW_SUM_SIGNED_BIG(pOutput, (overflow ? SUM_RES_GET_DSUM(&AVG_RES_GET_SUM(pInput)) : SUM_RES_GET_ISUM(&AVG_RES_GET_SUM(pInput))), overflow); + } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { + bool overflow = AVG_RES_GET_SUM_OVERFLOW(pInput, false, 0); + CHECK_OVERFLOW_SUM_UNSIGNED_BIG(pOutput, (overflow ? SUM_RES_GET_DSUM(&AVG_RES_GET_SUM(pInput)) : SUM_RES_GET_USUM(&AVG_RES_GET_SUM(pInput))), overflow); + } else if (IS_DECIMAL_TYPE(type)) { + AVG_RES_SET_INPUT_SCALE(pOutput, AVG_RES_GET_INPUT_SCALE(pInput)); + bool overflow = false; + SUM_RES_INC_DECIMAL_SUM(&AVG_RES_GET_DECIMAL_SUM(pOutput), &AVG_RES_GET_DECIMAL_SUM(pInput), TSDB_DATA_TYPE_DECIMAL); + if (overflow) return TSDB_CODE_DECIMAL_OVERFLOW; } else { - pOutput->sum.dsum += pInput->sum.dsum; + SUM_RES_INC_DSUM(&AVG_RES_GET_SUM(pOutput), SUM_RES_GET_DSUM(&AVG_RES_GET_SUM(pInput))); } - pOutput->count += pInput->count; + AVG_RES_INC_COUNT(pOutput, type, AVG_RES_GET_COUNT(pInput, true, type)); + return 0; } int32_t avgFunctionMerge(SqlFunctionCtx* pCtx) { @@ -431,15 +495,16 @@ int32_t avgFunctionMerge(SqlFunctionCtx* pCtx) { return TSDB_CODE_FUNC_FUNTION_PARA_TYPE; } - SAvgRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + void* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); int32_t start = pInput->startRowIndex; for (int32_t i = start; i < start + pInput->numOfRows; ++i) { if(colDataIsNull_s(pCol, i)) continue; char* data = colDataGetData(pCol, i); - SAvgRes* pInputInfo = (SAvgRes*)varDataVal(data); - avgTransferInfo(pInputInfo, pInfo); + void* pInputInfo = varDataVal(data); + int32_t code = avgTransferInfo(pCtx, pInputInfo, pInfo); + if (code != 0) return code; } SET_VAL(GET_RES_INFO(pCtx), 1, 1); @@ -514,20 +579,27 @@ int32_t avgInvertFunction(SqlFunctionCtx* pCtx) { int32_t avgCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx); - SAvgRes* pDBuf = GET_ROWCELL_INTERBUF(pDResInfo); + void* pDBuf = GET_ROWCELL_INTERBUF(pDResInfo); + int32_t type = AVG_RES_GET_TYPE(pDBuf, pDestCtx->inputType); SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx); - SAvgRes* pSBuf = GET_ROWCELL_INTERBUF(pSResInfo); - int16_t type = pDBuf->type == TSDB_DATA_TYPE_NULL ? pSBuf->type : pDBuf->type; + void* pSBuf = GET_ROWCELL_INTERBUF(pSResInfo); + type = (type == TSDB_DATA_TYPE_NULL) ? AVG_RES_GET_TYPE(pSBuf, pDestCtx->inputType) : type; if (IS_SIGNED_NUMERIC_TYPE(type)) { - CHECK_OVERFLOW_SUM_SIGNED(pDBuf, pSBuf->sum.isum) + CHECK_OVERFLOW_SUM_SIGNED(pDBuf, SUM_RES_GET_ISUM(&AVG_RES_GET_SUM(pSBuf))); } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { - CHECK_OVERFLOW_SUM_UNSIGNED(pDBuf, pSBuf->sum.usum) + CHECK_OVERFLOW_SUM_UNSIGNED(pDBuf, SUM_RES_GET_USUM(&AVG_RES_GET_SUM(pSBuf))); + } else if (IS_DECIMAL_TYPE(type)) { + bool overflow = false; + SUM_RES_INC_DECIMAL_SUM(&AVG_RES_GET_DECIMAL_SUM(pDBuf), &SUM_RES_GET_DECIMAL_SUM(&AVG_RES_GET_DECIMAL_SUM(pSBuf)), type); + if (overflow) { + + } } else { - pDBuf->sum.dsum += pSBuf->sum.dsum; + SUM_RES_INC_DSUM(&AVG_RES_GET_SUM(pDBuf), SUM_RES_GET_DSUM(&AVG_RES_GET_SUM(pSBuf))); } - pDBuf->count += pSBuf->count; + AVG_RES_INC_COUNT(pDBuf, pDestCtx->inputType, AVG_RES_GET_COUNT(pSBuf, true, pDestCtx->inputType)); return TSDB_CODE_SUCCESS; } @@ -535,24 +607,46 @@ int32_t avgCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(pCtx); - SAvgRes* pRes = GET_ROWCELL_INTERBUF(pEntryInfo); - int32_t type = pRes->type; + void* pRes = GET_ROWCELL_INTERBUF(pEntryInfo); + int32_t type = AVG_RES_GET_TYPE(pRes, pCtx->inputType); + int64_t count = AVG_RES_GET_COUNT(pRes, true, type); - if (pRes->count > 0) { - if(pRes->sum.overflow) { - // overflow flag set , use dsum - pRes->result = pRes->sum.dsum / ((double)pRes->count); + if (AVG_RES_GET_COUNT(pRes, true, pCtx->inputType) > 0) { + + if(AVG_RES_GET_SUM_OVERFLOW(pRes, true, pCtx->inputType)) { + AVG_RES_GET_AVG(pRes) = SUM_RES_GET_DSUM(&AVG_RES_GET_SUM(pRes)) / ((double)AVG_RES_GET_COUNT(pRes, false, 0)); }else if (IS_SIGNED_NUMERIC_TYPE(type)) { - pRes->result = pRes->sum.isum / ((double)pRes->count); + AVG_RES_GET_AVG(pRes) = SUM_RES_GET_ISUM(&AVG_RES_GET_SUM(pRes)) / ((double)AVG_RES_GET_COUNT(pRes, false, 0)); } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { - pRes->result = pRes->sum.usum / ((double)pRes->count); + AVG_RES_GET_AVG(pRes) = SUM_RES_GET_USUM(&AVG_RES_GET_SUM(pRes)) / ((double)AVG_RES_GET_COUNT(pRes, false, 0)); + } else if (IS_DECIMAL_TYPE(type)) { + int32_t slotId = pCtx->pExpr->base.resSchema.slotId; + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + SDataType sumDt = {.type = TSDB_DATA_TYPE_DECIMAL, + .bytes = tDataTypes[TSDB_DATA_TYPE_DECIMAL].bytes, + .precision = pCol->info.precision, + .scale = AVG_RES_GET_INPUT_SCALE(pRes)}; + SDataType countDt = { + .type = TSDB_DATA_TYPE_BIGINT, .bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .precision = 0, .scale = 0}; + SDataType avgDt = {.type = TSDB_DATA_TYPE_DECIMAL, + .bytes = tDataTypes[TSDB_DATA_TYPE_DECIMAL].bytes, + .precision = pCol->info.precision, + .scale = pCol->info.scale}; + int64_t count = AVG_RES_GET_COUNT(pRes, true, type); + int32_t code = + decimalOp(OP_TYPE_DIV, &sumDt, &countDt, &avgDt, &SUM_RES_GET_DECIMAL_SUM(&AVG_RES_GET_DECIMAL_SUM(pRes)), + &count, &AVG_RES_GET_DECIMAL_AVG(pRes)); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } else { - pRes->result = pRes->sum.dsum / ((double)pRes->count); + AVG_RES_GET_AVG(pRes) = SUM_RES_GET_DSUM(&AVG_RES_GET_SUM(pRes)) / ((double)AVG_RES_GET_COUNT(pRes, false, 0)); } } - - if (pRes->count == 0 || isinf(pRes->result) || isnan(pRes->result)) { + if (AVG_RES_GET_COUNT(pRes, true, pCtx->inputType) == 0) { pEntryInfo->numOfRes = 0; + } else if (!IS_DECIMAL_TYPE(pCtx->inputType)) { + if (isinf(AVG_RES_GET_AVG(pRes)) || isnan(AVG_RES_GET_AVG(pRes))) pEntryInfo->numOfRes = 0; } else { pEntryInfo->numOfRes = 1; } @@ -562,8 +656,8 @@ int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t avgPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); - SAvgRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - int32_t resultBytes = getAvgInfoSize(); + void* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + int32_t resultBytes = AVG_RES_GET_SIZE(pCtx->inputType); char* res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char)); int32_t code = TSDB_CODE_SUCCESS; if (NULL == res) { diff --git a/source/libs/function/src/detail/tminmax.c b/source/libs/function/src/detail/tminmax.c index 8712096033..782bff11f3 100644 --- a/source/libs/function/src/detail/tminmax.c +++ b/source/libs/function/src/detail/tminmax.c @@ -19,6 +19,7 @@ #include "tdatablock.h" #include "tfunctionInt.h" #include "tglobal.h" +#include "decimal.h" #define __COMPARE_ACQUIRED_MAX(i, end, bm, _data, ctx, val, pos) \ int32_t code = TSDB_CODE_SUCCESS; \ @@ -395,6 +396,40 @@ static int32_t doExtractVal(SColumnInfoData* pCol, int32_t i, int32_t end, SqlFu } break; } + case TSDB_DATA_TYPE_DECIMAL64: { + const Decimal64* pData = (const Decimal64*)pCol->pData; + const SDecimalOps* pOps = getDecimalOps(TSDB_DATA_TYPE_DECIMAL64); + int32_t code = 0; + for (; i < end; ++i) { + if (colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + if (pOps->gt(&pBuf->v, &pData[i], DECIMAL_WORD_NUM(Decimal64))) { + pBuf->v = DECIMAL64_GET_VALUE(&pData[i]); + if (pCtx->subsidiaries.num > 0) { + code = updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + if (TSDB_CODE_SUCCESS != code) return code; + } + } + } + } break; + case TSDB_DATA_TYPE_DECIMAL: { + int32_t code = 0; + const SDecimalOps* pOps = getDecimalOps(TSDB_DATA_TYPE_DECIMAL); + const Decimal128* pData = (const Decimal128*)pCol->pData; + for (; i < end; ++i) { + if (colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + if (pOps->gt(pBuf->dec, &pData[i], DECIMAL_WORD_NUM(Decimal128))) { + memcpy(pBuf->dec, pData + i, pCol->info.bytes); + if (pCtx->subsidiaries.num > 0) { + code = updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + if (TSDB_CODE_SUCCESS != code) return code; + } + } + } + } break; } } else { switch (pCol->info.type) { @@ -458,6 +493,40 @@ static int32_t doExtractVal(SColumnInfoData* pCol, int32_t i, int32_t end, SqlFu __COMPARE_ACQUIRED_MAX(i, end, pCol->nullbitmap, pData, pCtx, *(double*)&(pBuf->v), &pBuf->tuplePos) break; } + case TSDB_DATA_TYPE_DECIMAL64: { + const Decimal64* pData = (const Decimal64*)pCol->pData; + const SDecimalOps* pOps = getDecimalOps(TSDB_DATA_TYPE_DECIMAL64); + int32_t code = 0; + for (; i < end; ++i) { + if (colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + if (pOps->lt(&pBuf->v, &pData[i], DECIMAL_WORD_NUM(Decimal64))) { + pBuf->v = DECIMAL64_GET_VALUE(&pData[i]); + if (pCtx->subsidiaries.num > 0) { + code = updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + if (TSDB_CODE_SUCCESS != code) return code; + } + } + } + } break; + case TSDB_DATA_TYPE_DECIMAL: { + int32_t code = 0; + const SDecimalOps* pOps = getDecimalOps(TSDB_DATA_TYPE_DECIMAL); + const Decimal128* pData = (const Decimal128*)pCol->pData; + for (; i < end; ++i) { + if (colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + if (pOps->lt(pBuf->dec, &pData[i], DECIMAL_WORD_NUM(Decimal128))) { + memcpy(pBuf->dec, pData + i, pCol->info.bytes); + if (pCtx->subsidiaries.num > 0) { + code = updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + if (TSDB_CODE_SUCCESS != code) return code; + } + } + } + } break; case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: { @@ -549,11 +618,18 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc, int32_t* nElems) } int16_t index = 0; - void* tval = (isMinFunc) ? &pInput->pColumnDataAgg[0]->min : &pInput->pColumnDataAgg[0]->max; + void* tval = NULL; + if (IS_DECIMAL_TYPE(type)) { + tval = isMinFunc ? pInput->pColumnDataAgg[0]->decimal128Min : pInput->pColumnDataAgg[0]->decimal128Max; + } else { + tval = (isMinFunc) ? &pInput->pColumnDataAgg[0]->min : &pInput->pColumnDataAgg[0]->max; + } if (!pBuf->assign) { if (type == TSDB_DATA_TYPE_FLOAT) { GET_FLOAT_VAL(&pBuf->v) = GET_DOUBLE_VAL(tval); + } else if (type == TSDB_DATA_TYPE_DECIMAL) { + memcpy(pBuf->dec, tval, pCol->info.bytes); } else { pBuf->v = GET_INT64_VAL(tval); } @@ -562,7 +638,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc, int32_t* nElems) } else { if (IS_SIGNED_NUMERIC_TYPE(type)) { int64_t prev = 0; - GET_TYPED_DATA(prev, int64_t, type, &pBuf->v); + GET_TYPED_DATA(prev, int64_t, type, &pBuf->v, 0); int64_t val = GET_INT64_VAL(tval); if ((prev < val) ^ isMinFunc) { @@ -571,7 +647,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc, int32_t* nElems) } } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { uint64_t prev = 0; - GET_TYPED_DATA(prev, uint64_t, type, &pBuf->v); + GET_TYPED_DATA(prev, uint64_t, type, &pBuf->v, 0); uint64_t val = GET_UINT64_VAL(tval); if ((prev < val) ^ isMinFunc) { @@ -580,7 +656,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc, int32_t* nElems) } } else if (type == TSDB_DATA_TYPE_DOUBLE) { double prev = 0; - GET_TYPED_DATA(prev, double, type, &pBuf->v); + GET_TYPED_DATA(prev, double, type, &pBuf->v, 0); double val = GET_DOUBLE_VAL(tval); if ((prev < val) ^ isMinFunc) { @@ -589,13 +665,25 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc, int32_t* nElems) } } else if (type == TSDB_DATA_TYPE_FLOAT) { float prev = 0; - GET_TYPED_DATA(prev, float, type, &pBuf->v); + GET_TYPED_DATA(prev, float, type, &pBuf->v, 0); float val = GET_DOUBLE_VAL(tval); if ((prev < val) ^ isMinFunc) { GET_FLOAT_VAL(&pBuf->v) = val; code = saveRelatedTupleTag(pCtx, pInput, tval); } + } else if (type == TSDB_DATA_TYPE_DECIMAL64) { + const SDecimalOps* pOps = getDecimalOps(type); + if (pOps->lt(&pBuf->v, tval, DECIMAL_WORD_NUM(Decimal64)) ^ isMinFunc) { + DECIMAL64_SET_VALUE((Decimal64*)&pBuf->v, *(int64_t*)tval); + code =saveRelatedTupleTag(pCtx, pInput, tval); + } + } else if (type == TSDB_DATA_TYPE_DECIMAL) { + const SDecimalOps* pOps = getDecimalOps(type); + if (pOps->lt(pBuf->dec, tval, DECIMAL_WORD_NUM(Decimal128)) ^ isMinFunc) { + DECIMAL128_CLONE((Decimal128*)pBuf->dec, (Decimal128*)tval); + code =saveRelatedTupleTag(pCtx, pInput, tval); + } } } @@ -612,8 +700,8 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc, int32_t* nElems) int32_t threshold[] = { //NULL, BOOL, TINYINT, SMALLINT, INT, BIGINT, FLOAT, DOUBLE, VARCHAR, TIMESTAMP, NCHAR, INT32_MAX, INT32_MAX, 32, 16, 8, 4, 8, 4, INT32_MAX, INT32_MAX, INT32_MAX, - // UTINYINT,USMALLINT, UINT, UBIGINT, JSON, VARBINARY, DECIMAL, BLOB, MEDIUMBLOB, BINARY - 32, 16, 8, 4, INT32_MAX, INT32_MAX, INT32_MAX, INT32_MAX, INT32_MAX, INT32_MAX, + // UTINYINT,USMALLINT, UINT, UBIGINT, JSON, VARBINARY, DECIMAL, BLOB, MEDIUMBLOB, BINARY, Decimal64 + 32, 16, 8, 4, INT32_MAX, INT32_MAX, INT32_MAX, INT32_MAX, INT32_MAX, INT32_MAX, INT32_MAX, }; // clang-format on @@ -656,6 +744,12 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc, int32_t* nElems) (void)memcpy(pBuf->str, colDataGetData(pCol, i), varDataTLen(colDataGetData(pCol, i))); break; } + case TSDB_DATA_TYPE_DECIMAL64: + *(int64_t*)&pBuf->v = *(int64_t*)p; + break; + case TSDB_DATA_TYPE_DECIMAL: + (void)memcpy(pBuf->dec, p, pCol->info.bytes); + break; default: (void)memcpy(&pBuf->v, p, pCol->info.bytes); break; diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 755f37ac02..605126f19c 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -436,6 +436,7 @@ int32_t createFunctionWithSrcFunc(const char* pName, const SFunctionNode* pSrcFu (*ppFunc)->hasPk = pSrcFunc->hasPk; (*ppFunc)->pkBytes = pSrcFunc->pkBytes; + (*ppFunc)->pSrcFuncRef = pSrcFunc; (void)snprintf((*ppFunc)->functionName, sizeof((*ppFunc)->functionName), "%s", pName); (*ppFunc)->pParameterList = pParameterList; @@ -646,7 +647,7 @@ static int32_t fmCreateStateMergeFunc(SFunctionNode* pFunc, SFunctionNode** pSta SNodeList* pParams = NULL; int32_t code = nodesCloneList(pFunc->pParameterList, &pParams); if (!pParams) return code; - code = createFunction(funcMgtBuiltins[pFunc->funcId].pMergeFunc, pParams, pStateMergeFunc); + code = createFunctionWithSrcFunc(funcMgtBuiltins[pFunc->funcId].pMergeFunc, pFunc, pParams, pStateMergeFunc); if (TSDB_CODE_SUCCESS != code) { nodesDestroyList(pParams); return code; diff --git a/source/libs/function/src/tpercentile.c b/source/libs/function/src/tpercentile.c index 405049fe6f..a637ad7be7 100644 --- a/source/libs/function/src/tpercentile.c +++ b/source/libs/function/src/tpercentile.c @@ -143,7 +143,7 @@ int32_t findOnlyResult(tMemBucket *pMemBucket, double *result) { return TSDB_CODE_FUNC_PERCENTILE_ERROR; } - GET_TYPED_DATA(*result, double, pMemBucket->type, pPage->data); + GET_TYPED_DATA(*result, double, pMemBucket->type, pPage->data, pMemBucket->typeMod); return TSDB_CODE_SUCCESS; } } @@ -154,7 +154,7 @@ int32_t findOnlyResult(tMemBucket *pMemBucket, double *result) { int32_t tBucketIntHash(tMemBucket *pBucket, const void *value, int32_t *index) { int64_t v = 0; - GET_TYPED_DATA(v, int64_t, pBucket->type, value); + GET_TYPED_DATA(v, int64_t, pBucket->type, value, pBucket->typeMod); *index = -1; @@ -186,7 +186,7 @@ int32_t tBucketIntHash(tMemBucket *pBucket, const void *value, int32_t *index) { int32_t tBucketUintHash(tMemBucket *pBucket, const void *value, int32_t *index) { int64_t v = 0; - GET_TYPED_DATA(v, uint64_t, pBucket->type, value); + GET_TYPED_DATA(v, uint64_t, pBucket->type, value, pBucket->typeMod); *index = -1; @@ -268,7 +268,7 @@ static void resetSlotInfo(tMemBucket *pBucket) { } } -int32_t tMemBucketCreate(int32_t nElemSize, int16_t dataType, double minval, double maxval, bool hasWindowOrGroup, +int32_t tMemBucketCreate(int32_t nElemSize, int16_t dataType, STypeMod typeMod, double minval, double maxval, bool hasWindowOrGroup, tMemBucket **pBucket, int32_t numOfElements) { *pBucket = (tMemBucket *)taosMemoryCalloc(1, sizeof(tMemBucket)); if (*pBucket == NULL) { @@ -286,6 +286,7 @@ int32_t tMemBucketCreate(int32_t nElemSize, int16_t dataType, double minval, dou (*pBucket)->bytes = nElemSize; (*pBucket)->total = 0; (*pBucket)->times = 1; + (*pBucket)->typeMod = typeMod; (*pBucket)->maxCapacity = 200000; (*pBucket)->groupPagesMap = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); @@ -353,10 +354,11 @@ void tMemBucketDestroy(tMemBucket **pBucket) { taosMemoryFreeClear(*pBucket); } -int32_t tMemBucketUpdateBoundingBox(MinMaxEntry *r, const char *data, int32_t dataType) { +int32_t tMemBucketUpdateBoundingBox(MinMaxEntry *r, const char *data, tMemBucket* pBucket) { + int32_t dataType = pBucket->type; if (IS_SIGNED_NUMERIC_TYPE(dataType)) { int64_t v = 0; - GET_TYPED_DATA(v, int64_t, dataType, data); + GET_TYPED_DATA(v, int64_t, dataType, data, pBucket->typeMod); if (r->dMinVal > v) { r->dMinVal = v; @@ -367,7 +369,7 @@ int32_t tMemBucketUpdateBoundingBox(MinMaxEntry *r, const char *data, int32_t da } } else if (IS_UNSIGNED_NUMERIC_TYPE(dataType)) { uint64_t v = 0; - GET_TYPED_DATA(v, uint64_t, dataType, data); + GET_TYPED_DATA(v, uint64_t, dataType, data, pBucket->typeMod); if (r->u64MinVal > v) { r->u64MinVal = v; @@ -378,7 +380,7 @@ int32_t tMemBucketUpdateBoundingBox(MinMaxEntry *r, const char *data, int32_t da } } else if (IS_FLOAT_TYPE(dataType)) { double v = 0; - GET_TYPED_DATA(v, double, dataType, data); + GET_TYPED_DATA(v, double, dataType, data, pBucket->typeMod); if (r->dMinVal > v) { r->dMinVal = v; @@ -415,7 +417,7 @@ int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) { count += 1; tMemBucketSlot *pSlot = &pBucket->pSlots[index]; - code = tMemBucketUpdateBoundingBox(&pSlot->range, d, pBucket->type); + code = tMemBucketUpdateBoundingBox(&pSlot->range, d, pBucket); if (TSDB_CODE_SUCCESS != code) { return code; } @@ -572,8 +574,8 @@ int32_t getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction char *nextVal = thisVal + pMemBucket->bytes; double td = 1.0, nd = 1.0; - GET_TYPED_DATA(td, double, pMemBucket->type, thisVal); - GET_TYPED_DATA(nd, double, pMemBucket->type, nextVal); + GET_TYPED_DATA(td, double, pMemBucket->type, thisVal, pMemBucket->typeMod); + GET_TYPED_DATA(nd, double, pMemBucket->type, nextVal, pMemBucket->typeMod); *result = (1 - fraction) * td + fraction * nd; taosMemoryFreeClear(buffer); @@ -586,7 +588,7 @@ int32_t getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction } // try next round tMemBucket *tmpBucket = NULL; - int32_t code = tMemBucketCreate(pMemBucket->bytes, pMemBucket->type, pSlot->range.dMinVal, pSlot->range.dMaxVal, + int32_t code = tMemBucketCreate(pMemBucket->bytes, pMemBucket->type, pMemBucket->typeMod, pSlot->range.dMinVal, pSlot->range.dMaxVal, false, &tmpBucket, pSlot->info.size); if (TSDB_CODE_SUCCESS != code) { tMemBucketDestroy(&tmpBucket); diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 257ad3d8ea..d53430fb22 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -326,7 +326,7 @@ static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) { indexError("invalid length for node:%p, length: %d", node, LIST_LENGTH(nl->pNodeList)); SIF_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - SIF_ERR_RET(scalarGenerateSetFromList((void **)¶m->pFilter, node, nl->node.resType.type, 0)); + SIF_ERR_RET(scalarGenerateSetFromList((void **)¶m->pFilter, node, nl->node.resType.type, 0, 0)); if (taosHashPut(ctx->pRes, &node, POINTER_BYTES, param, sizeof(*param))) { taosHashCleanup(param->pFilter); param->pFilter = NULL; diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 1f5155a689..eb6e37457c 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -199,7 +199,14 @@ static int32_t valueNodeCopy(const SValueNode* pSrc, SValueNode* pDst) { memcpy(pDst->datum.p, pSrc->datum.p, len); break; } + case TSDB_DATA_TYPE_DECIMAL64: + COPY_SCALAR_FIELD(datum.d); + break; case TSDB_DATA_TYPE_DECIMAL: + pDst->datum.p = taosMemCalloc(1, pSrc->node.resType.bytes); + if (!pDst->datum.p) return terrno; + memcpy(pDst->datum.p, pSrc->datum.p, pSrc->node.resType.bytes); + break; case TSDB_DATA_TYPE_BLOB: case TSDB_DATA_TYPE_MEDIUMBLOB: default: @@ -234,6 +241,7 @@ static int32_t functionNodeCopy(const SFunctionNode* pSrc, SFunctionNode* pDst) COPY_SCALAR_FIELD(pkBytes); COPY_SCALAR_FIELD(hasOriginalFunc); COPY_SCALAR_FIELD(originalFuncId); + COPY_OBJECT_FIELD(srcFuncInputType, sizeof(SDataType)); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 012ef41699..5a318c12c8 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -4573,6 +4573,7 @@ static const char* jkFunctionPkBytes = "PkBytes"; static const char* jkFunctionIsMergeFunc = "IsMergeFunc"; static const char* jkFunctionMergeFuncOf = "MergeFuncOf"; static const char* jkFunctionTrimType = "TrimType"; +static const char* jkFunctionSrcFuncInputDT = "SrcFuncInputDataType"; static int32_t functionNodeToJson(const void* pObj, SJson* pJson) { const SFunctionNode* pNode = (const SFunctionNode*)pObj; @@ -4608,6 +4609,9 @@ static int32_t functionNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkFunctionTrimType, pNode->trimType); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkFunctionSrcFuncInputDT, dataTypeToJson, &pNode->srcFuncInputType); + } return code; } @@ -4645,6 +4649,9 @@ static int32_t jsonToFunctionNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkFunctionTrimType, pNode->trimType, code); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonToObject(pJson, jkFunctionSrcFuncInputDT, jsonToDataType, &pNode->srcFuncInputType); + } return code; } diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index 18553b1575..27646c8978 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -882,6 +882,11 @@ static int32_t datumToMsg(const void* pObj, STlvEncoder* pEncoder) { code = tlvEncodeBinary(pEncoder, VALUE_CODE_DATUM, pNode->datum.p, getJsonValueLen(pNode->datum.p)); break; case TSDB_DATA_TYPE_DECIMAL: + code = tlvEncodeBinary(pEncoder, VALUE_CODE_DATUM, pNode->datum.p, pNode->node.resType.bytes); + break; + case TSDB_DATA_TYPE_DECIMAL64: + code = tlvEncodeI64(pEncoder, VALUE_CODE_DATUM, pNode->datum.i); + break; case TSDB_DATA_TYPE_BLOB: // todo default: @@ -1000,6 +1005,17 @@ static int32_t msgToDatum(STlv* pTlv, void* pObj) { break; } case TSDB_DATA_TYPE_DECIMAL: + pNode->datum.p = taosMemoryCalloc(1, pNode->node.resType.bytes); + if (!pNode->datum.p) { + code = terrno; + break; + } + code = tlvDecodeBinary(pTlv, pNode->datum.p); + break; + case TSDB_DATA_TYPE_DECIMAL64: + code = tlvDecodeI64(pTlv, &pNode->datum.i); + *(int64_t*)&pNode->typeData = pNode->datum.i; + break; case TSDB_DATA_TYPE_BLOB: // todo default: @@ -1143,6 +1159,7 @@ enum { FUNCTION_CODE_IS_MERGE_FUNC, FUNCTION_CODE_MERGE_FUNC_OF, FUNCTION_CODE_TRIM_TYPE, + FUNCTION_SRC_FUNC_INPUT_TYPE, }; static int32_t functionNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { @@ -1179,6 +1196,9 @@ static int32_t functionNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { if (TSDB_CODE_SUCCESS == code) { code = tlvEncodeEnum(pEncoder, FUNCTION_CODE_TRIM_TYPE, pNode->trimType); } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, FUNCTION_SRC_FUNC_INPUT_TYPE, dataTypeInlineToMsg, &pNode->srcFuncInputType); + } return code; } @@ -1223,6 +1243,8 @@ static int32_t msgToFunctionNode(STlvDecoder* pDecoder, void* pObj) { case FUNCTION_CODE_TRIM_TYPE: code = tlvDecodeEnum(pTlv, &pNode->trimType, sizeof(pNode->trimType)); break; + case FUNCTION_SRC_FUNC_INPUT_TYPE: + code = tlvDecodeObjFromTlv(pTlv, msgToDataTypeInline, &pNode->srcFuncInputType); default: break; } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 4a72cdea21..6321e4381f 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -1082,7 +1082,7 @@ void nodesDestroyNode(SNode* pNode) { SValueNode* pValue = (SValueNode*)pNode; destroyExprNode((SExprNode*)pNode); taosMemoryFreeClear(pValue->literal); - if (IS_VAR_DATA_TYPE(pValue->node.resType.type)) { + if (IS_VAR_DATA_TYPE(pValue->node.resType.type) || pValue->node.resType.type == TSDB_DATA_TYPE_DECIMAL) { taosMemoryFreeClear(pValue->datum.p); } break; @@ -1651,6 +1651,7 @@ void nodesDestroyNode(SNode* pNode) { taosMemoryFreeClear(pQuery->pCmdMsg->pMsg); taosMemoryFreeClear(pQuery->pCmdMsg); } + taosMemoryFreeClear(pQuery->pResExtSchema); taosArrayDestroy(pQuery->pDbList); taosArrayDestroy(pQuery->pTableList); taosArrayDestroy(pQuery->pTargetTableList); @@ -2318,12 +2319,14 @@ void* nodesGetValueFromNode(SValueNode* pNode) { case TSDB_DATA_TYPE_UBIGINT: case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_DOUBLE: + case TSDB_DATA_TYPE_DECIMAL64: return (void*)&pNode->typeData; case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_JSON: case TSDB_DATA_TYPE_GEOMETRY: + case TSDB_DATA_TYPE_DECIMAL: return (void*)pNode->datum.p; default: break; @@ -2387,13 +2390,19 @@ int32_t nodesSetValueNodeValue(SValueNode* pNode, void* value) { case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: - case TSDB_DATA_TYPE_DECIMAL: case TSDB_DATA_TYPE_JSON: case TSDB_DATA_TYPE_BLOB: case TSDB_DATA_TYPE_MEDIUMBLOB: case TSDB_DATA_TYPE_GEOMETRY: pNode->datum.p = (char*)value; break; + case TSDB_DATA_TYPE_DECIMAL64: + pNode->datum.i = *(int64_t*)value; + pNode->typeData = *(int64_t*)value; + break; + case TSDB_DATA_TYPE_DECIMAL: + pNode->datum.p = (char*)value; + break; default: return TSDB_CODE_APP_ERROR; } @@ -3036,7 +3045,17 @@ int32_t nodesValueNodeToVariant(const SValueNode* pNode, SVariant* pVal) { code = terrno; } break; + case TSDB_DATA_TYPE_DECIMAL64: + pVal->d = pNode->datum.d; + break; case TSDB_DATA_TYPE_DECIMAL: + pVal->pz = taosMemoryCalloc(1, pVal->nLen); + if (!pVal->pz) { + code = terrno; + break; + } + memcpy(pVal->pz, pNode->datum.p, pVal->nLen); + break; case TSDB_DATA_TYPE_BLOB: // todo default: diff --git a/source/libs/parser/CMakeLists.txt b/source/libs/parser/CMakeLists.txt index 088cdc4368..50896a2715 100644 --- a/source/libs/parser/CMakeLists.txt +++ b/source/libs/parser/CMakeLists.txt @@ -33,7 +33,7 @@ target_include_directories( target_link_libraries( parser - PRIVATE os util nodes catalog function scalar geometry transport qcom + PRIVATE os util nodes catalog function scalar geometry transport qcom decimal ) if(${BUILD_TEST}) diff --git a/source/libs/parser/inc/parAst.h b/source/libs/parser/inc/parAst.h index 40c33f39a1..7a920ba7ea 100644 --- a/source/libs/parser/inc/parAst.h +++ b/source/libs/parser/inc/parAst.h @@ -192,6 +192,7 @@ SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* SDataType createDataType(uint8_t type); SDataType createVarLenDataType(uint8_t type, const SToken* pLen); +SDataType createDecimalDataType(uint8_t type, const SToken* pPrecisionToken, const SToken* pScaleToken); SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt); SNode* createAlterDatabaseOptions(SAstCreateContext* pCxt); diff --git a/source/libs/parser/inc/parInt.h b/source/libs/parser/inc/parInt.h index 4b5d106318..a8737056c8 100644 --- a/source/libs/parser/inc/parInt.h +++ b/source/libs/parser/inc/parInt.h @@ -36,7 +36,7 @@ int32_t parse(SParseContext* pParseCxt, SQuery** pQuery); int32_t collectMetaKey(SParseContext* pParseCxt, SQuery* pQuery, SParseMetaCache* pMetaCache); int32_t authenticate(SParseContext* pParseCxt, SQuery* pQuery, SParseMetaCache* pMetaCache); int32_t translate(SParseContext* pParseCxt, SQuery* pQuery, SParseMetaCache* pMetaCache); -int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema); +int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema, SExtSchema** pExtSchema); int32_t calculateConstant(SParseContext* pParseCxt, SQuery* pQuery); int32_t translatePostCreateStream(SParseContext* pParseCxt, SQuery* pQuery, SSDataBlock* pBlock); int32_t translatePostCreateSmaIndex(SParseContext* pParseCxt, SQuery* pQuery, SSDataBlock* pBlock); diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index ede31ec2a5..445883ab55 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -125,6 +125,7 @@ int32_t buildInvalidOperationMsgExt(SMsgBuf* pBuf, const char* pFormat, ...); int32_t buildSyntaxErrMsg(SMsgBuf* pBuf, const char* additionalInfo, const char* sourceStr); SSchema* getTableColumnSchema(const STableMeta* pTableMeta); +SSchemaExt* getTableColumnExtSchema(const STableMeta* pTableMeta); SSchema* getTableTagSchema(const STableMeta* pTableMeta); int32_t getNumOfColumns(const STableMeta* pTableMeta); int32_t getNumOfTags(const STableMeta* pTableMeta); @@ -186,6 +187,7 @@ int32_t getTsmaFromCache(SParseMetaCache* pMetaCache, const SName* pTsmaName, ST * @retval val range between [INT64_MIN, INT64_MAX] */ int64_t int64SafeSub(int64_t a, int64_t b); +STypeMod calcTypeMod(const SDataType* pType); #ifdef __cplusplus } diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index cea013bc2b..dccdbc26cb 100755 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -504,9 +504,9 @@ type_name(A) ::= MEDIUMBLOB. type_name(A) ::= BLOB. { A = createDataType(TSDB_DATA_TYPE_BLOB); } type_name(A) ::= VARBINARY NK_LP NK_INTEGER(B) NK_RP. { A = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &B); } type_name(A) ::= GEOMETRY NK_LP NK_INTEGER(B) NK_RP. { A = createVarLenDataType(TSDB_DATA_TYPE_GEOMETRY, &B); } -type_name(A) ::= DECIMAL. { A = createDataType(TSDB_DATA_TYPE_DECIMAL); } -type_name(A) ::= DECIMAL NK_LP NK_INTEGER NK_RP. { A = createDataType(TSDB_DATA_TYPE_DECIMAL); } -type_name(A) ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP. { A = createDataType(TSDB_DATA_TYPE_DECIMAL); } +// type_name(A) ::= DECIMAL. { A = createDataType(TSDB_DATA_TYPE_DECIMAL); } +type_name(A) ::= DECIMAL NK_LP NK_INTEGER(B) NK_RP. { A = createDecimalDataType(TSDB_DATA_TYPE_DECIMAL, &B, NULL); } +type_name(A) ::= DECIMAL NK_LP NK_INTEGER(B) NK_COMMA NK_INTEGER(C) NK_RP. { A = createDecimalDataType(TSDB_DATA_TYPE_DECIMAL, &B, &C); } %type type_name_default_len { SDataType } %destructor type_name_default_len { } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 4883c14972..1fe060f430 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -2496,6 +2496,15 @@ SDataType createVarLenDataType(uint8_t type, const SToken* pLen) { return dt; } +SDataType createDecimalDataType(uint8_t type, const SToken* pPrecisionToken, const SToken* pScaleToken) { + SDataType dt = {0}; + dt.precision = taosStr2UInt8(pPrecisionToken->z, NULL, 10); + dt.scale = pScaleToken ? taosStr2Int32(pScaleToken->z, NULL, 10) : 0; + dt.type = decimalTypeFromPrecision(dt.precision); + dt.bytes = tDataTypes[dt.type].bytes; + return dt; +} + SNode* createCreateTableStmt(SAstCreateContext* pCxt, bool ignoreExists, SNode* pRealTable, SNodeList* pCols, SNodeList* pTags, SNode* pOptions) { CHECK_PARSER_STATUS(pCxt); diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index bd463bfd9d..48637038c4 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -224,7 +224,7 @@ int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32 (void)memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length); } else { - (void)memcpy(&pVal->value.val, &(kv->value), kv->length); + valueSetDatum(&pVal->value, kv->type, &(kv->value), kv->length); } pVal->flag = CV_FLAG_VALUE; @@ -327,7 +327,7 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc TSDB_CHECK_NULL(pVal->value.pData, ret, lino, end, terrno); (void)memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length); } else { - (void)memcpy(&pVal->value.val, &(kv->value), kv->length); + valueSetDatum(&pVal->value, kv->type, &(kv->value), kv->length); } pVal->flag = CV_FLAG_VALUE; } diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 930ec66795..55b45ea471 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -19,6 +19,7 @@ #include "scalar.h" #include "tglobal.h" #include "ttime.h" +#include "decimal.h" typedef struct SInsertParseContext { SParseContext* pComCxt; @@ -1574,94 +1575,100 @@ static int32_t parseSchemaClauseTop(SInsertParseContext* pCxt, SVnodeModifyOpStm } static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, SToken* pToken, SSchema* pSchema, - int16_t timePrec, SColVal* pVal) { + const SSchemaExt* pExtSchema, int16_t timePrec, SColVal* pVal) { switch (pSchema->type) { case TSDB_DATA_TYPE_BOOL: { if ((pToken->type == TK_NK_BOOL || pToken->type == TK_NK_STRING) && (pToken->n != 0)) { if (IS_TRUE_STR(pToken->z, pToken->n)) { - pVal->value.val = TRUE_VALUE; + VALUE_SET_TRIVIAL_DATUM(&pVal->value, TRUE_VALUE); } else if (IS_FALSE_STR(pToken->z, pToken->n)) { - pVal->value.val = FALSE_VALUE; - } else if (TSDB_CODE_SUCCESS == toDoubleEx(pToken->z, pToken->n, pToken->type, (double*)&pVal->value.val)) { - *(int8_t*)(&pVal->value.val) = (*(double*)&pVal->value.val == 0 ? FALSE_VALUE : TRUE_VALUE); + VALUE_SET_TRIVIAL_DATUM(&pVal->value, FALSE_VALUE); + } else if (TSDB_CODE_SUCCESS == + toDoubleEx(pToken->z, pToken->n, pToken->type, (double*)&VALUE_GET_TRIVIAL_DATUM(&pVal->value))) { + int8_t v = (*(double*)&VALUE_GET_TRIVIAL_DATUM(&pVal->value) == 0 ? FALSE_VALUE : TRUE_VALUE); + valueSetDatum(&pVal->value, TSDB_DATA_TYPE_BOOL, &v, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); } else { return buildSyntaxErrMsg(&pCxt->msg, "invalid bool data", pToken->z); } } else if (pToken->type == TK_NK_INTEGER) { - pVal->value.val = ((taosStr2Int64(pToken->z, NULL, 10) == 0) ? FALSE_VALUE : TRUE_VALUE); + int8_t v = ((taosStr2Int64(pToken->z, NULL, 10) == 0) ? FALSE_VALUE : TRUE_VALUE); + VALUE_SET_TRIVIAL_DATUM(&pVal->value, v); } else if (pToken->type == TK_NK_FLOAT) { - pVal->value.val = ((taosStr2Double(pToken->z, NULL) == 0) ? FALSE_VALUE : TRUE_VALUE); + int8_t v = ((taosStr2Double(pToken->z, NULL) == 0) ? FALSE_VALUE : TRUE_VALUE); + VALUE_SET_TRIVIAL_DATUM(&pVal->value, v); } else if ((pToken->type == TK_NK_HEX || pToken->type == TK_NK_BIN) && - (TSDB_CODE_SUCCESS == toDoubleEx(pToken->z, pToken->n, pToken->type, (double*)&pVal->value.val))) { - *(int8_t*)(&pVal->value.val) = (*(double*)&pVal->value.val == 0 ? FALSE_VALUE : TRUE_VALUE); + (TSDB_CODE_SUCCESS == + toDoubleEx(pToken->z, pToken->n, pToken->type, (double*)&VALUE_GET_TRIVIAL_DATUM(&pVal->value)))) { + int8_t v = *(double*)&VALUE_GET_TRIVIAL_DATUM(&pVal->value) == 0 ? FALSE_VALUE : TRUE_VALUE; + valueSetDatum(&pVal->value, TSDB_DATA_TYPE_BOOL, &v, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); } else { return buildSyntaxErrMsg(&pCxt->msg, "invalid bool data", pToken->z); } break; } case TSDB_DATA_TYPE_TINYINT: { - int32_t code = toIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val); + int32_t code = toIntegerEx(pToken->z, pToken->n, pToken->type, &VALUE_GET_TRIVIAL_DATUM(&pVal->value)); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid tinyint data", pToken->z); - } else if (!IS_VALID_TINYINT(pVal->value.val)) { + } else if (!IS_VALID_TINYINT(VALUE_GET_TRIVIAL_DATUM(&pVal->value))) { return buildSyntaxErrMsg(&pCxt->msg, "tinyint data overflow", pToken->z); } break; } case TSDB_DATA_TYPE_UTINYINT: { - int32_t code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val); + int32_t code = toUIntegerEx(pToken->z, pToken->n, pToken->type, (uint64_t*)&VALUE_GET_TRIVIAL_DATUM(&pVal->value)); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned tinyint data", pToken->z); - } else if (pVal->value.val > UINT8_MAX) { + } else if (VALUE_GET_TRIVIAL_DATUM(&pVal->value) > UINT8_MAX) { return buildSyntaxErrMsg(&pCxt->msg, "unsigned tinyint data overflow", pToken->z); } break; } case TSDB_DATA_TYPE_SMALLINT: { - int32_t code = toIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val); + int32_t code = toIntegerEx(pToken->z, pToken->n, pToken->type, &VALUE_GET_TRIVIAL_DATUM(&pVal->value)); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid smallint data", pToken->z); - } else if (!IS_VALID_SMALLINT(pVal->value.val)) { + } else if (!IS_VALID_SMALLINT(VALUE_GET_TRIVIAL_DATUM(&pVal->value))) { return buildSyntaxErrMsg(&pCxt->msg, "smallint data overflow", pToken->z); } break; } case TSDB_DATA_TYPE_USMALLINT: { - int32_t code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val); + int32_t code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &VALUE_GET_TRIVIAL_DATUM(&pVal->value)); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned smallint data", pToken->z); - } else if (pVal->value.val > UINT16_MAX) { + } else if (VALUE_GET_TRIVIAL_DATUM(&pVal->value) > UINT16_MAX) { return buildSyntaxErrMsg(&pCxt->msg, "unsigned smallint data overflow", pToken->z); } break; } case TSDB_DATA_TYPE_INT: { - int32_t code = toIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val); + int32_t code = toIntegerEx(pToken->z, pToken->n, pToken->type, &VALUE_GET_TRIVIAL_DATUM(&pVal->value)); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid int data", pToken->z); - } else if (!IS_VALID_INT(pVal->value.val)) { + } else if (!IS_VALID_INT(VALUE_GET_TRIVIAL_DATUM(&pVal->value))) { return buildSyntaxErrMsg(&pCxt->msg, "int data overflow", pToken->z); } break; } case TSDB_DATA_TYPE_UINT: { - int32_t code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val); + int32_t code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &VALUE_GET_TRIVIAL_DATUM(&pVal->value)); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned int data", pToken->z); - } else if (pVal->value.val > UINT32_MAX) { + } else if (VALUE_GET_TRIVIAL_DATUM(&pVal->value) > UINT32_MAX) { return buildSyntaxErrMsg(&pCxt->msg, "unsigned int data overflow", pToken->z); } break; } case TSDB_DATA_TYPE_BIGINT: { - int32_t code = toIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val); + int32_t code = toIntegerEx(pToken->z, pToken->n, pToken->type, &VALUE_GET_TRIVIAL_DATUM(&pVal->value)); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid bigint data", pToken->z); } break; } case TSDB_DATA_TYPE_UBIGINT: { - int32_t code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val); + int32_t code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &VALUE_GET_TRIVIAL_DATUM(&pVal->value)); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned bigint data", pToken->z); } @@ -1677,7 +1684,7 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z); } float f = dv; - memcpy(&pVal->value.val, &f, sizeof(f)); + valueSetDatum(&pVal->value, TSDB_DATA_TYPE_FLOAT, &f, sizeof(f)); break; } case TSDB_DATA_TYPE_DOUBLE: { @@ -1689,7 +1696,7 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, if (isinf(dv) || isnan(dv)) { return buildSyntaxErrMsg(&pCxt->msg, "illegal double data", pToken->z); } - pVal->value.val = *(int64_t*)&dv; + VALUE_SET_TRIVIAL_DATUM(&pVal->value, (*(int64_t*)&dv)); break; } case TSDB_DATA_TYPE_BINARY: { @@ -1776,11 +1783,52 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_TIMESTAMP: { - if (parseTime(pSql, pToken, timePrec, &pVal->value.val, &pCxt->msg, pCxt->pComCxt->timezone) != TSDB_CODE_SUCCESS) { + if (parseTime(pSql, pToken, timePrec, &VALUE_GET_TRIVIAL_DATUM(&pVal->value), &pCxt->msg, + pCxt->pComCxt->timezone) != TSDB_CODE_SUCCESS) { return buildSyntaxErrMsg(&pCxt->msg, "invalid timestamp", pToken->z); } break; } + case TSDB_DATA_TYPE_DECIMAL: { + if (!pExtSchema) { + qError("Decimal type without ext schema info, cannot parse decimal values"); + return TSDB_CODE_PAR_INTERNAL_ERROR; + } + uint8_t precision = 0, scale = 0; + decimalFromTypeMod(pExtSchema->typeMod, &precision, &scale); + Decimal128 dec = {0}; + int32_t code = decimal128FromStr(pToken->z, pToken->n, precision, scale, &dec); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + + // precision check + // scale auto fit + + code = decimal128ToDataVal(&dec, &pVal->value); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + break; + } + case TSDB_DATA_TYPE_DECIMAL64: { + if (!pExtSchema) { + qError("Decimal type without ext schema info, cannot parse decimal values"); + return TSDB_CODE_PAR_INTERNAL_ERROR; + } + uint8_t precision = 0, scale = 0; + decimalFromTypeMod(pExtSchema->typeMod, &precision, &scale); + Decimal64 dec = {0}; + int32_t code = decimal64FromStr(pToken->z, pToken->n, precision, scale, &dec); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + code = decimal64ToDataVal(&dec, &pVal->value); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + break; + } default: return TSDB_CODE_FAILED; } @@ -1790,7 +1838,7 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, } static int32_t parseValueToken(SInsertParseContext* pCxt, const char** pSql, SToken* pToken, SSchema* pSchema, - int16_t timePrec, SColVal* pVal) { + const SSchemaExt* pExtSchema, int16_t timePrec, SColVal* pVal) { int32_t code = checkAndTrimValue(pToken, pCxt->tmpTokenBuf, &pCxt->msg, pSchema->type); if (TSDB_CODE_SUCCESS == code && isNullValue(pSchema->type, pToken)) { if (TSDB_DATA_TYPE_TIMESTAMP == pSchema->type && PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) { @@ -1805,7 +1853,7 @@ static int32_t parseValueToken(SInsertParseContext* pCxt, const char** pSql, STo if (pToken->n == 0 && IS_NUMERIC_TYPE(pSchema->type)) { return buildSyntaxErrMsg(&pCxt->msg, "invalid numeric data", pToken->z); } - code = parseValueTokenImpl(pCxt, pSql, pToken, pSchema, timePrec, pVal); + code = parseValueTokenImpl(pCxt, pSql, pToken, pSchema, pExtSchema, timePrec, pVal); } return code; @@ -1815,7 +1863,7 @@ static void clearColValArray(SArray* pCols) { int32_t num = taosArrayGetSize(pCols); for (int32_t i = 0; i < num; ++i) { SColVal* pCol = taosArrayGet(pCols, i); - if (IS_VAR_DATA_TYPE(pCol->value.type)) { + if (IS_VAR_DATA_TYPE(pCol->value.type) || pCol->value.type == TSDB_DATA_TYPE_DECIMAL) { taosMemoryFreeClear(pCol->value.pData); } } @@ -1905,8 +1953,9 @@ static int32_t processCtbTagsAfterCtbName(SInsertParseContext* pCxt, SVnodeModif static int32_t doGetStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, const char** ppSql, SStbRowsDataContext* pStbRowsCxt, SToken* pToken, const SBoundColInfo* pCols, - const SSchema* pSchemas, SToken* tagTokens, SSchema** tagSchemas, int* pNumOfTagTokens, - bool* bFoundTbName, bool* setCtbName, SBoundColInfo* ctbCols) { + const SSchema* pSchemas, const SSchemaExt* pExtSchemas, SToken* tagTokens, + SSchema** tagSchemas, int* pNumOfTagTokens, bool* bFoundTbName, bool* setCtbName, + SBoundColInfo* ctbCols) { int32_t code = TSDB_CODE_SUCCESS; SArray* pTagNames = pStbRowsCxt->aTagNames; SArray* pTagVals = pStbRowsCxt->aTagVals; @@ -1984,9 +2033,10 @@ static int32_t doGetStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* return buildInvalidOperationMsg(&pCxt->msg, "not support mixed bind and non-bind values"); } if (pCols->pColIndex[i] < numOfCols) { - const SSchema* pSchema = &pSchemas[pCols->pColIndex[i]]; + const SSchema* pSchema = &pSchemas[pCols->pColIndex[i]]; + const SSchemaExt* pExtSchema = pExtSchemas + pCols->pColIndex[i]; SColVal* pVal = taosArrayGet(pStbRowsCxt->aColVals, pCols->pColIndex[i]); - code = parseValueToken(pCxt, ppSql, pToken, (SSchema*)pSchema, precision, pVal); + code = parseValueToken(pCxt, ppSql, pToken, (SSchema*)pSchema, pExtSchema, precision, pVal); if (TK_NK_VARIABLE == pToken->type) { code = buildInvalidOperationMsg(&pCxt->msg, "not expected row value"); } @@ -2034,6 +2084,7 @@ static int32_t getStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS bool* setCtbName, SBoundColInfo* ctbCols) { SBoundColInfo* pCols = &pStbRowsCxt->boundColsInfo; SSchema* pSchemas = getTableColumnSchema(pStbRowsCxt->pStbMeta); + SSchemaExt* pExtSchemas = getTableColumnExtSchema(pStbRowsCxt->pStbMeta); bool bFoundTbName = false; const char* pOrigSql = *ppSql; @@ -2043,7 +2094,7 @@ static int32_t getStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS SSchema* tagSchemas[TSDB_MAX_TAGS] = {0}; int numOfTagTokens = 0; - code = doGetStbRowValues(pCxt, pStmt, ppSql, pStbRowsCxt, pToken, pCols, pSchemas, tagTokens, tagSchemas, + code = doGetStbRowValues(pCxt, pStmt, ppSql, pStbRowsCxt, pToken, pCols, pSchemas, pExtSchemas, tagTokens, tagSchemas, &numOfTagTokens, &bFoundTbName, setCtbName, ctbCols); if (code != TSDB_CODE_SUCCESS) { @@ -2238,8 +2289,9 @@ static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pSt static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataCxt* pTableCxt, bool* pGotRow, SToken* pToken) { - SBoundColInfo* pCols = &pTableCxt->boundColsInfo; - SSchema* pSchemas = getTableColumnSchema(pTableCxt->pMeta); + SBoundColInfo* pCols = &pTableCxt->boundColsInfo; + SSchema* pSchemas = getTableColumnSchema(pTableCxt->pMeta); + const SSchemaExt* pExtSchemas = getTableColumnExtSchema(pTableCxt->pMeta); int32_t code = TSDB_CODE_SUCCESS; // 1. set the parsed value from sql string @@ -2252,8 +2304,9 @@ static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataC break; } - SSchema* pSchema = &pSchemas[pCols->pColIndex[i]]; - SColVal* pVal = taosArrayGet(pTableCxt->pValues, pCols->pColIndex[i]); + SSchema* pSchema = &pSchemas[pCols->pColIndex[i]]; + const SSchemaExt* pExtSchema = pExtSchemas + pCols->pColIndex[i]; + SColVal* pVal = taosArrayGet(pTableCxt->pValues, pCols->pColIndex[i]); if (pToken->type == TK_NK_QUESTION) { pCxt->isStmtBind = true; @@ -2273,7 +2326,7 @@ static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataC } if (TSDB_CODE_SUCCESS == code) { - code = parseValueToken(pCxt, pSql, pToken, pSchema, getTableInfo(pTableCxt->pMeta).precision, pVal); + code = parseValueToken(pCxt, pSql, pToken, pSchema, pExtSchema, getTableInfo(pTableCxt->pMeta).precision, pVal); } } diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index 5306d483ca..9bb98eb223 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -1060,7 +1060,7 @@ int32_t buildBoundFields(int32_t numOfBound, int16_t* boundColumns, SSchema* pSc schema = &pSchema[boundColumns[i]]; tstrncpy((*fields)[i].name, schema->name, 65); (*fields)[i].type = schema->type; - (*fields)[i].bytes = schema->bytes; + (*fields)[i].bytes = calcTypeBytesFromSchemaBytes(schema->type, schema->bytes, true); } } diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 80be418353..fdc9815de7 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -384,7 +384,7 @@ int32_t insGetTableDataCxt(SHashObj* pHash, void* id, int32_t idLen, STableMeta* static void destroyColVal(void* p) { SColVal* pVal = p; if (TSDB_DATA_TYPE_NCHAR == pVal->value.type || TSDB_DATA_TYPE_GEOMETRY == pVal->value.type || - TSDB_DATA_TYPE_VARBINARY == pVal->value.type) { + TSDB_DATA_TYPE_VARBINARY == pVal->value.type || TSDB_DATA_TYPE_DECIMAL == pVal->value.type) { taosMemoryFreeClear(pVal->value.pData); } } @@ -907,13 +907,32 @@ static bool findFileds(SSchema* pSchema, TAOS_FIELD* fields, int numFields) { return false; } -int32_t checkSchema(SSchema* pColSchema, int8_t* fields, char* errstr, int32_t errstrLen) { +int32_t checkSchema(SSchema* pColSchema, SSchemaExt* pColExtSchema, int8_t* fields, char* errstr, int32_t errstrLen) { if (*fields != pColSchema->type) { if (errstr != NULL) snprintf(errstr, errstrLen, "column type not equal, name:%s, schema type:%s, data type:%s", pColSchema->name, tDataTypes[pColSchema->type].name, tDataTypes[*fields].name); return TSDB_CODE_INVALID_PARA; } + + if (IS_DECIMAL_TYPE(pColSchema->type)) { + uint8_t precision = 0, scale = 0; + decimalFromTypeMod(pColExtSchema->typeMod, &precision, &scale); + uint8_t precisionData = 0, scaleData = 0; + int32_t bytes = *(int32_t*)(fields + sizeof(int8_t)); + extractDecimalTypeInfoFromBytes(&bytes, &precisionData, &scaleData); + if (precision != precisionData || scale != scaleData) { + if (errstr != NULL) + snprintf(errstr, errstrLen, + "column decimal type not equal, name:%s, schema type:%s, precision:%d, scale:%d, data type:%s, " + "precision:%d, scale:%d", + pColSchema->name, tDataTypes[pColSchema->type].name, precision, scale, tDataTypes[*fields].name, + precisionData, scaleData); + return TSDB_CODE_INVALID_PARA; + } + return 0; + } + if (IS_VAR_DATA_TYPE(pColSchema->type) && *(int32_t*)(fields + sizeof(int8_t)) > pColSchema->bytes) { if (errstr != NULL) snprintf(errstr, errstrLen, @@ -935,7 +954,7 @@ int32_t checkSchema(SSchema* pColSchema, int8_t* fields, char* errstr, int32_t e } #define PRCESS_DATA(i, j) \ - ret = checkSchema(pColSchema, fields, errstr, errstrLen); \ + ret = checkSchema(pColSchema, pColExtSchema, fields, errstr, errstrLen); \ if (ret != 0) { \ goto end; \ } \ @@ -1034,7 +1053,8 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate char* pStart = p; - SSchema* pSchema = getTableColumnSchema(pTableCxt->pMeta); + SSchema* pSchema = getTableColumnSchema(pTableCxt->pMeta); + SSchemaExt* pExtSchemas = getTableColumnExtSchema(pTableCxt->pMeta); SBoundColInfo* boundInfo = &pTableCxt->boundColsInfo; if (tFields != NULL && numFields != numOfCols) { @@ -1048,12 +1068,14 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate int32_t len = TMIN(numOfCols, boundInfo->numOfBound); for (int j = 0; j < len; j++) { SSchema* pColSchema = &pSchema[j]; + SSchemaExt* pColExtSchema = &pExtSchemas[j]; PRCESS_DATA(j, j) } } else { for (int i = 0; i < numFields; i++) { for (int j = 0; j < boundInfo->numOfBound; j++) { SSchema* pColSchema = &pSchema[j]; + SSchemaExt* pColExtSchema = &pExtSchemas[j]; char* fieldName = NULL; if (raw) { fieldName = ((SSchemaWrapper*)tFields)->pSchema[i].name; diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index 215804da15..1acbeadcfc 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -89,6 +89,7 @@ static SKeyword keywordTable[] = { {"DATABASE", TK_DATABASE}, {"DATABASES", TK_DATABASES}, {"DBS", TK_DBS}, + {"DECIMAL", TK_DECIMAL}, {"DELETE", TK_DELETE}, {"DELETE_MARK", TK_DELETE_MARK}, {"DESC", TK_DESC}, diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 20a8724796..71d58e3077 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -33,6 +33,7 @@ #include "tcol.h" #include "tglobal.h" #include "ttime.h" +#include "decimal.h" #define generateDealNodeErrMsg(pCxt, code, ...) \ (pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, code, ##__VA_ARGS__), DEAL_RES_ERROR) @@ -612,7 +613,7 @@ static int32_t rewriteDropTableWithMetaCache(STranslateContext* pCxt) { int32_t metaSize = sizeof(STableMeta) + sizeof(SSchema) * (pMeta->tableInfo.numOfColumns + pMeta->tableInfo.numOfTags); int32_t schemaExtSize = - (useCompress(pMeta->tableType) && pMeta->schemaExt) ? sizeof(SSchemaExt) * pMeta->tableInfo.numOfColumns : 0; + (withExtSchema(pMeta->tableType) && pMeta->schemaExt) ? sizeof(SSchemaExt) * pMeta->tableInfo.numOfColumns : 0; const char* pTbName = (const char*)pMeta + metaSize + schemaExtSize; SName name = {0}; @@ -1357,7 +1358,7 @@ static bool hasPkInTable(const STableMeta* pTableMeta) { } static void setColumnInfoBySchema(const SRealTableNode* pTable, const SSchema* pColSchema, int32_t tagFlag, - SColumnNode* pCol) { + SColumnNode* pCol, const SSchemaExt* pExtSchema) { tstrncpy(pCol->dbName, pTable->table.dbName, TSDB_DB_NAME_LEN); tstrncpy(pCol->tableAlias, pTable->table.tableAlias, TSDB_TABLE_NAME_LEN); tstrncpy(pCol->tableName, pTable->table.tableName, TSDB_TABLE_NAME_LEN); @@ -1381,6 +1382,8 @@ static void setColumnInfoBySchema(const SRealTableNode* pTable, const SSchema* p pCol->tableHasPk = hasPkInTable(pTable->pMeta); pCol->isPk = (pCol->tableHasPk) && (pColSchema->flags & COL_IS_KEY); pCol->numOfPKs = pTable->pMeta->tableInfo.numOfPKs; + + if (pExtSchema) fillTypeFromTypeMod(&pCol->node.resType, pExtSchema->typeMod); } static int32_t setColumnInfoByExpr(STempTableNode* pTable, SExprNode* pExpr, SColumnNode** pColRef, bool joinSrc) { @@ -1478,7 +1481,10 @@ static int32_t createColumnsByTable(STranslateContext* pCxt, const STableNode* p if (TSDB_CODE_SUCCESS != code) { return generateSyntaxErrMsg(&pCxt->msgBuf, code); } - setColumnInfoBySchema((SRealTableNode*)pTable, pMeta->schema + i, (i - pMeta->tableInfo.numOfColumns), pCol); + SSchemaExt* pSchemaExt = + pMeta->schemaExt ? (i >= pMeta->tableInfo.numOfColumns ? NULL : (pMeta->schemaExt + i)) : NULL; + setColumnInfoBySchema((SRealTableNode*)pTable, pMeta->schema + i, (i - pMeta->tableInfo.numOfColumns), pCol, + pSchemaExt); setColumnPrimTs(pCxt, pCol, pTable); code = nodesListStrictAppend(pList, (SNode*)pCol); } @@ -1546,7 +1552,7 @@ static int32_t findAndSetColumn(STranslateContext* pCxt, SColumnNode** pColRef, return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, pCol->colName); } - setColumnInfoBySchema((SRealTableNode*)pTable, pMeta->schema, -1, pCol); + setColumnInfoBySchema((SRealTableNode*)pTable, pMeta->schema, -1, pCol, NULL); pCol->isPrimTs = true; *pFound = true; return TSDB_CODE_SUCCESS; @@ -1555,7 +1561,9 @@ static int32_t findAndSetColumn(STranslateContext* pCxt, SColumnNode** pColRef, for (int32_t i = 0; i < nums; ++i) { if (0 == strcmp(pCol->colName, pMeta->schema[i].name) && !invisibleColumn(pCxt->pParseCxt->enableSysInfo, pMeta->tableType, pMeta->schema[i].flags)) { - setColumnInfoBySchema((SRealTableNode*)pTable, pMeta->schema + i, (i - pMeta->tableInfo.numOfColumns), pCol); + + SSchemaExt* pSchemaExt = pMeta->schemaExt ? (i >= pMeta->tableInfo.numOfColumns ? NULL : (pMeta->schemaExt + i)) : NULL; + setColumnInfoBySchema((SRealTableNode*)pTable, pMeta->schema + i, (i - pMeta->tableInfo.numOfColumns), pCol, pSchemaExt); setColumnPrimTs(pCxt, pCol, pTable); *pFound = true; break; @@ -2438,7 +2446,7 @@ static int32_t rewriteCountStar(STranslateContext* pCxt, SFunctionNode* pCount) if (TSDB_CODE_SUCCESS == code) { if (NULL != pTable && QUERY_NODE_REAL_TABLE == nodeType(pTable)) { - setColumnInfoBySchema((SRealTableNode*)pTable, ((SRealTableNode*)pTable)->pMeta->schema, -1, pCol); + setColumnInfoBySchema((SRealTableNode*)pTable, ((SRealTableNode*)pTable)->pMeta->schema, -1, pCol, NULL); } else { code = rewriteCountStarAsCount1(pCxt, pCount); } @@ -2464,7 +2472,7 @@ static int32_t rewriteCountNotNullValue(STranslateContext* pCxt, SFunctionNode* SColumnNode* pCol = NULL; code = nodesMakeNode(QUERY_NODE_COLUMN, (SNode**)&pCol); if (TSDB_CODE_SUCCESS == code) { - setColumnInfoBySchema((SRealTableNode*)pTable, ((SRealTableNode*)pTable)->pMeta->schema, -1, pCol); + setColumnInfoBySchema((SRealTableNode*)pTable, ((SRealTableNode*)pTable)->pMeta->schema, -1, pCol, NULL); NODES_DESTORY_LIST(pCount->pParameterList); code = nodesListMakeAppend(&pCount->pParameterList, (SNode*)pCol); } @@ -2493,7 +2501,7 @@ static int32_t rewriteCountTbname(STranslateContext* pCxt, SFunctionNode* pCount SColumnNode* pCol = NULL; code = nodesMakeNode(QUERY_NODE_COLUMN, (SNode**)&pCol); if (TSDB_CODE_SUCCESS == code) { - setColumnInfoBySchema((SRealTableNode*)pTable, ((SRealTableNode*)pTable)->pMeta->schema, -1, pCol); + setColumnInfoBySchema((SRealTableNode*)pTable, ((SRealTableNode*)pTable)->pMeta->schema, -1, pCol, NULL); NODES_DESTORY_LIST(pCount->pParameterList); code = nodesListMakeAppend(&pCount->pParameterList, (SNode*)pCol); } @@ -2510,7 +2518,8 @@ static bool hasInvalidFuncNesting(SFunctionNode* pFunc) { static int32_t getFuncInfo(STranslateContext* pCxt, SFunctionNode* pFunc) { // the time precision of the function execution environment pFunc->dual = pCxt->dual; - pFunc->node.resType.precision = getPrecisionFromCurrStmt(pCxt->pCurrStmt, TSDB_TIME_PRECISION_MILLI); + if (!IS_DECIMAL_TYPE(pFunc->node.resType.type)) + pFunc->node.resType.precision = getPrecisionFromCurrStmt(pCxt->pCurrStmt, TSDB_TIME_PRECISION_MILLI); int32_t code = fmGetFuncInfo(pFunc, pCxt->msgBuf.buf, pCxt->msgBuf.len); if (TSDB_CODE_FUNC_NOT_BUILTIN_FUNTION == code) { code = getUdfInfo(pCxt, pFunc); @@ -3372,6 +3381,7 @@ static int32_t createCastFunc(STranslateContext* pCxt, SNode* pExpr, SDataType d return TSDB_CODE_OUT_OF_MEMORY; } code = getFuncInfo(pCxt, pFunc); + pFunc->node.resType = dt; if (TSDB_CODE_SUCCESS != code) { nodesClearList(pFunc->pParameterList); pFunc->pParameterList = NULL; @@ -3428,6 +3438,12 @@ static int32_t selectCommonType(SDataType* commonType, const SDataType* newType) } if (commonType->type == newType->type) { + if (IS_DECIMAL_TYPE(commonType->type)) { + if ((commonType->precision - commonType->scale) < (newType->precision - newType->scale)) { + commonType->precision = newType->precision; + commonType->scale = newType->scale; + } + } commonType->bytes = TMAX(commonType->bytes, newType->bytes); return TSDB_CODE_SUCCESS; } @@ -3438,6 +3454,11 @@ static int32_t selectCommonType(SDataType* commonType, const SDataType* newType) } else if ((resultType == TSDB_DATA_TYPE_NCHAR) && (IS_MATHABLE_TYPE(commonType->type) || IS_MATHABLE_TYPE(newType->type))) { commonType->bytes = TMAX(TMAX(commonType->bytes, newType->bytes), QUERY_NUMBER_MAX_DISPLAY_LEN * TSDB_NCHAR_SIZE); + } else if (IS_DECIMAL_TYPE(resultType)) { + if ((commonType->precision - commonType->scale) < (newType->precision - newType->scale)) { + commonType->precision = newType->precision; + commonType->scale = newType->scale; + } } else { commonType->bytes = TMAX(TMAX(commonType->bytes, newType->bytes), TYPE_BYTES[resultType]); } @@ -5518,7 +5539,7 @@ static int32_t createTags(STranslateContext* pCxt, SNodeList** pOutput) { if (TSDB_CODE_SUCCESS != code) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_OUT_OF_MEMORY); } - setColumnInfoBySchema(pTable, pTagsSchema + i, 1, pCol); + setColumnInfoBySchema(pTable, pTagsSchema + i, 1, pCol, NULL); if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(pOutput, (SNode*)pCol)) { NODES_DESTORY_LIST(*pOutput); return TSDB_CODE_OUT_OF_MEMORY; @@ -9458,7 +9479,7 @@ static int32_t columnDefNodeToField(SNodeList* pList, SArray** pArray, bool calB } else { field.bytes = pCol->dataType.bytes; } - + field.typeMod = calcTypeMod(&pCol->dataType); tstrncpy(field.name, pCol->colName, TSDB_COL_NAME_LEN); if (pCol->pOptions) { setColEncode(&field.compress, columnEncodeVal(((SColumnOptions*)pCol->pOptions)->encode)); @@ -9471,6 +9492,9 @@ static int32_t columnDefNodeToField(SNodeList* pList, SArray** pArray, bool calB if (pCol->pOptions && ((SColumnOptions*)pCol->pOptions)->bPrimaryKey) { field.flags |= COL_IS_KEY; } + if (field.typeMod != 0) { + field.flags |= COL_HAS_TYPE_MOD; + } if (NULL == taosArrayPush(*pArray, &field)) { code = terrno; break; @@ -9587,6 +9611,18 @@ static int32_t checkTableRollupOption(STranslateContext* pCxt, SNodeList* pFuncs return TSDB_CODE_SUCCESS; } +static int32_t checkDecimalDataType(STranslateContext* pCxt, SDataType pDataType) { + if (IS_DECIMAL_TYPE(pDataType.type)) { + if (pDataType.precision < TSDB_DECIMAL_MIN_PRECISION || pDataType.precision > TSDB_DECIMAL_MAX_PRECISION || + pDataType.precision < pDataType.scale) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, + "Invalid column type DECIMAL(%d,%d) , invalid precision or scale", + pDataType.precision, pDataType.scale); + } + } + return TSDB_CODE_SUCCESS; +} + static int32_t checkTableTagsSchema(STranslateContext* pCxt, SHashObj* pHash, SNodeList* pTags) { int32_t ntags = LIST_LENGTH(pTags); if (0 == ntags) { @@ -9623,6 +9659,11 @@ static int32_t checkTableTagsSchema(STranslateContext* pCxt, SHashObj* pHash, SN } else { break; } + if (TSDB_CODE_SUCCESS == code) { + if (IS_DECIMAL_TYPE(pTag->dataType.type)) { + code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, "Decimal type is not allowed for tag"); + } + } } if (TSDB_CODE_SUCCESS == code && tagsSize > TSDB_MAX_TAGS_LEN) { @@ -9694,6 +9735,10 @@ static int32_t checkTableColsSchema(STranslateContext* pCxt, SHashObj* pHash, in } } + if (TSDB_CODE_SUCCESS == code && IS_DECIMAL_TYPE(pCol->dataType.type)) { + code = checkDecimalDataType(pCxt, pCol->dataType); + } + if (TSDB_CODE_SUCCESS == code) { code = taosHashPut(pHash, pCol->colName, len, &pCol, POINTER_BYTES); } @@ -9907,6 +9952,9 @@ static void toSchema(const SColumnDefNode* pCol, col_id_t colId, SSchema* pSchem if (pCol->pOptions && ((SColumnOptions*)pCol->pOptions)->bPrimaryKey) { flags |= COL_IS_KEY; } + if (IS_DECIMAL_TYPE(pCol->dataType.type)) { + flags |= COL_HAS_TYPE_MOD; + } pSchema->colId = colId; pSchema->type = pCol->dataType.type; pSchema->bytes = calcTypeBytes(pCol->dataType); @@ -10392,9 +10440,15 @@ static int32_t buildAlterSuperTableReq(STranslateContext* pCxt, SAlterTableStmt* if (NULL == pAlterReq->pFields) { return terrno; } + pAlterReq->pTypeMods = taosArrayInit(2, sizeof(STypeMod)); + if (!pAlterReq->pTypeMods) return terrno; + STypeMod typeMod = calcTypeMod(&pStmt->dataType); switch (pStmt->alterType) { case TSDB_ALTER_TABLE_ADD_COLUMN: + if (NULL == taosArrayPush(pAlterReq->pTypeMods, &typeMod)) { + return terrno; + } // fall through case TSDB_ALTER_TABLE_ADD_TAG: case TSDB_ALTER_TABLE_DROP_TAG: case TSDB_ALTER_TABLE_DROP_COLUMN: @@ -10463,6 +10517,7 @@ static int32_t buildAlterSuperTableReq(STranslateContext* pCxt, SAlterTableStmt* } } if (NULL == taosArrayPush(pAlterReq->pFields, &field)) return terrno; + if (NULL == taosArrayPush(pAlterReq->pTypeMods, &typeMod)) return terrno; break; } default: @@ -12295,11 +12350,16 @@ static int32_t adjustDataTypeOfProjections(STranslateContext* pCxt, const STable } SSchema* pSchemas = getTableColumnSchema(pMeta); + const SSchemaExt* pExtSchemas = getTableColumnExtSchema(pMeta); int32_t index = 0; SNode* pProj = NULL; FOREACH(pProj, pProjections) { SSchema* pSchema = pSchemas + index++; SDataType dt = {.type = pSchema->type, .bytes = pSchema->bytes}; + if (IS_DECIMAL_TYPE(pSchema->type)) { + STypeMod typeMod = (pExtSchemas + (index - 1))->typeMod; + extractTypeFromTypeMod(pSchema->type, typeMod, &dt.precision, &dt.scale, NULL); + } if (!dataTypeEqual(&dt, &((SExprNode*)pProj)->resType)) { SNode* pFunc = NULL; int32_t code = createCastFunc(pCxt, pProj, dt, &pFunc); @@ -14297,12 +14357,17 @@ static int32_t translateSubquery(STranslateContext* pCxt, SNode* pNode) { return code; } -static int32_t extractQueryResultSchema(const SNodeList* pProjections, int32_t* numOfCols, SSchema** pSchema) { +static int32_t extractQueryResultSchema(const SNodeList* pProjections, int32_t* numOfCols, SSchema** pSchema, SExtSchema** ppExtSchemas) { *numOfCols = LIST_LENGTH(pProjections); *pSchema = taosMemoryCalloc((*numOfCols), sizeof(SSchema)); if (NULL == (*pSchema)) { return terrno; } + if (ppExtSchemas) *ppExtSchemas = taosMemoryCalloc(*numOfCols, sizeof(SExtSchema)); + if (ppExtSchemas && *ppExtSchemas == NULL) { + taosMemoryFreeClear(*pSchema); + return terrno; + } SNode* pNode; int32_t index = 0; @@ -14314,6 +14379,9 @@ static int32_t extractQueryResultSchema(const SNodeList* pProjections, int32_t* } else { (*pSchema)[index].type = pExpr->resType.type; (*pSchema)[index].bytes = pExpr->resType.bytes; + if (ppExtSchemas) { + (*ppExtSchemas)[index].typeMod = calcTypeMod(&pExpr->resType); + } } (*pSchema)[index].colId = index + 1; if ('\0' != pExpr->userAlias[0]) { @@ -14343,7 +14411,7 @@ static int32_t extractExplainResultSchema(int32_t* numOfCols, SSchema** pSchema) static int32_t extractDescribeResultSchema(STableMeta* pMeta, int32_t* numOfCols, SSchema** pSchema) { *numOfCols = DESCRIBE_RESULT_COLS; - if (pMeta && useCompress(pMeta->tableType)) *numOfCols = DESCRIBE_RESULT_COLS_COMPRESS; + if (pMeta && withExtSchema(pMeta->tableType)) *numOfCols = DESCRIBE_RESULT_COLS_COMPRESS; *pSchema = taosMemoryCalloc((*numOfCols), sizeof(SSchema)); if (NULL == (*pSchema)) { return terrno; @@ -14365,7 +14433,7 @@ static int32_t extractDescribeResultSchema(STableMeta* pMeta, int32_t* numOfCols (*pSchema)[3].bytes = DESCRIBE_RESULT_NOTE_LEN; tstrncpy((*pSchema)[3].name, "note", TSDB_COL_NAME_LEN); - if (pMeta && useCompress(pMeta->tableType)) { + if (pMeta && withExtSchema(pMeta->tableType)) { (*pSchema)[4].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[4].bytes = DESCRIBE_RESULT_COPRESS_OPTION_LEN; tstrncpy((*pSchema)[4].name, "encode", TSDB_COL_NAME_LEN); @@ -14488,7 +14556,7 @@ static int32_t extractCompactDbResultSchema(int32_t* numOfCols, SSchema** pSchem return TSDB_CODE_SUCCESS; } -int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema) { +int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema, SExtSchema** ppExtSchemas) { if (NULL == pRoot) { return TSDB_CODE_SUCCESS; } @@ -14496,7 +14564,7 @@ int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pS switch (nodeType(pRoot)) { case QUERY_NODE_SELECT_STMT: case QUERY_NODE_SET_OPERATOR: - return extractQueryResultSchema(getProjectList(pRoot), numOfCols, pSchema); + return extractQueryResultSchema(getProjectList(pRoot), numOfCols, pSchema, ppExtSchemas); case QUERY_NODE_EXPLAIN_STMT: return extractExplainResultSchema(numOfCols, pSchema); case QUERY_NODE_DESCRIBE_STMT: { @@ -15283,6 +15351,16 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt* return code; } } + if (IS_DECIMAL_TYPE(pColDef->dataType.type)) { + if (!req.pExtSchemas) { + req.pExtSchemas = taosMemoryCalloc(pStmt->pCols->length, sizeof(SExtSchema)); + if (NULL == req.pExtSchemas) { + tdDestroySVCreateTbReq(&req); + return terrno; + } + } + req.pExtSchemas[index].typeMod = calcTypeMod(&pColDef->dataType); + } ++index; } pBatch->info = *pVgroupInfo; @@ -16829,6 +16907,12 @@ static int32_t buildAddColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, S int8_t code = setColCompressByOption(pReq->type, columnEncodeVal(pStmt->pColOptions->encode), columnCompressVal(pStmt->pColOptions->compress), columnLevelVal(pStmt->pColOptions->compressLevel), true, &pReq->compress); + if (code != TSDB_CODE_SUCCESS) return code; + } + pReq->typeMod = 0; + if (IS_DECIMAL_TYPE(pStmt->dataType.type)) { + pReq->typeMod = decimalCalcTypeMod(pStmt->dataType.precision, pStmt->dataType.scale); + pReq->flags |= COL_HAS_TYPE_MOD; } return TSDB_CODE_SUCCESS; @@ -17969,7 +18053,8 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) { if (pQuery->haveResultSet) { taosMemoryFreeClear(pQuery->pResSchema); - if (TSDB_CODE_SUCCESS != extractResultSchema(pQuery->pRoot, &pQuery->numOfResCols, &pQuery->pResSchema)) { + taosMemoryFreeClear(pQuery->pResExtSchema); + if (TSDB_CODE_SUCCESS != extractResultSchema(pQuery->pRoot, &pQuery->numOfResCols, &pQuery->pResSchema, &pQuery->pResExtSchema)) { return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 2b02d18193..e69b9b585d 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -18,6 +18,7 @@ #include "querynodes.h" #include "tarray.h" #include "tlog.h" +#include "decimal.h" #define USER_AUTH_KEY_MAX_LEN TSDB_USER_LEN + TSDB_TABLE_FNAME_LEN + 2 @@ -295,6 +296,8 @@ int32_t buildSyntaxErrMsg(SMsgBuf* pBuf, const char* additionalInfo, const char* SSchema* getTableColumnSchema(const STableMeta* pTableMeta) { return (SSchema*)pTableMeta->schema; } +SSchemaExt* getTableColumnExtSchema(const STableMeta* pTableMeta) { return pTableMeta->schemaExt; } + static SSchema* getOneColumnSchema(const STableMeta* pTableMeta, int32_t colIndex) { SSchema* pSchema = (SSchema*)pTableMeta->schema; return &pSchema[colIndex]; @@ -1070,7 +1073,7 @@ int32_t getTableNameFromCache(SParseMetaCache* pMetaCache, const SName* pName, c int32_t metaSize = sizeof(STableMeta) + sizeof(SSchema) * (pMeta->tableInfo.numOfColumns + pMeta->tableInfo.numOfTags); int32_t schemaExtSize = - (useCompress(pMeta->tableType) && pMeta->schemaExt) ? sizeof(SSchemaExt) * pMeta->tableInfo.numOfColumns : 0; + (withExtSchema(pMeta->tableType) && pMeta->schemaExt) ? sizeof(SSchemaExt) * pMeta->tableInfo.numOfColumns : 0; const char* pTableName = (const char*)pMeta + metaSize + schemaExtSize; tstrncpy(pTbName, pTableName, TSDB_TABLE_NAME_LEN); } @@ -1401,7 +1404,7 @@ STableCfg* tableCfgDup(STableCfg* pCfg) { pNew->pSchemas = pSchema; SSchemaExt* pSchemaExt = NULL; - if (useCompress(pCfg->tableType) && pCfg->pSchemaExt) { + if (withExtSchema(pCfg->tableType) && pCfg->pSchemaExt) { int32_t schemaExtSize = pCfg->numOfColumns * sizeof(SSchemaExt); pSchemaExt = taosMemoryMalloc(schemaExtSize); if (!pSchemaExt) goto err; @@ -1503,3 +1506,10 @@ int64_t int64SafeSub(int64_t a, int64_t b) { } return res; } + +STypeMod calcTypeMod(const SDataType* pType) { + if (IS_DECIMAL_TYPE(pType->type)) { + return decimalCalcTypeMod(pType->precision, pType->scale); + } + return 0; +} diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 6d2822f038..cdf2ecb52a 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -398,7 +398,7 @@ void qDestroyParseContext(SParseContext* pCxt) { void qDestroyQuery(SQuery* pQueryNode) { nodesDestroyNode((SNode*)pQueryNode); } int32_t qExtractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema) { - return extractResultSchema(pRoot, numOfCols, pSchema); + return extractResultSchema(pRoot, numOfCols, pSchema, NULL); } int32_t qSetSTableIdForRsma(SNode* pStmt, int64_t uid) { diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index bc8566cc0c..6ba8be1838 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -5718,6 +5718,7 @@ static int32_t tableCountScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLo } static SSortLogicNode* sortNonPriKeySatisfied(SLogicNode* pNode) { + return NULL; if (QUERY_NODE_LOGIC_PLAN_SORT != nodeType(pNode)) { return NULL; } diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 20f4dd6327..7fe52ca304 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -576,7 +576,7 @@ int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst) { int32_t metaSize = sizeof(STableMeta) + numOfField * sizeof(SSchema); int32_t schemaExtSize = 0; - if (useCompress(pSrc->tableType) && pSrc->schemaExt) { + if (withExtSchema(pSrc->tableType) && pSrc->schemaExt) { schemaExtSize = pSrc->tableInfo.numOfColumns * sizeof(SSchemaExt); } *pDst = taosMemoryMalloc(metaSize + schemaExtSize); @@ -584,7 +584,7 @@ int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst) { return terrno; } memcpy(*pDst, pSrc, metaSize); - if (useCompress(pSrc->tableType) && pSrc->schemaExt) { + if (withExtSchema(pSrc->tableType) && pSrc->schemaExt) { (*pDst)->schemaExt = (SSchemaExt*)((char*)*pDst + metaSize); memcpy((*pDst)->schemaExt, pSrc->schemaExt, schemaExtSize); } else { diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index ee41909109..9d7676376a 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -532,7 +532,7 @@ int32_t queryCreateTableMetaFromMsg(STableMetaRsp *msg, bool isStb, STableMeta * QUERY_PARAM_CHECK(pMeta); int32_t total = msg->numOfColumns + msg->numOfTags; int32_t metaSize = sizeof(STableMeta) + sizeof(SSchema) * total; - int32_t schemaExtSize = (useCompress(msg->tableType) && msg->pSchemaExt) ? sizeof(SSchemaExt) * msg->numOfColumns : 0; + int32_t schemaExtSize = (withExtSchema(msg->tableType) && msg->pSchemaExt) ? sizeof(SSchemaExt) * msg->numOfColumns : 0; STableMeta *pTableMeta = taosMemoryCalloc(1, metaSize + schemaExtSize); if (NULL == pTableMeta) { @@ -553,7 +553,7 @@ int32_t queryCreateTableMetaFromMsg(STableMetaRsp *msg, bool isStb, STableMeta * pTableMeta->tableInfo.numOfColumns = msg->numOfColumns; memcpy(pTableMeta->schema, msg->pSchemas, sizeof(SSchema) * total); - if (useCompress(msg->tableType) && msg->pSchemaExt) { + if (withExtSchema(msg->tableType) && msg->pSchemaExt) { pTableMeta->schemaExt = pSchemaExt; memcpy(pSchemaExt, msg->pSchemaExt, schemaExtSize); } else { @@ -588,7 +588,7 @@ int32_t queryCreateTableMetaExFromMsg(STableMetaRsp *msg, bool isStb, STableMeta QUERY_PARAM_CHECK(pMeta); int32_t total = msg->numOfColumns + msg->numOfTags; int32_t metaSize = sizeof(STableMeta) + sizeof(SSchema) * total; - int32_t schemaExtSize = (useCompress(msg->tableType) && msg->pSchemaExt) ? sizeof(SSchemaExt) * msg->numOfColumns : 0; + int32_t schemaExtSize = (withExtSchema(msg->tableType) && msg->pSchemaExt) ? sizeof(SSchemaExt) * msg->numOfColumns : 0; int32_t tbNameSize = strlen(msg->tbName) + 1; STableMeta *pTableMeta = taosMemoryCalloc(1, metaSize + schemaExtSize + tbNameSize); @@ -610,7 +610,7 @@ int32_t queryCreateTableMetaExFromMsg(STableMetaRsp *msg, bool isStb, STableMeta pTableMeta->tableInfo.numOfColumns = msg->numOfColumns; TAOS_MEMCPY(pTableMeta->schema, msg->pSchemas, sizeof(SSchema) * total); - if (useCompress(msg->tableType) && msg->pSchemaExt) { + if (withExtSchema(msg->tableType) && msg->pSchemaExt) { pTableMeta->schemaExt = pSchemaExt; TAOS_MEMCPY(pSchemaExt, msg->pSchemaExt, schemaExtSize); } else { diff --git a/source/libs/scalar/CMakeLists.txt b/source/libs/scalar/CMakeLists.txt index 1fe0f9a18d..985fb2c2d3 100644 --- a/source/libs/scalar/CMakeLists.txt +++ b/source/libs/scalar/CMakeLists.txt @@ -16,6 +16,7 @@ target_link_libraries(scalar PRIVATE qcom PRIVATE parser PRIVATE geometry + PRIVATE decimal ) if(${BUILD_TEST}) diff --git a/source/libs/scalar/inc/sclInt.h b/source/libs/scalar/inc/sclInt.h index 8caa3edf42..457880c172 100644 --- a/source/libs/scalar/inc/sclInt.h +++ b/source/libs/scalar/inc/sclInt.h @@ -25,9 +25,10 @@ extern "C" { #include "function.h" typedef struct SOperatorValueType { - int32_t opResType; - int32_t selfType; - int32_t peerType; + int32_t opResType; + int32_t selfType; + int32_t peerType; + STypeMod selfTypeMod; } SOperatorValueType; typedef struct SScalarCtx { @@ -138,9 +139,10 @@ int32_t sclConvertValueToSclParam(SValueNode* pValueNode, SScalarParam* out, int int32_t sclCreateColumnInfoData(SDataType* pType, int32_t numOfRows, SScalarParam* pParam); int32_t sclConvertToTsValueNode(int8_t precision, SValueNode* valueNode); -#define GET_PARAM_TYPE(_c) ((_c)->columnData ? (_c)->columnData->info.type : (_c)->hashValueType) +#define GET_PARAM_TYPE(_c) ((_c)->columnData ? (_c)->columnData->info.type : (_c)->filterValueType) #define GET_PARAM_BYTES(_c) ((_c)->columnData->info.bytes) #define GET_PARAM_PRECISON(_c) ((_c)->columnData->info.precision) +#define GET_PARAM_SCALE(_c) ((_c)->columnData->info.scale) void sclFreeParam(SScalarParam* param); int32_t doVectorCompare(SScalarParam* pLeft, SScalarParam *pLeftVar, SScalarParam* pRight, SScalarParam *pOut, int32_t startIndex, int32_t numOfRows, diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index a711b78868..990216c01f 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ #include +#include "decimal.h" #include "nodes.h" #include "os.h" #include "tglobal.h" @@ -136,7 +137,8 @@ __compar_fn_t gDataCompare[] = { setChkInBytes2, setChkInBytes4, setChkInBytes8, comparestrRegexMatch, comparestrRegexNMatch, setChkNotInBytes1, setChkNotInBytes2, setChkNotInBytes4, setChkNotInBytes8, compareChkNotInString, comparestrPatternNMatch, comparewcsPatternNMatch, - comparewcsRegexMatch, comparewcsRegexNMatch, compareLenBinaryVal + comparewcsRegexMatch, comparewcsRegexNMatch, compareLenBinaryVal, compareDecimal64, + compareDecimal128, setChkInDecimalHash, setChkNotInDecimalHash, }; __compar_fn_t gInt8SignCompare[] = {compareInt8Val, compareInt8Int16, compareInt8Int32, @@ -209,6 +211,10 @@ int32_t filterGetCompFuncIdx(int32_t type, int32_t optr, int8_t *comparFn, bool *comparFn = 0; code = TSDB_CODE_QRY_JSON_IN_ERROR; break; + case TSDB_DATA_TYPE_DECIMAL64: + case TSDB_DATA_TYPE_DECIMAL: + *comparFn = 33; + break; default: *comparFn = 0; break; @@ -242,6 +248,10 @@ int32_t filterGetCompFuncIdx(int32_t type, int32_t optr, int8_t *comparFn, bool *comparFn = 0; code = TSDB_CODE_QRY_JSON_IN_ERROR; break; + case TSDB_DATA_TYPE_DECIMAL64: + case TSDB_DATA_TYPE_DECIMAL: + *comparFn = 34; + break; default: *comparFn = 0; break; @@ -350,7 +360,12 @@ int32_t filterGetCompFuncIdx(int32_t type, int32_t optr, int8_t *comparFn, bool case TSDB_DATA_TYPE_UBIGINT: *comparFn = 14; break; - + case TSDB_DATA_TYPE_DECIMAL64: + *comparFn = 31; + break; + case TSDB_DATA_TYPE_DECIMAL: + *comparFn = 32; + break; default: *comparFn = 0; break; @@ -362,7 +377,7 @@ int32_t filterGetCompFuncIdx(int32_t type, int32_t optr, int8_t *comparFn, bool int32_t filterGetCompFunc(__compar_fn_t *func, int32_t type, int32_t optr) { int8_t compFuncIdx = 0; int32_t code = filterGetCompFuncIdx(type, optr, &compFuncIdx, true); - *func = gDataCompare[compFuncIdx]; + if (TSDB_CODE_SUCCESS == code) *func = gDataCompare[compFuncIdx]; return code; } @@ -2225,7 +2240,7 @@ int32_t fltInitValFieldData(SFilterInfo *info) { } if (unit->compare.optr == OP_TYPE_IN) { - FLT_ERR_RET(scalarGenerateSetFromList((void **)&fi->data, fi->desc, type, 0)); + FLT_ERR_RET(scalarGenerateSetFromList((void **)&fi->data, fi->desc, type, 0, 0)); if (fi->data == NULL) { fltError("failed to convert in param"); FLT_ERR_RET(TSDB_CODE_APP_ERROR); @@ -3908,6 +3923,8 @@ typedef enum { FLT_SCL_DATUM_KIND_FLOAT64, FLT_SCL_DATUM_KIND_VARCHAR, FLT_SCL_DATUM_KIND_NCHAR, + FLT_SCL_DATUM_KIND_DECIMAL64, + FLT_SCL_DATUM_KIND_DECIMAL, FLT_SCL_DATUM_KIND_MAX, } SFltSclDatumKind; @@ -3918,6 +3935,7 @@ typedef struct { uint64_t u; // for uint double d; // for double uint8_t *pData; // for varchar, nchar, len prefixed + Decimal dec; // for decimal }; SDataType type; // TODO: original data type, may not be used? } SFltSclDatum; @@ -3938,6 +3956,14 @@ int32_t fltSclCompareWithFloat64(SFltSclDatum *val1, SFltSclDatum *val2) { case FLT_SCL_DATUM_KIND_FLOAT64: { return compareDoubleVal(&val1->d, &val2->d); } + case FLT_SCL_DATUM_KIND_DECIMAL64: { + double d = doubleFromDecimal64(&val1->i, val1->type.precision, val1->type.scale); + return compareDoubleVal(&d, &val2->d); + } + case FLT_SCL_DATUM_KIND_DECIMAL: { + double d = doubleFromDecimal128(&val1->dec, val1->type.precision, val1->type.scale); + return compareDoubleVal(&d, &val2->d); + } // TODO: varchar, nchar default: qError("not supported comparsion. kind1 %d, kind2 %d", val1->kind, val2->kind); @@ -3979,12 +4005,25 @@ int32_t fltSclCompareWithUInt64(SFltSclDatum *val1, SFltSclDatum *val2) { } } +int32_t fltSclCompareWithDecimal(void* pData1, const SDataType* pType1, void* pData2, const SDataType* pType2) { + SDecimalCompareCtx ctx1 = {.pData = pData1, .type =pType1->type, .typeMod = decimalCalcTypeMod(pType1->precision, pType1->scale)}, + ctx2 = {.pData = pData2, .type = pType2->type, .typeMod = decimalCalcTypeMod(pType2->precision, pType2->scale)}; + if (decimalCompare(OP_TYPE_GREATER_THAN, &ctx1, &ctx2)) return 1; + if (decimalCompare(OP_TYPE_EQUAL, &ctx1, &ctx2)) return 0; + return -1; +} + int32_t fltSclCompareDatum(SFltSclDatum *val1, SFltSclDatum *val2) { if (val2->kind == FLT_SCL_DATUM_KIND_NULL || val2->kind == FLT_SCL_DATUM_KIND_MIN || val2->kind == FLT_SCL_DATUM_KIND_MAX) { return (val1->kind < val2->kind) ? -1 : ((val1->kind > val2->kind) ? 1 : 0); } + if (val1->kind == FLT_SCL_DATUM_KIND_NULL || val1->kind == FLT_SCL_DATUM_KIND_MIN || + val1->kind == FLT_SCL_DATUM_KIND_MAX) { + return (val1->kind < val2->kind) ? -1 : ((val1->kind > val2->kind) ? 1 : 0); + } + switch (val2->kind) { case FLT_SCL_DATUM_KIND_UINT64: { return fltSclCompareWithUInt64(val1, val2); @@ -3995,7 +4034,14 @@ int32_t fltSclCompareDatum(SFltSclDatum *val1, SFltSclDatum *val2) { case FLT_SCL_DATUM_KIND_FLOAT64: { return fltSclCompareWithFloat64(val1, val2); } - // TODO: varchar/nchar + case FLT_SCL_DATUM_KIND_DECIMAL64: { + void* pData1 = val1->kind == FLT_SCL_DATUM_KIND_DECIMAL64 ? (void*)&val1->i : (void*)&val1->dec; + return fltSclCompareWithDecimal(pData1, &val1->type, &val2->i, &val2->type); + } + case FLT_SCL_DATUM_KIND_DECIMAL: { + void* pData1 = val1->kind == FLT_SCL_DATUM_KIND_DECIMAL64 ? (void*)&val1->i : (void*)&val1->dec; + return fltSclCompareWithDecimal(pData1, &val1->type, &val2->dec, &val2->type); + } default: qError("not supported kind when compare datum. kind2 : %d", val2->kind); return 0; @@ -4143,7 +4189,80 @@ int32_t fltSclGetOrCreateColumnRange(SColumnNode *colNode, SArray *colRangeList, return TSDB_CODE_SUCCESS; } -int32_t fltSclBuildDatumFromValueNode(SFltSclDatum *datum, SValueNode *valNode) { +static int32_t fltSclBuildDecimalDatumFromValueNode(SFltSclDatum* datum, SColumnNode* pColNode, SValueNode* valNode) { + datum->type = pColNode->node.resType; + SDataType valDt = valNode->node.resType; + if (valNode->isNull) { + datum->kind = FLT_SCL_DATUM_KIND_NULL; + } else { + void* pInput = NULL; + switch (valNode->node.resType.type) { + case TSDB_DATA_TYPE_NULL: + datum->kind = FLT_SCL_DATUM_KIND_NULL; + FLT_RET(0); + case TSDB_DATA_TYPE_BOOL: + pInput = &valNode->datum.b; + break; + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + pInput = &valNode->datum.i; + break; + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_UBIGINT: + pInput = &valNode->datum.u; + break; + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: + datum->kind = FLT_SCL_DATUM_KIND_FLOAT64; + datum->type = valDt; + datum->d = valNode->datum.d; + FLT_RET(0); + case TSDB_DATA_TYPE_VARCHAR: + datum->kind = FLT_SCL_DATUM_KIND_FLOAT64; + datum->type.type = TSDB_DATA_TYPE_DOUBLE; + datum->type.bytes = DOUBLE_BYTES; + datum->d = taosStr2Double(valNode->literal, NULL); + FLT_RET(0); + case TSDB_DATA_TYPE_DECIMAL64: + datum->kind = FLT_SCL_DATUM_KIND_DECIMAL64; + datum->type = valDt; + datum->i = valNode->datum.i; + FLT_RET(0); + case TSDB_DATA_TYPE_DECIMAL: + datum->kind = FLT_SCL_DATUM_KIND_DECIMAL; + datum->type = valDt; + datum->dec = *(Decimal*)valNode->datum.p; + FLT_RET(0); + default: + qError("not supported type %d when build decimal datum from value node", valNode->node.resType.type); + return TSDB_CODE_INVALID_PARA; + } + + void *pData = NULL; + if (datum->type.type == TSDB_DATA_TYPE_DECIMAL64) { + pData = &datum->i; + datum->kind = FLT_SCL_DATUM_KIND_DECIMAL64; + } else if (datum->type.type == TSDB_DATA_TYPE_DECIMAL) { + pData = &datum->dec; + datum->kind = FLT_SCL_DATUM_KIND_DECIMAL; + } + if (datum->kind == FLT_SCL_DATUM_KIND_DECIMAL64 || datum->kind == FLT_SCL_DATUM_KIND_DECIMAL) { + int32_t code = convertToDecimal(pInput, &valDt, pData, &datum->type); + if (TSDB_CODE_SUCCESS != code) return code; + } + } + FLT_RET(0); +} + +int32_t fltSclBuildDatumFromValueNode(SFltSclDatum *datum, SColumnNode* pColNode, SValueNode *valNode) { + if (IS_DECIMAL_TYPE(pColNode->node.resType.type)) { + return fltSclBuildDecimalDatumFromValueNode(datum, pColNode, valNode); + } datum->type = valNode->node.resType; if (valNode->isNull) { @@ -4192,7 +4311,7 @@ int32_t fltSclBuildDatumFromValueNode(SFltSclDatum *datum, SValueNode *valNode) return TSDB_CODE_SUCCESS; } -int32_t fltSclBuildDatumFromBlockSmaValue(SFltSclDatum *datum, uint8_t type, int64_t val) { +int32_t fltSclBuildDatumFromBlockSmaValue(SFltSclDatum *datum, uint8_t type, void* val) { switch (type) { case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: @@ -4201,7 +4320,7 @@ int32_t fltSclBuildDatumFromBlockSmaValue(SFltSclDatum *datum, uint8_t type, int case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_TIMESTAMP: { datum->kind = FLT_SCL_DATUM_KIND_INT64; - datum->i = val; + datum->i = *(int64_t*)val; break; } case TSDB_DATA_TYPE_UTINYINT: @@ -4209,15 +4328,24 @@ int32_t fltSclBuildDatumFromBlockSmaValue(SFltSclDatum *datum, uint8_t type, int case TSDB_DATA_TYPE_UINT: case TSDB_DATA_TYPE_UBIGINT: { datum->kind = FLT_SCL_DATUM_KIND_UINT64; - datum->u = *(uint64_t *)&val; + datum->u = *(uint64_t *)val; break; } case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_DOUBLE: { datum->kind = FLT_SCL_DATUM_KIND_FLOAT64; - datum->d = *(double *)&val; + datum->d = *(double *)val; break; } + case TSDB_DATA_TYPE_DECIMAL64: + datum->kind = FLT_SCL_DATUM_KIND_DECIMAL64; + datum->u = *(uint64_t *)val; + break; + case TSDB_DATA_TYPE_DECIMAL: + datum->kind = FLT_SCL_DATUM_KIND_DECIMAL; + datum->dec = *(Decimal *)val; + break; + // TODO:varchar/nchar/json default: { datum->kind = FLT_SCL_DATUM_KIND_NULL; @@ -4254,11 +4382,14 @@ int32_t fltSclBuildRangeFromBlockSma(SFltSclColumnRange *colRange, SColumnDataAg FLT_ERR_RET(terrno); } } + int8_t type = colRange->colNode->node.resType.type; SFltSclDatum min = {0}; - FLT_ERR_RET(fltSclBuildDatumFromBlockSmaValue(&min, colRange->colNode->node.resType.type, pAgg->min)); + min.type = colRange->colNode->node.resType; + FLT_ERR_RET(fltSclBuildDatumFromBlockSmaValue(&min, type, COL_AGG_GET_MIN_PTR(pAgg, type))); SFltSclPoint minPt = {.excl = false, .start = true, .val = min}; SFltSclDatum max = {0}; - FLT_ERR_RET(fltSclBuildDatumFromBlockSmaValue(&max, colRange->colNode->node.resType.type, pAgg->max)); + max.type = min.type; + FLT_ERR_RET(fltSclBuildDatumFromBlockSmaValue(&max, type, COL_AGG_GET_MAX_PTR(pAgg, type))); SFltSclPoint maxPt = {.excl = false, .start = false, .val = max}; if (NULL == taosArrayPush(points, &minPt)) { FLT_ERR_RET(terrno); @@ -4718,12 +4849,19 @@ EDealRes fltReviseRewriter(SNode **pNode, void *pContext) { stat->scalarMode = true; return DEAL_RES_CONTINUE; } + if (IS_DECIMAL_TYPE(valueNode->node.resType.type)) { + stat->scalarMode = true; + return DEAL_RES_CONTINUE; + } return DEAL_RES_CONTINUE; } if (QUERY_NODE_COLUMN == nodeType(*pNode)) { SColumnNode *colNode = (SColumnNode *)*pNode; stat->precision = colNode->node.resType.precision; + if (IS_DECIMAL_TYPE(colNode->node.resType.type)) { + stat->scalarMode = true; + } return DEAL_RES_CONTINUE; } @@ -4738,7 +4876,8 @@ EDealRes fltReviseRewriter(SNode **pNode, void *pContext) { uint8_t type = valueNode->node.resType.type; SNode *node = NULL; FOREACH(node, listNode->pNodeList) { - if (type != ((SValueNode *)node)->node.resType.type) { + uint8_t nodeT = ((SExprNode*)node)->resType.type; + if (type != nodeT || IS_DECIMAL_TYPE(nodeT)) { stat->scalarMode = true; return DEAL_RES_CONTINUE; } @@ -4902,11 +5041,69 @@ _return: FLT_RET(code); } +static int32_t fltSclBuildRangePointsForInOper(SFltSclOperator* oper, SArray* points) { + SNodeListNode *listNode = (SNodeListNode *)oper->valNode; + SFltSclDatum minDatum = {.kind = FLT_SCL_DATUM_KIND_INT64, .i = INT64_MAX, .type = oper->colNode->node.resType}; + SFltSclDatum maxDatum = {.kind = FLT_SCL_DATUM_KIND_INT64, .i = INT64_MIN, .type = oper->colNode->node.resType}; + SNode* nodeItem = NULL; + FOREACH(nodeItem, listNode->pNodeList) { + SValueNode *valueNode = (SValueNode *)nodeItem; + SFltSclDatum valDatum; + FLT_ERR_RET(fltSclBuildDatumFromValueNode(&valDatum, oper->colNode, valueNode)); + if (valDatum.kind == FLT_SCL_DATUM_KIND_NULL) { + continue; + } + if (IS_DECIMAL_TYPE(oper->colNode->node.resType.type)) { + if (IS_DECIMAL_TYPE(valDatum.type.type)) { + double v = valDatum.type.type == TSDB_DATA_TYPE_DECIMAL64 + ? doubleFromDecimal64(&valDatum.i, valDatum.type.precision, valDatum.type.scale) + : doubleFromDecimal128(&valDatum.dec, valDatum.type.precision, valDatum.type.scale); + if (minDatum.kind == FLT_SCL_DATUM_KIND_FLOAT64) { + minDatum.d = TMIN(v, minDatum.d); + maxDatum.d = TMAX(v, maxDatum.d); + } else if (minDatum.kind == FLT_SCL_DATUM_KIND_INT64) { + minDatum.d = v; + maxDatum.d = v; + minDatum.kind = FLT_SCL_DATUM_KIND_FLOAT64; + maxDatum.kind = FLT_SCL_DATUM_KIND_FLOAT64; + } + } else if (valDatum.kind == FLT_SCL_DATUM_KIND_FLOAT64) { + if (minDatum.kind == FLT_SCL_DATUM_KIND_INT64) { + minDatum.kind = FLT_SCL_DATUM_KIND_FLOAT64; + maxDatum.kind = FLT_SCL_DATUM_KIND_FLOAT64; + minDatum.d = TMIN(valDatum.d, minDatum.d); + maxDatum.d = TMAX(valDatum.d, maxDatum.d); + } else { + minDatum.d = TMIN(valDatum.d, minDatum.d); + maxDatum.d = TMAX(valDatum.d, maxDatum.d); + } + } + continue; + } + if(valueNode->node.resType.type == TSDB_DATA_TYPE_FLOAT || valueNode->node.resType.type == TSDB_DATA_TYPE_DOUBLE) { + minDatum.i = TMIN(minDatum.i, valDatum.d); + maxDatum.i = TMAX(maxDatum.i, valDatum.d); + } else { + minDatum.i = TMIN(minDatum.i, valDatum.i); + maxDatum.i = TMAX(maxDatum.i, valDatum.i); + } + } + SFltSclPoint startPt = {.start = true, .excl = false, .val = minDatum}; + SFltSclPoint endPt = {.start = false, .excl = false, .val = maxDatum}; + if (NULL == taosArrayPush(points, &startPt)) { + FLT_ERR_RET(terrno); + } + if (NULL == taosArrayPush(points, &endPt)) { + FLT_ERR_RET(terrno); + } + FLT_RET(0); +} + int32_t fltSclBuildRangePoints(SFltSclOperator *oper, SArray *points) { switch (oper->type) { case OP_TYPE_GREATER_THAN: { SFltSclDatum start; - FLT_ERR_RET(fltSclBuildDatumFromValueNode(&start, oper->valNode)); + FLT_ERR_RET(fltSclBuildDatumFromValueNode(&start, oper->colNode, oper->valNode)); SFltSclPoint startPt = {.start = true, .excl = true, .val = start}; SFltSclDatum end = {.kind = FLT_SCL_DATUM_KIND_MAX, .type = oper->colNode->node.resType}; SFltSclPoint endPt = {.start = false, .excl = false, .val = end}; @@ -4920,7 +5117,7 @@ int32_t fltSclBuildRangePoints(SFltSclOperator *oper, SArray *points) { } case OP_TYPE_GREATER_EQUAL: { SFltSclDatum start; - FLT_ERR_RET(fltSclBuildDatumFromValueNode(&start, oper->valNode)); + FLT_ERR_RET(fltSclBuildDatumFromValueNode(&start, oper->colNode, oper->valNode)); SFltSclPoint startPt = {.start = true, .excl = false, .val = start}; SFltSclDatum end = {.kind = FLT_SCL_DATUM_KIND_MAX, .type = oper->colNode->node.resType}; SFltSclPoint endPt = {.start = false, .excl = false, .val = end}; @@ -4934,7 +5131,7 @@ int32_t fltSclBuildRangePoints(SFltSclOperator *oper, SArray *points) { } case OP_TYPE_LOWER_THAN: { SFltSclDatum end; - FLT_ERR_RET(fltSclBuildDatumFromValueNode(&end, oper->valNode)); + FLT_ERR_RET(fltSclBuildDatumFromValueNode(&end, oper->colNode, oper->valNode)); SFltSclPoint endPt = {.start = false, .excl = true, .val = end}; SFltSclDatum start = {.kind = FLT_SCL_DATUM_KIND_MIN, .type = oper->colNode->node.resType}; SFltSclPoint startPt = {.start = true, .excl = false, .val = start}; @@ -4948,7 +5145,7 @@ int32_t fltSclBuildRangePoints(SFltSclOperator *oper, SArray *points) { } case OP_TYPE_LOWER_EQUAL: { SFltSclDatum end; - FLT_ERR_RET(fltSclBuildDatumFromValueNode(&end, oper->valNode)); + FLT_ERR_RET(fltSclBuildDatumFromValueNode(&end, oper->colNode, oper->valNode)); SFltSclPoint endPt = {.start = false, .excl = false, .val = end}; SFltSclDatum start = {.kind = FLT_SCL_DATUM_KIND_MIN, .type = oper->colNode->node.resType}; SFltSclPoint startPt = {.start = true, .excl = false, .val = start}; @@ -4962,7 +5159,7 @@ int32_t fltSclBuildRangePoints(SFltSclOperator *oper, SArray *points) { } case OP_TYPE_EQUAL: { SFltSclDatum valDatum; - FLT_ERR_RET(fltSclBuildDatumFromValueNode(&valDatum, oper->valNode)); + FLT_ERR_RET(fltSclBuildDatumFromValueNode(&valDatum, oper->colNode, oper->valNode)); SFltSclPoint startPt = {.start = true, .excl = false, .val = valDatum}; SFltSclPoint endPt = {.start = false, .excl = false, .val = valDatum}; if (NULL == taosArrayPush(points, &startPt)) { @@ -4975,7 +5172,7 @@ int32_t fltSclBuildRangePoints(SFltSclOperator *oper, SArray *points) { } case OP_TYPE_NOT_EQUAL: { SFltSclDatum valDatum; - FLT_ERR_RET(fltSclBuildDatumFromValueNode(&valDatum, oper->valNode)); + FLT_ERR_RET(fltSclBuildDatumFromValueNode(&valDatum, oper->colNode, oper->valNode)); { SFltSclDatum start = {.kind = FLT_SCL_DATUM_KIND_MIN, .type = oper->colNode->node.resType}; SFltSclPoint startPt = {.start = true, .excl = false, .val = start}; @@ -5026,31 +5223,8 @@ int32_t fltSclBuildRangePoints(SFltSclOperator *oper, SArray *points) { break; } case OP_TYPE_IN: { - SNodeListNode *listNode = (SNodeListNode *)oper->valNode; - SFltSclDatum minDatum = {.kind = FLT_SCL_DATUM_KIND_INT64, .i = INT64_MAX, .type = oper->colNode->node.resType}; - SFltSclDatum maxDatum = {.kind = FLT_SCL_DATUM_KIND_INT64, .i = INT64_MIN, .type = oper->colNode->node.resType}; - SNode* nodeItem = NULL; - FOREACH(nodeItem, listNode->pNodeList) { - SValueNode *valueNode = (SValueNode *)nodeItem; - SFltSclDatum valDatum; - FLT_ERR_RET(fltSclBuildDatumFromValueNode(&valDatum, valueNode)); - if(valueNode->node.resType.type == TSDB_DATA_TYPE_FLOAT || valueNode->node.resType.type == TSDB_DATA_TYPE_DOUBLE) { - minDatum.i = TMIN(minDatum.i, valDatum.d); - maxDatum.i = TMAX(maxDatum.i, valDatum.d); - } else { - minDatum.i = TMIN(minDatum.i, valDatum.i); - maxDatum.i = TMAX(maxDatum.i, valDatum.i); - } - } - SFltSclPoint startPt = {.start = true, .excl = false, .val = minDatum}; - SFltSclPoint endPt = {.start = false, .excl = false, .val = maxDatum}; - if (NULL == taosArrayPush(points, &startPt)) { - FLT_ERR_RET(terrno); - } - if (NULL == taosArrayPush(points, &endPt)) { - FLT_ERR_RET(terrno); - } - break; + FLT_ERR_RET(fltSclBuildRangePointsForInOper(oper, points)); + break; } default: { qError("not supported operator type : %d when build range points", oper->type); @@ -5075,7 +5249,11 @@ static int32_t fltSclProcessCNF(SFilterInfo *pInfo, SArray *sclOpListCNF, SArray if (NULL == points) { FLT_ERR_RET(terrno); } - FLT_ERR_RET(fltSclBuildRangePoints(sclOper, points)); + int32_t code = fltSclBuildRangePoints(sclOper, points); + if (code != 0) { + taosArrayDestroy(points); + FLT_ERR_RET(code); + } if (taosArrayGetSize(colRange->points) != 0) { SArray *merged = taosArrayInit(4, sizeof(SFltSclPoint)); if (NULL == merged) { @@ -5198,16 +5376,25 @@ int32_t fltOptimizeNodes(SFilterInfo *pInfo, SNode **pNode, SFltTreeStat *pStat) } FLT_ERR_JRET(fltSclProcessCNF(pInfo, sclOpList, colRangeList)); pInfo->sclCtx.fltSclRange = colRangeList; + colRangeList = NULL; +_return: for (int32_t i = 0; i < taosArrayGetSize(sclOpList); ++i) { SFltSclOperator *sclOp = taosArrayGet(sclOpList, i); if (NULL == sclOp) { - FLT_ERR_JRET(TSDB_CODE_OUT_OF_RANGE); + code = TSDB_CODE_OUT_OF_RANGE; + break; } nodesDestroyNode((SNode *)sclOp->colNode); nodesDestroyNode((SNode *)sclOp->valNode); } -_return: + + for (int32_t i = 0; i < taosArrayGetSize(colRangeList); ++i) { + SFltSclColumnRange *colRange = taosArrayGet(colRangeList, i); + nodesDestroyNode((SNode *)colRange->colNode); + taosArrayDestroy(colRange->points); + } + taosArrayDestroy(colRangeList); taosArrayDestroy(sclOpList); return code; } diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 7dd3a8e9e3..d78f8ad44c 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -10,6 +10,7 @@ #include "tdatablock.h" #include "ttime.h" #include "tudf.h" +#include "decimal.h" int32_t scalarGetOperatorParamNum(EOperatorType type) { if (OP_TYPE_IS_NULL == type || OP_TYPE_IS_NOT_NULL == type || OP_TYPE_IS_TRUE == type || @@ -117,7 +118,7 @@ _return: } // processType = 0 means all type. 1 means number, 2 means var, 3 means float, 4 means var&integer -int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type, int8_t processType) { +int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type, STypeMod typeMod, int8_t processType) { SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(type), true, false); if (NULL == pObj) { sclError("taosHashInit failed, size:%d", 256); @@ -154,6 +155,7 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type, int8_ } } else { out.columnData->info.bytes = tDataTypes[type].bytes; + extractTypeFromTypeMod(type, typeMod, &out.columnData->info.precision, &out.columnData->info.scale, NULL); } int32_t overflow = 0; @@ -378,8 +380,9 @@ int32_t sclInitParam(SNode *node, SScalarParam *param, SScalarCtx *ctx, int32_t SCL_RET(TSDB_CODE_QRY_INVALID_INPUT); } - int32_t type = ctx->type.selfType; - SNode* nodeItem = NULL; + int32_t type = ctx->type.selfType; + STypeMod typeMod = 0; + SNode *nodeItem = NULL; FOREACH(nodeItem, nodeList->pNodeList) { SValueNode *valueNode = (SValueNode *)nodeItem; int32_t tmp = vectorGetConvertType(type, valueNode->node.resType.type); @@ -391,18 +394,26 @@ int32_t sclInitParam(SNode *node, SScalarParam *param, SScalarCtx *ctx, int32_t if (IS_NUMERIC_TYPE(type)){ ctx->type.peerType = type; } + // Currently, all types of node list can't be decimal types. + // Decimal op double/float/nchar/varchar types will convert these types to double type. + // All other types do not need scale info, so here we use self scale + // When decimal types are supported in value list, we need to check convertability of different decimal types. + // And the new decimal scale will also be calculated. + if (IS_DECIMAL_TYPE(type)) + typeMod = decimalCalcTypeMod(TSDB_DECIMAL_MAX_PRECISION, getScaleFromTypeMod(type, ctx->type.selfTypeMod)); type = ctx->type.peerType; if (IS_VAR_DATA_TYPE(ctx->type.selfType) && IS_NUMERIC_TYPE(type)){ - SCL_ERR_RET(scalarGenerateSetFromList((void **)¶m->pHashFilter, node, type, 1)); - SCL_ERR_RET(scalarGenerateSetFromList((void **)¶m->pHashFilterOthers, node, ctx->type.selfType, 2)); + SCL_ERR_RET(scalarGenerateSetFromList((void **)¶m->pHashFilter, node, type, typeMod, 1)); + SCL_ERR_RET(scalarGenerateSetFromList((void **)¶m->pHashFilterOthers, node, ctx->type.selfType, typeMod, 2)); } else if (IS_INTEGER_TYPE(ctx->type.selfType) && IS_FLOAT_TYPE(type)){ - SCL_ERR_RET(scalarGenerateSetFromList((void **)¶m->pHashFilter, node, type, 3)); - SCL_ERR_RET(scalarGenerateSetFromList((void **)¶m->pHashFilterOthers, node, ctx->type.selfType, 4)); + SCL_ERR_RET(scalarGenerateSetFromList((void **)¶m->pHashFilter, node, type, typeMod, 2)); + SCL_ERR_RET(scalarGenerateSetFromList((void **)¶m->pHashFilterOthers, node, ctx->type.selfType, typeMod, 4)); } else { - SCL_ERR_RET(scalarGenerateSetFromList((void **)¶m->pHashFilter, node, type, 0)); + SCL_ERR_RET(scalarGenerateSetFromList((void **)¶m->pHashFilter, node, type, typeMod, 0)); } - param->hashValueType = type; + param->filterValueTypeMod = typeMod; + param->filterValueType = type; param->colAlloced = true; if (taosHashPut(ctx->pRes, &node, POINTER_BYTES, param, sizeof(*param))) { taosHashCleanup(param->pHashFilter); @@ -553,7 +564,7 @@ _return: SCL_RET(code); } -int32_t sclGetNodeType(SNode *pNode, SScalarCtx *ctx, int32_t *type) { +int32_t sclGetNodeType(SNode *pNode, SScalarCtx *ctx, int32_t *type, STypeMod* pTypeMod) { if (NULL == pNode) { *type = -1; return TSDB_CODE_SUCCESS; @@ -563,16 +574,19 @@ int32_t sclGetNodeType(SNode *pNode, SScalarCtx *ctx, int32_t *type) { case QUERY_NODE_VALUE: { SValueNode *valueNode = (SValueNode *)pNode; *type = valueNode->node.resType.type; + *pTypeMod = typeGetTypeModFromDataType(&valueNode->node.resType); return TSDB_CODE_SUCCESS; } case QUERY_NODE_NODE_LIST: { SNodeListNode *nodeList = (SNodeListNode *)pNode; *type = nodeList->node.resType.type; + *pTypeMod = typeGetTypeModFromDataType(&nodeList->node.resType); return TSDB_CODE_SUCCESS; } case QUERY_NODE_COLUMN: { SColumnNode *colNode = (SColumnNode *)pNode; *type = colNode->node.resType.type; + *pTypeMod = typeGetTypeModFromDataType(&colNode->node.resType); return TSDB_CODE_SUCCESS; } case QUERY_NODE_FUNCTION: @@ -584,18 +598,20 @@ int32_t sclGetNodeType(SNode *pNode, SScalarCtx *ctx, int32_t *type) { SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } *type = (int32_t)(res->columnData->info.type); + *pTypeMod = typeGetTypeModFromColInfo(&res->columnData->info); return TSDB_CODE_SUCCESS; } } *type = -1; + *pTypeMod = 0; return TSDB_CODE_SUCCESS; } int32_t sclSetOperatorValueType(SOperatorNode *node, SScalarCtx *ctx) { ctx->type.opResType = node->node.resType.type; - SCL_ERR_RET(sclGetNodeType(node->pLeft, ctx, &(ctx->type.selfType))); - SCL_ERR_RET(sclGetNodeType(node->pRight, ctx, &(ctx->type.peerType))); + SCL_ERR_RET(sclGetNodeType(node->pLeft, ctx, &(ctx->type.selfType), &ctx->type.selfTypeMod)); + SCL_ERR_RET(sclGetNodeType(node->pRight, ctx, &(ctx->type.peerType), &ctx->type.selfTypeMod)); SCL_RET(TSDB_CODE_SUCCESS); } @@ -877,7 +893,7 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o int32_t ind = (i >= params[m].numOfRows) ? (params[m].numOfRows - 1) : i; char *p = colDataGetData(params[m].columnData, ind); - GET_TYPED_DATA(value, bool, params[m].columnData->info.type, p); + GET_TYPED_DATA(value, bool, params[m].columnData->info.type, p, typeGetTypeModFromColInfo(¶ms[m].columnData->info)); if (LOGIC_COND_TYPE_AND == node->condType && (false == value)) { complete = true; @@ -1256,6 +1272,16 @@ EDealRes sclRewriteFunction(SNode **pNode, SScalarCtx *ctx) { return DEAL_RES_ERROR; } (void)memcpy(res->datum.p, output.columnData->pData, varDataTLen(output.columnData->pData)); + } else if (type == TSDB_DATA_TYPE_DECIMAL) { + res->datum.p = taosMemoryCalloc(1, DECIMAL128_BYTES); + if (!res->datum.p) { + sclError("calloc %d failed", DECIMAL128_BYTES); + sclFreeParam(&output); + nodesDestroyNode((SNode*)res); + ctx->code = terrno; + return DEAL_RES_ERROR; + } + (void)memcpy(res->datum.p, output.columnData->pData, DECIMAL128_BYTES); } else { ctx->code = nodesSetValueNodeValue(res, output.columnData->pData); if (ctx->code) { @@ -1684,11 +1710,17 @@ _return: } static int32_t sclGetMinusOperatorResType(SOperatorNode *pOp) { - if (!IS_MATHABLE_TYPE(((SExprNode *)(pOp->pLeft))->resType.type)) { + const SDataType* pDt = &((SExprNode*)(pOp->pLeft))->resType; + if (!IS_MATHABLE_TYPE(pDt->type)) { return TSDB_CODE_TSC_INVALID_OPERATION; } - pOp->node.resType.type = TSDB_DATA_TYPE_DOUBLE; - pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; + + if (IS_DECIMAL_TYPE(pDt->type)) { + pOp->node.resType = *pDt; + } else { + pOp->node.resType.type = TSDB_DATA_TYPE_DOUBLE; + pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; + } return TSDB_CODE_SUCCESS; } @@ -1699,6 +1731,7 @@ static int32_t sclGetMathOperatorResType(SOperatorNode *pOp) { SDataType ldt = ((SExprNode *)(pOp->pLeft))->resType; SDataType rdt = ((SExprNode *)(pOp->pRight))->resType; + bool hasDecimalType = IS_DECIMAL_TYPE(ldt.type) || IS_DECIMAL_TYPE(rdt.type); if ((TSDB_DATA_TYPE_TIMESTAMP == ldt.type && TSDB_DATA_TYPE_TIMESTAMP == rdt.type) || TSDB_DATA_TYPE_VARBINARY == ldt.type || TSDB_DATA_TYPE_VARBINARY == rdt.type || @@ -1711,8 +1744,12 @@ static int32_t sclGetMathOperatorResType(SOperatorNode *pOp) { pOp->node.resType.type = TSDB_DATA_TYPE_TIMESTAMP; pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes; } else { - pOp->node.resType.type = TSDB_DATA_TYPE_DOUBLE; - pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; + if (hasDecimalType) { + return decimalGetRetType(&ldt, &rdt, pOp->opType, &pOp->node.resType); + } else { + pOp->node.resType.type = TSDB_DATA_TYPE_DOUBLE; + pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; + } } return TSDB_CODE_SUCCESS; } @@ -1780,6 +1817,7 @@ static int32_t sclGetBitwiseOperatorResType(SOperatorNode *pOp) { if (TSDB_DATA_TYPE_VARBINARY == ldt.type || TSDB_DATA_TYPE_VARBINARY == rdt.type) { return TSDB_CODE_TSC_INVALID_OPERATION; } + if (IS_DECIMAL_TYPE(ldt.type) || IS_DECIMAL_TYPE(rdt.type)) return TSDB_CODE_TSC_INVALID_OPERATION; pOp->node.resType.type = TSDB_DATA_TYPE_BIGINT; pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; return TSDB_CODE_SUCCESS; diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 280a469153..e6f4baf513 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -9,6 +9,7 @@ #include "tjson.h" #include "ttime.h" #include "filter.h" +#include "decimal.h" typedef float (*_float_fn)(float); typedef float (*_float_fn_2)(float, float); @@ -1217,9 +1218,9 @@ int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu int32_t subPos = 0; int32_t subLen = INT16_MAX; - GET_TYPED_DATA(subPos, int32_t, GET_PARAM_TYPE(&pInput[1]), colDataGetData(pInputData[1], colIdx[1])); + GET_TYPED_DATA(subPos, int32_t, GET_PARAM_TYPE(&pInput[1]), colDataGetData(pInputData[1], colIdx[1]), typeGetTypeModFromColInfo(&pInput[1].columnData->info)); if (inputNum == 3) { - GET_TYPED_DATA(subLen, int32_t, GET_PARAM_TYPE(&pInput[2]), colDataGetData(pInputData[2], colIdx[2])); + GET_TYPED_DATA(subLen, int32_t, GET_PARAM_TYPE(&pInput[2]), colDataGetData(pInputData[2], colIdx[2]), typeGetTypeModFromColInfo(&pInput[2].columnData->info)); } if (subPos == 0 || subLen < 1) { @@ -1336,7 +1337,7 @@ int32_t charFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp if (colDataIsNull_s(pInput[j].columnData, i)) { continue; } else if (IS_NUMERIC_TYPE(GET_PARAM_TYPE(&pInput[j]))) { - GET_TYPED_DATA(num, int32_t, GET_PARAM_TYPE(&pInput[j]), colDataGetData(pInput[j].columnData, colIdx)); + GET_TYPED_DATA(num, int32_t, GET_PARAM_TYPE(&pInput[j]), colDataGetData(pInput[j].columnData, colIdx), typeGetTypeModFromColInfo(&pInput[j].columnData->info)); getAsciiChar(num, &output); } else if (TSDB_DATA_TYPE_BINARY == GET_PARAM_TYPE(&pInput[j])) { num = taosStr2Int32(varDataVal(colDataGetData(pInput[j].columnData, colIdx)), NULL, 10); @@ -1477,7 +1478,8 @@ int32_t positionFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p int32_t trimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { // trim space uint8_t trimType = 0; - GET_TYPED_DATA(trimType, int32_t, GET_PARAM_TYPE(&pInput[inputNum - 1]), pInput[inputNum - 1].columnData->pData); + GET_TYPED_DATA(trimType, int32_t, GET_PARAM_TYPE(&pInput[inputNum - 1]), pInput[inputNum - 1].columnData->pData, + typeGetTypeModFromColInfo(&pInput[inputNum - 1].columnData->info)); switch (trimType) { case TRIM_TYPE_LEADING: { SCL_ERR_RET(doTrimFunction(pInput, inputNum, pOutput, tltrimspace, tltrim)); @@ -1672,7 +1674,9 @@ int32_t substrIdxFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam * char *delimStr = varDataVal(colDataGetData(pInputData[1], colIdx2)); int32_t delimLen = varDataLen(colDataGetData(pInputData[1], colIdx2)); bool needFreeDelim = false; - GET_TYPED_DATA(count, int32_t, GET_PARAM_TYPE(&pInput[2]), colDataGetData(pInputData[2], (pInput[2].numOfRows == 1) ? 0 : k)); + GET_TYPED_DATA(count, int32_t, GET_PARAM_TYPE(&pInput[2]), + colDataGetData(pInputData[2], (pInput[2].numOfRows == 1) ? 0 : k), + typeGetTypeModFromColInfo(&pInputData[2]->info)); int32_t startPosBytes; int32_t endPosBytes; @@ -1734,7 +1738,8 @@ int32_t repeatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu if (colDataIsNull_s(pInput[1].columnData, i)) { continue; } - GET_TYPED_DATA(tmpCount, int32_t, GET_PARAM_TYPE(&pInput[1]), colDataGetData(pInput[1].columnData, i)); + GET_TYPED_DATA(tmpCount, int32_t, GET_PARAM_TYPE(&pInput[1]), colDataGetData(pInput[1].columnData, i), + typeGetTypeModFromColInfo(&pInput[1].columnData->info)); maxCount = TMAX(maxCount, tmpCount); } pInputData[0] = pInput[0].columnData; @@ -1760,7 +1765,8 @@ int32_t repeatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu continue; } int32_t count = 0; - GET_TYPED_DATA(count, int32_t, GET_PARAM_TYPE(&pInput[1]), colDataGetData(pInput[1].columnData, i)); + GET_TYPED_DATA(count, int32_t, GET_PARAM_TYPE(&pInput[1]), colDataGetData(pInput[1].columnData, i), + typeGetTypeModFromColInfo(&pInput[1].columnData->info)); if (count <= 0) { varDataSetLen(output, 0); SCL_ERR_JRET(colDataSetVal(pOutputData, i, outputBuf, false)); @@ -1782,7 +1788,8 @@ int32_t repeatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu continue; } int32_t count = 0; - GET_TYPED_DATA(count, int32_t, GET_PARAM_TYPE(&pInput[1]), colDataGetData(pInput[1].columnData, i)); + GET_TYPED_DATA(count, int32_t, GET_PARAM_TYPE(&pInput[1]), colDataGetData(pInput[1].columnData, i), + typeGetTypeModFromColInfo(&pInput[1].columnData->info)); if (count <= 0) { varDataSetLen(output, 0); SCL_ERR_JRET(colDataSetVal(pOutputData, i, outputBuf, false)); @@ -1805,7 +1812,8 @@ int32_t repeatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu continue; } int32_t count = 0; - GET_TYPED_DATA(count, int32_t, GET_PARAM_TYPE(&pInput[1]), colDataGetData(pInput[1].columnData, 0)); + GET_TYPED_DATA(count, int32_t, GET_PARAM_TYPE(&pInput[1]), colDataGetData(pInput[1].columnData, 0), + typeGetTypeModFromColInfo(&pInput[1].columnData->info)); if (count <= 0) { varDataSetLen(output, 0); SCL_ERR_JRET(colDataSetVal(pOutputData, i, outputBuf, false)); @@ -1867,7 +1875,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp convBuf[len] = 0; *(int8_t *)output = taosStr2Int8(convBuf, NULL, 10); } else { - GET_TYPED_DATA(*(int8_t *)output, int8_t, inputType, input); + GET_TYPED_DATA(*(int8_t *)output, int8_t, inputType, input, typeGetTypeModFromColInfo(&pInput[0].columnData->info)); } break; } @@ -1885,7 +1893,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp convBuf[len] = 0; *(int16_t *)output = taosStr2Int16(convBuf, NULL, 10); } else { - GET_TYPED_DATA(*(int16_t *)output, int16_t, inputType, input); + GET_TYPED_DATA(*(int16_t *)output, int16_t, inputType, input, typeGetTypeModFromColInfo(&pInput[0].columnData->info)); } break; } @@ -1904,7 +1912,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp convBuf[len] = 0; *(int32_t *)output = taosStr2Int32(convBuf, NULL, 10); } else { - GET_TYPED_DATA(*(int32_t *)output, int32_t, inputType, input); + GET_TYPED_DATA(*(int32_t *)output, int32_t, inputType, input, typeGetTypeModFromColInfo(&pInput[0].columnData->info)); } break; } @@ -1922,7 +1930,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp convBuf[len] = 0; *(int64_t *)output = taosStr2Int64(convBuf, NULL, 10); } else { - GET_TYPED_DATA(*(int64_t *)output, int64_t, inputType, input); + GET_TYPED_DATA(*(int64_t *)output, int64_t, inputType, input, typeGetTypeModFromColInfo(&pInput[0].columnData->info)); } break; } @@ -1940,7 +1948,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp convBuf[len] = 0; *(uint8_t *)output = taosStr2UInt8(convBuf, NULL, 10); } else { - GET_TYPED_DATA(*(uint8_t *)output, uint8_t, inputType, input); + GET_TYPED_DATA(*(uint8_t *)output, uint8_t, inputType, input, typeGetTypeModFromColInfo(&pInput[0].columnData->info)); } break; } @@ -1958,7 +1966,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp convBuf[len] = 0; *(uint16_t *)output = taosStr2UInt16(convBuf, NULL, 10); } else { - GET_TYPED_DATA(*(uint16_t *)output, uint16_t, inputType, input); + GET_TYPED_DATA(*(uint16_t *)output, uint16_t, inputType, input, typeGetTypeModFromColInfo(&pInput[0].columnData->info)); } break; } @@ -1976,7 +1984,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp convBuf[len] = 0; *(uint32_t *)output = taosStr2UInt32(convBuf, NULL, 10); } else { - GET_TYPED_DATA(*(uint32_t *)output, uint32_t, inputType, input); + GET_TYPED_DATA(*(uint32_t *)output, uint32_t, inputType, input, typeGetTypeModFromColInfo(&pInput[0].columnData->info)); } break; } @@ -1995,7 +2003,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp convBuf[len] = 0; *(uint64_t *)output = taosStr2UInt64(convBuf, NULL, 10); } else { - GET_TYPED_DATA(*(uint64_t *)output, uint64_t, inputType, input); + GET_TYPED_DATA(*(uint64_t *)output, uint64_t, inputType, input, typeGetTypeModFromColInfo(&pInput[0].columnData->info)); } break; } @@ -2013,7 +2021,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp convBuf[len] = 0; *(float *)output = taosStr2Float(convBuf, NULL); } else { - GET_TYPED_DATA(*(float *)output, float, inputType, input); + GET_TYPED_DATA(*(float *)output, float, inputType, input, typeGetTypeModFromColInfo(&pInput[0].columnData->info)); } break; } @@ -2031,7 +2039,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp convBuf[len] = 0; *(double *)output = taosStr2Double(convBuf, NULL); } else { - GET_TYPED_DATA(*(double *)output, double, inputType, input); + GET_TYPED_DATA(*(double *)output, double, inputType, input, typeGetTypeModFromColInfo(&pInput[0].columnData->info)); } break; } @@ -2049,7 +2057,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp convBuf[len] = 0; *(bool *)output = taosStr2Int8(convBuf, NULL, 10); } else { - GET_TYPED_DATA(*(bool *)output, bool, inputType, input); + GET_TYPED_DATA(*(bool *)output, bool, inputType, input, typeGetTypeModFromColInfo(&pInput[0].columnData->info)); } break; } @@ -2057,7 +2065,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp int64_t timeVal; if (inputType == TSDB_DATA_TYPE_BINARY || inputType == TSDB_DATA_TYPE_NCHAR) { int64_t timePrec; - GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData); + GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData, typeGetTypeModFromColInfo(&pInput[1].columnData->info)); int32_t ret = convertStringToTimestamp(inputType, input, timePrec, &timeVal, pInput->tz, pInput->charsetCxt); if (ret != TSDB_CODE_SUCCESS) { *(int64_t *)output = 0; @@ -2065,7 +2073,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp *(int64_t *)output = timeVal; } } else { - GET_TYPED_DATA(*(int64_t *)output, int64_t, inputType, input); + GET_TYPED_DATA(*(int64_t *)output, int64_t, inputType, input, typeGetTypeModFromColInfo(&pInput[0].columnData->info)); } break; } @@ -2091,7 +2099,14 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp varDataSetLen(output, len); } else { int32_t outputSize = (outputLen - VARSTR_HEADER_SIZE) < bufSize ? (outputLen - VARSTR_HEADER_SIZE + 1): bufSize; - NUM_TO_STRING(inputType, input, outputSize, buf); + if (IS_DECIMAL_TYPE(inputType)) { + if (outputType == TSDB_DATA_TYPE_GEOMETRY) return TSDB_CODE_FUNC_FUNTION_PARA_TYPE; + uint8_t inputPrec = GET_PARAM_PRECISON(&pInput[0]), inputScale = GET_PARAM_SCALE(&pInput[0]); + code = decimalToStr(input, inputType, inputPrec, inputScale, buf, outputSize); + if (code != 0) goto _end; + } else { + NUM_TO_STRING(inputType, input, outputSize, buf); + } int32_t len = (int32_t)strlen(buf); len = (outputLen - VARSTR_HEADER_SIZE) > len ? len : (outputLen - VARSTR_HEADER_SIZE); (void)memcpy(varDataVal(output), buf, len); @@ -2137,7 +2152,13 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp (void)memcpy(output, input, len + VARSTR_HEADER_SIZE); varDataSetLen(output, len); } else { - NUM_TO_STRING(inputType, input, bufSize, buf); + if (IS_DECIMAL_TYPE(inputType)) { + uint8_t inputPrec = GET_PARAM_PRECISON(&pInput[0]), inputScale = GET_PARAM_SCALE(&pInput[0]); + code = decimalToStr(input, inputType, inputPrec, inputScale, buf, bufSize); + if (code != 0) goto _end; + } else { + NUM_TO_STRING(inputType, input, bufSize, buf); + } len = (int32_t)strlen(buf); len = outputCharLen > len ? len : outputCharLen; bool ret = taosMbsToUcs4(buf, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len, pInput->charsetCxt); @@ -2155,6 +2176,31 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp break; } + case TSDB_DATA_TYPE_DECIMAL64: + case TSDB_DATA_TYPE_DECIMAL: { + SDataType iT = GET_COL_DATA_TYPE(pInput[0].columnData->info), oT = GET_COL_DATA_TYPE(pOutput->columnData->info); + if (inputType == TSDB_DATA_TYPE_NCHAR) { + int32_t len = taosUcs4ToMbs((TdUcs4 *)(varDataVal(input)), varDataLen(input), convBuf, pInput->charsetCxt); + if (len < 0) { + code = TSDB_CODE_SCALAR_CONVERT_ERROR; + goto _end; + } + convBuf[len] = 0; + iT.bytes = len; + code = convertToDecimal(convBuf, &iT, output, &oT); + } else { + if (IS_VAR_DATA_TYPE(iT.type)) { + iT.bytes = varDataLen(input); + code = convertToDecimal(varDataVal(input), &iT, output, &oT); + } else { + code = convertToDecimal(input, &iT, output, &oT); + } + } + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + goto _end; + } + } break; default: { code = TSDB_CODE_FAILED; goto _end; @@ -2205,7 +2251,7 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam * int64_t quot = 0; long mod = 0; - GET_TYPED_DATA(timeVal, int64_t, type, input); + GET_TYPED_DATA(timeVal, int64_t, type, input, typeGetTypeModFromColInfo(&pInput[0].columnData->info)); switch (pInput->columnData[0].info.precision) { case TSDB_TIME_PRECISION_MILLI: { @@ -2278,7 +2324,8 @@ int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarP int32_t type = GET_PARAM_TYPE(pInput); int64_t timePrec; int32_t idx = (inputNum == 2) ? 1 : 2; - GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[idx]), pInput[idx].columnData->pData); + GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[idx]), pInput[idx].columnData->pData, + typeGetTypeModFromColInfo(&pInput[idx].columnData->info)); for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { if (colDataIsNull_s(pInput[0].columnData, i)) { @@ -2457,16 +2504,19 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara bool ignoreTz = true; char timezoneStr[20] = {0}; - GET_TYPED_DATA(timeUnit, int64_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData); + GET_TYPED_DATA(timeUnit, int64_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData, + typeGetTypeModFromColInfo(&pInput[1].columnData->info)); int32_t timePrecIdx = 2, timeZoneIdx = 3; if (inputNum == 5) { timePrecIdx += 1; timeZoneIdx += 1; - GET_TYPED_DATA(ignoreTz, bool, GET_PARAM_TYPE(&pInput[2]), pInput[2].columnData->pData); + GET_TYPED_DATA(ignoreTz, bool, GET_PARAM_TYPE(&pInput[2]), pInput[2].columnData->pData, + typeGetTypeModFromColInfo(&pInput[2].columnData->info)); } - GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[timePrecIdx]), pInput[timePrecIdx].columnData->pData); + GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[timePrecIdx]), pInput[timePrecIdx].columnData->pData, + typeGetTypeModFromColInfo(&pInput[timePrecIdx].columnData->info)); (void)memcpy(timezoneStr, varDataVal(pInput[timeZoneIdx].columnData->pData), varDataLen(pInput[timeZoneIdx].columnData->pData)); for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { @@ -2484,9 +2534,9 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara continue; } } else if (type == TSDB_DATA_TYPE_BIGINT) { /* unix timestamp */ - GET_TYPED_DATA(timeVal, int64_t, type, input); + GET_TYPED_DATA(timeVal, int64_t, type, input, typeGetTypeModFromColInfo(&pInput[0].columnData->info)); } else if (type == TSDB_DATA_TYPE_TIMESTAMP) { /* timestamp column*/ - GET_TYPED_DATA(timeVal, int64_t, type, input); + GET_TYPED_DATA(timeVal, int64_t, type, input, typeGetTypeModFromColInfo(&pInput[0].columnData->info)); } char buf[20] = {0}; @@ -2510,10 +2560,13 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { int64_t timeUnit = -1, timePrec, timeVal[2] = {0}; if (inputNum == 4) { - GET_TYPED_DATA(timeUnit, int64_t, GET_PARAM_TYPE(&pInput[2]), pInput[2].columnData->pData); - GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[3]), pInput[3].columnData->pData); + GET_TYPED_DATA(timeUnit, int64_t, GET_PARAM_TYPE(&pInput[2]), pInput[2].columnData->pData, + typeGetTypeModFromColInfo(&pInput[2].columnData->info)); + GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[3]), pInput[3].columnData->pData, + typeGetTypeModFromColInfo(&pInput[3].columnData->info)); } else { - GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[2]), pInput[2].columnData->pData); + GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[2]), pInput[2].columnData->pData, + typeGetTypeModFromColInfo(&pInput[2].columnData->info)); } int64_t factor = TSDB_TICK_PER_SECOND(timePrec); @@ -2544,7 +2597,7 @@ int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p break; } } else if (type == TSDB_DATA_TYPE_BIGINT || type == TSDB_DATA_TYPE_TIMESTAMP) { /* unix timestamp or ts column*/ - GET_TYPED_DATA(timeVal[k], int64_t, type, input[k]); + GET_TYPED_DATA(timeVal[k], int64_t, type, input[k], typeGetTypeModFromColInfo(&pInput[k].columnData->info)); if (type == TSDB_DATA_TYPE_TIMESTAMP) { int64_t timeValSec = timeVal[k] / factor; if (timeValSec < 1000000000) { @@ -2643,7 +2696,8 @@ int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p int32_t nowFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { int64_t timePrec; - GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[0]), pInput[0].columnData->pData); + GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[0]), pInput[0].columnData->pData, + typeGetTypeModFromColInfo(&pInput[0].columnData->info)); int64_t ts = taosGetTimestamp(timePrec); for (int32_t i = 0; i < pInput->numOfRows; ++i) { @@ -2655,7 +2709,8 @@ int32_t nowFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu int32_t todayFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { int64_t timePrec; - GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[0]), pInput[0].columnData->pData); + GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[0]), pInput[0].columnData->pData, + typeGetTypeModFromColInfo(&pInput[0].columnData->info)); int64_t ts = taosGetTimestampToday(timePrec, pInput->tz); for (int32_t i = 0; i < pInput->numOfRows; ++i) { @@ -2691,7 +2746,8 @@ int32_t weekdayFunctionImpl(SScalarParam *pInput, int32_t inputNum, SScalarParam int64_t timePrec, timeVal = 0; - GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData); + GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData, + typeGetTypeModFromColInfo(&pInput[1].columnData->info)); for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { if (colDataIsNull_s(pInput[0].columnData, i)) { @@ -2708,9 +2764,9 @@ int32_t weekdayFunctionImpl(SScalarParam *pInput, int32_t inputNum, SScalarParam continue; } } else if (type == TSDB_DATA_TYPE_BIGINT) { /* unix timestamp */ - GET_TYPED_DATA(timeVal, int64_t, type, input); + GET_TYPED_DATA(timeVal, int64_t, type, input, typeGetTypeModFromColInfo(&pInput[0].columnData->info)); } else if (type == TSDB_DATA_TYPE_TIMESTAMP) { /* timestamp column*/ - GET_TYPED_DATA(timeVal, int64_t, type, input); + GET_TYPED_DATA(timeVal, int64_t, type, input, typeGetTypeModFromColInfo(&pInput[0].columnData->info)); } struct STm tm; TAOS_CHECK_RETURN(taosTs2Tm(timeVal, timePrec, &tm, pInput->tz)); @@ -2818,9 +2874,9 @@ int32_t weekFunctionImpl(SScalarParam *pInput, int32_t inputNum, SScalarParam *p continue; } } else if (type == TSDB_DATA_TYPE_BIGINT) { /* unix timestamp */ - GET_TYPED_DATA(timeVal, int64_t, type, input); + GET_TYPED_DATA(timeVal, int64_t, type, input, typeGetTypeModFromColInfo(&pInput[0].columnData->info)); } else if (type == TSDB_DATA_TYPE_TIMESTAMP) { /* timestamp column*/ - GET_TYPED_DATA(timeVal, int64_t, type, input); + GET_TYPED_DATA(timeVal, int64_t, type, input, typeGetTypeModFromColInfo(&pInput[0].columnData->info)); } struct STm tm; SCL_ERR_RET(taosTs2Tm(timeVal, prec, &tm, pInput->tz)); @@ -2837,18 +2893,22 @@ int32_t weekFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp int64_t timePrec; int32_t mode = 0; if (inputNum == 2) { - GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData); + GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData, + typeGetTypeModFromColInfo(&pInput[1].columnData->info)); return weekFunctionImpl(pInput, inputNum, pOutput, timePrec, mode); } else { - GET_TYPED_DATA(mode, int32_t , GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData); - GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[2]), pInput[2].columnData->pData); + GET_TYPED_DATA(mode, int32_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData, + typeGetTypeModFromColInfo(&pInput[1].columnData->info)); + GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[2]), pInput[2].columnData->pData, + typeGetTypeModFromColInfo(&pInput[2].columnData->info)); return weekFunctionImpl(pInput, inputNum, pOutput, timePrec, mode); } } int32_t weekofyearFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { int64_t timePrec; - GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData); + GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData, + typeGetTypeModFromColInfo(&pInput[1].columnData->info)); return weekFunctionImpl(pInput, inputNum, pOutput, timePrec, 3); } @@ -2907,7 +2967,8 @@ int32_t randFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp // for constant seed, only set seed once if ((pInput[0].numOfRows == 1 && i == 0) || (pInput[0].numOfRows != 1)) { if (!IS_NULL_TYPE(GET_PARAM_TYPE(&pInput[0])) && !colDataIsNull_s(pInput[0].columnData, i)) { - GET_TYPED_DATA(seed, int32_t, GET_PARAM_TYPE(&pInput[0]), colDataGetData(pInput[0].columnData, i)); + GET_TYPED_DATA(seed, int32_t, GET_PARAM_TYPE(&pInput[0]), colDataGetData(pInput[0].columnData, i), + typeGetTypeModFromColInfo(&pInput[0].columnData->info)); taosSeedRand(seed); } } @@ -2996,7 +3057,8 @@ static int32_t doScalarFunction2(SScalarParam *pInput, int32_t inputNum, SScalar continue; } double in2; - GET_TYPED_DATA(in2, double, GET_PARAM_TYPE(&pInput[1]), colDataGetData(pInputData[1], colIdx2)); + GET_TYPED_DATA(in2, double, GET_PARAM_TYPE(&pInput[1]), colDataGetData(pInputData[1], colIdx2), + typeGetTypeModFromColInfo(&pInputData[1]->info)); switch (GET_PARAM_TYPE(&pInput[0])) { case TSDB_DATA_TYPE_DOUBLE: { double *in = (double *)pInputData[0]->pData; @@ -3268,6 +3330,17 @@ int32_t sumScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam * double *in = (double *)pInputData->pData; *out += in[i]; } + } else if (type == TSDB_DATA_TYPE_DECIMAL) { + Decimal128* pOut = (Decimal128*)pOutputData->pData; + if (type == TSDB_DATA_TYPE_DECIMAL64) { + const Decimal64* pIn = (Decimal64*)pInputData->pData; + const SDecimalOps *pOps = getDecimalOps(type); + pOps->add(pOut, pIn + i, DECIMAL_WORD_NUM(Decimal64)); + } else if (type == TSDB_DATA_TYPE_DECIMAL) { + const Decimal128* pIn = (Decimal128*)pInputData->pData; + const SDecimalOps *pOps = getDecimalOps(type); + pOps->add(pOut, pIn + i, DECIMAL_WORD_NUM(Decimal128)); + } } } @@ -3380,6 +3453,24 @@ static int32_t doMinMaxScalarFunction(SScalarParam *pInput, int32_t inputNum, SS } break; } + case TSDB_DATA_TYPE_DECIMAL64: { + Decimal64* p1 = (Decimal64*)pInputData->pData; + Decimal64* p2 = (Decimal64*)pOutputData->pData; + const SDecimalOps *pOps = getDecimalOps(type); + if (pOps->gt(p1 + i, p2, DECIMAL_WORD_NUM(Decimal64)) ^ isMinFunc) { + *p2 = p1[i]; + } + break; + } + case TSDB_DATA_TYPE_DECIMAL: { + Decimal128 *p1 = (Decimal128 *)pInputData->pData; + Decimal128 *p2 = (Decimal128 *)pOutputData->pData; + const SDecimalOps *pOps = getDecimalOps(type); + if (pOps->gt(p1 + i, p2, DECIMAL_WORD_NUM(Decimal128)) ^ isMinFunc) { + *p2 = p1[i]; + } + break; + } } } @@ -3472,7 +3563,7 @@ int32_t avgScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam * } case TSDB_DATA_TYPE_FLOAT: { float *in = (float *)pInputData->pData; - float *out = (float *)pOutputData->pData; + double *out = (double *)pOutputData->pData; *out += in[i]; count++; break; @@ -3484,6 +3575,22 @@ int32_t avgScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam * count++; break; } + case TSDB_DATA_TYPE_DECIMAL64: { + const Decimal64 *in = (Decimal64 *)pInputData->pData; + Decimal128 *out = (Decimal128 *)pOutputData->pData; + const SDecimalOps *pOps = getDecimalOps(type); + // check overflow + pOps->add(out, in + i, DECIMAL_WORD_NUM(Decimal64)); + count++; + } break; + case TSDB_DATA_TYPE_DECIMAL: { + const Decimal128 *in = (Decimal128 *)pInputData->pData; + Decimal128 *out = (Decimal128 *)pOutputData->pData; + const SDecimalOps *pOps = getDecimalOps(type); + // check overflow + pOps->add(out, in + i, DECIMAL_WORD_NUM(Decimal128)); + count++; + } break; } } @@ -3499,6 +3606,20 @@ int32_t avgScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam * } else if (IS_FLOAT_TYPE(type)) { double *out = (double *)pOutputData->pData; *(double *)out = *out / (double)count; + } else if (IS_DECIMAL_TYPE(type)) { + Decimal128 *out = (Decimal128 *)pOutputData->pData; + SDataType sumDt = {.type = TSDB_DATA_TYPE_DECIMAL, + .bytes = DECIMAL128_BYTES, + .precision = TSDB_DECIMAL128_MAX_PRECISION, + .scale = pInputData->info.scale}; + SDataType countDt = { + .type = TSDB_DATA_TYPE_BIGINT, .bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .precision = 0, .scale = 0}; + SDataType avgDt = {.type = TSDB_DATA_TYPE_DECIMAL, + .bytes = tDataTypes[TSDB_DATA_TYPE_DECIMAL].bytes, + .precision = pOutputData->info.precision, + .scale = pOutputData->info.scale}; + int32_t code = decimalOp(OP_TYPE_DIV, &sumDt, &countDt, &avgDt, out, &count, out); + if (code != 0) return code; } } @@ -3634,8 +3755,10 @@ int32_t leastSQRScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarPa double startVal, stepVal; double matrix[2][3] = {0}; - GET_TYPED_DATA(startVal, double, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData); - GET_TYPED_DATA(stepVal, double, GET_PARAM_TYPE(&pInput[2]), pInput[2].columnData->pData); + GET_TYPED_DATA(startVal, double, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData, + typeGetTypeModFromColInfo(&pInput[1].columnData->info)); + GET_TYPED_DATA(stepVal, double, GET_PARAM_TYPE(&pInput[2]), pInput[2].columnData->pData, + typeGetTypeModFromColInfo(&pInput[2].columnData->info)); int32_t type = GET_PARAM_TYPE(pInput); int64_t count = 0; @@ -3811,7 +3934,7 @@ int32_t percentileScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalar break; } char *in = pInputData->pData; - GET_TYPED_DATA(val, double, type, in); + GET_TYPED_DATA(val, double, type, in, typeGetTypeModFromColInfo(&pInputData->info)); } if (hasNull) { @@ -3848,7 +3971,7 @@ int32_t spreadScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara char *in = pInputData->pData; double val = 0; - GET_TYPED_DATA(val, double, type, in); + GET_TYPED_DATA(val, double, type, in, typeGetTypeModFromColInfo(&pInputData->info)); if (val < GET_DOUBLE_VAL(&min)) { SET_DOUBLE_VAL(&min, val); @@ -4319,7 +4442,7 @@ int32_t histogramScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarP char *data = colDataGetData(pInputData, i); double v; - GET_TYPED_DATA(v, double, type, data); + GET_TYPED_DATA(v, double, type, data, typeGetTypeModFromColInfo(&pInputData->info)); for (int32_t k = 0; k < numOfBins; ++k) { if (v > bins[k].lower && v <= bins[k].upper) { @@ -4500,7 +4623,7 @@ static int32_t greatestLeastImpl(SScalarParam *pInput, int32_t inputNum, SScalar if (oldType != outputType) { pCovertParams[j].covertParam = (SScalarParam){0}; setTzCharset(&pCovertParams[j].covertParam, pParam->tz, pParam->charsetCxt); - SCL_ERR_JRET(vectorConvertSingleCol(pParam, &pCovertParams[j].covertParam, outputType, 0, pParam->numOfRows)); + SCL_ERR_JRET(vectorConvertSingleCol(pParam, &pCovertParams[j].covertParam, outputType, 0, 0, pParam->numOfRows)); pCovertParams[j].param = &pCovertParams[j].covertParam; pCovertParams[j].converted = true; } else { diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 742759db07..1c3fc3625d 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -28,6 +28,7 @@ #include "ttime.h" #include "ttypes.h" #include "geosWrapper.h" +#include "decimal.h" #define LEFT_COL ((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void *)pLeftCol : pLeftCol->pData)) #define RIGHT_COL ((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void *)pRightCol : pRightCol->pData)) @@ -40,55 +41,65 @@ #define IS_HELPER_NULL(col, i) colDataIsNull_s(col, i) || IS_JSON_NULL(col->info.type, colDataGetVarData(col, i)) bool noConvertBeforeCompare(int32_t leftType, int32_t rightType, int32_t optr) { - return IS_NUMERIC_TYPE(leftType) && IS_NUMERIC_TYPE(rightType) && - (optr >= OP_TYPE_GREATER_THAN && optr <= OP_TYPE_NOT_EQUAL); + return !IS_DECIMAL_TYPE(leftType) && !IS_DECIMAL_TYPE(rightType) && IS_NUMERIC_TYPE(leftType) && + IS_NUMERIC_TYPE(rightType) && (optr >= OP_TYPE_GREATER_THAN && optr <= OP_TYPE_NOT_EQUAL); } +bool compareForType(__compar_fn_t fp, int32_t optr, SColumnInfoData* pColL, int32_t idxL, SColumnInfoData* pColR, int32_t idxR); +bool compareForTypeWithColAndHash(__compar_fn_t fp, int32_t optr, SColumnInfoData *pColL, int32_t idxL, + const void *hashData, int32_t hashType, STypeMod hashTypeMod); + +static int32_t vectorMathBinaryOpForDecimal(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t step, + int32_t i, EOperatorType op); + +static int32_t vectorMathUnaryOpForDecimal(SScalarParam *pCol, SScalarParam *pOut, int32_t step, int32_t i, + EOperatorType op); + int32_t convertNumberToNumber(const void *inData, void *outData, int8_t inType, int8_t outType) { switch (outType) { case TSDB_DATA_TYPE_BOOL: { - GET_TYPED_DATA(*((bool *)outData), bool, inType, inData); + GET_TYPED_DATA(*((bool *)outData), bool, inType, inData, 0); break; } case TSDB_DATA_TYPE_TINYINT: { - GET_TYPED_DATA(*((int8_t *)outData), int8_t, inType, inData); + GET_TYPED_DATA(*((int8_t *)outData), int8_t, inType, inData, 0); break; } case TSDB_DATA_TYPE_SMALLINT: { - GET_TYPED_DATA(*((int16_t *)outData), int16_t, inType, inData); + GET_TYPED_DATA(*((int16_t *)outData), int16_t, inType, inData, 0); break; } case TSDB_DATA_TYPE_INT: { - GET_TYPED_DATA(*((int32_t *)outData), int32_t, inType, inData); + GET_TYPED_DATA(*((int32_t *)outData), int32_t, inType, inData, 0); break; } case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_TIMESTAMP: { - GET_TYPED_DATA(*((int64_t *)outData), int64_t, inType, inData); + GET_TYPED_DATA(*((int64_t *)outData), int64_t, inType, inData, 0); break; } case TSDB_DATA_TYPE_UTINYINT: { - GET_TYPED_DATA(*((uint8_t *)outData), uint8_t, inType, inData); + GET_TYPED_DATA(*((uint8_t *)outData), uint8_t, inType, inData, 0); break; } case TSDB_DATA_TYPE_USMALLINT: { - GET_TYPED_DATA(*((uint16_t *)outData), uint16_t, inType, inData); + GET_TYPED_DATA(*((uint16_t *)outData), uint16_t, inType, inData, 0); break; } case TSDB_DATA_TYPE_UINT: { - GET_TYPED_DATA(*((uint32_t *)outData), uint32_t, inType, inData); + GET_TYPED_DATA(*((uint32_t *)outData), uint32_t, inType, inData, 0); break; } case TSDB_DATA_TYPE_UBIGINT: { - GET_TYPED_DATA(*((uint64_t *)outData), uint64_t, inType, inData); + GET_TYPED_DATA(*((uint64_t *)outData), uint64_t, inType, inData, 0); break; } case TSDB_DATA_TYPE_FLOAT: { - GET_TYPED_DATA(*((float *)outData), float, inType, inData); + GET_TYPED_DATA(*((float *)outData), float, inType, inData, 0); break; } case TSDB_DATA_TYPE_DOUBLE: { - GET_TYPED_DATA(*((double *)outData), double, inType, inData); + GET_TYPED_DATA(*((double *)outData), double, inType, inData, 0); break; } default: { @@ -239,6 +250,16 @@ static FORCE_INLINE int32_t varToTimestamp(char *buf, SScalarParam *pOut, int32_ SCL_RET(code); } +static FORCE_INLINE int32_t varToDecimal(char* buf, SScalarParam* pOut, int32_t rowIndex, int32_t* overflow) { + Decimal *pDec = (Decimal *)colDataGetData(pOut->columnData, rowIndex); + int32_t code = decimalFromStr(buf, strlen(buf), pOut->columnData->info.precision, pOut->columnData->info.scale, pDec); + if (TSDB_CODE_SUCCESS != code) { + if (overflow) *overflow = code == TSDB_CODE_DECIMAL_OVERFLOW; + SCL_RET(code); + } + SCL_RET(code); +} + static FORCE_INLINE int32_t varToSigned(char *buf, SScalarParam *pOut, int32_t rowIndex, int32_t *overflow) { if (overflow) { int64_t minValue = tDataTypes[pOut->columnData->info.type].minValue; @@ -499,6 +520,8 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) { } else if (TSDB_DATA_TYPE_VARBINARY == pCtx->outType) { func = varToVarbinary; vton = true; + } else if (IS_DECIMAL_TYPE(pCtx->outType)) { + func = varToDecimal; } else { sclError("invalid convert outType:%d, inType:%d", pCtx->outType, pCtx->inType); SCL_ERR_RET(TSDB_CODE_APP_ERROR); @@ -726,7 +749,7 @@ int32_t vectorConvertToVarData(SSclVectorConvCtx *pCtx) { } int64_t value = 0; - GET_TYPED_DATA(value, int64_t, pCtx->inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, int64_t, pCtx->inType, colDataGetData(pInputCol, i), typeGetTypeModFromColInfo(&pInputCol->info)); int32_t len = tsnprintf(varDataVal(tmp), sizeof(tmp) - VARSTR_HEADER_SIZE, "%" PRId64, value); varDataLen(tmp) = len; if (pCtx->outType == TSDB_DATA_TYPE_NCHAR) { @@ -743,7 +766,7 @@ int32_t vectorConvertToVarData(SSclVectorConvCtx *pCtx) { } uint64_t value = 0; - GET_TYPED_DATA(value, uint64_t, pCtx->inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, uint64_t, pCtx->inType, colDataGetData(pInputCol, i), typeGetTypeModFromColInfo(&pInputCol->info)); int32_t len = tsnprintf(varDataVal(tmp), sizeof(tmp) - VARSTR_HEADER_SIZE, "%" PRIu64, value); varDataLen(tmp) = len; if (pCtx->outType == TSDB_DATA_TYPE_NCHAR) { @@ -760,7 +783,7 @@ int32_t vectorConvertToVarData(SSclVectorConvCtx *pCtx) { } double value = 0; - GET_TYPED_DATA(value, double, pCtx->inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, double, pCtx->inType, colDataGetData(pInputCol, i), typeGetTypeModFromColInfo(&pInputCol->info)); int32_t len = tsnprintf(varDataVal(tmp), sizeof(tmp) - VARSTR_HEADER_SIZE, "%lf", value); varDataLen(tmp) = len; if (pCtx->outType == TSDB_DATA_TYPE_NCHAR) { @@ -809,7 +832,7 @@ int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut, int64_t maxValue = tDataTypes[cCtx.outType].maxValue; double value = 0; - GET_TYPED_DATA(value, double, cCtx.inType, colDataGetData(pInputCol, 0)); + GET_TYPED_DATA(value, double, cCtx.inType, colDataGetData(pInputCol, 0), typeGetTypeModFromColInfo(&pInputCol->info)); if (value > maxValue) { *overflow = 1; @@ -825,7 +848,7 @@ int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut, uint64_t maxValue = (uint64_t)tDataTypes[cCtx.outType].maxValue; double value = 0; - GET_TYPED_DATA(value, double, cCtx.inType, colDataGetData(pInputCol, 0)); + GET_TYPED_DATA(value, double, cCtx.inType, colDataGetData(pInputCol, 0), typeGetTypeModFromColInfo(&pInputCol->info)); if (value > maxValue) { *overflow = 1; @@ -849,7 +872,7 @@ int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut, } bool value = 0; - GET_TYPED_DATA(value, bool, cCtx.inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, bool, cCtx.inType, colDataGetData(pInputCol, i), typeGetTypeModFromColInfo(&pInputCol->info)); colDataSetInt8(pOutputCol, i, (int8_t *)&value); } break; @@ -862,7 +885,7 @@ int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut, } int8_t value = 0; - GET_TYPED_DATA(value, int8_t, cCtx.inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, int8_t, cCtx.inType, colDataGetData(pInputCol, i), typeGetTypeModFromColInfo(&pInputCol->info)); colDataSetInt8(pOutputCol, i, (int8_t *)&value); } break; @@ -875,7 +898,7 @@ int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut, } int16_t value = 0; - GET_TYPED_DATA(value, int16_t, cCtx.inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, int16_t, cCtx.inType, colDataGetData(pInputCol, i), typeGetTypeModFromColInfo(&pInputCol->info)); colDataSetInt16(pOutputCol, i, (int16_t *)&value); } break; @@ -888,7 +911,7 @@ int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut, } int32_t value = 0; - GET_TYPED_DATA(value, int32_t, cCtx.inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, int32_t, cCtx.inType, colDataGetData(pInputCol, i), typeGetTypeModFromColInfo(&pInputCol->info)); colDataSetInt32(pOutputCol, i, (int32_t *)&value); } break; @@ -902,7 +925,7 @@ int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut, } int64_t value = 0; - GET_TYPED_DATA(value, int64_t, cCtx.inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, int64_t, cCtx.inType, colDataGetData(pInputCol, i), typeGetTypeModFromColInfo(&pInputCol->info)); colDataSetInt64(pOutputCol, i, (int64_t *)&value); } break; @@ -915,7 +938,7 @@ int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut, } uint8_t value = 0; - GET_TYPED_DATA(value, uint8_t, cCtx.inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, uint8_t, cCtx.inType, colDataGetData(pInputCol, i), typeGetTypeModFromColInfo(&pInputCol->info)); colDataSetInt8(pOutputCol, i, (int8_t *)&value); } break; @@ -928,7 +951,7 @@ int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut, } uint16_t value = 0; - GET_TYPED_DATA(value, uint16_t, cCtx.inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, uint16_t, cCtx.inType, colDataGetData(pInputCol, i), typeGetTypeModFromColInfo(&pInputCol->info)); colDataSetInt16(pOutputCol, i, (int16_t *)&value); } break; @@ -941,7 +964,7 @@ int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut, } uint32_t value = 0; - GET_TYPED_DATA(value, uint32_t, cCtx.inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, uint32_t, cCtx.inType, colDataGetData(pInputCol, i), typeGetTypeModFromColInfo(&pInputCol->info)); colDataSetInt32(pOutputCol, i, (int32_t *)&value); } break; @@ -954,7 +977,7 @@ int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut, } uint64_t value = 0; - GET_TYPED_DATA(value, uint64_t, cCtx.inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, uint64_t, cCtx.inType, colDataGetData(pInputCol, i), typeGetTypeModFromColInfo(&pInputCol->info)); colDataSetInt64(pOutputCol, i, (int64_t *)&value); } break; @@ -967,7 +990,7 @@ int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut, } float value = 0; - GET_TYPED_DATA(value, float, cCtx.inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, float, cCtx.inType, colDataGetData(pInputCol, i), typeGetTypeModFromColInfo(&pInputCol->info)); colDataSetFloat(pOutputCol, i, (float *)&value); } break; @@ -980,7 +1003,7 @@ int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut, } double value = 0; - GET_TYPED_DATA(value, double, cCtx.inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, double, cCtx.inType, colDataGetData(pInputCol, i), typeGetTypeModFromColInfo(&pInputCol->info)); colDataSetDouble(pOutputCol, i, (double *)&value); } break; @@ -991,6 +1014,22 @@ int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut, case TSDB_DATA_TYPE_GEOMETRY: { return vectorConvertToVarData(&cCtx); } + case TSDB_DATA_TYPE_DECIMAL: { + for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { + if (colDataIsNull_f(pInputCol->nullbitmap, i)) { + colDataSetNULL(pOutputCol, i); + continue; + } + + Decimal value = {0}; + SDataType inputType = GET_COL_DATA_TYPE(pInputCol->info), outputType = GET_COL_DATA_TYPE(pOutputCol->info); + int32_t code = convertToDecimal(colDataGetData(pInputCol, i), &inputType, &value, &outputType); + if (TSDB_CODE_SUCCESS != code) return code; + code = colDataSetVal(pOutputCol, i, (const char*)&value, false); + if (TSDB_CODE_SUCCESS != code) return code; + } + break; + } default: sclError("invalid convert output type:%d", cCtx.outType); return TSDB_CODE_APP_ERROR; @@ -1000,53 +1039,55 @@ int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut, } int8_t gConvertTypes[TSDB_DATA_TYPE_MAX][TSDB_DATA_TYPE_MAX] = { - /* NULL BOOL TINY SMAL INT BIG FLOA DOUB VARC TIME NCHA UTIN USMA UINT UBIG JSON VARB DECI BLOB MEDB GEOM*/ - /*NULL*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - /*BOOL*/ 0, 0, 2, 3, 4, 5, 6, 7, 5, 9, 5, 11, 12, 13, 14, 0, -1, 0, 0, 0, -1, - /*TINY*/ 0, 0, 0, 3, 4, 5, 6, 7, 5, 9, 5, 3, 4, 5, 7, 0, -1, 0, 0, 0, -1, - /*SMAL*/ 0, 0, 0, 0, 4, 5, 6, 7, 5, 9, 5, 3, 4, 5, 7, 0, -1, 0, 0, 0, -1, - /*INT */ 0, 0, 0, 0, 0, 5, 6, 7, 5, 9, 5, 4, 4, 5, 7, 0, -1, 0, 0, 0, -1, - /*BIGI*/ 0, 0, 0, 0, 0, 0, 6, 7, 5, 9, 5, 5, 5, 5, 7, 0, -1, 0, 0, 0, -1, - /*FLOA*/ 0, 0, 0, 0, 0, 0, 0, 7, 6, 6, 6, 6, 6, 6, 6, 0, -1, 0, 0, 0, -1, - /*DOUB*/ 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 7, 7, 0, -1, 0, 0, 0, -1, - /*VARC*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8, 7, 7, 7, 7, 0, 16, 0, 0, 0, 20, - /*TIME*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 7, 0, -1, 0, 0, 0, -1, - /*NCHA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 0, 16, 0, 0, 0, -1, - /*UTIN*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 13, 14, 0, -1, 0, 0, 0, -1, - /*USMA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 14, 0, -1, 0, 0, 0, -1, - /*UINT*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, -1, 0, 0, 0, -1, - /*UBIG*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, -1, - /*JSON*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, -1, - /*VARB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1, -1, -1, - /*DECI*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, -1, - /*BLOB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, -1, - /*MEDB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, -1, - /*GEOM*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, 0 + /*NULL BOOL TINY SMAL INT BIG FLOA DOUB VARC TIME NCHA UTIN USMA UINT UBIG JSON VARB DECI BLOB MEDB GEOM DEC64*/ + /*NULL*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + /*BOOL*/ 0, 0, 2, 3, 4, 5, 6, 7, 5, 9, 5, 11, 12, 13, 14, 0, -1, 17, 0, 0, -1, 17, + /*TINY*/ 0, 0, 0, 3, 4, 5, 6, 7, 5, 9, 5, 3, 4, 5, 7, 0, -1, 17, 0, 0, -1, 17, + /*SMAL*/ 0, 0, 0, 0, 4, 5, 6, 7, 5, 9, 5, 3, 4, 5, 7, 0, -1, 17, 0, 0, -1, 17, + /*INT */ 0, 0, 0, 0, 0, 5, 6, 7, 5, 9, 5, 4, 4, 5, 7, 0, -1, 17, 0, 0, -1, 17, + /*BIGI*/ 0, 0, 0, 0, 0, 0, 6, 7, 5, 9, 5, 5, 5, 5, 7, 0, -1, 17, 0, 0, -1, 17, + /*FLOA*/ 0, 0, 0, 0, 0, 0, 0, 7, 6, 6, 6, 6, 6, 6, 6, 0, -1, 7, 0, 0, -1, 7, + /*DOUB*/ 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 7, 7, 0, -1, 7, 0, 0, -1, 7, + /*VARC*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8, 7, 7, 7, 7, 0, 16, 7, 0, 0, 20, 7, + /*TIME*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 7, 0, -1, 17, 0, 0, -1, 17, + /*NCHA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 0, 16, 7, 0, 0, -1, 7, + /*UTIN*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 13, 14, 0, -1, 17, 0, 0, -1, 17, + /*USMA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 14, 0, -1, 17, 0, 0, -1, 17, + /*UINT*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, -1, 17, 0, 0, -1, 17, + /*UBIG*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 17, 0, 0, -1, 17, + /*JSON*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1, 0, 0, -1, -1, + /*VARB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1, -1, -1, -1, + /*DECI*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, -1, -1, -1, 17, + /*BLOB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, -1, -1, + /*MEDB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, -1, -1, + /*GEOM*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, 0, -1, + /*DEC64*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, 0, 0, }; int8_t gDisplyTypes[TSDB_DATA_TYPE_MAX][TSDB_DATA_TYPE_MAX] = { - /* NULL BOOL TINY SMAL INT BIGI FLOA DOUB VARC TIM NCHA UTIN USMA UINT UBIG JSON VARB DECI BLOB MEDB GEOM*/ - /*NULL*/ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, -1, -1, -1, 20, - /*BOOL*/ 0, 1, 2, 3, 4, 5, 6, 7, 8, 5, 10, 11, 12, 13, 14, -1, -1, -1, -1, -1, -1, - /*TINY*/ 0, 0, 2, 3, 4, 5, 8, 8, 8, 5, 10, 3, 4, 5, 8, -1, -1, -1, -1, -1, -1, - /*SMAL*/ 0, 0, 0, 3, 4, 5, 8, 8, 8, 5, 10, 3, 4, 5, 8, -1, -1, -1, -1, -1, -1, - /*INT */ 0, 0, 0, 0, 4, 5, 8, 8, 8, 5, 10, 4, 4, 5, 8, -1, -1, -1, -1, -1, -1, - /*BIGI*/ 0, 0, 0, 0, 0, 5, 8, 8, 8, 5, 10, 5, 5, 5, 8, -1, -1, -1, -1, -1, -1, - /*FLOA*/ 0, 0, 0, 0, 0, 0, 6, 7, 8, 8, 10, 8, 8, 8, 8, -1, -1, -1, -1, -1, -1, - /*DOUB*/ 0, 0, 0, 0, 0, 0, 0, 7, 8, 8, 10, 8, 8, 8, 8, -1, -1, -1, -1, -1, -1, - /*VARC*/ 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 10, 8, 8, 8, 8, -1, 16, -1, -1, -1, -1, - /*TIME*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 10, 5, 5, 5, 8, -1, -1, -1, -1, -1, -1, - /*NCHA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 10, 10, 10, 10, -1, -1, -1, -1, -1, -1, - /*UTINY*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 12, 13, 14, -1, -1, -1, -1, -1, -1, - /*USMA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 13, 14, -1, -1, -1, -1, -1, -1, - /*UINT*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 14, -1, -1, -1, -1, -1, -1, - /*UBIG*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, -1, -1, -1, -1, -1, -1, - /*JSON*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, -1, -1, -1, -1, -1, - /*VARB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, -1, -1, -1, -1, - /*DECI*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1, -1, -1, - /*BLOB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1, -1, - /*MEDB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1, - /*GEOM*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20 + /*NULL BOOL TINY SMAL INT BIGI FLOA DOUB VARC TIM NCHA UTIN USMA UINT UBIG JSON VARB DECI BLOB MEDB GEOM DEC64*/ + /*NULL*/ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, -1, -1, 20, 21, + /*BOOL*/ 0, 1, 2, 3, 4, 5, 6, 7, 8, 5, 10, 11, 12, 13, 14, -1, -1, 17, -1, -1, -1, 17, + /*TINY*/ 0, 0, 2, 3, 4, 5, 8, 8, 8, 5, 10, 3, 4, 5, 8, -1, -1, 17, -1, -1, -1, 17, + /*SMAL*/ 0, 0, 0, 3, 4, 5, 8, 8, 8, 5, 10, 3, 4, 5, 8, -1, -1, 17, -1, -1, -1, 17, + /*INT */ 0, 0, 0, 0, 4, 5, 8, 8, 8, 5, 10, 4, 4, 5, 8, -1, -1, 17, -1, -1, -1, 17, + /*BIGI*/ 0, 0, 0, 0, 0, 5, 8, 8, 8, 5, 10, 5, 5, 5, 8, -1, -1, 17, -1, -1, -1, 17, + /*FLOA*/ 0, 0, 0, 0, 0, 0, 6, 7, 8, 8, 10, 8, 8, 8, 8, -1, -1, 7, -1, -1, -1, 7, + /*DOUB*/ 0, 0, 0, 0, 0, 0, 0, 7, 8, 8, 10, 8, 8, 8, 8, -1, -1, 7, -1, -1, -1, 7, + /*VARC*/ 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 10, 8, 8, 8, 8, -1, 16, 7, -1, -1, -1, 7, + /*TIME*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 10, 5, 5, 5, 8, -1, -1, 17, -1, -1, -1, 17, + /*NCHA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 10, 10, 10, 10, -1, -1, 7, -1, -1, -1, 7, + /*UTINY*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 12, 13, 14, -1, -1, 17, -1, -1, -1, 17, + /*USMA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 13, 14, -1, -1, 17, -1, -1, -1, -1, + /*UINT*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 14, -1, -1, 17, -1, -1, -1, -1, + /*UBIG*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, -1, -1, 17, -1, -1, -1, -1, + /*JSON*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, -1, -1, -1, -1, -1, -1, + /*VARB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, -1, -1, -1, -1, -1, + /*DECI*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1, -1, 17, + /*BLOB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1, -1, -1, + /*MEDB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1, -1, + /*GEOM*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, -1, + /*DEC64*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, }; int32_t vectorGetConvertType(int32_t type1, int32_t type2) { @@ -1061,8 +1102,23 @@ int32_t vectorGetConvertType(int32_t type1, int32_t type2) { return gConvertTypes[type2][type1]; } -int32_t vectorConvertSingleCol(SScalarParam *input, SScalarParam *output, int32_t type, int32_t startIndex, - int32_t numOfRows) { +STypeMod getConvertTypeMod(int32_t type, const SColumnInfo* pCol1, const SColumnInfo* pCol2) { + if (IS_DECIMAL_TYPE(type)) { + if (IS_DECIMAL_TYPE(pCol1->type) && (!pCol2 || !IS_DECIMAL_TYPE(pCol2->type))) { + return decimalCalcTypeMod(GET_DEICMAL_MAX_PRECISION(type), pCol1->scale); + } else if (pCol2 && IS_DECIMAL_TYPE(pCol2->type) && !IS_DECIMAL_TYPE(pCol1->type)) { + return decimalCalcTypeMod(GET_DEICMAL_MAX_PRECISION(type), pCol2->scale); + } else if (IS_DECIMAL_TYPE(pCol1->type) && pCol2 && IS_DECIMAL_TYPE(pCol2->type)) { + return decimalCalcTypeMod(GET_DEICMAL_MAX_PRECISION(type), TMAX(pCol1->scale, pCol2->scale)); + } else { + return 0; + } + } + return 0; +} + +int32_t vectorConvertSingleCol(SScalarParam *input, SScalarParam *output, int32_t type, STypeMod typeMod, + int32_t startIndex, int32_t numOfRows) { if (input->columnData == NULL && (input->pHashFilter != NULL || input->pHashFilterOthers != NULL)){ return TSDB_CODE_SUCCESS; } @@ -1071,6 +1127,11 @@ int32_t vectorConvertSingleCol(SScalarParam *input, SScalarParam *output, int32_ SDataType t = {.type = type}; t.bytes = (IS_VAR_DATA_TYPE(t.type) && input->columnData) ? input->columnData->info.bytes:tDataTypes[type].bytes; t.precision = (IS_TIMESTAMP_TYPE(t.type) && input->columnData) ? input->columnData->info.precision : TSDB_TIME_PRECISION_MILLI; + if (IS_DECIMAL_TYPE(type)) { + extractTypeFromTypeMod(type, typeMod, &t.precision, &t.scale, NULL); + // We do not change scale here for decimal types. + if (IS_DECIMAL_TYPE(input->columnData->info.type)) t.scale = input->columnData->info.scale; + } int32_t code = sclCreateColumnInfoData(&t, input->numOfRows, output); if (code != TSDB_CODE_SUCCESS) { @@ -1093,8 +1154,9 @@ int32_t vectorConvertCols(SScalarParam *pLeft, SScalarParam *pRight, SScalarPara return TSDB_CODE_SUCCESS; } - int8_t type = 0; - int32_t code = 0; + int8_t type = 0; + int32_t code = 0; + STypeMod outTypeMod = 0; SScalarParam *param1 = pLeft, *paramOut1 = pLeftOut; SScalarParam *param2 = pRight, *paramOut2 = pRightOut; @@ -1117,14 +1179,15 @@ int32_t vectorConvertCols(SScalarParam *pLeft, SScalarParam *pRight, SScalarPara terrno = TSDB_CODE_SCALAR_CONVERT_ERROR; return TSDB_CODE_SCALAR_CONVERT_ERROR; } + outTypeMod = getConvertTypeMod(type, ¶m1->columnData->info, param2->columnData ? ¶m2->columnData->info : NULL); } if (type != GET_PARAM_TYPE(param1)) { - SCL_ERR_RET(vectorConvertSingleCol(param1, paramOut1, type, startIndex, numOfRows)); + SCL_ERR_RET(vectorConvertSingleCol(param1, paramOut1, type, outTypeMod, startIndex, numOfRows)); } if (type != GET_PARAM_TYPE(param2)) { - SCL_ERR_RET(vectorConvertSingleCol(param2, paramOut2, type, startIndex, numOfRows)); + SCL_ERR_RET(vectorConvertSingleCol(param2, paramOut2, type, outTypeMod, startIndex, numOfRows)); } return TSDB_CODE_SUCCESS; @@ -1195,8 +1258,9 @@ static int32_t vectorConvertVarToDouble(SScalarParam *pInput, int32_t *converted SColumnInfoData *pCol = pInput->columnData; int32_t code = TSDB_CODE_SUCCESS; *pOutputCol = NULL; - if (IS_VAR_DATA_TYPE(pCol->info.type) && pCol->info.type != TSDB_DATA_TYPE_JSON && pCol->info.type != TSDB_DATA_TYPE_VARBINARY) { - SCL_ERR_RET(vectorConvertSingleCol(pInput, &output, TSDB_DATA_TYPE_DOUBLE, -1, -1)); + bool isVarChar = IS_VAR_DATA_TYPE(pCol->info.type) && pCol->info.type != TSDB_DATA_TYPE_JSON && pCol->info.type != TSDB_DATA_TYPE_VARBINARY; + if (isVarChar || IS_DECIMAL_TYPE(pCol->info.type)) { + SCL_ERR_RET(vectorConvertSingleCol(pInput, &output, TSDB_DATA_TYPE_DOUBLE, 0, -1, -1)); *converted = VECTOR_DO_CONVERT; *pOutputCol = output.columnData; SCL_RET(code); @@ -1224,11 +1288,9 @@ int32_t vectorMathAdd(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *p int32_t code = TSDB_CODE_SUCCESS; int32_t leftConvert = 0, rightConvert = 0; - SColumnInfoData *pLeftCol = NULL; - SColumnInfoData *pRightCol = NULL; - SCL_ERR_JRET(vectorConvertVarToDouble(pLeft, &leftConvert, &pLeftCol)); - SCL_ERR_JRET(vectorConvertVarToDouble(pRight, &rightConvert, &pRightCol)); - if(checkOperatorRestypeIsTimestamp(OP_TYPE_ADD, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight))) { // timestamp plus duration + SColumnInfoData *pLeftCol = pLeft->columnData; + SColumnInfoData *pRightCol = pRight->columnData; + if(pOutputCol->info.type == TSDB_DATA_TYPE_TIMESTAMP) { // timestamp plus duration int64_t *output = (int64_t *)pOutputCol->pData; _getBigintValue_fn_t getVectorBigintValueFnLeft; _getBigintValue_fn_t getVectorBigintValueFnRight; @@ -1258,7 +1320,11 @@ int32_t vectorMathAdd(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *p *output = leftRes + rightRes; } } + } else if (IS_DECIMAL_TYPE(pOutputCol->info.type)) { + SCL_ERR_JRET(vectorMathBinaryOpForDecimal(pLeft, pRight, pOut, step, i, OP_TYPE_ADD)); } else { + SCL_ERR_JRET(vectorConvertVarToDouble(pLeft, &leftConvert, &pLeftCol)); + SCL_ERR_JRET(vectorConvertVarToDouble(pRight, &rightConvert, &pRightCol)); double *output = (double *)pOutputCol->pData; _getDoubleValue_fn_t getVectorDoubleValueFnLeft; _getDoubleValue_fn_t getVectorDoubleValueFnRight; @@ -1357,10 +1423,10 @@ int32_t vectorMathSub(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *p int32_t leftConvert = 0, rightConvert = 0; SColumnInfoData *pLeftCol = NULL; SColumnInfoData *pRightCol = NULL; - SCL_ERR_JRET(vectorConvertVarToDouble(pLeft, &leftConvert, &pLeftCol)); - SCL_ERR_JRET(vectorConvertVarToDouble(pRight, &rightConvert, &pRightCol)); - if (checkOperatorRestypeIsTimestamp(OP_TYPE_SUB, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight))) { // timestamp minus duration + if (pOutputCol->info.type == TSDB_DATA_TYPE_TIMESTAMP) { // timestamp minus duration + SCL_ERR_JRET(vectorConvertVarToDouble(pLeft, &leftConvert, &pLeftCol)); + SCL_ERR_JRET(vectorConvertVarToDouble(pRight, &rightConvert, &pRightCol)); int64_t *output = (int64_t *)pOutputCol->pData; _getBigintValue_fn_t getVectorBigintValueFnLeft; _getBigintValue_fn_t getVectorBigintValueFnRight; @@ -1386,7 +1452,11 @@ int32_t vectorMathSub(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *p *output = leftRes - rightRes; } } + } else if (pOutputCol->info.type == TSDB_DATA_TYPE_DECIMAL) { + SCL_ERR_JRET(vectorMathBinaryOpForDecimal(pLeft, pRight, pOut, step, i, OP_TYPE_SUB)); } else { + SCL_ERR_JRET(vectorConvertVarToDouble(pLeft, &leftConvert, &pLeftCol)); + SCL_ERR_JRET(vectorConvertVarToDouble(pRight, &rightConvert, &pRightCol)); double *output = (double *)pOutputCol->pData; _getDoubleValue_fn_t getVectorDoubleValueFnLeft; _getDoubleValue_fn_t getVectorDoubleValueFnRight; @@ -1457,31 +1527,35 @@ int32_t vectorMathMultiply(SScalarParam *pLeft, SScalarParam *pRight, SScalarPar int32_t leftConvert = 0, rightConvert = 0; SColumnInfoData *pLeftCol = NULL; SColumnInfoData *pRightCol = NULL; - SCL_ERR_JRET(vectorConvertVarToDouble(pLeft, &leftConvert, &pLeftCol)); - SCL_ERR_JRET(vectorConvertVarToDouble(pRight, &rightConvert, &pRightCol)); + if (pOutputCol->info.type == TSDB_DATA_TYPE_DECIMAL) { + SCL_ERR_JRET(vectorMathBinaryOpForDecimal(pLeft, pRight, pOut, step, i, OP_TYPE_MULTI)); + } else { + SCL_ERR_JRET(vectorConvertVarToDouble(pLeft, &leftConvert, &pLeftCol)); + SCL_ERR_JRET(vectorConvertVarToDouble(pRight, &rightConvert, &pRightCol)); - _getDoubleValue_fn_t getVectorDoubleValueFnLeft; - _getDoubleValue_fn_t getVectorDoubleValueFnRight; - SCL_ERR_JRET(getVectorDoubleValueFn(pLeftCol->info.type, &getVectorDoubleValueFnLeft)); - SCL_ERR_JRET(getVectorDoubleValueFn(pRightCol->info.type, &getVectorDoubleValueFnRight)); + _getDoubleValue_fn_t getVectorDoubleValueFnLeft; + _getDoubleValue_fn_t getVectorDoubleValueFnRight; + SCL_ERR_JRET(getVectorDoubleValueFn(pLeftCol->info.type, &getVectorDoubleValueFnLeft)); + SCL_ERR_JRET(getVectorDoubleValueFn(pRightCol->info.type, &getVectorDoubleValueFnRight)); - double *output = (double *)pOutputCol->pData; - if (pLeft->numOfRows == pRight->numOfRows) { - for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { - if (IS_NULL) { - colDataSetNULL(pOutputCol, i); - continue; // TODO set null or ignore + double *output = (double *)pOutputCol->pData; + if (pLeft->numOfRows == pRight->numOfRows) { + for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { + if (IS_NULL) { + colDataSetNULL(pOutputCol, i); + continue; // TODO set null or ignore + } + double leftRes = 0; + double rightRes = 0; + SCL_ERR_JRET(getVectorDoubleValueFnLeft(LEFT_COL, i, &leftRes)); + SCL_ERR_JRET(getVectorDoubleValueFnRight(RIGHT_COL, i, &rightRes)); + *output = leftRes * rightRes; } - double leftRes = 0; - double rightRes = 0; - SCL_ERR_JRET(getVectorDoubleValueFnLeft(LEFT_COL, i, &leftRes)); - SCL_ERR_JRET(getVectorDoubleValueFnRight(RIGHT_COL, i, &rightRes)); - *output = leftRes * rightRes; + } else if (pLeft->numOfRows == 1) { + SCL_ERR_JRET(vectorMathMultiplyHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i)); + } else if (pRight->numOfRows == 1) { + SCL_ERR_JRET(vectorMathMultiplyHelper(pLeftCol, pRightCol, pOutputCol, pLeft->numOfRows, step, i)); } - } else if (pLeft->numOfRows == 1) { - SCL_ERR_JRET(vectorMathMultiplyHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i)); - } else if (pRight->numOfRows == 1) { - SCL_ERR_JRET(vectorMathMultiplyHelper(pLeftCol, pRightCol, pOutputCol, pLeft->numOfRows, step, i)); } _return: @@ -1501,37 +1575,21 @@ int32_t vectorMathDivide(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam int32_t leftConvert = 0, rightConvert = 0; SColumnInfoData *pLeftCol = NULL; SColumnInfoData *pRightCol = NULL; - SCL_ERR_JRET(vectorConvertVarToDouble(pLeft, &leftConvert, &pLeftCol)); - SCL_ERR_JRET(vectorConvertVarToDouble(pRight, &rightConvert, &pRightCol)); + if (pOutputCol->info.type == TSDB_DATA_TYPE_DECIMAL) { + SCL_ERR_JRET(vectorMathBinaryOpForDecimal(pLeft, pRight, pOut, step, i, OP_TYPE_DIV)); + } else { + SCL_ERR_JRET(vectorConvertVarToDouble(pLeft, &leftConvert, &pLeftCol)); + SCL_ERR_JRET(vectorConvertVarToDouble(pRight, &rightConvert, &pRightCol)); - _getDoubleValue_fn_t getVectorDoubleValueFnLeft; - _getDoubleValue_fn_t getVectorDoubleValueFnRight; - SCL_ERR_JRET(getVectorDoubleValueFn(pLeftCol->info.type, &getVectorDoubleValueFnLeft)); - SCL_ERR_JRET(getVectorDoubleValueFn(pRightCol->info.type, &getVectorDoubleValueFnRight)); + _getDoubleValue_fn_t getVectorDoubleValueFnLeft; + _getDoubleValue_fn_t getVectorDoubleValueFnRight; + SCL_ERR_JRET(getVectorDoubleValueFn(pLeftCol->info.type, &getVectorDoubleValueFnLeft)); + SCL_ERR_JRET(getVectorDoubleValueFn(pRightCol->info.type, &getVectorDoubleValueFnRight)); - double *output = (double *)pOutputCol->pData; - if (pLeft->numOfRows == pRight->numOfRows) { - for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { - if (IS_NULL) { // divide by 0 check - colDataSetNULL(pOutputCol, i); - continue; - } - double rightRes = 0; - SCL_ERR_JRET((getVectorDoubleValueFnRight(RIGHT_COL, i, &rightRes))); - if (rightRes == 0) { - colDataSetNULL(pOutputCol, i); - continue; - } - double leftRes = 0; - SCL_ERR_JRET(getVectorDoubleValueFnLeft(LEFT_COL, i, &leftRes)); - *output = leftRes / rightRes; - } - } else if (pLeft->numOfRows == 1) { - if (IS_HELPER_NULL(pLeftCol, 0)) { // Set pLeft->numOfRows NULL value - colDataSetNNULL(pOutputCol, 0, pRight->numOfRows); - } else { - for (; i >= 0 && i < pRight->numOfRows; i += step, output += 1) { - if (IS_HELPER_NULL(pRightCol, i)) { // divide by 0 check + double *output = (double *)pOutputCol->pData; + if (pLeft->numOfRows == pRight->numOfRows) { + for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { + if (IS_NULL) { // divide by 0 check colDataSetNULL(pOutputCol, i); continue; } @@ -1542,29 +1600,49 @@ int32_t vectorMathDivide(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam continue; } double leftRes = 0; - SCL_ERR_JRET(getVectorDoubleValueFnLeft(LEFT_COL, 0, &leftRes)); + SCL_ERR_JRET(getVectorDoubleValueFnLeft(LEFT_COL, i, &leftRes)); *output = leftRes / rightRes; } - } - } else if (pRight->numOfRows == 1) { - if (IS_HELPER_NULL(pRightCol, 0)) { // Set pLeft->numOfRows NULL value (divde by 0 check) - colDataSetNNULL(pOutputCol, 0, pLeft->numOfRows); - } else { - double rightRes = 0; - SCL_ERR_JRET((getVectorDoubleValueFnRight(RIGHT_COL, 0, &rightRes))); - if (rightRes == 0) { - colDataSetNNULL(pOutputCol, 0, pLeft->numOfRows); + } else if (pLeft->numOfRows == 1) { + if (IS_HELPER_NULL(pLeftCol, 0)) { // Set pLeft->numOfRows NULL value + colDataSetNNULL(pOutputCol, 0, pRight->numOfRows); } else { - for (; i >= 0 && i < pLeft->numOfRows; i += step, output += 1) { - if (IS_HELPER_NULL(pLeftCol, i)) { + for (; i >= 0 && i < pRight->numOfRows; i += step, output += 1) { + if (IS_HELPER_NULL(pRightCol, i)) { // divide by 0 check + colDataSetNULL(pOutputCol, i); + continue; + } + double rightRes = 0; + SCL_ERR_JRET((getVectorDoubleValueFnRight(RIGHT_COL, i, &rightRes))); + if (rightRes == 0) { colDataSetNULL(pOutputCol, i); continue; } double leftRes = 0; - SCL_ERR_JRET(getVectorDoubleValueFnLeft(LEFT_COL, i, &leftRes)); + SCL_ERR_JRET(getVectorDoubleValueFnLeft(LEFT_COL, 0, &leftRes)); *output = leftRes / rightRes; } } + } else if (pRight->numOfRows == 1) { + if (IS_HELPER_NULL(pRightCol, 0)) { // Set pLeft->numOfRows NULL value (divde by 0 check) + colDataSetNNULL(pOutputCol, 0, pLeft->numOfRows); + } else { + double rightRes = 0; + SCL_ERR_JRET((getVectorDoubleValueFnRight(RIGHT_COL, 0, &rightRes))); + if (rightRes == 0) { + colDataSetNNULL(pOutputCol, 0, pLeft->numOfRows); + } else { + for (; i >= 0 && i < pLeft->numOfRows; i += step, output += 1) { + if (IS_HELPER_NULL(pLeftCol, i)) { + colDataSetNULL(pOutputCol, i); + continue; + } + double leftRes = 0; + SCL_ERR_JRET(getVectorDoubleValueFnLeft(LEFT_COL, i, &leftRes)); + *output = leftRes / rightRes; + } + } + } } } @@ -1585,37 +1663,40 @@ int32_t vectorMathRemainder(SScalarParam *pLeft, SScalarParam *pRight, SScalarPa int32_t leftConvert = 0, rightConvert = 0; SColumnInfoData *pLeftCol = NULL; SColumnInfoData *pRightCol = NULL; - SCL_ERR_JRET(vectorConvertVarToDouble(pLeft, &leftConvert, &pLeftCol)); - SCL_ERR_JRET(vectorConvertVarToDouble(pRight, &rightConvert, &pRightCol)); + if (pOutputCol->info.type == TSDB_DATA_TYPE_DECIMAL) { + SCL_ERR_JRET(vectorMathBinaryOpForDecimal(pLeft, pRight, pOut, step, i, OP_TYPE_REM)); + } else { + SCL_ERR_JRET(vectorConvertVarToDouble(pLeft, &leftConvert, &pLeftCol)); + SCL_ERR_JRET(vectorConvertVarToDouble(pRight, &rightConvert, &pRightCol)); - _getDoubleValue_fn_t getVectorDoubleValueFnLeft; - _getDoubleValue_fn_t getVectorDoubleValueFnRight; - SCL_ERR_JRET(getVectorDoubleValueFn(pLeftCol->info.type, &getVectorDoubleValueFnLeft)); - SCL_ERR_JRET(getVectorDoubleValueFn(pRightCol->info.type, &getVectorDoubleValueFnRight)); + _getDoubleValue_fn_t getVectorDoubleValueFnLeft; + _getDoubleValue_fn_t getVectorDoubleValueFnRight; + SCL_ERR_JRET(getVectorDoubleValueFn(pLeftCol->info.type, &getVectorDoubleValueFnLeft)); + SCL_ERR_JRET(getVectorDoubleValueFn(pRightCol->info.type, &getVectorDoubleValueFnRight)); - double *output = (double *)pOutputCol->pData; + double *output = (double *)pOutputCol->pData; - int32_t numOfRows = TMAX(pLeft->numOfRows, pRight->numOfRows); - for (; i < numOfRows && i >= 0; i += step, output += 1) { - int32_t leftidx = pLeft->numOfRows == 1 ? 0 : i; - int32_t rightidx = pRight->numOfRows == 1 ? 0 : i; - if (IS_HELPER_NULL(pLeftCol, leftidx) || IS_HELPER_NULL(pRightCol, rightidx)) { - colDataSetNULL(pOutputCol, i); - continue; + int32_t numOfRows = TMAX(pLeft->numOfRows, pRight->numOfRows); + for (; i < numOfRows && i >= 0; i += step, output += 1) { + int32_t leftidx = pLeft->numOfRows == 1 ? 0 : i; + int32_t rightidx = pRight->numOfRows == 1 ? 0 : i; + if (IS_HELPER_NULL(pLeftCol, leftidx) || IS_HELPER_NULL(pRightCol, rightidx)) { + colDataSetNULL(pOutputCol, i); + continue; + } + + double lx = 0; + double rx = 0; + SCL_ERR_JRET(getVectorDoubleValueFnLeft(LEFT_COL, leftidx, &lx)); + SCL_ERR_JRET(getVectorDoubleValueFnRight(RIGHT_COL, rightidx, &rx)); + if (isnan(lx) || isinf(lx) || isnan(rx) || isinf(rx) || FLT_EQUAL(rx, 0)) { + colDataSetNULL(pOutputCol, i); + continue; + } + + *output = lx - ((int64_t)(lx / rx)) * rx; } - - double lx = 0; - double rx = 0; - SCL_ERR_JRET(getVectorDoubleValueFnLeft(LEFT_COL, leftidx, &lx)); - SCL_ERR_JRET(getVectorDoubleValueFnRight(RIGHT_COL, rightidx, &rx)); - if (isnan(lx) || isinf(lx) || isnan(rx) || isinf(rx) || FLT_EQUAL(rx, 0)) { - colDataSetNULL(pOutputCol, i); - continue; - } - - *output = lx - ((int64_t)(lx / rx)) * rx; } - _return: doReleaseVec(pLeftCol, leftConvert); doReleaseVec(pRightCol, rightConvert); @@ -1633,20 +1714,24 @@ int32_t vectorMathMinus(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam int32_t leftConvert = 0; SColumnInfoData *pLeftCol = NULL; - SCL_ERR_JRET(vectorConvertVarToDouble(pLeft, &leftConvert, &pLeftCol)); + if (IS_DECIMAL_TYPE(pOutputCol->info.type)) { + SCL_ERR_JRET(vectorMathUnaryOpForDecimal(pLeft, pOut, step, i, OP_TYPE_MINUS)); + } else { + SCL_ERR_JRET(vectorConvertVarToDouble(pLeft, &leftConvert, &pLeftCol)); - _getDoubleValue_fn_t getVectorDoubleValueFnLeft; - SCL_ERR_JRET(getVectorDoubleValueFn(pLeftCol->info.type, &getVectorDoubleValueFnLeft)); + _getDoubleValue_fn_t getVectorDoubleValueFnLeft; + SCL_ERR_JRET(getVectorDoubleValueFn(pLeftCol->info.type, &getVectorDoubleValueFnLeft)); - double *output = (double *)pOutputCol->pData; - for (; i < pLeft->numOfRows && i >= 0; i += step, output += 1) { - if (IS_HELPER_NULL(pLeftCol, i)) { - colDataSetNULL(pOutputCol, i); - continue; + double *output = (double *)pOutputCol->pData; + for (; i < pLeft->numOfRows && i >= 0; i += step, output += 1) { + if (IS_HELPER_NULL(pLeftCol, i)) { + colDataSetNULL(pOutputCol, i); + continue; + } + double result = 0; + SCL_ERR_JRET(getVectorDoubleValueFnLeft(LEFT_COL, i, &result)); + *output = (result == 0) ? 0 : -result; } - double result = 0; - SCL_ERR_JRET(getVectorDoubleValueFnLeft(LEFT_COL, i, &result)); - *output = (result == 0) ? 0 : -result; } _return: @@ -1768,10 +1853,7 @@ int32_t doVectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarPa int32_t leftIndex = (i >= pLeft->numOfRows) ? 0 : i; int32_t rightIndex = (i >= pRight->numOfRows) ? 0 : i; - char *pLeftData = colDataGetData(pLeft->columnData, leftIndex); - char *pRightData = colDataGetData(pRight->columnData, rightIndex); - - pRes[i] = filterDoCompare(fp, optr, pLeftData, pRightData); + pRes[i] = compareForType(fp, optr, pLeft->columnData, leftIndex, pRight->columnData, rightIndex); if (pRes[i]) { ++(*num); } @@ -1786,9 +1868,7 @@ int32_t doVectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarPa pRes[i] = false; continue; } - char *pLeftData = colDataGetData(pLeft->columnData, leftIndex); - char *pRightData = colDataGetData(pRight->columnData, rightIndex); - pRes[i] = filterDoCompare(fp, optr, pLeftData, pRightData); + pRes[i] = compareForType(fp, optr, pLeft->columnData, leftIndex, pRight->columnData, rightIndex); if (pRes[i]) { ++(*num); } @@ -1884,8 +1964,8 @@ int32_t doVectorCompare(SScalarParam *pLeft, SScalarParam *pLeftVar, SScalarPara continue; } - char *pLeftData = colDataGetData(pLeft->columnData, i); - bool res = filterDoCompare(fp, optr, pLeftData, pRight->pHashFilter); + bool res = compareForTypeWithColAndHash(fp, optr, pLeft->columnData, i, pRight->pHashFilter, + pRight->filterValueType, pRight->filterValueTypeMod); if (pLeftVar != NULL && taosHashGetSize(pRight->pHashFilterOthers) > 0){ do{ if (optr == OP_TYPE_IN && res){ @@ -1894,8 +1974,8 @@ int32_t doVectorCompare(SScalarParam *pLeft, SScalarParam *pLeftVar, SScalarPara if (optr == OP_TYPE_NOT_IN && !res){ break; } - pLeftData = colDataGetData(pLeftVar->columnData, i); - res = filterDoCompare(fpVar, optr, pLeftData, pRight->pHashFilterOthers); + res = compareForTypeWithColAndHash(fpVar, optr, pLeftVar->columnData, i, pRight->pHashFilterOthers, + pRight->filterValueType, pRight->filterValueTypeMod); }while(0); } colDataSetInt8(pOut->columnData, i, (int8_t *)&res); @@ -2027,7 +2107,7 @@ int32_t vectorIsTrue(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pO } { bool v = false; - GET_TYPED_DATA(v, bool, pOut->columnData->info.type, colDataGetData(pOut->columnData, i)); + GET_TYPED_DATA(v, bool, pOut->columnData->info.type, colDataGetData(pOut->columnData, i), typeGetTypeModFromColInfo(&pOut->columnData->info)); if (v) { ++pOut->numOfQualified; } @@ -2196,3 +2276,111 @@ bool checkOperatorRestypeIsTimestamp(EOperatorType opType, int32_t lType, int32_ } return false; } + +static int32_t vectorMathOpOneRowForDecimal(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t step, + int32_t i, EOperatorType op, SScalarParam *pOneRowParam) { + SScalarParam *pNotOneRowParam = pLeft == pOneRowParam ? pRight : pLeft; + Decimal *output = (Decimal *)pOut->columnData->pData; + int32_t code = 0; + SDataType leftType = GET_COL_DATA_TYPE(pLeft->columnData->info), + rightType = GET_COL_DATA_TYPE(pRight->columnData->info), + outType = GET_COL_DATA_TYPE(pOut->columnData->info); + if (IS_HELPER_NULL(pOneRowParam->columnData, 0)) { + colDataSetNNULL(pOut->columnData, 0, pNotOneRowParam->numOfRows); + } + Decimal oneRowData = {0}; + SDataType oneRowType = outType; + oneRowType.precision = TSDB_DECIMAL_MAX_PRECISION; + if (pLeft == pOneRowParam) { + oneRowType.scale = leftType.scale; + code = convertToDecimal(colDataGetData(pLeft->columnData, 0), &leftType, &oneRowData, &oneRowType); + } else { + oneRowType.scale = rightType.scale; + code = convertToDecimal(colDataGetData(pRight->columnData, 0), &rightType, &oneRowData, &oneRowType); + } + if (code != 0) return code; + + for (; i < pNotOneRowParam->numOfRows && i >= 0 && TSDB_CODE_SUCCESS == code; i += step, output += 1) { + if (IS_HELPER_NULL(pNotOneRowParam->columnData, i)) { + colDataSetNULL(pOut->columnData, i); + continue; + } + if (pOneRowParam == pLeft) { + code = + decimalOp(op, &oneRowType, &rightType, &outType, &oneRowData, colDataGetData(pRight->columnData, i), output); + } else { + code = decimalOp(op, &leftType, &oneRowType, &outType, colDataGetData(pLeft->columnData, i), &oneRowData, output); + } + } + return code; +} + +static int32_t vectorMathUnaryOpForDecimal(SScalarParam *pCol, SScalarParam *pOut, int32_t step, int32_t i, + EOperatorType op) { + int32_t code = 0; + SColumnInfoData *pOutputCol = pOut->columnData; + char *pDec = pOutputCol->pData; + for (; i < pCol->numOfRows && i >= 0; i += step, pDec += tDataTypes[pOutputCol->info.type].bytes) { + if (IS_HELPER_NULL(pCol->columnData, i)) { + colDataSetNULL(pOutputCol, i); + continue; + } + SDataType colDt = GET_COL_DATA_TYPE(pCol->columnData->info), outDt = GET_COL_DATA_TYPE(pOutputCol->info); + + code = decimalOp(op, &colDt, NULL, &outDt, colDataGetData(pCol->columnData, i), NULL, pDec); + } + return code; +} + +static int32_t vectorMathBinaryOpForDecimal(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t step, + int32_t i, EOperatorType op) { + Decimal *output = (Decimal *)pOut->columnData->pData; + int32_t code = 0; + SDataType leftType = GET_COL_DATA_TYPE(pLeft->columnData->info), + rightType = GET_COL_DATA_TYPE(pRight->columnData->info), + outType = GET_COL_DATA_TYPE(pOut->columnData->info); + if (pLeft->numOfRows == pRight->numOfRows) { + for (; i < pRight->numOfRows && i >= 0 && TSDB_CODE_SUCCESS == code; i += step, output += 1) { + if (IS_NULL) { + colDataSetNULL(pOut->columnData, i); + continue; + } + code = decimalOp(op, &leftType, &rightType, &outType, colDataGetData(pLeft->columnData, i), + colDataGetData(pRight->columnData, i), output); + } + } else if (pLeft->numOfRows == 1) { + code = vectorMathOpOneRowForDecimal(pLeft, pRight, pOut, step, i, op, pLeft); + } else if (pRight->numOfRows == 1) { + code = vectorMathOpOneRowForDecimal(pLeft, pRight, pOut, step, i, op, pRight); + } + return code; +} + +bool compareForType(__compar_fn_t fp, int32_t optr, SColumnInfoData* pColL, int32_t idxL, SColumnInfoData* pColR, int32_t idxR) { + void* pLeftData = colDataGetData(pColL, idxL), *pRightData = colDataGetData(pColR, idxR); + if (IS_DECIMAL_TYPE(pColL->info.type) || IS_DECIMAL_TYPE(pColR->info.type)) { + SDecimalCompareCtx ctxL = {.pData = pLeftData, + .type = pColL->info.type, + .typeMod = typeGetTypeModFromColInfo(&pColL->info)}, + ctxR = {.pData = pRightData, + .type = pColR->info.type, + .typeMod = typeGetTypeModFromColInfo(&pColR->info)}; + return filterDoCompare(fp, optr, &ctxL, &ctxR); + } else { + return filterDoCompare(fp, optr, pLeftData, pRightData); + } +} + +bool compareForTypeWithColAndHash(__compar_fn_t fp, int32_t optr, SColumnInfoData *pColL, int32_t idxL, + const void *pHashData, int32_t hashType, STypeMod hashTypeMod) { + void * pLeftData = colDataGetData(pColL, idxL); + if (IS_DECIMAL_TYPE(pColL->info.type) || IS_DECIMAL_TYPE(hashType)) { + SDecimalCompareCtx ctxL = {.pData = pLeftData, + .type = pColL->info.type, + .typeMod = typeGetTypeModFromColInfo(&pColL->info)}, + ctxR = {.pData = (void *)pHashData, .type = hashType, .typeMod = hashTypeMod}; + return filterDoCompare(fp, optr, &ctxL, &ctxR); + } else { + return filterDoCompare(fp, optr, pLeftData, (void*)pHashData); + } +} diff --git a/source/util/CMakeLists.txt b/source/util/CMakeLists.txt index d606d83712..934c356c64 100644 --- a/source/util/CMakeLists.txt +++ b/source/util/CMakeLists.txt @@ -40,14 +40,14 @@ if(TD_LINUX) util PUBLIC os common PUBLIC lz4_static fast-lzma2 pcre2-8 - PUBLIC api cjson geos_c TSZ + PUBLIC api cjson geos_c TSZ decimal ) else() target_link_libraries( util PUBLIC os common PUBLIC lz4_static pcre2-8 - PUBLIC api cjson geos_c TSZ + PUBLIC api cjson geos_c TSZ decimal ) endif() diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index 18229f2e81..9dc667c41e 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -25,6 +25,7 @@ #include "types.h" #include "osString.h" #include "ttimer.h" +#include "decimal.h" int32_t setChkInBytes1(const void *pLeft, const void *pRight) { return NULL != taosHashGet((SHashObj *)pRight, pLeft, 1) ? 1 : 0; @@ -58,6 +59,15 @@ int32_t setChkNotInBytes8(const void *pLeft, const void *pRight) { return NULL == taosHashGet((SHashObj *)pRight, pLeft, 8) ? 1 : 0; } +int32_t setChkInDecimalHash(const void* pLeft, const void* pRight) { + const SDecimalCompareCtx *pCtxL = pLeft, *pCtxR = pRight; + return NULL != taosHashGet((SHashObj *)(pCtxR->pData), pCtxL->pData, tDataTypes[pCtxL->type].bytes) ? 1 : 0; +} + +int32_t setChkNotInDecimalHash(const void* pLeft, const void* pRight) { + return NULL == taosHashGet((SHashObj *)pRight, pLeft, 16) ? 1 : 0; +} + int32_t compareChkInString(const void *pLeft, const void *pRight) { return NULL != taosHashGet((SHashObj *)pRight, pLeft, varDataTLen(pLeft)) ? 1 : 0; } @@ -1034,6 +1044,46 @@ int32_t compareUint64Uint32(const void *pLeft, const void *pRight) { return 0; } +int32_t compareDecimal64(const void* pleft, const void* pright) { + if (decimal64Compare(OP_TYPE_GREATER_THAN, pleft, pright)) return 1; + if (decimal64Compare(OP_TYPE_LOWER_THAN, pleft, pright)) return -1; + return 0; +} + +int32_t compareDecimal128(const void* pleft, const void* pright) { + if (decimalCompare(OP_TYPE_GREATER_THAN, pleft, pright)) return 1; + if (decimalCompare(OP_TYPE_LOWER_THAN, pleft, pright)) return -1; + return 0; +} + +int32_t compareDecimal64SameScale(const void* pleft, const void* pright) { + const SDecimalOps* pOps = getDecimalOps(TSDB_DATA_TYPE_DECIMAL64); + if (pOps->gt(pleft, pright, DECIMAL_WORD_NUM(Decimal64))) return 1; + if (pOps->lt(pleft, pright, DECIMAL_WORD_NUM(Decimal64))) return -1; + return 0; +} + +int32_t compareDecimal64SameScaleDesc(const void* pLeft, const void* pRight) { + const SDecimalOps* pOps = getDecimalOps(TSDB_DATA_TYPE_DECIMAL64); + if (pOps->lt(pLeft, pRight, DECIMAL_WORD_NUM(Decimal64))) return 1; + if (pOps->gt(pLeft, pRight, DECIMAL_WORD_NUM(Decimal64))) return -1; + return 0; +} + +int32_t compareDecimal128SameScale(const void* pleft, const void* pright) { + const SDecimalOps* pOps = getDecimalOps(TSDB_DATA_TYPE_DECIMAL); + if (pOps->gt(pleft, pright, DECIMAL_WORD_NUM(Decimal))) return 1; + if (pOps->lt(pleft, pright, DECIMAL_WORD_NUM(Decimal))) return -1; + return 0; +} + +int32_t compareDecimal128SameScaleDesc(const void* pLeft, const void* pRight) { + const SDecimalOps* pOps = getDecimalOps(TSDB_DATA_TYPE_DECIMAL); + if (pOps->lt(pLeft, pRight, DECIMAL_WORD_NUM(Decimal))) return 1; + if (pOps->gt(pLeft, pRight, DECIMAL_WORD_NUM(Decimal))) return -1; + return 0; +} + int32_t compareJsonValDesc(const void *pLeft, const void *pRight) { return compareJsonVal(pRight, pLeft); } /* @@ -1778,6 +1828,10 @@ __compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order) { return (order == TSDB_ORDER_ASC) ? compareLenPrefixedWStr : compareLenPrefixedWStrDesc; case TSDB_DATA_TYPE_JSON: return (order == TSDB_ORDER_ASC) ? compareJsonVal : compareJsonValDesc; + case TSDB_DATA_TYPE_DECIMAL64: + return (order == TSDB_ORDER_ASC) ? compareDecimal64SameScale : compareDecimal64SameScaleDesc; + case TSDB_DATA_TYPE_DECIMAL: + return (order == TSDB_ORDER_ASC) ? compareDecimal128SameScale : compareDecimal128SameScaleDesc; default: return (order == TSDB_ORDER_ASC) ? compareInt32Val : compareInt32ValDesc; } diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 6ffb5b635a..efc20f1665 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -1637,7 +1637,7 @@ int32_t tsDecompressBigint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int int8_t alvl = tsGetCompressL2Level(l2, lvl); \ return compressL2Dict[l2].comprFn(pIn, nIn, pOut, nOut, type, alvl); \ } else { \ - uTrace("dencode:%s, decompress:%s, level:%d, type:%s", "disabled", compressL2Dict[l1].name, lvl, \ + uTrace("dencode:%s, decompress:%s, level:%d, type:%s", "disabled", compressL2Dict[l2].name, lvl, \ tDataTypes[type].name); \ return compressL2Dict[l2].decomprFn(pIn, nIn, pOut, nOut, type); \ } \ @@ -1831,6 +1831,25 @@ int32_t tsDecompressBigint2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, in FUNC_COMPRESS_IMPL(pIn, nIn, nEle, pOut, nOut, cmprAlg, pBuf, nBuf, TSDB_DATA_TYPE_BIGINT, 0); } +int32_t tsCompressDecimal64(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg, + void *pBuf, int32_t nBuf) { + FUNC_COMPRESS_IMPL(pIn, nIn, nEle, pOut, nOut, cmprAlg, pBuf, nBuf, TSDB_DATA_TYPE_DECIMAL64, 1); +} +int32_t tsDecompressDecimal64(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg, + void *pBuf, int32_t nBuf) { + + FUNC_COMPRESS_IMPL(pIn, nIn, nEle, pOut, nOut, cmprAlg, pBuf, nBuf, TSDB_DATA_TYPE_DECIMAL64, 0); +} + +int32_t tsCompressDecimal128(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg, + void *pBuf, int32_t nBuf) { + FUNC_COMPRESS_IMPL(pIn, nIn, nEle, pOut, nOut, cmprAlg, pBuf, nBuf, TSDB_DATA_TYPE_DECIMAL, 1); +} +int32_t tsDecompressDecimal128(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg, + void *pBuf, int32_t nBuf) { + FUNC_COMPRESS_IMPL(pIn, nIn, nEle, pOut, nOut, cmprAlg, pBuf, nBuf, TSDB_DATA_TYPE_DECIMAL, 0); +} + void tcompressDebug(uint32_t cmprAlg, uint8_t *l1Alg, uint8_t *l2Alg, uint8_t *level) { DEFINE_VAR(cmprAlg) *l1Alg = l1; @@ -1903,6 +1922,12 @@ int32_t getWordLength(char type) { case TSDB_DATA_TYPE_TINYINT: wordLength = CHAR_BYTES; break; + case TSDB_DATA_TYPE_DECIMAL64: + wordLength = DECIMAL64_BYTES; + break; + case TSDB_DATA_TYPE_DECIMAL: + wordLength = DECIMAL128_BYTES; + break; default: uError("Invalid decompress integer type:%d", type); return TSDB_CODE_INVALID_PARA; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 84d2416af2..5daca53911 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -120,6 +120,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_OUT_OF_BUFFER, "Out of buffer") TAOS_DEFINE_ERROR(TSDB_CODE_INTERNAL_ERROR, "Internal error") TAOS_DEFINE_ERROR(TSDB_CODE_TIME_ERROR, "Internal error in time") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_DISK_ID, "Internal error invalid disk id") +TAOS_DEFINE_ERROR(TSDB_CODE_DECIMAL_OVERFLOW, "Decimal value overflow") //client TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_OPERATION, "Invalid operation") diff --git a/source/util/src/thashutil.c b/source/util/src/thashutil.c index aab0fd454b..bcfe35477e 100644 --- a/source/util/src/thashutil.c +++ b/source/util/src/thashutil.c @@ -240,6 +240,10 @@ int32_t taosDoubleEqual(const void *a, const void *b, size_t UNUSED_PARAM(sz)) { return getComparFunc(TSDB_DATA_TYPE_DOUBLE, -1)(a, b); } +int32_t taosDecimalEqual(const void* a, const void* b, size_t UNUSED_PARAM(sz)) { + return 0; +} + _equal_fn_t taosGetDefaultEqualFunction(int32_t type) { _equal_fn_t fn = NULL; switch (type) { @@ -249,6 +253,10 @@ _equal_fn_t taosGetDefaultEqualFunction(int32_t type) { case TSDB_DATA_TYPE_DOUBLE: fn = taosDoubleEqual; break; + case TSDB_DATA_TYPE_DECIMAL64: + case TSDB_DATA_TYPE_DECIMAL: + fn = memcmp; + break; default: fn = memcmp; break; diff --git a/source/util/src/tutil.c b/source/util/src/tutil.c index 4ce04b1201..a9820e4f3a 100644 --- a/source/util/src/tutil.c +++ b/source/util/src/tutil.c @@ -278,7 +278,7 @@ char *paGetToken(char *string, char **token, int32_t *tokenLen) { return string; } -int64_t strnatoi(char *num, int32_t len) { +int64_t strnatoi(const char *num, int32_t len) { int64_t ret = 0, i, dig, base = 1; if (len > (int32_t)strlen(num)) { diff --git a/source/util/test/CMakeLists.txt b/source/util/test/CMakeLists.txt index 768e465fea..9158e07e72 100644 --- a/source/util/test/CMakeLists.txt +++ b/source/util/test/CMakeLists.txt @@ -30,7 +30,7 @@ ENDIF() # ADD_EXECUTABLE(trefTest ./trefTest.c) # TARGET_LINK_LIBRARIES(trefTest util common) # ENDIF () -INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/src/util/inc) +INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/source/util/inc) INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/include/common) add_executable(heapTest "heapTest.cpp") diff --git a/source/util/test/memPoolTest.cpp b/source/util/test/memPoolTest.cpp index a8da96711d..8af8b01368 100644 --- a/source/util/test/memPoolTest.cpp +++ b/source/util/test/memPoolTest.cpp @@ -2299,15 +2299,6 @@ TEST(functionsTest, internalFunc) { #endif - - - - - - - - - int main(int argc, char** argv) { taosSeedRand(taosGetTimestampSec()); mptInit(); @@ -2317,4 +2308,6 @@ int main(int argc, char** argv) { + + #pragma GCC diagnosti diff --git a/tests/army/query/function/test_resinfo.py b/tests/army/query/function/test_resinfo.py index 5a59ed45cd..8f0128842e 100644 --- a/tests/army/query/function/test_resinfo.py +++ b/tests/army/query/function/test_resinfo.py @@ -26,7 +26,7 @@ from frame.sql import * from frame.caseBase import * from frame import * -initial_hash_resinfoInt = "fbfd69d6f0aa6e015a7b5475b33ee8c8" +initial_hash_resinfoInt = "eae723d1ecdd18993a11d43d1b00316d" initial_hash_resinfo = "172d04aa7af0d8cd2e4d9df284079958" class TDTestCase(TBase): diff --git a/tests/ci/scan_file_path.py b/tests/ci/scan_file_path.py index 9244d37456..8ab48ecba2 100644 --- a/tests/ci/scan_file_path.py +++ b/tests/ci/scan_file_path.py @@ -64,7 +64,7 @@ os.makedirs(log_file_path, exist_ok=True) scan_log_file = f"{log_file_path}/scan_log.txt" logger.add(scan_log_file, rotation="10MB", retention="7 days", level="DEBUG") -#if error happens, open this to debug +# if error happens, open this to debug # print(self_path,work_path,TD_project_path,log_file_path,change_file_list) # scan result base path @@ -75,7 +75,7 @@ scan_result_base_path = f"{log_file_path}/clang_scan_result/" # compile_commands_path = f"{work_path}/debugNoSan/compile_commands.json" compile_commands_path = f"{TD_project_path}/debug/compile_commands.json" -#if error happens, open this to debug +# if error happens, open this to debug # print(f"compile_commands_path:{compile_commands_path}") # # replace the docerk worf path with real work path in compile_commands.json @@ -116,8 +116,15 @@ class CommandExecutor: def scan_files_path(source_file_path): # scan_dir_list = ["source", "include", "docs/examples", "tests/script/api", "src/plugins"] scan_dir_list = ["source", "include", "docs/examples", "src/plugins"] - scan_skip_file_list = ["/root/charles/TDinternal/community/tools/taosws-rs/target/release/build/openssl-sys-7811e597b848e397/out/openssl-build/install/include/openssl", - "/test/", "contrib", "debug", "deps", "/root/charles/TDinternal/community/source/libs/parser/src/sql.c", "/root/charles/TDinternal/community/source/client/jni/windows/win32/bridge/AccessBridgeCalls.c"] + scan_skip_file_list = [ + "/root/charles/TDinternal/community/tools/taosws-rs/target/release/build/openssl-sys-7811e597b848e397/out/openssl-build/install/include/openssl", + "/test/", + "contrib", + "debug", + "deps", + "/root/charles/TDinternal/community/source/libs/parser/src/sql.c", + "/root/charles/TDinternal/community/source/client/jni/windows/win32/bridge/AccessBridgeCalls.c", + ] for root, dirs, files in os.walk(source_file_path): for file in files: if any(item in root for item in scan_dir_list): @@ -129,7 +136,17 @@ def scan_files_path(source_file_path): def input_files(change_files): # scan_dir_list = ["source", "include", "docs/examples", "tests/script/api", "src/plugins"] scan_dir_list = ["source", "include", "docs/examples", "src/plugins"] - scan_skip_file_list = ["tools/taosws-rs/target/release/build/openssl-sys-7811e597b848e397/out/openssl-build/install/include/openssl", "/test/", "contrib", "debug", "deps", "source/libs/parser/src/sql.c", "source/libs/azure", "source/client/jni/windows/win32/bridge/AccessBridgeCalls.c"] + scan_skip_file_list = [ + "tools/taosws-rs/target/release/build/openssl-sys-7811e597b848e397/out/openssl-build/install/include/openssl", + "/test/", + "contrib", + "debug", + "deps", + "source/libs/parser/src/sql.c", + "source/libs/azure", + "source/client/jni/windows/win32/bridge/AccessBridgeCalls.c", + "source/libs/decimal/", + ] with open(change_files, 'r') as file: for line in file: file_name = line.strip() diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 397c690ac7..06184e546a 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -323,6 +323,16 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsma2.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsma2.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsma2.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/decimal.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/decimal.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/decimal.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/decimal.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/decimal.py -Q 1 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/decimal2.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/decimal2.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/decimal2.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/decimal2.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/decimal2.py -Q 1 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tbnameIn.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tbnameIn.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tbnameIn.py -Q 2 diff --git a/tests/system-test/2-query/decimal.py b/tests/system-test/2-query/decimal.py new file mode 100644 index 0000000000..eb03207041 --- /dev/null +++ b/tests/system-test/2-query/decimal.py @@ -0,0 +1,2435 @@ +import math +from random import randrange +import random +import time +import threading +import secrets +import numpy + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from decimal import * +from multiprocessing import Value, Lock +from functools import cmp_to_key + +class AtomicCounter: + def __init__(self, initial_value=0): + self._value = Value('i', initial_value) + self._lock = Lock() + + def fetch_add(self, delta = 1): + with self._lock: + old_value = self._value.value + self._value.value += delta + return old_value + +getcontext().prec = 40 + +def get_decimal(val, scale: int) -> Decimal: + if val == 'NULL': + return None + getcontext().prec = 100 + return Decimal(val).quantize(Decimal("1." + "0" * scale), ROUND_HALF_UP) + +syntax_error = -2147473920 +invalid_column = -2147473918 +invalid_compress_level = -2147483084 +invalid_encode_param = -2147483087 +invalid_operation = -2147483136 +scalar_convert_err = -2147470768 + + +decimal_test_query = True +decimal_insert_validator_test = True +operator_test_round = 1 +tb_insert_rows = 1000 +binary_op_with_const_test = True +binary_op_with_col_test = True +unary_op_test = True +binary_op_in_where_test = True +test_decimal_funcs = False +cast_func_test_round = 10 + +class DecimalTypeGeneratorConfig: + def __init__(self): + self.enable_weight_overflow: bool = False + self.weightOverflowRatio: float = 0.001 + self.enable_scale_overflow: bool = True + self.scale_overflow_ratio = 0.1 + self.enable_positive_sign = False + self.with_corner_case = True + self.corner_case_ratio = 0.1 + self.positive_ratio = 0.7 + self.prec = 38 + self.scale = 10 + + +class DecimalStringRandomGenerator: + def __init__(self): + self.corner_cases = ["0", "NULL", "0.", ".0", "000.000000"] + self.ratio_base: int = 1000000 + + def possible(self, possibility: float) -> bool: + return random.randint(0, self.ratio_base) < possibility * self.ratio_base + + def generate_sign(self, positive_ratio: float) -> str: + if self.possible(positive_ratio): + return "+" + return "-" + + def generate_digit(self) -> str: + return str(random.randint(0, 9)) + + def current_should_generate_corner_case(self, corner_case_ratio: float) -> bool: + return self.possible(corner_case_ratio) + + def generate_corner_case(self, config: DecimalTypeGeneratorConfig) -> str: + if self.possible(0.8): + return random.choice(self.corner_cases) + else: + res = self.generate_digit() * (config.prec - config.scale) + if self.possible(0.8): + res += "." + if self.possible(0.8): + res += self.generate_digit() * config.scale + return res + + ## 写入大整数的例子, 如10000000000, scale解析时可能为负数 + ## Generate decimal with E/e + def generate(self, config: DecimalTypeGeneratorConfig) -> str: + ret: str = "" + sign = self.generate_sign(config.positive_ratio) + if config.with_corner_case and self.current_should_generate_corner_case( + config.corner_case_ratio + ): + ret += self.generate_corner_case(config) + else: + if config.enable_positive_sign or sign != "+": + ret += sign + weight = random.randint(1, config.prec - config.scale) + scale = random.randint(1, config.scale) + for i in range(weight): + ret += self.generate_digit() + + if config.enable_weight_overflow and self.possible( + config.weightOverflowRatio + ): + extra_weight = ( + config.prec + - weight + + 1 + + random.randint(1, self.get_max_prec(config.prec)) + ) + while extra_weight > 0: + ret += self.generate_digit() + extra_weight -= 1 + ret += "." + for i in range(scale): + ret += self.generate_digit() + if config.enable_scale_overflow and self.possible( + config.scale_overflow_ratio + ): + extra_scale = ( + config.scale + - scale + + 1 + + random.randint(1, self.get_max_prec(config.prec)) + ) + while extra_scale > 0: + ret += self.generate_digit() + extra_scale -= 1 + return ret + + def get_max_prec(self, prec): + if prec <= 18: + return 18 + else: + return 38 + + +class DecimalColumnAggregator: + def __init__(self): + self.max: Decimal = Decimal("0") + self.min: Decimal = Decimal("0") + self.count: int = 0 + self.sum: Decimal = Decimal("0") + self.null_num: int = 0 + self.none_num: int = 0 + self.first = None + self.last = None + + def add_value(self, value: str, scale: int): + self.count += 1 + if value == "NULL": + self.null_num += 1 + elif value == "None": + self.none_num += 1 + else: + v: Decimal = get_decimal(value, scale) + if self.first is None: + self.first = v + self.last = v + self.sum += v + if v > self.max: + self.max = v + if v < self.min: + self.min = v + +atomic_counter = AtomicCounter(0) + +class TaosShell: + def __init__(self): + self.counter_ = atomic_counter.fetch_add() + self.queryResult = [] + self.tmp_file_path = "/tmp/taos_shell_result" + + def get_file_path(self): + return f"{self.tmp_file_path}_{self.counter_}" + + def read_result(self): + with open(self.get_file_path(), "r") as f: + lines = f.readlines() + lines = lines[1:] + for line in lines: + col = 0 + vals: list[str] = line.split(",") + if len(self.queryResult) == 0: + self.queryResult = [[] for i in range(len(vals))] + for val in vals: + self.queryResult[col].append(val.strip().strip('"')) + col += 1 + + def query(self, sql: str): + with open(self.get_file_path(), "a+") as f: + f.truncate(0) + self.queryResult = [] + try: + command = f'taos -s "{sql} >> {self.get_file_path()}"' + result = subprocess.run( + command, shell=True, check=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE + ) + self.read_result() + except Exception as e: + tdLog.exit(f"Command '{sql}' failed with error: {e.stderr.decode('utf-8')}") + self.queryResult = [] + raise + return self.queryResult + +class DecimalColumnExpr: + def __init__(self, format: str, executor): + self.format_: str = format + self.executor_ = executor + self.params_ = () + self.res_type_: DataType = None + self.query_col: Column = None + + def __str__(self): + return f"({self.format_})".format(*self.params_) + + def execute(self, params): + return self.executor_(self, params) + + def get_query_col_val(self, tbname, i): + return self.query_col.get_val_for_execute(tbname, i) + + def get_val(self, tbname: str, idx: int): + params = () + for p in self.params_: + params = params + (p.get_val(tbname, idx),) + return self.execute(params) + + def convert_to_res_type(self, val: Decimal) -> Decimal: + if self.res_type_.is_decimal_type(): + return get_decimal(val, self.res_type_.scale()) + elif self.res_type_.type == TypeEnum.DOUBLE: + return float(val) + + def get_input_types(self) -> List: + pass + + def should_skip_for_decimal(self, cols: list)->bool: + return False + + def check_query_results(self, query_col_res: List, tbname: str): + query_len = len(query_col_res) + pass + + def check_for_filtering(self, query_col_res: List, tbname: str): + j: int = -1 + for i in range(len(query_col_res)): + j += 1 + v_from_query = query_col_res[i] + while True: + params = () + for p in self.params_: + if isinstance(p, Column) or isinstance(p, DecimalColumnExpr): + p = p.get_val_for_execute(tbname, j) + params = params + (p,) + v_from_calc_in_py = self.execute(params) + + if not v_from_calc_in_py: + j += 1 + continue + else: + break + dec_from_query = Decimal(v_from_query) + dec_from_calc = self.get_query_col_val(tbname, j) + if dec_from_query != dec_from_calc: + tdLog.exit(f"filter with {self} failed, query got: {dec_from_query}, expect {dec_from_calc}, param: {params}") + else: + pass + #tdLog.info(f"filter with {self} succ, query got: {dec_from_query}, expect {dec_from_calc}, param: {params}") + + def check(self, query_col_res: List, tbname: str): + for i in range(len(query_col_res)): + v_from_query = query_col_res[i] + params = () + for p in self.params_: + if isinstance(p, Column) or isinstance(p, DecimalColumnExpr): + p = p.get_val_for_execute(tbname, i) + params = params + (p,) + v_from_calc_in_py = self.execute(params) + + if v_from_calc_in_py == 'NULL' or v_from_query == 'NULL': + if v_from_calc_in_py != v_from_query: + tdLog.exit(f"query with expr: {self} calc in py got: {v_from_calc_in_py}, query got: {v_from_query}") + #tdLog.debug(f"query with expr: {self} calc got same result: NULL") + continue + failed = False + if self.res_type_.type == TypeEnum.BOOL: + query_res = bool(int(v_from_query)) + calc_res = bool(int(v_from_calc_in_py)) + failed = query_res != calc_res + elif isinstance(v_from_calc_in_py, float): + query_res = float(v_from_query) + calc_res = float(v_from_calc_in_py) + failed = not math.isclose(query_res, calc_res, abs_tol=1e-7) + else: + query_res = Decimal(v_from_query) + calc_res = Decimal(v_from_calc_in_py) + failed = query_res != calc_res + if failed: + tdLog.exit( + f"check decimal column failed for expr: {self}, input: {[t.__str__() for t in self.get_input_types()]}, res_type: {self.res_type_}, params: {params}, query: {v_from_query}, expect {calc_res}, but get {query_res}" + ) + else: + pass + #tdLog.info( f"op succ: {self}, in: {[t.__str__() for t in self.get_input_types()]}, res: {self.res_type_}, params: {params}, insert:{v_from_calc_in_py} query:{v_from_query}, py calc: {calc_res}") + + ## format_params are already been set + def generate_res_type(self): + pass + + def generate(self, format_params) -> str: + self.params_ = format_params + self.generate_res_type() + return self.__str__() + + +class TypeEnum: + BOOL = 1 + TINYINT = 2 + SMALLINT = 3 + INT = 4 + BIGINT = 5 + FLOAT = 6 + DOUBLE = 7 + VARCHAR = 8 + TIMESTAMP = 9 + NCHAR = 10 + UTINYINT = 11 + USMALLINT = 12 + UINT = 13 + UBIGINT = 14 + JSON = 15 + VARBINARY = 16 + DECIMAL = 17 + BINARY = 8 + GEOMETRY = 20 + DECIMAL64 = 21 + + @staticmethod + def get_type_prec(type: int): + type_prec = [0, 1, 3, 5, 10, 19, 38, 38, 0, 19, 10, 3, 5, 10, 20, 0, 0, 0, 0, 0, 0, 0] + return type_prec[type] + + @staticmethod + def get_type_str(type: int): + type_str = [ + "", + "BOOL", + "TINYINT", + "SMALLINT", + "INT", + "BIGINT", + "FLOAT", + "DOUBLE", + "VARCHAR", + "TIMESTAMP", + "NCHAR", + "TINYINT UNSIGNED", + "SMALLINT UNSIGNED", + "INT UNSIGNED", + "BIGINT UNSIGNED", + "JSON", + "VARBINARY", + "DECIMAL", + "", + "", + "GEOMETRY", + "DECIMAL", + ] + return type_str[type] + + +class DataType: + def __init__(self, type: int, length: int = 0, type_mod: int = 0): + self.type: int = type + self.length = length + self.type_mod = type_mod + + def __str__(self): + if self.type_mod != 0: + return f"{TypeEnum.get_type_str(self.type)}({self.prec()}, {self.scale()})" + if self.length: + return f"{TypeEnum.get_type_str(self.type)}({self.length})" + return TypeEnum.get_type_str(self.type) + + def __eq__(self, other): + return self.type == other.type and self.length == other.length and self.type_mod == other.type_mod + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash((self.type, self.length)) + + def __repr__(self): + return f"DataType({self.type}, {self.length}, {self.type_mod})" + + def is_decimal_type(self): + return self.type == TypeEnum.DECIMAL or self.type == TypeEnum.DECIMAL64 + + def is_varchar_type(self): + return self.type == TypeEnum.VARCHAR or self.type == TypeEnum.NCHAR or self.type == TypeEnum.VARBINARY or self.type == TypeEnum.JSON or self.type == TypeEnum.BINARY + + def is_real_type(self): + return self.type == TypeEnum.FLOAT or self.type == TypeEnum.DOUBLE + + def prec(self): + return 0 + + def scale(self): + return 0 + + ## TODO generate NULL, None + def generate_value(self) -> str: + if self.type == TypeEnum.BOOL: + return ['true', 'false'][secrets.randbelow(2)] + if self.type == TypeEnum.TINYINT: + return str(secrets.randbelow(256) - 128) + if self.type == TypeEnum.SMALLINT: + return str(secrets.randbelow(65536) - 32768) + if self.type == TypeEnum.INT: + return str(secrets.randbelow(4294967296) - 2147483648) + if self.type == TypeEnum.BIGINT: + return str(secrets.randbelow(9223372036854775808) - 4611686018427387904) + if self.type == TypeEnum.FLOAT or self.type == TypeEnum.DOUBLE: + return str(random.random()) + if ( + self.type == TypeEnum.VARCHAR + or self.type == TypeEnum.NCHAR + or self.type == TypeEnum.VARBINARY + ): + return f"'{str(random.random())[0:self.length]}'" + if self.type == TypeEnum.TIMESTAMP: + return str(secrets.randbelow(9223372036854775808)) + if self.type == TypeEnum.UTINYINT: + return str(secrets.randbelow(256)) + if self.type == TypeEnum.USMALLINT: + return str(secrets.randbelow(65536)) + if self.type == TypeEnum.UINT: + return str(secrets.randbelow(4294967296)) + if self.type == TypeEnum.UBIGINT: + return str(secrets.randbelow(9223372036854775808)) + if self.type == TypeEnum.JSON: + return f'{{"key": "{secrets.token_urlsafe(10)}"}}' + if self.type == TypeEnum.GEOMETRY: + return "'POINT(1.0 1.0)'" + raise Exception(f"unsupport type {self.type}") + + def check(self, values, offset: int): + return True + + def get_typed_val_for_execute(self, val, const_col = False): + if self.type == TypeEnum.DOUBLE: + return float(val) + elif self.type == TypeEnum.BOOL: + if val == "true": + return 1 + else: + return 0 + elif self.type == TypeEnum.FLOAT: + if const_col: + val = float(str(numpy.float32(val))) + else: + val = float(numpy.float32(val)) + elif self.type == TypeEnum.DECIMAL or self.type == TypeEnum.DECIMAL64: + return get_decimal(val, self.scale()) + elif isinstance(val, str): + val = val.strip("'") + return val + + def get_typed_val(self, val): + if self.type == TypeEnum.FLOAT: + return float(str(numpy.float32(val))) + elif self.type == TypeEnum.DOUBLE: + return float(val) + return val + + @staticmethod + def get_decimal_types() -> list: + return [TypeEnum.DECIMAL64, TypeEnum.DECIMAL] + + @staticmethod + def get_decimal_op_types()-> list: + return [ + TypeEnum.BOOL, + TypeEnum.TINYINT, + TypeEnum.SMALLINT, + TypeEnum.INT, + TypeEnum.BIGINT, + TypeEnum.FLOAT, + TypeEnum.DOUBLE, + TypeEnum.VARCHAR, + TypeEnum.NCHAR, + TypeEnum.UTINYINT, + TypeEnum.USMALLINT, + TypeEnum.UINT, + TypeEnum.UBIGINT, + TypeEnum.DECIMAL, + TypeEnum.DECIMAL64, + ] + +class DecimalType(DataType): + DECIMAL_MAX_PRECISION = 38 + DECIMAL64_MAX_PRECISION = 18 + def __init__(self, type, precision: int, scale: int): + self.precision_ = precision + self.scale_ = scale + if type == TypeEnum.DECIMAL64: + bytes = 8 + else: + bytes = 16 + super().__init__(type, bytes, self.get_decimal_type_mod()) + self.decimal_generator: DecimalStringRandomGenerator = DecimalStringRandomGenerator() + self.generator_config: DecimalTypeGeneratorConfig = DecimalTypeGeneratorConfig() + #self.generator_config.with_corner_case = False + self.generator_config.prec = precision + self.generator_config.scale = scale + self.aggregator: DecimalColumnAggregator = DecimalColumnAggregator() + self.values: List[str] = [] + + def get_decimal_type_mod(self) -> int: + return self.precision_ * 100 + self.scale() + + def set_prec(self, prec: int): + self.precision_ = prec + self.type_mod = self.get_decimal_type_mod() + + def set_scale(self, scale: int): + self.scale_ = scale + self.type_mod = self.get_decimal_type_mod() + + def prec(self): + return self.precision_ + + def scale(self): + return self.scale_ + + def __str__(self): + return f"DECIMAL({self.precision_}, {self.scale()})" + + def __eq__(self, other: DataType): + return self.precision_ == other.prec() and self.scale() == other.scale() + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash((self.precision_, self.scale())) + + def __repr__(self): + return f"DecimalType({self.precision_}, {self.scale()})" + + def generate_value(self) -> str: + val = self.decimal_generator.generate(self.generator_config) + self.aggregator.add_value(val, self.scale()) ## convert to Decimal first + # self.values.append(val) ## save it into files maybe + return val + + def get_typed_val(self, val): + if val == "NULL": + return None + return get_decimal(val, self.scale()) + + def get_typed_val_for_execute(self, val, const_col = False): + return self.get_typed_val(val) + + @staticmethod + def default_compression() -> str: + return "zstd" + + @staticmethod + def default_encode() -> str: + return "disabled" + + def check(self, values, offset: int): + val_from_query = values + val_insert = self.values[offset:] + for v_from_query, v_from_insert in zip(val_from_query, val_insert): + if v_from_insert == "NULL": + if v_from_query.strip() != "NULL": + tdLog.debug( + f"val_insert: {val_insert} val_from_query: {val_from_query}" + ) + tdLog.exit(f"insert NULL, query not NULL: {v_from_query}") + else: + continue + try: + dec_query: Decimal = Decimal(v_from_query) + dec_insert: Decimal = Decimal(v_from_insert) + dec_insert = get_decimal(dec_insert, self.scale()) + except Exception as e: + tdLog.exit(f"failed to convert {v_from_query} or {v_from_insert} to decimal, {e}") + return False + if dec_query != dec_insert: + tdLog.exit( + f"check decimal column failed for insert: {v_from_insert}, query: {v_from_query}, expect {dec_insert}, but get {dec_query}" + ) + return False + else: + tdLog.debug( + f"check decimal succ, insert:{v_from_insert} query:{v_from_query}, py dec: {dec_insert}" + ) + + + @staticmethod + def decimal_type_from_other_type(other: DataType): + prec = 0 + return DecimalType(other.type, other.length, other.type_mod) + +class Column: + def __init__(self, type: DataType): + self.type_: DataType = type + self.name_: str = "" + self.saved_vals:dict[str:[]] = {} + + def is_constant_col(self): + return '' in self.saved_vals.keys() + + def get_typed_val(self, val): + return self.type_.get_typed_val(val) + + def get_typed_val_for_execute(self, val, const_col = False): + return self.type_.get_typed_val_for_execute(val, const_col) + + def get_constant_val(self): + return self.get_typed_val(self.saved_vals[''][0]) + + def get_constant_val_for_execute(self): + return self.get_typed_val_for_execute(self.saved_vals[''][0], const_col=True) + + def __str__(self): + if self.is_constant_col(): + return str(self.get_constant_val()) + return self.name_ + + def get_val_for_execute(self, tbname: str, idx: int): + if self.is_constant_col(): + return self.get_constant_val_for_execute() + if len(self.saved_vals) > 1: + for key in self.saved_vals.keys(): + l = len(self.saved_vals[key]) + if idx < l: + return self.get_typed_val_for_execute(self.saved_vals[key][idx]) + else: + idx -= l + return self.get_typed_val_for_execute(self.saved_vals[tbname][idx]) + + def get_cardinality(self, tbname): + if self.is_constant_col(): + return 1 + elif len(self.saved_vals) > 1: + return len(self.saved_vals['t0']) + else: + return len(self.saved_vals[tbname]) + + @staticmethod + def comp_key(key1, key2): + if key1 is None: + return -1 + if key2 is None: + return 1 + return key1 - key2 + + def get_ordered_result(self, tbname: str, asc: bool) -> list: + if tbname in self.saved_vals: + return sorted( + [ + get_decimal(val, self.type_.scale()) + for val in self.saved_vals[tbname] + ], + reverse=not asc, + key=cmp_to_key(Column.comp_key) + ) + else: + res = [] + for val in self.saved_vals.values(): + res.extend(val) + return sorted( + [get_decimal(val, self.type_.scale()) for val in res], reverse=not asc, + key=cmp_to_key(Column.comp_key) + ) + + def get_group_num(self, tbname, ignore_null=False) -> int: + if tbname in self.saved_vals: + s = set(get_decimal(val, self.type_.scale()) for val in self.saved_vals[tbname]) + if ignore_null: + s.remove(None) + return len(s) + else: + res = set() + for vals in self.saved_vals.values(): + for v in vals: + res.add(get_decimal(v, self.type_.scale())) + if ignore_null: + res.remove(None) + return len(res) + + ## tbName: for normal table, pass the tbname, for child table, pass the child table name + def generate_value(self, tbName: str = '', save: bool = True): + val = self.type_.generate_value() + if save: + if tbName not in self.saved_vals: + self.saved_vals[tbName] = [] + ## for constant columns, always replace the last val + if self.is_constant_col(): + self.saved_vals[tbName] = [val] + else: + self.saved_vals[tbName].append(val) + return val + + def get_type_str(self) -> str: + return str(self.type_) + + def set_name(self, name: str): + self.name_ = name + + def check(self, values, offset: int): + return self.type_.check(values, offset) + + def construct_type_value(self, val: str): + if ( + self.type_.type == TypeEnum.BINARY + or self.type_.type == TypeEnum.VARCHAR + or self.type_.type == TypeEnum.NCHAR + or self.type_.type == TypeEnum.VARBINARY + or self.type_.type == TypeEnum.JSON + ): + return f"'{val}'" + else: + return val + + @staticmethod + def get_decimal_unsupported_types() -> list: + return [ + TypeEnum.JSON, + TypeEnum.GEOMETRY, + TypeEnum.VARBINARY, + ] + + @staticmethod + def get_decimal_oper_const_cols() -> list: + types_unable_to_be_const = [ + TypeEnum.TINYINT, + TypeEnum.SMALLINT, + TypeEnum.INT, + TypeEnum.UINT, + TypeEnum.USMALLINT, + TypeEnum.UTINYINT, + TypeEnum.UBIGINT, + ] + return Column.get_all_type_columns( + Column.get_decimal_unsupported_types() + + Column.get_decimal_types() + + types_unable_to_be_const + ) + + @staticmethod + def get_decimal_types() -> List: + return [TypeEnum.DECIMAL, TypeEnum.DECIMAL64] + + @staticmethod + def get_all_type_columns(types_to_exclude: List[TypeEnum] = []) -> List: + all_types = [ + Column(DataType(TypeEnum.BOOL)), + Column(DataType(TypeEnum.TINYINT)), + Column(DataType(TypeEnum.SMALLINT)), + Column(DataType(TypeEnum.INT)), + Column(DataType(TypeEnum.BIGINT)), + Column(DataType(TypeEnum.FLOAT)), + Column(DataType(TypeEnum.DOUBLE)), + Column(DataType(TypeEnum.VARCHAR, 255)), + Column(DataType(TypeEnum.TIMESTAMP)), + Column(DataType(TypeEnum.NCHAR, 255)), + Column(DataType(TypeEnum.UTINYINT)), + Column(DataType(TypeEnum.USMALLINT)), + Column(DataType(TypeEnum.UINT)), + Column(DataType(TypeEnum.UBIGINT)), + Column(DataType(TypeEnum.JSON)), + Column(DataType(TypeEnum.VARBINARY, 255)), + Column(DecimalType(TypeEnum.DECIMAL, 38, 10)), + Column(DataType(TypeEnum.BINARY, 255)), + Column(DataType(TypeEnum.GEOMETRY, 10240)), + Column(DecimalType(TypeEnum.DECIMAL64, 18, 4)), + ] + ret = [] + for c in all_types: + found = False + for type in types_to_exclude: + if c.type_.type == type: + found = True + break + if not found: + ret.append(c) + return ret + + +class DecimalColumnTableCreater: + def __init__( + self, + conn, + dbName: str, + tbName: str, + columns: List[Column], + tags_cols: List[Column] = [], + col_prefix: str = "c", + tag_prefix: str = "t", + ): + self.conn = conn + self.dbName = dbName + self.tbName = tbName + self.tags_cols = tags_cols + self.columns: List[Column] = columns + self.col_prefix = col_prefix + self.tag_prefix = tag_prefix + + def create(self): + if len(self.tags_cols) > 0: + table = "stable" + else: + table = "table" + sql = f"create {table} {self.dbName}.{self.tbName} (ts timestamp" + for i, column in enumerate(self.columns): + tbname = f"{self.col_prefix}{i+1}" + sql += f", {tbname} {column.get_type_str()}" + column.set_name(tbname) + if self.tags_cols: + sql += ") tags(" + for i, tag in enumerate(self.tags_cols): + tagname = f"{self.tag_prefix}{i+1}" + sql += f"{tagname} {tag.get_type_str()}" + tag.set_name(tagname) + if i != len(self.tags_cols) - 1: + sql += ", " + sql += ")" + self.conn.execute(sql, queryTimes=1) + + def create_child_table( + self, + ctbPrefix: str, + ctbNum: int, + tag_cols: List[Column], + tag_values: List[str], + ): + for i in range(ctbNum): + tbname = f"{ctbPrefix}{i}" + sql = f"create table {self.dbName}.{tbname} using {self.dbName}.{self.tbName} tags(" + for j, tag in enumerate(tag_cols): + sql += f"{tag.construct_type_value(tag_values[j])}" + if j != len(tag_cols) - 1: + sql += ", " + sql += ")" + self.conn.execute(sql, queryTimes=1) + + +class TableInserter: + def __init__( + self, + conn, + dbName: str, + tbName: str, + columns: List[Column], + tags_cols: List[Column] = [], + ): + self.conn: TDSql = conn + self.dbName = dbName + self.tbName = tbName + self.tag_cols = tags_cols + self.columns = columns + + def insert(self, rows: int, start_ts: int, step: int, flush_database: bool = False): + pre_insert = f"insert into {self.dbName}.{self.tbName} values" + sql = pre_insert + t = datetime.now() + for i in range(rows): + sql += f"({start_ts + i * step}" + for column in self.columns: + sql += f", {column.generate_value(self.tbName)}" + sql += ")" + if i != rows - 1: + sql += ", " + local_flush_database = i % 5000 == 0 + if len(sql) > 1000: + # tdLog.debug(f"insert into with sql{sql}") + if flush_database and local_flush_database: + self.conn.execute(f"flush database {self.dbName}", queryTimes=1) + self.conn.execute(sql, queryTimes=1) + t1 = datetime.now() + if (t1 - t).seconds > 1: + TaosShell().query(f"select last(c1), last(c2) from {self.dbName}.{self.tbName}") + t = t1 + sql = pre_insert + if len(sql) > len(pre_insert): + # tdLog.debug(f"insert into with sql{sql}") + if flush_database: + self.conn.execute(f"flush database {self.dbName}", queryTimes=1) + self.conn.execute(sql, queryTimes=1) + TaosShell().query(f"select last(c1), last(c2) from {self.dbName}.{self.tbName}") + +class DecimalCastTypeGenerator: + def __init__(self, input_type: DataType): + self.input_type_: DataType = input_type + + def get_possible_output_types(self) -> List[int]: + if not self.input_type_.is_decimal_type(): + return DataType.get_decimal_types() + else: + return DataType.get_decimal_op_types() + + def do_generate_type(self, dt: int) ->DataType: + if dt == TypeEnum.DECIMAL: + prec = random.randint(1, DecimalType.DECIMAL_MAX_PRECISION) + return DecimalType(dt, prec, random.randint(0, prec)) + elif dt == TypeEnum.DECIMAL64: + prec = random.randint(1, DecimalType.DECIMAL64_MAX_PRECISION) + return DecimalType(dt, prec, random.randint(0, prec)) + elif dt == TypeEnum.BINARY or dt == TypeEnum.VARCHAR: + return DataType(dt, random.randint(16, 255), 0) + else: + return DataType(dt, 0, 0) + + def generate(self, num: int) -> List[DataType]: + res: list[DataType] = [] + for _ in range(num): + dt = random.choice(self.get_possible_output_types()) + dt = self.do_generate_type(dt) + res.append(dt) + res = list(set(res)) + return res + +class TableDataValidator: + def __init__(self, columns: List[Column], tbName: str, dbName: str, tbIdx: int = 0): + self.columns = columns + self.tbName = tbName + self.dbName = dbName + self.tbIdx = tbIdx + + def validate(self): + if not decimal_insert_validator_test: + return + sql = f"select * from {self.dbName}.{self.tbName}" + res = TaosShell().query(sql) + row_num = len(res[1]) + colIdx = 1 + for col in self.columns: + if ( + col.type_.type == TypeEnum.DECIMAL + or col.type_.type == TypeEnum.DECIMAL64 + ): + col.check(res[colIdx], row_num * self.tbIdx) + colIdx += 1 + +class DecimalFunction(DecimalColumnExpr): + def __init__(self, format, executor, name: str): + super().__init__(format, executor) + self.func_name_ = name + + def is_agg_func(self, op: str) ->bool: + return False + + def get_func_res(self): + return None + + def get_input_types(self): + return [self.query_col] + + @staticmethod + def get_decimal_agg_funcs() -> List: + return [ + DecimalMinFunction(), + DecimalMaxFunction(), + DecimalSumFunction(), + DecimalAvgFunction(), + DecimalCountFunction(), + DecimalLastRowFunction(), + DecimalLastFunction(), + DecimalFirstFunction(), + ] + + def check_results(self, query_col_res: List) -> bool: + return False + + def check_for_agg_func(self, query_col_res: List, tbname: str, func): + col_expr = self.query_col + for i in range(col_expr.get_cardinality(tbname)): + col_val = col_expr.get_val_for_execute(tbname, i) + self.execute((col_val,)) + if not self.check_results(query_col_res): + tdLog.exit(f"check failed for {self}, query got: {query_col_res}, expect {self.get_func_res()}") + else: + tdLog.info(f"check expr: {func} with val: {col_val} got result: {query_col_res}, expect: {self.get_func_res()}") + +class DecimalCastFunction(DecimalFunction): + def __init__(self): + super().__init__("cast({0} as {1})", DecimalCastFunction.execute_cast, "cast") + + def should_skip_for_decimal(self, cols: list)->bool: + return False + + def check_results(self, query_col_res: List) -> bool: + return False + + def generate_res_type(self)->DataType: + self.query_col = self.params_[0] + self.res_type_ = self.params_[1] + return self.res_type_ + + def check(self, res: list, tbname: str): + calc_res = [] + params = [] + for i in range(self.query_col.get_cardinality(tbname)): + val = self.query_col.get_val_for_execute(tbname, i) + params.append(val) + try: + calc_val = self.execute(val) + except OverflowError as e: + tdLog.info(f"execute {self} overflow for param: {val}") + calc_res = [] + break + calc_res.append(calc_val) + if len(calc_res) != len(res): + tdLog.exit(f"check result for {self} failed len got: {len(res)}, expect: {len(calc_res)}") + if len(calc_res) == 0: + return True + for v, calc_v, p in zip(res, calc_res, params): + query_v = self.execute_cast(v) + if isinstance(calc_v, float): + eq = math.isclose(query_v, calc_v, rel_tol=1e-7) + elif isinstance(calc_v, numpy.float32): + eq = math.isclose(query_v, calc_v, abs_tol=1e-6, rel_tol=1e-6) + elif isinstance(p, float) or isinstance(p, str): + eq = math.isclose(query_v, calc_v, rel_tol=1e-7) + else: + eq = query_v == calc_v + if not eq: + tdLog.exit(f"check result for {self} failed with param: {p} query got: {v}, expect: {calc_v}") + return True + + def execute_cast(self, val): + if val is None or val == 'NULL': + return None + if self.res_type_.type == TypeEnum.BOOL: + return Decimal(val) != 0 + elif self.res_type_.type == TypeEnum.TINYINT: + dec = Decimal(val).quantize(Decimal("1"), ROUND_HALF_UP) + return int(dec) & 0xFF + elif self.res_type_.type == TypeEnum.SMALLINT: + dec = Decimal(val).quantize(Decimal("1"), ROUND_HALF_UP) + return int(dec) & 0xFFFF + elif self.res_type_.type == TypeEnum.INT: + dec = Decimal(val).quantize(Decimal("1"), ROUND_HALF_UP) + return int(dec) & 0xFFFFFFFF + elif self.res_type_.type == TypeEnum.BIGINT or self.res_type_.type == TypeEnum.TIMESTAMP: + dec = Decimal(val).quantize(Decimal("1"), ROUND_HALF_UP) + return int(dec) & 0xFFFFFFFFFFFFFFFF + elif self.res_type_.type == TypeEnum.FLOAT: + return numpy.float32(val) + elif self.res_type_.type == TypeEnum.DOUBLE: + return float(val) + elif self.res_type_.type == TypeEnum.VARCHAR or self.res_type_.type == TypeEnum.NCHAR: + if Decimal(val) == 0: + return "0" + return str(val)[0:self.res_type_.length] + elif self.res_type_.type == TypeEnum.UTINYINT: + dec = Decimal(val).quantize(Decimal("1"), ROUND_HALF_UP) + return int(dec) & 0xFF + elif self.res_type_.type == TypeEnum.USMALLINT: + dec = Decimal(val).quantize(Decimal("1"), ROUND_HALF_UP) + return int(dec) & 0xFFFF + elif self.res_type_.type == TypeEnum.UINT: + dec = Decimal(val).quantize(Decimal("1"), ROUND_HALF_UP) + return int(dec) & 0xFFFFFFFF + elif self.res_type_.type == TypeEnum.UBIGINT: + dec = Decimal(val).quantize(Decimal("1"), ROUND_HALF_UP) + return int(dec) & 0xFFFFFFFFFFFFFFFF + elif self.res_type_.is_decimal_type(): + max: Decimal = Decimal( + "9" * (self.res_type_.prec() - self.res_type_.scale()) + + "." + + "9" * self.res_type_.scale() + ) + if max < get_decimal(val, self.res_type_.scale()): + raise OverflowError() + try: + return get_decimal(val, self.res_type_.scale()) + except Exception as e: + tdLog.exit(f"failed to cast {val} to {self.res_type_}, {e}") + else: + raise Exception(f"cast unsupported type {self.res_type_.type}") + +class DecimalAggFunction(DecimalFunction): + def __init__(self, format, executor, name: str): + super().__init__(format, executor, name) + + def is_agg_func(self, op: str)-> bool: + return True + + def should_skip_for_decimal(self, cols: list)-> bool: + col: Column = cols[0] + if col.type_.is_decimal_type(): + return False + return True + + def check_results(self, query_col_res): + if len(query_col_res) == 0: + tdLog.info(f"query got no output: {self}, py calc: {self.get_func_res()}") + return True + else: + return self.get_func_res() == Decimal(query_col_res[0]) + +class DecimalLastRowFunction(DecimalAggFunction): + def __init__(self): + super().__init__("last_row({0})", DecimalLastRowFunction.execute_last_row, "last_row") + self.res_ = None + def get_func_res(self): + decimal_type:DecimalType = self.query_col.type_ + return decimal_type.aggregator.last + def generate_res_type(self): + self.res_type_ = self.query_col.type_ + def execute_last_row(self, params): + if params[0] is not None: + self.res_ = Decimal(params[0]) + +class DecimalCacheLastRowFunction(DecimalAggFunction): + def __init__(self): + super().__init__("_cache_last_row({0})", DecimalCacheLastRowFunction.execute_cache_last_row, "_cache_last_row") + def get_func_res(self): + return 1 + def generate_res_type(self): + self.res_type_ = self.query_col.type_ + def execute_cache_last_row(self, params): + return 1 + +class DecimalCacheLastFunction(DecimalAggFunction): + pass + +class DecimalFirstFunction(DecimalAggFunction): + def __init__(self): + super().__init__("first({0})", DecimalFirstFunction.execute_first, "first") + self.res_ = None + def get_func_res(self): + decimal_type: DecimalType = self.query_col.type_ + return decimal_type.aggregator.first + def generate_res_type(self): + self.res_type_ = self.query_col.type_ + def execute_first(self, params): + pass + +class DecimalLastFunction(DecimalAggFunction): + def __init__(self): + super().__init__("last({0})", DecimalLastFunction.execute_last, "last") + self.res_ = None + def get_func_res(self): + decimal_type:DecimalType = self.query_col.type_ + return decimal_type.aggregator.last + def generate_res_type(self): + self.res_type_ = self.query_col.type_ + def execute_last(self, params): + pass + +class DecimalHyperloglogFunction(DecimalAggFunction): + pass + +class DecimalSampleFunction(DecimalAggFunction): + pass + +class DecimalTailFunction(DecimalAggFunction): + pass + +class DecimalUniqueFunction(DecimalAggFunction): + pass + +class DecimalModeFunction(DecimalAggFunction): + pass + +class DecimalCountFunction(DecimalAggFunction): + def __init__(self): + super().__init__("count({0})", DecimalCountFunction.execute_count, "count") + def get_func_res(self): + decimal_type: DecimalType = self.query_col.type_ + return decimal_type.aggregator.count - decimal_type.aggregator.null_num + def generate_res_type(self): + self.res_type_ = DataType(TypeEnum.BIGINT, 8, 0) + + def execute_count(self, params): + return 1 + +class DecimalMinFunction(DecimalAggFunction): + def __init__(self): + super().__init__("min({0})", DecimalMinFunction.execute_min, "min") + self.min_: Decimal = None + + def get_func_res(self) -> Decimal: + decimal_type: DecimalType = self.query_col.type_ + return decimal_type.aggregator.min + return self.min_ + + def generate_res_type(self) -> DataType: + self.res_type_ = self.query_col.type_ + + def execute_min(self, params): + if params[0] is None: + return + if self.min_ is None: + self.min_ = Decimal(params[0]) + else: + self.min_ = min(self.min_, Decimal(params[0])) + return self.min_ + +class DecimalMaxFunction(DecimalAggFunction): + def __init__(self): + super().__init__("max({0})", DecimalMaxFunction.execute_max, "max") + self.max_: Decimal = None + + def get_func_res(self) -> Decimal: + decimal_type: DecimalType = self.query_col.type_ + return decimal_type.aggregator.max + + def generate_res_type(self) -> DataType: + self.res_type_ = self.query_col.type_ + + def execute_max(self, params): + if params[0] is None: + return + if self.max_ is None: + self.max_ = Decimal(params[0]) + else: + self.max_ = max(self.max_, Decimal(params[0])) + return self.max_ + +class DecimalSumFunction(DecimalAggFunction): + def __init__(self): + super().__init__("sum({0})", DecimalSumFunction.execute_sum, "sum") + self.sum_:Decimal = None + def get_func_res(self) -> Decimal: + decimal_type: DecimalType = self.query_col.type_ + return decimal_type.aggregator.sum + return self.sum_ + def generate_res_type(self) -> DataType: + self.res_type_ = self.query_col.type_ + self.res_type_.set_prec(DecimalType.DECIMAL_MAX_PRECISION) + def execute_sum(self, params): + if params[0] is None: + return + if self.sum_ is None: + self.sum_ = Decimal(params[0]) + else: + self.sum_ += Decimal(params[0]) + return self.sum_ + +class DecimalAvgFunction(DecimalAggFunction): + def __init__(self): + super().__init__("avg({0})", DecimalAvgFunction.execute_avg, "avg") + self.count_: Decimal = 0 + self.sum_: Decimal = None + def get_func_res(self) -> Decimal: + decimal_type: DecimalType = self.query_col.type_ + return get_decimal( + decimal_type.aggregator.sum + / (decimal_type.aggregator.count - decimal_type.aggregator.null_num), + self.res_type_.scale(), + ) + def generate_res_type(self) -> DataType: + sum_type = self.query_col.type_ + sum_type.set_prec(DecimalType.DECIMAL_MAX_PRECISION) + count_type = DataType(TypeEnum.BIGINT, 8, 0) + self.res_type_ = DecimalBinaryOperator.calc_decimal_prec_scale(sum_type, count_type, "/") + def execute_avg(self, params): + if params[0] is None: + return + if self.sum_ is None: + self.sum_ = Decimal(params[0]) + else: + self.sum_ += Decimal(params[0]) + self.count_ += 1 + return self.get_func_res() + +class DecimalBinaryOperator(DecimalColumnExpr): + def __init__(self, format, executor, op: str): + super().__init__(format, executor) + self.op_ = op + self.left_type_: DataType = None + self.right_type_: DataType = None + + def __str__(self): + return super().__str__() + + def generate(self, format_params) -> str: + return super().generate(format_params) + + def should_skip_for_decimal(self, cols: list): + left_col = cols[0] + right_col = cols[1] + if not left_col.type_.is_decimal_type() and not right_col.type_.is_decimal_type(): + return True + if self.op_ != "%": + return False + ## TODO wjm why skip decimal % float/double? it's wrong now. + left_is_real = left_col.type_.is_real_type() or left_col.type_.is_varchar_type() + right_is_real = right_col.type_.is_real_type() or right_col.type_.is_varchar_type() + if left_is_real or right_is_real: + return True + return False + + @staticmethod + def check_null(params): + if params[0] is None or params[1] is None: + return True + else: + return False + + @staticmethod + def is_compare_op(op: str)-> bool: + return op in ["==", "!=", ">", "<", ">=", "<="] + + @staticmethod + def calc_decimal_prec_scale(left: DataType, right: DataType, op: str) -> DecimalType: + left_prec = 0 + left_scale = 0 + right_prec = 0 + right_scale = 0 + if not left.is_decimal_type(): + left_prec = TypeEnum.get_type_prec(left.type) + else: + left_prec = left.prec() + left_scale = left.scale() + if not right.is_decimal_type(): + right_prec = TypeEnum.get_type_prec(right.type) + else: + right_prec = right.prec() + right_scale = right.scale() + + out_prec = 0 + out_scale = 0 + if op in ['+', '-']: + out_scale = max(left_scale, right_scale) + out_prec = max(left_prec - left_scale, right_prec - right_scale) + out_scale + 1 + elif op == '*': + out_scale = left_scale + right_scale + out_prec = left_prec + right_prec + 1 + elif op == '/': + out_scale = max(left_scale + right_prec + 1, 6) + out_prec = left_prec - left_scale + right_scale + out_scale + elif op == '%': + out_scale = max(left_scale, right_scale) + out_prec = min(left_prec - left_scale, right_prec - right_scale) + out_scale + else: + raise Exception(f"unknown op for binary operators: {op}") + + if out_prec > 38: + min_scale = min(6, out_scale) + delta = out_prec - 38 + out_prec = 38 + out_scale = max(min_scale, out_scale - delta) + return DecimalType(TypeEnum.DECIMAL, out_prec, out_scale) + + def generate_res_type(self): + left_type = self.params_[0].type_ + right_type = self.params_[1].type_ + self.left_type_ = left_type + self.right_type_ = right_type + if DecimalBinaryOperator.is_compare_op(self.op_): + self.res_type_ = DataType(TypeEnum.BOOL) + else: + ret_double_types = [TypeEnum.VARCHAR, TypeEnum.BINARY, TypeEnum.DOUBLE, TypeEnum.FLOAT, TypeEnum.NCHAR] + if left_type.type in ret_double_types or right_type.type in ret_double_types: + self.res_type_ = DataType(TypeEnum.DOUBLE) + else: + self.res_type_ = DecimalBinaryOperator.calc_decimal_prec_scale(left_type, right_type, self.op_) + + def get_input_types(self)-> list: + return [self.left_type_, self.right_type_] + + def get_convert_type(self, params): + ret_float = False + if isinstance(params[0], float) or isinstance(params[1], float): + ret_float = True + left = params[0] + right = params[1] + if isinstance(params[0], str): + ret_float = True + if isinstance(params[1], str): + ret_float = True + return (left, right), ret_float + + def execute_plus(self, params): + if DecimalBinaryOperator.check_null(params): + return 'NULL' + (left, right), ret_float = self.get_convert_type(params) + if self.res_type_.type == TypeEnum.DOUBLE: + return float(left) + float(right) + else: + return self.convert_to_res_type(Decimal(left) + Decimal(right)) + + def execute_minus(self, params): + if DecimalBinaryOperator.check_null(params): + return 'NULL' + (left, right), ret_float = self.get_convert_type(params) + if self.res_type_.type == TypeEnum.DOUBLE: + return float(left) - float(right) + else: + return self.convert_to_res_type(Decimal(left) - Decimal(right)) + + def execute_mul(self, params): + if DecimalBinaryOperator.check_null(params): + return 'NULL' + (left, right), ret_float = self.get_convert_type(params) + if self.res_type_.type == TypeEnum.DOUBLE: + return float(left) * float(right) + else: + return self.convert_to_res_type(Decimal(left) * Decimal(right)) + + def execute_div(self, params): + if DecimalBinaryOperator.check_null(params): + return 'NULL' + (left, right), _ = self.get_convert_type(params) + if self.res_type_.type == TypeEnum.DOUBLE: + if right == 0: + return 'NULL' + return float(left) / float(right) + else: + return self.convert_to_res_type(Decimal(left) / Decimal(right)) + + def execute_mod(self, params): + if DecimalBinaryOperator.check_null(params): + return 'NULL' + (left, right), _ = self.get_convert_type(params) + if self.res_type_.type == TypeEnum.DOUBLE: + return self.convert_to_res_type(Decimal(left) % Decimal(right)) + else: + return self.convert_to_res_type(Decimal(left) % Decimal(right)) + + def execute_eq(self, params): + if DecimalBinaryOperator.check_null(params): + return False + (left, right), convert_float = self.get_convert_type(params) + if convert_float: + return float(left) == float(right) + else: + return Decimal(left) == Decimal(right) + + def execute_eq_filtering(self, params): + if self.execute_eq(params): + return True + + def execute_ne(self, params): + if DecimalBinaryOperator.check_null(params): + return False + (left, right), convert_float = self.get_convert_type(params) + if convert_float: + return float(left) != float(right) + else: + return Decimal(left) != Decimal(right) + + def execute_gt(self, params): + if DecimalBinaryOperator.check_null(params): + return False + (left, right), convert_float = self.get_convert_type(params) + if convert_float: + return float(left) > float(right) + else: + return Decimal(left) > Decimal(right) + + def execute_lt(self, params): + if DecimalBinaryOperator.check_null(params): + return False + (left, right), convert_float = self.get_convert_type(params) + if convert_float: + return float(left) < float(right) + else: + return Decimal(left) < Decimal(right) + + def execute_ge(self, params): + if DecimalBinaryOperator.check_null(params): + return False + (left, right), convert_float = self.get_convert_type(params) + if convert_float: + return float(left) >= float(right) + else: + return Decimal(left) >= Decimal(right) + + def execute_le(self, params): + if DecimalBinaryOperator.check_null(params): + return False + (left, right), convert_float = self.get_convert_type(params) + if convert_float: + return float(left) <= float(right) + else: + return Decimal(left) <= Decimal(right) + + @staticmethod + def get_all_binary_ops() -> List[DecimalColumnExpr]: + return [ + DecimalBinaryOperator(" {0} + {1} ", DecimalBinaryOperator.execute_plus, "+"), + DecimalBinaryOperator(" {0} - {1} ", DecimalBinaryOperator.execute_minus, "-"), + DecimalBinaryOperator(" {0} * {1} ", DecimalBinaryOperator.execute_mul, "*"), + DecimalBinaryOperator(" {0} / {1} ", DecimalBinaryOperator.execute_div, "/"), + DecimalBinaryOperator(" {0} % {1} ", DecimalBinaryOperator.execute_mod, "%"), + DecimalBinaryOperator(" {0} == {1} ", DecimalBinaryOperator.execute_eq, "=="), + DecimalBinaryOperator(" {0} != {1} ", DecimalBinaryOperator.execute_ne, "!="), + DecimalBinaryOperator(" {0} > {1} ", DecimalBinaryOperator.execute_gt, ">"), + DecimalBinaryOperator(" {0} < {1} ", DecimalBinaryOperator.execute_lt, "<"), + DecimalBinaryOperator(" {0} >= {1} ", DecimalBinaryOperator.execute_ge, ">="), + DecimalBinaryOperator(" {0} <= {1} ", DecimalBinaryOperator.execute_le, "<="), + ] + + @staticmethod + def get_all_filtering_binary_compare_ops() -> List[DecimalColumnExpr]: + return [ + DecimalBinaryOperator(" {0} == {1} ", DecimalBinaryOperator.execute_eq, "=="), + DecimalBinaryOperator(" {0} != {1} ", DecimalBinaryOperator.execute_ne, "!="), + DecimalBinaryOperator(" {0} > {1} ", DecimalBinaryOperator.execute_gt, ">"), + DecimalBinaryOperator(" {0} < {1} ", DecimalBinaryOperator.execute_lt, "<"), + DecimalBinaryOperator(" {0} >= {1} ", DecimalBinaryOperator.execute_ge, ">="), + DecimalBinaryOperator(" {0} <= {1} ", DecimalBinaryOperator.execute_le, "<="), + ] + + def execute(self, params): + return super().execute(params) + +class DecimalUnaryOperator(DecimalColumnExpr): + def __init__(self, format, executor, op: str): + super().__init__(format, executor) + self.op_ = op + self.col_type_: DataType = None + + def should_skip_for_decimal(self, cols: list): + col:Column = cols[0] + if not col.type_.is_decimal_type(): + return True + return False + + @staticmethod + def get_all_unary_ops() -> List[DecimalColumnExpr]: + return [ + DecimalUnaryOperator(" -{0} ", DecimalUnaryOperator.execute_minus, "-"), + ] + + def get_input_types(self)-> list: + return [self.col_type_] + + def generate_res_type(self): + self.res_type_ = self.col_type_ = self.params_[0].type_ + + def execute_minus(self, params) -> Decimal: + if params[0] is None: + return 'NULL' + return -Decimal(params[0]) + +class DecimalBinaryOperatorIn(DecimalBinaryOperator): + def __init__(self, op: str): + super().__init__(op) + + def execute(self, left, right): + if self.op_.lower()() == "in": + return left in right + if self.op_.lower() == "not in": + return left not in right + + +class TDTestCase: + updatecfgDict = { + "asynclog": 0, + "ttlUnit": 1, + "ttlPushInterval": 5, + "ratioOfVnodeStreamThrea": 4, + "debugFlag": 143, + } + + def __init__(self): + self.vgroups = 4 + self.ctbNum = 10 + self.rowsPerTbl = 10000 + self.duraion = "1h" + self.norm_tb_columns = [] + self.tags = [] + self.stable_name = "meters" + self.norm_table_name = "nt" + self.col_prefix = "c" + self.c_table_prefix = "t" + self.tag_name_prefix = "t" + self.db_name = "test" + self.c_table_num = 10 + self.no_decimal_col_tb_name = "tt" + self.stb_columns = [] + self.stream_name = "stream1" + self.stream_out_stb = "stream_out_stb" + self.tsma_name = "tsma1" + self.query_test_round = 10000 + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def check_desc_for_one_ctb( + self, ctbPrefix: str, columns: List[Column], tags: List[Column] = [] + ): + ctb_idx = randrange(self.c_table_num) + return self.check_desc(f"{ctbPrefix}{ctb_idx}", columns, tags) + + def check_desc(self, tbname: str, columns: List[Column], tags: List[Column] = []): + sql = f"desc {self.db_name}.{tbname}" + tdSql.query(sql, queryTimes=1) + results = tdSql.queryResult + for i, col in enumerate(columns): + if col.type_.type == TypeEnum.DECIMAL: + if results[i + 1][1] != col.type_.__str__(): + tdLog.info(str(results)) + tdLog.exit( + f"check desc failed for table: {tbname} column {results[i+1][0]} type is {results[i+1][1]}, expect {col.type_}" + ) + if results[i + 1][4] != DecimalType.default_encode(): + tdLog.exit( + f"check desc failed for table: {tbname} column {results[i+1][0]} encode is {results[i+1][5]}, expect {DecimalType.default_encode()}" + ) + if results[i + 1][5] != DecimalType.default_compression(): + tdLog.exit( + f"check desc failed for table: {tbname} column {results[i+1][0]} compression is {results[i+1][4]}, expect {DecimalType.default_compression()}" + ) + if tbname == self.stable_name: + self.check_desc_for_one_ctb(self.c_table_prefix, columns, tags) + + def check_show_create_table( + self, tbname: str, cols: List[Column], tags: List[Column] = [] + ): + sql = f"show create table {self.db_name}.{tbname}" + tdSql.query(sql, queryTimes=1) + create_table_sql = tdSql.queryResult[0][1] + decimal_idx = 0 + results = re.findall(r"DECIMAL\((\d+),(\d+)\)", create_table_sql) + for i, col in enumerate(cols): + if ( + col.type_.type == TypeEnum.DECIMAL + or col.type_.type == TypeEnum.DECIMAL64 + ): + result_type = DecimalType( + col.type_.type, + int(results[decimal_idx][0]), + int(results[decimal_idx][1]), + ) + if result_type != col.type_: + tdLog.exit( + f"check show create table failed for: {tbname} column {i} type is {result_type}, expect {col.type}" + ) + decimal_idx += 1 + + def test_add_drop_columns_with_decimal(self, tbname: str, columns: List[Column]): + is_stb = tbname == self.stable_name + ## alter table add column + create_c99_sql = ( + f"alter table {self.db_name}.{tbname} add column c99 decimal(37, 19)" + ) + columns.append(Column(DecimalType(TypeEnum.DECIMAL, 37, 19))) + tdSql.execute(create_c99_sql, queryTimes=1, show=True) + self.check_desc(tbname, columns) + ## alter table add column with compression + create_c100_sql = f'ALTER TABLE {self.db_name}.{tbname} ADD COLUMN c100 decimal(36, 18) COMPRESS "zstd"' + tdSql.execute(create_c100_sql, queryTimes=1, show=True) + columns.append(Column(DecimalType(TypeEnum.DECIMAL, 36, 18))) + self.check_desc(tbname, columns) + + ## drop non decimal column + drop_c6_sql = f"alter table {self.db_name}.{tbname} drop column c6" + tdSql.execute(drop_c6_sql, queryTimes=1, show=True) + c6 = columns.pop(5) + self.check_desc(tbname, columns) + ## drop decimal column and not last column + drop_c99_sql = f"alter table {self.db_name}.{tbname} drop column c99" + tdSql.execute(drop_c99_sql, queryTimes=1, show=True) + c99 = columns.pop(len(columns) - 2) + self.check_desc(tbname, columns) + ## drop decimal column and last column + drop_c100_sql = f"alter table {self.db_name}.{tbname} drop column c100" + tdSql.execute(drop_c100_sql, queryTimes=1, show=True) + c100 = columns.pop(len(columns) - 1) + self.check_desc(tbname, columns) + + ## create decimal back + tdSql.execute(create_c99_sql, queryTimes=1, show=True) + tdSql.execute(create_c100_sql, queryTimes=1, show=True) + columns.append(c99) + columns.append(c100) + self.check_desc(tbname, columns) + + def test_decimal_column_ddl(self): + ## create decimal type table, normal/super table, decimal64/decimal128 + tdLog.printNoPrefix("-------- test create decimal column") + self.norm_tb_columns: List[Column] = [ + Column(DecimalType(TypeEnum.DECIMAL, 10, 2)), + Column(DecimalType(TypeEnum.DECIMAL, 20, 4)), + Column(DecimalType(TypeEnum.DECIMAL, 30, 8)), + Column(DecimalType(TypeEnum.DECIMAL, 38, 10)), + Column(DataType(TypeEnum.TINYINT)), + Column(DataType(TypeEnum.INT)), + Column(DataType(TypeEnum.BIGINT)), + Column(DataType(TypeEnum.DOUBLE)), + Column(DataType(TypeEnum.FLOAT)), + Column(DataType(TypeEnum.VARCHAR, 255)), + ] + self.tags: List[Column] = [ + Column(DataType(TypeEnum.INT)), + Column(DataType(TypeEnum.VARCHAR, 255)), + ] + self.stb_columns: List[Column] = [ + Column(DecimalType(TypeEnum.DECIMAL, 10, 2)), + Column(DecimalType(TypeEnum.DECIMAL, 20, 4)), + Column(DecimalType(TypeEnum.DECIMAL, 30, 8)), + Column(DecimalType(TypeEnum.DECIMAL, 38, 10)), + Column(DataType(TypeEnum.TINYINT)), + Column(DataType(TypeEnum.INT)), + Column(DataType(TypeEnum.BIGINT)), + Column(DataType(TypeEnum.DOUBLE)), + Column(DataType(TypeEnum.FLOAT)), + Column(DataType(TypeEnum.VARCHAR, 255)), + ] + DecimalColumnTableCreater( + tdSql, + self.db_name, + self.stable_name, + self.stb_columns, + self.tags, + col_prefix=self.col_prefix, + tag_prefix=self.tag_name_prefix, + ).create() + self.check_show_create_table("meters", self.stb_columns, self.tags) + + DecimalColumnTableCreater( + tdSql, self.db_name, self.norm_table_name, self.norm_tb_columns + ).create() + self.check_desc(self.norm_table_name, self.norm_tb_columns) + self.check_show_create_table(self.norm_table_name, self.norm_tb_columns) + + ## TODO add more values for all rows + tag_values = ["1", "t1"] + DecimalColumnTableCreater( + tdSql, self.db_name, self.stable_name, self.stb_columns + ).create_child_table( + self.c_table_prefix, self.c_table_num, self.tags, tag_values + ) + self.check_desc("meters", self.stb_columns, self.tags) + self.check_desc("t1", self.stb_columns, self.tags) + + ## invalid precision/scale + invalid_precision_scale = [ + ("decimal(-1, 2)", syntax_error), + ("decimal(39, 2)", invalid_column), + ("decimal(10, -1)", syntax_error), + ("decimal(10, 39)", invalid_column), + ("decimal(10, 2.5)", syntax_error), + ("decimal(10.5, 2)", syntax_error), + ("decimal(10.5, 2.5)", syntax_error), + ("decimal(0, 2)", invalid_column), + ("decimal(0)", invalid_column), + ("decimal", syntax_error), + ("decimal()", syntax_error), + ] + for i in invalid_precision_scale: + sql = f"create table {self.db_name}.invalid_decimal_precision_scale (ts timestamp, c1 {i[0]})" + tdSql.error(sql, i[1]) + + ## can't create decimal tag + sql = ( + "create stable %s.invalid_decimal_tag (ts timestamp) tags (t1 decimal(10, 2))" + % (self.db_name) + ) + tdSql.error(sql, invalid_column) + + ## alter table add/drop column + self.test_add_drop_columns_with_decimal( + self.norm_table_name, self.norm_tb_columns + ) + self.test_add_drop_columns_with_decimal(self.stable_name, self.stb_columns) + + ## drop index from stb + ### These ops will override the previous stbobjs and meta entries, so test it + + ## TODO test encode and compress for decimal type + sql = f'ALTER TABLE {self.db_name}.{self.norm_table_name} ADD COLUMN c101 decimal(37, 19) ENCODE "simple8b" COMPRESS "zstd"' + tdSql.error(sql, invalid_encode_param) + sql = f'ALTER TABLE {self.db_name}.{self.norm_table_name} ADD COLUMN c101 decimal(37, 19) ENCODE "delta-i" COMPRESS "zstd"' + tdSql.error(sql, invalid_encode_param) + sql = f'ALTER TABLE {self.db_name}.{self.norm_table_name} ADD COLUMN c101 decimal(37, 19) ENCODE "delta-d" COMPRESS "zstd"' + tdSql.error(sql, invalid_encode_param) + sql = f'ALTER TABLE {self.db_name}.{self.norm_table_name} ADD COLUMN c101 decimal(37, 19) ENCODE "bit-packing" COMPRESS "zstd"' + tdSql.error(sql, invalid_encode_param) + + def test_insert_decimal_values(self): + tdLog.debug("start to insert decimal values") + for i in range(self.c_table_num): + TableInserter( + tdSql, + self.db_name, + f"{self.c_table_prefix}{i}", + self.stb_columns, + self.tags, + ).insert(tb_insert_rows, 1537146000000, 500) + + for i in range(self.c_table_num): + TableDataValidator( + self.stb_columns, self.c_table_prefix + str(i), self.db_name, i + ).validate() + + TableInserter( + tdSql, self.db_name, self.norm_table_name, self.norm_tb_columns + ).insert(tb_insert_rows, 1537146000000, 500, flush_database=True) + TableDataValidator( + self.norm_tb_columns, self.norm_table_name, self.db_name + ).validate() + + tdSql.execute("flush database %s" % (self.db_name), queryTimes=1) + for i in range(self.c_table_num): + TableDataValidator( + self.stb_columns, self.c_table_prefix + str(i), self.db_name, i + ).validate() + TableDataValidator( + self.norm_tb_columns, self.norm_table_name, self.db_name + ).validate() + + ## insert null/None for decimal type + + ## insert with column format + + def no_decimal_table_test(self): + columns = [ + Column(DataType(TypeEnum.TINYINT)), + Column(DataType(TypeEnum.INT)), + Column(DataType(TypeEnum.BIGINT)), + Column(DataType(TypeEnum.DOUBLE)), + Column(DataType(TypeEnum.FLOAT)), + Column(DataType(TypeEnum.VARCHAR, 255)), + ] + DecimalColumnTableCreater( + tdSql, self.db_name, self.no_decimal_col_tb_name, columns, [] + ).create() + TableInserter(tdSql, self.db_name, self.no_decimal_col_tb_name, columns).insert( + 10000, 1537146000000, 500, flush_database=True + ) + ## TODO wjm test non support decimal version upgrade to decimal support version, and add decimal column + + ## Test metaentry compatibility problem for decimal type + ## How to test it? + ## Create table with no decimal type, the metaentries should not have extschma, and add decimal column, the metaentries should have extschema for all columns. + sql = f"ALTER TABLE {self.db_name}.{self.no_decimal_col_tb_name} ADD COLUMN c200 decimal(37, 19)" + tdSql.execute(sql, queryTimes=1) ## now meta entry has ext schemas + columns.append(Column(DecimalType(TypeEnum.DECIMAL, 37, 19))) + self.check_desc(self.no_decimal_col_tb_name, columns) + + ## After drop this only decimal column, the metaentries should not have extschema for all columns. + sql = ( + f"ALTER TABLE {self.db_name}.{self.no_decimal_col_tb_name} DROP COLUMN c200" + ) + tdSql.execute(sql, queryTimes=1) ## now meta entry has no ext schemas + columns.pop(len(columns) - 1) + self.check_desc(self.no_decimal_col_tb_name, columns) + sql = f"ALTER TABLE {self.db_name}.{self.no_decimal_col_tb_name} ADD COLUMN c200 int" + tdSql.execute(sql, queryTimes=1) ## meta entry has no ext schemas + columns.append(Column(DataType(TypeEnum.INT))) + self.check_desc(self.no_decimal_col_tb_name, columns) + + self.test_add_drop_columns_with_decimal(self.no_decimal_col_tb_name, columns) + + def test_decimal_ddl(self): + tdSql.execute("create database test cachemodel 'both'", queryTimes=1) + self.test_decimal_column_ddl() + + def test_decimal_and_stream(self): + create_stream = f"CREATE STREAM {self.stream_name} FILL_HISTORY 1 INTO {self.db_name}.{self.stream_out_stb} AS SELECT _wstart, count(c1), avg(c2), sum(c3) FROM {self.db_name}.{self.stable_name} INTERVAL(10s)" + tdSql.execute(create_stream, queryTimes=1, show=True) + self.wait_query_result( + f"select count(*) from {self.db_name}.{self.stream_out_stb}", [(50,)], 30 + ) + ## test combine functions + create_stream = "CREATE STREAM stream2 trigger at_once watermark 100s INTO test.stream_out_stb_2 AS SELECT _wstart, count(*), min(c2), max(c2), last(c1), last(c3), avg(c2), sum(c3), min(c1), max(c1), avg(c1) FROM test.nt session(ts, 5s)" + tdSql.execute(create_stream, queryTimes=1, show=True) + cols_vals = [] + ts = datetime.now() + rows = [] + row = [] + for col in self.norm_tb_columns: + v = col.generate_value(self.norm_table_name) + cols_vals.append(v) + row.append(v) + rows.append(row) + row = [] + sql = f"insert into test.nt values('{ts}', {' ,'.join(cols_vals)})" + tdSql.execute(sql, queryTimes=1, show=True) + ts = ts + timedelta(seconds=6) + cols_vals = [] + for col in self.norm_tb_columns: + v = col.generate_value(self.norm_table_name) + cols_vals.append(v) + row.append(v) + rows.append(row) + row = [] + sql = f"insert into test.nt values('{ts}', {' ,'.join(cols_vals)})" + tdSql.execute(sql, queryTimes=1, show=True) + ts = ts - timedelta(seconds=4) + ## waiting for the last two windows been calculated + time.sleep(10) + cols_vals = [] + for col in self.norm_tb_columns: + v = col.generate_value(self.norm_table_name) + cols_vals.append(v) + row.append(v) + rows.append(row) + sql = f"insert into test.nt values('{ts}', {' ,'.join(cols_vals)})" + tdSql.execute(sql, queryTimes=1, show=True) + self.wait_query_result("select `count(*)` from test.stream_out_stb_2", [(3,)], 10) + res = TaosShell().query("select * from test.stream_out_stb_2") + if len(res) != 12: ## groupid + tdLog.exit(f"expect 12 columns but got: {len(res)}") + c1 = self.norm_tb_columns[0] + c2 = self.norm_tb_columns[1] + c3 = self.norm_tb_columns[2] + c1_vals = [get_decimal(v, c1.type_.scale()) for v in [row[0] for row in rows]] + c2_vals = [get_decimal(v, c2.type_.scale()) for v in [row[1] for row in rows]] + c3_vals = [get_decimal(v, c3.type_.scale()) for v in [row[2] for row in rows]] + min_c2 = Decimal(res[2][0]) + if min_c2 != min(c2_vals): + tdLog.exit(f"expect min(c2) = {min(c2_vals)} got: {min_c2}") + max_c2 = Decimal(res[3][0]) + if max_c2 != max(c2_vals): + tdLog.exit(f"expect max(c2) = {max(c2_vals)} got: {max_c2}") + + + def test_decimal_and_tsma(self): + create_tsma = f"CREATE TSMA {self.tsma_name} ON {self.db_name}.{self.stable_name} FUNCTION(count(c1), min(c2), max(c3), avg(C3)) INTERVAL(1m)" + tdSql.execute(create_tsma, queryTimes=1, show=True) + self.wait_query_result( + f"select count(*) from {self.db_name}.{self.tsma_name}_tsma_res_stb_", + [(9 * self.c_table_num,)], + 30, + ) + + def test_decimal_and_view(self): + c1 = self.norm_tb_columns[0] + create_view_sql = f'create view {self.db_name}.view1 as select {c1} as c1, cast({c1} as decimal(38, 10)) as c2 from {self.db_name}.{self.norm_table_name}' + tdSql.execute(create_view_sql) + res = TaosShell().query(f'select c1 from {self.db_name}.view1') + if len(res[0]) != c1.get_cardinality(self.norm_table_name): + tdLog.exit(f"query from view1 got rows: {len(res)} expect: {c1.get_cardinality(self.norm_table_name)}") + for i in range(len(res[0])): + v_query = res[0][i] + v_insert = c1.get_val_for_execute(self.norm_table_name, i) + if (v_insert is None and v_query == 'NULL') or Decimal(v_query) == v_insert: + continue + else: + tdLog.exit(f"query from view got different results: {v_query}, expect: {v_insert}") + #self.check_desc("view1", [c1, Column(DecimalType(TypeEnum.DECIMAL, 38, 10))]) + + def run(self): + self.test_decimal_ddl() + self.no_decimal_table_test() + self.test_insert_decimal_values() + self.test_query_decimal() + #self.test_decimal_and_tsma() + #self.test_decimal_and_view() + #self.test_decimal_and_stream() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + def wait_query_result(self, sql: str, expect_result, times): + for i in range(times): + tdLog.info(f"wait query result for {sql}, times: {i}") + tdSql.query(sql, queryTimes=1) + results = tdSql.queryResult + if results != expect_result: + time.sleep(1) + continue + return True + tdLog.exit( + f"wait query result timeout for {sql} failed after {times} time, expect {expect_result}, but got {results}" + ) + + def wait_query_at_least_rows(self, sql: str, rows, wait_times): + for i in range(wait_times): + tdLog.info(f"wait query rows at least for {sql}, times: {i}") + tdSql.query(sql, queryTimes=1, show=True) + results = tdSql.queryResult + if len(results) < rows: + time.sleep(1) + continue + return True + tdLog.exit( + f"wait query rows at least for {sql} failed after {wait_times} times, expect at least {rows} rows, but got {len(results)} rows" + ) + + def check_decimal_binary_expr_with_col_results( + self, dbname, tbname, tb_cols: List[Column], exprs: List[DecimalColumnExpr] + ): + if not binary_op_with_col_test: + return + for expr in exprs: + for col in tb_cols: + if col.name_ == '': + continue + for col2 in tb_cols: + if col2.name_ =='': + continue + if expr.should_skip_for_decimal([col, col2]): + continue + select_expr = expr.generate((col, col2)) + sql = f"select {select_expr} from {dbname}.{tbname}" + res = TaosShell().query(sql) + if len(res) > 0: + expr.check(res[0], tbname) + else: + tdLog.info(f"sql: {sql} got no output") + + def check_decimal_binary_expr_with_const_col_results_for_one_expr( + self, + dbname, + tbname, + tb_cols: List[Column], + expr: DecimalColumnExpr, + get_constant_cols_func, + ): + constant_cols = get_constant_cols_func() + for col in tb_cols: + if col.name_ == '': + continue + left_is_decimal = col.type_.is_decimal_type() + for const_col in constant_cols: + right_is_decimal = const_col.type_.is_decimal_type() + if expr.should_skip_for_decimal([col, const_col]): + continue + const_col.generate_value() + select_expr = expr.generate((const_col, col)) + sql = f"select {select_expr} from {dbname}.{tbname}" + shell = TaosShell() + res = shell.query(sql) + if len(res) > 0: + expr.check(res[0], tbname) + select_expr = expr.generate((col, const_col)) + sql = f"select {select_expr} from {dbname}.{tbname}" + res = shell.query(sql) + if len(res) > 0: + expr.check(res[0], tbname) + else: + tdLog.info(f"sql: {sql} got no output") + + def check_decimal_binary_expr_with_const_col_results( + self, + dbname, + tbname, + tb_cols: List[Column], + get_constant_cols_func, + get_exprs_func, + ): + exprs: List[DecimalColumnExpr] = get_exprs_func() + if not binary_op_with_const_test: + return + ts: list[threading.Thread] = [] + for expr in exprs: + t = self.run_in_thread2( + self.check_decimal_binary_expr_with_const_col_results_for_one_expr, + (dbname, tbname, tb_cols, expr, get_constant_cols_func), + ) + ts.append(t) + for t in ts: + t.join() + + def check_decimal_unary_expr_results(self, dbname, tbname, tb_cols: List[Column], exprs: List[DecimalColumnExpr]): + if not unary_op_test: + return + for expr in exprs: + for col in tb_cols: + if col.name_ == '': + continue + if expr.should_skip_for_decimal([col]): + continue + select_expr = expr.generate([col]) + sql = f"select {select_expr} from {dbname}.{tbname}" + res = TaosShell().query(sql) + if len(res) > 0: + expr.check(res[0], tbname) + else: + tdLog.info(f"sql: {sql} got no output") + + def run_in_thread(self, times, func, params) -> threading.Thread: + threads: List[threading.Thread] = [] + for i in range(times): + t = threading.Thread(target=func, args=params) + t.start() + threads.append(t) + for t in threads: + t.join() + + def run_in_thread2(self, func, params) -> threading.Thread: + t = threading.Thread(target=func, args=params) + t.start() + return t + + ## test others unsupported types operator with decimal + def test_decimal_unsupported_types(self): + unsupported_type = [ + TypeEnum.JSON, + TypeEnum.GEOMETRY, + TypeEnum.VARBINARY, + ] + all_type_columns: List[Column] = Column.get_all_type_columns([TypeEnum.JSON]) + tbname = "test_decimal_unsupported_types" + tag_cols = [Column(DataType(TypeEnum.JSON))] + tb_creater = DecimalColumnTableCreater( tdSql, self.db_name, tbname, all_type_columns, tag_cols) + tb_creater.create() + tb_creater.create_child_table(tbname, 10, tag_cols, ['{"k1": "v1"}']) + for i in range(10): + TableInserter(tdSql, self.db_name, f'{tbname}{i}', all_type_columns, tag_cols).insert(100, 1537146000000, 500, flush_database=True) + for col in all_type_columns: + ## only test decimal cols + if not col.type_.is_decimal_type(): + continue + for unsupported_col in all_type_columns: + ## only test unsupported cols + if not unsupported_col.type_.type in unsupported_type: + continue + for binary_op in DecimalBinaryOperator.get_all_binary_ops(): + select_expr = binary_op.generate((col, unsupported_col)) + sql = f"select {select_expr} from {self.db_name}.{tbname}" + select_expr_reverse = binary_op.generate((unsupported_col, col)) + sql_reverse = ( + f"select {select_expr_reverse} from {self.db_name}.{tbname}" + ) + tdLog.info( + f"select expr: {select_expr} with type: {col.type_} and {unsupported_col.type_} should err" + ) + err = tdSql.error(sql) + if tdSql.errno != invalid_operation and tdSql.errno != scalar_convert_err: + tdLog.exit(f"expected err not occured for sql: {sql}, expect: {invalid_operation} or {scalar_convert_err}, but got {tdSql.errno}") + + tdLog.info( + f"select expr: {select_expr} with type: {unsupported_col.type_} and {col.type_} should err" + ) + err = tdSql.error(sql_reverse) + if tdSql.errno != invalid_operation and tdSql.errno != scalar_convert_err: + tdLog.exit(f"expected err not occured for sql: {sql}, expect: {invalid_operation} or {scalar_convert_err}, but got {tdSql.errno}") + + def test_decimal_operators(self): + tdLog.debug("start to test decimal operators") + self.test_decimal_unsupported_types() + ## tables: meters, nt + ## columns: c1, c2, c3, c4, c5, c7, c8, c9, c10, c99, c100 + binary_operators = DecimalBinaryOperator.get_all_binary_ops() + + ## decimal operator with constants of all other types + self.run_in_thread( + operator_test_round, + self.check_decimal_binary_expr_with_const_col_results, + ( + self.db_name, + self.norm_table_name, + self.norm_tb_columns, + Column.get_decimal_oper_const_cols, + DecimalBinaryOperator.get_all_binary_ops, + ), + ) + + ## test decimal column op decimal column + for i in range(operator_test_round): + self.check_decimal_binary_expr_with_col_results( + self.db_name, self.norm_table_name, self.norm_tb_columns, binary_operators) + + unary_operators = DecimalUnaryOperator.get_all_unary_ops() + self.check_decimal_unary_expr_results( + self.db_name, + self.norm_table_name, + self.norm_tb_columns, + unary_operators,) + + + def test_decimal_last_first_func(self): + pass + + def test_query_decimal_with_sma(self): + pass + + def check_decimal_where_with_binary_expr_with_const_col_results( + self, + dbname, + tbname, + tb_cols: List[Column], + constant_cols: List[Column], + exprs: List[DecimalColumnExpr], + ): + if not binary_op_in_where_test: + return + for expr in exprs: + tdLog.info(f"start to test decimal where filtering with const cols for expr: {expr.format_}") + for col in tb_cols: + if col.name_ == '': + continue + for const_col in constant_cols: + if expr.should_skip_for_decimal([col, const_col]): + continue + const_col.generate_value() + select_expr = expr.generate((const_col, col)) + if const_col.type_.is_decimal_type(): + expr.query_col = col + else: + expr.query_col = col + sql = f"select {expr.query_col} from {dbname}.{tbname} where {select_expr}" + res = TaosShell().query(sql) + ##TODO wjm no need to check len(res) for filtering test, cause we need to check for every row in the table to check if the filtering is working + if len(res) > 0: + expr.check_for_filtering(res[0], tbname) + select_expr = expr.generate((col, const_col)) + sql = f"select {expr.query_col} from {dbname}.{tbname} where {select_expr}" + res = TaosShell().query(sql) + if len(res) > 0: + expr.check_for_filtering(res[0], tbname) + else: + tdLog.info(f"sql: {sql} got no output") + + def check_decimal_where_with_binary_expr_with_col_results( + self, dbname, tbname, tb_cols: List[Column], exprs: List[DecimalColumnExpr] + ): + if not binary_op_in_where_test: + return + for expr in exprs: + tdLog.info(f"start to test decimal where filtering with cols for expr{expr.format_}") + for col in tb_cols: + if col.name_ == '': + continue + for col2 in tb_cols: + if col2.name_ == '': + continue + if expr.should_skip_for_decimal([col, col2]): + continue + select_expr = expr.generate((col, col2)) + if col.type_.is_decimal_type(): + expr.query_col = col + else: + expr.query_col = col2 + sql = f"select {expr.query_col} from {dbname}.{tbname} where {select_expr}" + res = TaosShell().query(sql) + if len(res) > 0: + expr.check_for_filtering(res[0], tbname) + else: + tdLog.info(f"sql: {sql} got no output") + select_expr = expr.generate((col2, col)) + sql = f"select {expr.query_col} from {dbname}.{tbname} where {select_expr}" + res = TaosShell().query(sql) + if len(res) > 0: + expr.check_for_filtering(res[0], tbname) + else: + tdLog.info(f"sql: {sql} got no output") + + def test_query_decimal_where_clause(self): + tdLog.info("start to test decimal where filtering") + binary_compare_ops = DecimalBinaryOperator.get_all_filtering_binary_compare_ops() + const_cols = Column.get_decimal_oper_const_cols() + for i in range(operator_test_round): + self.check_decimal_where_with_binary_expr_with_const_col_results( + self.db_name, + self.norm_table_name, + self.norm_tb_columns, + const_cols, + binary_compare_ops, + ) + + for i in range(operator_test_round): + self.check_decimal_where_with_binary_expr_with_col_results( + self.db_name, + self.norm_table_name, + self.norm_tb_columns, + binary_compare_ops) + + ## TODO wjm + ## 3. (dec op const col) op const col + ## 4. (dec op dec) op const col + ## 5. (dec op const col) op dec + ## 6. (dec op dec) op dec + + def test_query_with_order_by_for_tb(self, tbname: str, cols: list): + for col in cols: + if col.type_.is_decimal_type() and col.name_ != '': + self.test_query_with_order_by(col, tbname) + + def test_query_with_order_by(self, order_col: Column, tbname): + sql = f"select {order_col} from {self.db_name}.{tbname} order by {order_col} asc" + query_res = TaosShell().query(sql)[0] + calculated_ordered_res = order_col.get_ordered_result(tbname, True) + for v_from_query, v_from_calc in zip(query_res, calculated_ordered_res): + if v_from_calc is None: + if v_from_query != 'NULL': + tdLog.exit(f"query result: {v_from_query} not equal to calculated result: NULL") + elif Decimal(v_from_query) != v_from_calc: + tdLog.exit(f"query result: {v_from_query} not equal to calculated result: {v_from_calc}") + + + def test_query_decimal_order_clause(self): + self.test_query_with_order_by_for_tb(self.norm_table_name, self.norm_tb_columns) + self.test_query_with_order_by_for_tb(self.stable_name, self.stb_columns) + + def test_query_decimal_group_by_decimal(self, tbname: str, cols: list): + for col in cols: + if col.type_.is_decimal_type() and col.name_ != '': + sql = f"select count(*) from {self.db_name}.{tbname} group by {col}" + query_res = TaosShell().query(sql)[0] + calculated_grouped_res = col.get_group_num(tbname) + if len(query_res) != calculated_grouped_res: + tdLog.exit(f"query result: {len(query_res)} not equal to calculated result: {calculated_grouped_res}") + + def test_query_decimal_group_by_clause(self): + self.test_query_decimal_group_by_decimal(self.norm_table_name, self.norm_tb_columns) + self.test_query_decimal_group_by_decimal(self.stable_name, self.stb_columns) + + def test_query_decimal_group_by_with_having(self, tbname, cols: list): + for col in cols: + if col.type_.is_decimal_type() and col.name_ != '': + sql = f"select count(*) from {self.db_name}.{tbname} group by {col} having {col} is not null" + query_res = TaosShell().query(sql)[0] + calculated_grouped_res = col.get_group_num(tbname, ignore_null=True) + if len(query_res) != calculated_grouped_res: + tdLog.exit(f"query result: {len(query_res)} not equal to calculated result: {calculated_grouped_res}") + + def test_query_decimal_having_clause(self): + self.test_query_decimal_group_by_with_having(self.norm_table_name, self.norm_tb_columns) + self.test_query_decimal_group_by_with_having(self.stable_name, self.stb_columns) + + def test_query_decimal_interval_fill(self): + pass + + def test_query_decimal_partition_by(self): + pass + + def test_query_decimal_case_when(self): + sql = "select case when cast(1 as decimal(10, 4)) >= 1 then cast(88888888.88 as decimal(10,2)) else cast(3.333 as decimal(10,3)) end" + res = TaosShell().query(sql)[0] + if res[0] != "88888888.88": + tdLog.exit(f"query result for sql: {sql}: {res[0]} not equal to expected result: 88888888.88") + sql = "select case when cast(1 as decimal(10, 4)) > 1 then cast(88888888.88 as decimal(10,2)) else cast(3.333 as decimal(10,3)) end" + res = TaosShell().query(sql)[0] + if res[0] != "3.33": + tdLog.exit(f"query result for sql: {sql}: {res[0]} not equal to expected result: 3.33") + + sql = "select case when cast(1 as decimal(10, 4)) > 1 then cast(88888888.88 as decimal(10,2)) else 1.23 end" + res = TaosShell().query(sql)[0] + if float(res[0]) != float(1.23): + tdLog.exit(f"query result for sql: {sql}: {res[0]} not equal to expected result: 1.23") + + sql = "select case when cast(1 as decimal(10, 4)) >= 1 then cast(88888888.88 as decimal(10,2)) else 1.23 end" + res = TaosShell().query(sql)[0] + if float(res[0]) != float(88888888.88): + tdLog.exit(f"query result for sql: {sql}: {res[0]} not equal to expected result: 88888888.88") + + sql = "select case when cast(1 as decimal(10, 4)) >= 1 then cast(88888888.88 as decimal(10,2)) else '1.23' end" + res = TaosShell().query(sql)[0] + if float(res[0]) != 88888888.88: + tdLog.exit(f"query result for sql: {sql}: {res[0]} not equal to expected result: 88888888.88") + + sql = "select case when cast(1 as decimal(10, 4)) > 1 then cast(88888888.88 as decimal(10,2)) else '1.23' end" + res = TaosShell().query(sql)[0] + if float(res[0]) != 1.23: + tdLog.exit(f"query result for sql: {sql}: {res[0]} not equal to expected result: 88888888.88") + + sql = "select case when cast(1 as decimal(10, 4)) > 1 then cast(88888888.88 as decimal(10,2)) else 'abcd' end" + res = TaosShell().query(sql)[0] + if float(res[0]) != 0: + tdLog.exit(f"query result for sql: {sql}: {res[0]} not equal to expected result: 0") + + def test_decimal_agg_funcs(self, dbname, tbname, tb_cols: List[Column], get_agg_funcs_func): + agg_funcs: List[DecimalFunction] = get_agg_funcs_func() + for func in agg_funcs: + for col in tb_cols: + if col.name_ == '' or func.should_skip_for_decimal([col]): + continue + func.query_col = col + select_expr = func.generate([col]) + sql = f"select {select_expr} from {dbname}.{tbname}" + res = TaosShell().query(sql) + if len(res) > 0: + res = res[0] + func.check_for_agg_func(res, tbname, func) + + def test_decimal_cast_func(self, dbname, tbname, tb_cols: List[Column]): + for col in tb_cols: + if col.name_ == '': + continue + to_types: list[DataType] = DecimalCastTypeGenerator(col.type_).generate(cast_func_test_round) + for t in to_types: + cast_func = DecimalCastFunction() + expr = cast_func.generate([col, t]) + sql = f"select {expr} from {dbname}.{tbname}" + res = TaosShell().query(sql) + if len(res) > 0: + res = res[0] + cast_func.check(res, tbname) + + def test_decimal_functions(self): + if not test_decimal_funcs: + return + self.test_decimal_agg_funcs( + self.db_name, + self.norm_table_name, + self.norm_tb_columns, + DecimalFunction.get_decimal_agg_funcs, + ) + ##self.test_decimal_agg_funcs( self.db_name, self.stable_name, self.stb_columns, DecimalFunction.get_decimal_agg_funcs) + self.test_decimal_cast_func(self.db_name, self.norm_table_name, self.norm_tb_columns) + + def test_query_decimal(self): + if not decimal_test_query: + return + self.test_decimal_operators() + #self.test_query_decimal_where_clause() + #self.test_decimal_functions() + #self.test_query_decimal_with_sma() + #self.test_query_decimal_order_clause() + #self.test_query_decimal_case_when() + #self.test_query_decimal_group_by_clause() + #self.test_query_decimal_having_clause() + + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/decimal2.py b/tests/system-test/2-query/decimal2.py new file mode 100644 index 0000000000..f3ee5b3c75 --- /dev/null +++ b/tests/system-test/2-query/decimal2.py @@ -0,0 +1,2444 @@ +import math +from random import randrange +import random +import time +import threading +import secrets +import numpy + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from decimal import * +from multiprocessing import Value, Lock +from functools import cmp_to_key + +class AtomicCounter: + def __init__(self, initial_value=0): + self._value = Value('i', initial_value) + self._lock = Lock() + + def fetch_add(self, delta = 1): + with self._lock: + old_value = self._value.value + self._value.value += delta + return old_value + +getcontext().prec = 40 + +def get_decimal(val, scale: int) -> Decimal: + if val == 'NULL': + return None + getcontext().prec = 100 + return Decimal(val).quantize(Decimal("1." + "0" * scale), ROUND_HALF_UP) + +syntax_error = -2147473920 +invalid_column = -2147473918 +invalid_compress_level = -2147483084 +invalid_encode_param = -2147483087 +invalid_operation = -2147483136 +scalar_convert_err = -2147470768 + + +decimal_test_query = True +decimal_insert_validator_test = False +operator_test_round = 1 +tb_insert_rows = 1000 +binary_op_with_const_test = False +binary_op_with_col_test = False +unary_op_test = False +binary_op_in_where_test = True +test_decimal_funcs = True +cast_func_test_round = 10 + +class DecimalTypeGeneratorConfig: + def __init__(self): + self.enable_weight_overflow: bool = False + self.weightOverflowRatio: float = 0.001 + self.enable_scale_overflow: bool = True + self.scale_overflow_ratio = 0.1 + self.enable_positive_sign = False + self.with_corner_case = True + self.corner_case_ratio = 0.1 + self.positive_ratio = 0.7 + self.prec = 38 + self.scale = 10 + + +class DecimalStringRandomGenerator: + def __init__(self): + self.corner_cases = ["0", "NULL", "0.", ".0", "000.000000"] + self.ratio_base: int = 1000000 + + def possible(self, possibility: float) -> bool: + return random.randint(0, self.ratio_base) < possibility * self.ratio_base + + def generate_sign(self, positive_ratio: float) -> str: + if self.possible(positive_ratio): + return "+" + return "-" + + def generate_digit(self) -> str: + return str(random.randint(0, 9)) + + def current_should_generate_corner_case(self, corner_case_ratio: float) -> bool: + return self.possible(corner_case_ratio) + + def generate_corner_case(self, config: DecimalTypeGeneratorConfig) -> str: + if self.possible(0.8): + return random.choice(self.corner_cases) + else: + res = self.generate_digit() * (config.prec - config.scale) + if self.possible(0.8): + res += "." + if self.possible(0.8): + res += self.generate_digit() * config.scale + return res + + ## 写入大整数的例子, 如10000000000, scale解析时可能为负数 + ## Generate decimal with E/e + def generate(self, config: DecimalTypeGeneratorConfig) -> str: + ret: str = "" + sign = self.generate_sign(config.positive_ratio) + if config.with_corner_case and self.current_should_generate_corner_case( + config.corner_case_ratio + ): + ret += self.generate_corner_case(config) + else: + if config.enable_positive_sign or sign != "+": + ret += sign + weight = random.randint(1, config.prec - config.scale) + scale = random.randint(1, config.scale) + for i in range(weight): + ret += self.generate_digit() + + if config.enable_weight_overflow and self.possible( + config.weightOverflowRatio + ): + extra_weight = ( + config.prec + - weight + + 1 + + random.randint(1, self.get_max_prec(config.prec)) + ) + while extra_weight > 0: + ret += self.generate_digit() + extra_weight -= 1 + ret += "." + for i in range(scale): + ret += self.generate_digit() + if config.enable_scale_overflow and self.possible( + config.scale_overflow_ratio + ): + extra_scale = ( + config.scale + - scale + + 1 + + random.randint(1, self.get_max_prec(config.prec)) + ) + while extra_scale > 0: + ret += self.generate_digit() + extra_scale -= 1 + return ret + + def get_max_prec(self, prec): + if prec <= 18: + return 18 + else: + return 38 + + +class DecimalColumnAggregator: + def __init__(self): + self.max: Decimal = Decimal("0") + self.min: Decimal = Decimal("0") + self.count: int = 0 + self.sum: Decimal = Decimal("0") + self.null_num: int = 0 + self.none_num: int = 0 + self.first = None + self.last = None + + def add_value(self, value: str, scale: int): + self.count += 1 + if value == "NULL": + self.null_num += 1 + elif value == "None": + self.none_num += 1 + else: + v: Decimal = get_decimal(value, scale) + if self.first is None: + self.first = v + self.last = v + self.sum += v + if v > self.max: + self.max = v + if v < self.min: + self.min = v + +atomic_counter = AtomicCounter(0) + +class TaosShell: + def __init__(self): + self.counter_ = atomic_counter.fetch_add() + self.queryResult = [] + self.tmp_file_path = "/tmp/taos_shell_result" + + def get_file_path(self): + return f"{self.tmp_file_path}_{self.counter_}" + + def read_result(self): + with open(self.get_file_path(), "r") as f: + lines = f.readlines() + lines = lines[1:] + for line in lines: + col = 0 + vals: list[str] = line.split(",") + if len(self.queryResult) == 0: + self.queryResult = [[] for i in range(len(vals))] + for val in vals: + self.queryResult[col].append(val.strip().strip('"')) + col += 1 + + def query(self, sql: str): + with open(self.get_file_path(), "a+") as f: + f.truncate(0) + self.queryResult = [] + try: + command = f'taos -s "{sql} >> {self.get_file_path()}"' + result = subprocess.run( + command, shell=True, check=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE + ) + self.read_result() + except Exception as e: + tdLog.exit(f"Command '{sql}' failed with error: {e.stderr.decode('utf-8')}") + self.queryResult = [] + raise + return self.queryResult + +class DecimalColumnExpr: + def __init__(self, format: str, executor): + self.format_: str = format + self.executor_ = executor + self.params_ = () + self.res_type_: DataType = None + self.query_col: Column = None + + def __str__(self): + return f"({self.format_})".format(*self.params_) + + def execute(self, params): + return self.executor_(self, params) + + def get_query_col_val(self, tbname, i): + return self.query_col.get_val_for_execute(tbname, i) + + def get_val(self, tbname: str, idx: int): + params = () + for p in self.params_: + params = params + (p.get_val(tbname, idx),) + return self.execute(params) + + def convert_to_res_type(self, val: Decimal) -> Decimal: + if self.res_type_.is_decimal_type(): + return get_decimal(val, self.res_type_.scale()) + elif self.res_type_.type == TypeEnum.DOUBLE: + return float(val) + + def get_input_types(self) -> List: + pass + + def should_skip_for_decimal(self, cols: list)->bool: + return False + + def check_query_results(self, query_col_res: List, tbname: str): + query_len = len(query_col_res) + pass + + def check_for_filtering(self, query_col_res: List, tbname: str): + j: int = -1 + for i in range(len(query_col_res)): + j += 1 + v_from_query = query_col_res[i] + while True: + params = () + for p in self.params_: + if isinstance(p, Column) or isinstance(p, DecimalColumnExpr): + p = p.get_val_for_execute(tbname, j) + params = params + (p,) + v_from_calc_in_py = self.execute(params) + + if not v_from_calc_in_py: + j += 1 + continue + else: + break + dec_from_query = Decimal(v_from_query) + dec_from_calc = self.get_query_col_val(tbname, j) + if dec_from_query != dec_from_calc: + tdLog.exit(f"filter with {self} failed, query got: {dec_from_query}, expect {dec_from_calc}, param: {params}") + else: + pass + #tdLog.info(f"filter with {self} succ, query got: {dec_from_query}, expect {dec_from_calc}, param: {params}") + + def check(self, query_col_res: List, tbname: str): + for i in range(len(query_col_res)): + v_from_query = query_col_res[i] + params = () + for p in self.params_: + if isinstance(p, Column) or isinstance(p, DecimalColumnExpr): + p = p.get_val_for_execute(tbname, i) + params = params + (p,) + v_from_calc_in_py = self.execute(params) + + if v_from_calc_in_py == 'NULL' or v_from_query == 'NULL': + if v_from_calc_in_py != v_from_query: + tdLog.exit(f"query with expr: {self} calc in py got: {v_from_calc_in_py}, query got: {v_from_query}") + #tdLog.debug(f"query with expr: {self} calc got same result: NULL") + continue + failed = False + if self.res_type_.type == TypeEnum.BOOL: + query_res = bool(int(v_from_query)) + calc_res = bool(int(v_from_calc_in_py)) + failed = query_res != calc_res + elif isinstance(v_from_calc_in_py, float): + query_res = float(v_from_query) + calc_res = float(v_from_calc_in_py) + failed = not math.isclose(query_res, calc_res, abs_tol=1e-7) + else: + query_res = Decimal(v_from_query) + calc_res = Decimal(v_from_calc_in_py) + failed = query_res != calc_res + if failed: + tdLog.exit( + f"check decimal column failed for expr: {self}, input: {[t.__str__() for t in self.get_input_types()]}, res_type: {self.res_type_}, params: {params}, query: {v_from_query}, expect {calc_res}, but get {query_res}" + ) + else: + pass + #tdLog.info( f"op succ: {self}, in: {[t.__str__() for t in self.get_input_types()]}, res: {self.res_type_}, params: {params}, insert:{v_from_calc_in_py} query:{v_from_query}, py calc: {calc_res}") + + ## format_params are already been set + def generate_res_type(self): + pass + + def generate(self, format_params) -> str: + self.params_ = format_params + self.generate_res_type() + return self.__str__() + + +class TypeEnum: + BOOL = 1 + TINYINT = 2 + SMALLINT = 3 + INT = 4 + BIGINT = 5 + FLOAT = 6 + DOUBLE = 7 + VARCHAR = 8 + TIMESTAMP = 9 + NCHAR = 10 + UTINYINT = 11 + USMALLINT = 12 + UINT = 13 + UBIGINT = 14 + JSON = 15 + VARBINARY = 16 + DECIMAL = 17 + BINARY = 8 + GEOMETRY = 20 + DECIMAL64 = 21 + + @staticmethod + def get_type_prec(type: int): + type_prec = [0, 1, 3, 5, 10, 19, 38, 38, 0, 19, 10, 3, 5, 10, 20, 0, 0, 0, 0, 0, 0, 0] + return type_prec[type] + + @staticmethod + def get_type_str(type: int): + type_str = [ + "", + "BOOL", + "TINYINT", + "SMALLINT", + "INT", + "BIGINT", + "FLOAT", + "DOUBLE", + "VARCHAR", + "TIMESTAMP", + "NCHAR", + "TINYINT UNSIGNED", + "SMALLINT UNSIGNED", + "INT UNSIGNED", + "BIGINT UNSIGNED", + "JSON", + "VARBINARY", + "DECIMAL", + "", + "", + "GEOMETRY", + "DECIMAL", + ] + return type_str[type] + + +class DataType: + def __init__(self, type: int, length: int = 0, type_mod: int = 0): + self.type: int = type + self.length = length + self.type_mod = type_mod + + def __str__(self): + if self.type_mod != 0: + return f"{TypeEnum.get_type_str(self.type)}({self.prec()}, {self.scale()})" + if self.length: + return f"{TypeEnum.get_type_str(self.type)}({self.length})" + return TypeEnum.get_type_str(self.type) + + def __eq__(self, other): + return self.type == other.type and self.length == other.length and self.type_mod == other.type_mod + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash((self.type, self.length)) + + def __repr__(self): + return f"DataType({self.type}, {self.length}, {self.type_mod})" + + def is_decimal_type(self): + return self.type == TypeEnum.DECIMAL or self.type == TypeEnum.DECIMAL64 + + def is_varchar_type(self): + return self.type == TypeEnum.VARCHAR or self.type == TypeEnum.NCHAR or self.type == TypeEnum.VARBINARY or self.type == TypeEnum.JSON or self.type == TypeEnum.BINARY + + def is_real_type(self): + return self.type == TypeEnum.FLOAT or self.type == TypeEnum.DOUBLE + + def prec(self): + return 0 + + def scale(self): + return 0 + + ## TODO generate NULL, None + def generate_value(self) -> str: + if self.type == TypeEnum.BOOL: + return ['true', 'false'][secrets.randbelow(2)] + if self.type == TypeEnum.TINYINT: + return str(secrets.randbelow(256) - 128) + if self.type == TypeEnum.SMALLINT: + return str(secrets.randbelow(65536) - 32768) + if self.type == TypeEnum.INT: + return str(secrets.randbelow(4294967296) - 2147483648) + if self.type == TypeEnum.BIGINT: + return str(secrets.randbelow(9223372036854775808) - 4611686018427387904) + if self.type == TypeEnum.FLOAT or self.type == TypeEnum.DOUBLE: + return str(random.random()) + if ( + self.type == TypeEnum.VARCHAR + or self.type == TypeEnum.NCHAR + or self.type == TypeEnum.VARBINARY + ): + return f"'{str(random.random())[0:self.length]}'" + if self.type == TypeEnum.TIMESTAMP: + return str(secrets.randbelow(9223372036854775808)) + if self.type == TypeEnum.UTINYINT: + return str(secrets.randbelow(256)) + if self.type == TypeEnum.USMALLINT: + return str(secrets.randbelow(65536)) + if self.type == TypeEnum.UINT: + return str(secrets.randbelow(4294967296)) + if self.type == TypeEnum.UBIGINT: + return str(secrets.randbelow(9223372036854775808)) + if self.type == TypeEnum.JSON: + return f'{{"key": "{secrets.token_urlsafe(10)}"}}' + if self.type == TypeEnum.GEOMETRY: + return "'POINT(1.0 1.0)'" + raise Exception(f"unsupport type {self.type}") + + def check(self, values, offset: int): + return True + + def get_typed_val_for_execute(self, val, const_col = False): + if self.type == TypeEnum.DOUBLE: + return float(val) + elif self.type == TypeEnum.BOOL: + if val == "true": + return 1 + else: + return 0 + elif self.type == TypeEnum.FLOAT: + if const_col: + val = float(str(numpy.float32(val))) + else: + val = float(numpy.float32(val)) + elif self.type == TypeEnum.DECIMAL or self.type == TypeEnum.DECIMAL64: + return get_decimal(val, self.scale()) + elif isinstance(val, str): + val = val.strip("'") + return val + + def get_typed_val(self, val): + if self.type == TypeEnum.FLOAT: + return float(str(numpy.float32(val))) + elif self.type == TypeEnum.DOUBLE: + return float(val) + return val + + @staticmethod + def get_decimal_types() -> list: + return [TypeEnum.DECIMAL64, TypeEnum.DECIMAL] + + @staticmethod + def get_decimal_op_types()-> list: + return [ + TypeEnum.BOOL, + TypeEnum.TINYINT, + TypeEnum.SMALLINT, + TypeEnum.INT, + TypeEnum.BIGINT, + TypeEnum.FLOAT, + TypeEnum.DOUBLE, + TypeEnum.VARCHAR, + TypeEnum.NCHAR, + TypeEnum.UTINYINT, + TypeEnum.USMALLINT, + TypeEnum.UINT, + TypeEnum.UBIGINT, + TypeEnum.DECIMAL, + TypeEnum.DECIMAL64, + ] + +class DecimalType(DataType): + DECIMAL_MAX_PRECISION = 38 + DECIMAL64_MAX_PRECISION = 18 + def __init__(self, type, precision: int, scale: int): + self.precision_ = precision + self.scale_ = scale + if type == TypeEnum.DECIMAL64: + bytes = 8 + else: + bytes = 16 + super().__init__(type, bytes, self.get_decimal_type_mod()) + self.decimal_generator: DecimalStringRandomGenerator = DecimalStringRandomGenerator() + self.generator_config: DecimalTypeGeneratorConfig = DecimalTypeGeneratorConfig() + #self.generator_config.with_corner_case = False + self.generator_config.prec = precision + self.generator_config.scale = scale + self.aggregator: DecimalColumnAggregator = DecimalColumnAggregator() + self.values: List[str] = [] + + def get_decimal_type_mod(self) -> int: + return self.precision_ * 100 + self.scale() + + def set_prec(self, prec: int): + self.precision_ = prec + self.type_mod = self.get_decimal_type_mod() + + def set_scale(self, scale: int): + self.scale_ = scale + self.type_mod = self.get_decimal_type_mod() + + def prec(self): + return self.precision_ + + def scale(self): + return self.scale_ + + def __str__(self): + return f"DECIMAL({self.precision_}, {self.scale()})" + + def __eq__(self, other: DataType): + return self.precision_ == other.prec() and self.scale() == other.scale() + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash((self.precision_, self.scale())) + + def __repr__(self): + return f"DecimalType({self.precision_}, {self.scale()})" + + def generate_value(self) -> str: + val = self.decimal_generator.generate(self.generator_config) + self.aggregator.add_value(val, self.scale()) ## convert to Decimal first + # self.values.append(val) ## save it into files maybe + return val + + def get_typed_val(self, val): + if val == "NULL": + return None + return get_decimal(val, self.scale()) + + def get_typed_val_for_execute(self, val, const_col = False): + return self.get_typed_val(val) + + @staticmethod + def default_compression() -> str: + return "zstd" + + @staticmethod + def default_encode() -> str: + return "disabled" + + def check(self, values, offset: int): + val_from_query = values + val_insert = self.values[offset:] + for v_from_query, v_from_insert in zip(val_from_query, val_insert): + if v_from_insert == "NULL": + if v_from_query.strip() != "NULL": + tdLog.debug( + f"val_insert: {val_insert} val_from_query: {val_from_query}" + ) + tdLog.exit(f"insert NULL, query not NULL: {v_from_query}") + else: + continue + try: + dec_query: Decimal = Decimal(v_from_query) + dec_insert: Decimal = Decimal(v_from_insert) + dec_insert = get_decimal(dec_insert, self.scale()) + except Exception as e: + tdLog.exit(f"failed to convert {v_from_query} or {v_from_insert} to decimal, {e}") + return False + if dec_query != dec_insert: + tdLog.exit( + f"check decimal column failed for insert: {v_from_insert}, query: {v_from_query}, expect {dec_insert}, but get {dec_query}" + ) + return False + else: + tdLog.debug( + f"check decimal succ, insert:{v_from_insert} query:{v_from_query}, py dec: {dec_insert}" + ) + + + @staticmethod + def decimal_type_from_other_type(other: DataType): + prec = 0 + return DecimalType(other.type, other.length, other.type_mod) + +class Column: + def __init__(self, type: DataType): + self.type_: DataType = type + self.name_: str = "" + self.saved_vals:dict[str:[]] = {} + + def is_constant_col(self): + return '' in self.saved_vals.keys() + + def get_typed_val(self, val): + return self.type_.get_typed_val(val) + + def get_typed_val_for_execute(self, val, const_col = False): + return self.type_.get_typed_val_for_execute(val, const_col) + + def get_constant_val(self): + return self.get_typed_val(self.saved_vals[''][0]) + + def get_constant_val_for_execute(self): + return self.get_typed_val_for_execute(self.saved_vals[''][0], const_col=True) + + def __str__(self): + if self.is_constant_col(): + return str(self.get_constant_val()) + return self.name_ + + def get_val_for_execute(self, tbname: str, idx: int): + if self.is_constant_col(): + return self.get_constant_val_for_execute() + if len(self.saved_vals) > 1: + for key in self.saved_vals.keys(): + l = len(self.saved_vals[key]) + if idx < l: + return self.get_typed_val_for_execute(self.saved_vals[key][idx]) + else: + idx -= l + return self.get_typed_val_for_execute(self.saved_vals[tbname][idx]) + + def get_cardinality(self, tbname): + if self.is_constant_col(): + return 1 + elif len(self.saved_vals) > 1: + return len(self.saved_vals['t0']) + else: + return len(self.saved_vals[tbname]) + + @staticmethod + def comp_key(key1, key2): + if key1 is None: + return -1 + if key2 is None: + return 1 + return key1 - key2 + + def get_ordered_result(self, tbname: str, asc: bool) -> list: + if tbname in self.saved_vals: + return sorted( + [ + get_decimal(val, self.type_.scale()) + for val in self.saved_vals[tbname] + ], + reverse=not asc, + key=cmp_to_key(Column.comp_key) + ) + else: + res = [] + for val in self.saved_vals.values(): + res.extend(val) + return sorted( + [get_decimal(val, self.type_.scale()) for val in res], reverse=not asc, + key=cmp_to_key(Column.comp_key) + ) + + def get_group_num(self, tbname, ignore_null=False) -> int: + if tbname in self.saved_vals: + s = set(get_decimal(val, self.type_.scale()) for val in self.saved_vals[tbname]) + if ignore_null: + s.remove(None) + return len(s) + else: + res = set() + for vals in self.saved_vals.values(): + for v in vals: + res.add(get_decimal(v, self.type_.scale())) + if ignore_null: + res.remove(None) + return len(res) + + ## tbName: for normal table, pass the tbname, for child table, pass the child table name + def generate_value(self, tbName: str = '', save: bool = True): + val = self.type_.generate_value() + if save: + if tbName not in self.saved_vals: + self.saved_vals[tbName] = [] + ## for constant columns, always replace the last val + if self.is_constant_col(): + self.saved_vals[tbName] = [val] + else: + self.saved_vals[tbName].append(val) + return val + + def get_type_str(self) -> str: + return str(self.type_) + + def set_name(self, name: str): + self.name_ = name + + def check(self, values, offset: int): + return self.type_.check(values, offset) + + def construct_type_value(self, val: str): + if ( + self.type_.type == TypeEnum.BINARY + or self.type_.type == TypeEnum.VARCHAR + or self.type_.type == TypeEnum.NCHAR + or self.type_.type == TypeEnum.VARBINARY + or self.type_.type == TypeEnum.JSON + ): + return f"'{val}'" + else: + return val + + @staticmethod + def get_decimal_unsupported_types() -> list: + return [ + TypeEnum.JSON, + TypeEnum.GEOMETRY, + TypeEnum.VARBINARY, + ] + + @staticmethod + def get_decimal_oper_const_cols() -> list: + types_unable_to_be_const = [ + TypeEnum.TINYINT, + TypeEnum.SMALLINT, + TypeEnum.INT, + TypeEnum.UINT, + TypeEnum.USMALLINT, + TypeEnum.UTINYINT, + TypeEnum.UBIGINT, + ] + return Column.get_all_type_columns( + Column.get_decimal_unsupported_types() + + Column.get_decimal_types() + + types_unable_to_be_const + ) + + @staticmethod + def get_decimal_types() -> List: + return [TypeEnum.DECIMAL, TypeEnum.DECIMAL64] + + @staticmethod + def get_all_type_columns(types_to_exclude: List[TypeEnum] = []) -> List: + all_types = [ + Column(DataType(TypeEnum.BOOL)), + Column(DataType(TypeEnum.TINYINT)), + Column(DataType(TypeEnum.SMALLINT)), + Column(DataType(TypeEnum.INT)), + Column(DataType(TypeEnum.BIGINT)), + Column(DataType(TypeEnum.FLOAT)), + Column(DataType(TypeEnum.DOUBLE)), + Column(DataType(TypeEnum.VARCHAR, 255)), + Column(DataType(TypeEnum.TIMESTAMP)), + Column(DataType(TypeEnum.NCHAR, 255)), + Column(DataType(TypeEnum.UTINYINT)), + Column(DataType(TypeEnum.USMALLINT)), + Column(DataType(TypeEnum.UINT)), + Column(DataType(TypeEnum.UBIGINT)), + Column(DataType(TypeEnum.JSON)), + Column(DataType(TypeEnum.VARBINARY, 255)), + Column(DecimalType(TypeEnum.DECIMAL, 38, 10)), + Column(DataType(TypeEnum.BINARY, 255)), + Column(DataType(TypeEnum.GEOMETRY, 10240)), + Column(DecimalType(TypeEnum.DECIMAL64, 18, 4)), + ] + ret = [] + for c in all_types: + found = False + for type in types_to_exclude: + if c.type_.type == type: + found = True + break + if not found: + ret.append(c) + return ret + + +class DecimalColumnTableCreater: + def __init__( + self, + conn, + dbName: str, + tbName: str, + columns: List[Column], + tags_cols: List[Column] = [], + col_prefix: str = "c", + tag_prefix: str = "t", + ): + self.conn = conn + self.dbName = dbName + self.tbName = tbName + self.tags_cols = tags_cols + self.columns: List[Column] = columns + self.col_prefix = col_prefix + self.tag_prefix = tag_prefix + + def create(self): + if len(self.tags_cols) > 0: + table = "stable" + else: + table = "table" + sql = f"create {table} {self.dbName}.{self.tbName} (ts timestamp" + for i, column in enumerate(self.columns): + tbname = f"{self.col_prefix}{i+1}" + sql += f", {tbname} {column.get_type_str()}" + column.set_name(tbname) + if self.tags_cols: + sql += ") tags(" + for i, tag in enumerate(self.tags_cols): + tagname = f"{self.tag_prefix}{i+1}" + sql += f"{tagname} {tag.get_type_str()}" + tag.set_name(tagname) + if i != len(self.tags_cols) - 1: + sql += ", " + sql += ")" + self.conn.execute(sql, queryTimes=1) + + def create_child_table( + self, + ctbPrefix: str, + ctbNum: int, + tag_cols: List[Column], + tag_values: List[str], + ): + for i in range(ctbNum): + tbname = f"{ctbPrefix}{i}" + sql = f"create table {self.dbName}.{tbname} using {self.dbName}.{self.tbName} tags(" + for j, tag in enumerate(tag_cols): + sql += f"{tag.construct_type_value(tag_values[j])}" + if j != len(tag_cols) - 1: + sql += ", " + sql += ")" + self.conn.execute(sql, queryTimes=1) + + +class TableInserter: + def __init__( + self, + conn, + dbName: str, + tbName: str, + columns: List[Column], + tags_cols: List[Column] = [], + ): + self.conn: TDSql = conn + self.dbName = dbName + self.tbName = tbName + self.tag_cols = tags_cols + self.columns = columns + + def insert(self, rows: int, start_ts: int, step: int, flush_database: bool = False): + pre_insert = f"insert into {self.dbName}.{self.tbName} values" + sql = pre_insert + t = datetime.now() + for i in range(rows): + sql += f"({start_ts + i * step}" + for column in self.columns: + sql += f", {column.generate_value(self.tbName)}" + sql += ")" + if i != rows - 1: + sql += ", " + local_flush_database = i % 5000 == 0 + if len(sql) > 1000: + # tdLog.debug(f"insert into with sql{sql}") + if flush_database and local_flush_database: + self.conn.execute(f"flush database {self.dbName}", queryTimes=1) + self.conn.execute(sql, queryTimes=1) + t1 = datetime.now() + if (t1 - t).seconds > 1: + TaosShell().query(f"select last(c1), last(c2) from {self.dbName}.{self.tbName}") + t = t1 + sql = pre_insert + if len(sql) > len(pre_insert): + # tdLog.debug(f"insert into with sql{sql}") + if flush_database: + self.conn.execute(f"flush database {self.dbName}", queryTimes=1) + self.conn.execute(sql, queryTimes=1) + TaosShell().query(f"select last(c1), last(c2) from {self.dbName}.{self.tbName}") + +class DecimalCastTypeGenerator: + def __init__(self, input_type: DataType): + self.input_type_: DataType = input_type + + def get_possible_output_types(self) -> List[int]: + if not self.input_type_.is_decimal_type(): + return DataType.get_decimal_types() + else: + return DataType.get_decimal_op_types() + + def do_generate_type(self, dt: int) ->DataType: + if dt == TypeEnum.DECIMAL: + prec = random.randint(1, DecimalType.DECIMAL_MAX_PRECISION) + return DecimalType(dt, prec, random.randint(0, prec)) + elif dt == TypeEnum.DECIMAL64: + prec = random.randint(1, DecimalType.DECIMAL64_MAX_PRECISION) + return DecimalType(dt, prec, random.randint(0, prec)) + elif dt == TypeEnum.BINARY or dt == TypeEnum.VARCHAR: + return DataType(dt, random.randint(16, 255), 0) + else: + return DataType(dt, 0, 0) + + def generate(self, num: int) -> List[DataType]: + res: list[DataType] = [] + for _ in range(num): + dt = random.choice(self.get_possible_output_types()) + dt = self.do_generate_type(dt) + res.append(dt) + res = list(set(res)) + return res + +class TableDataValidator: + def __init__(self, columns: List[Column], tbName: str, dbName: str, tbIdx: int = 0): + self.columns = columns + self.tbName = tbName + self.dbName = dbName + self.tbIdx = tbIdx + + def validate(self): + if not decimal_insert_validator_test: + return + sql = f"select * from {self.dbName}.{self.tbName}" + res = TaosShell().query(sql) + row_num = len(res[1]) + colIdx = 1 + for col in self.columns: + if ( + col.type_.type == TypeEnum.DECIMAL + or col.type_.type == TypeEnum.DECIMAL64 + ): + col.check(res[colIdx], row_num * self.tbIdx) + colIdx += 1 + +class DecimalFunction(DecimalColumnExpr): + def __init__(self, format, executor, name: str): + super().__init__(format, executor) + self.func_name_ = name + + def is_agg_func(self, op: str) ->bool: + return False + + def get_func_res(self): + return None + + def get_input_types(self): + return [self.query_col] + + @staticmethod + def get_decimal_agg_funcs() -> List: + return [ + DecimalMinFunction(), + DecimalMaxFunction(), + DecimalSumFunction(), + DecimalAvgFunction(), + DecimalCountFunction(), + DecimalLastRowFunction(), + DecimalLastFunction(), + DecimalFirstFunction(), + ] + + def check_results(self, query_col_res: List) -> bool: + return False + + def check_for_agg_func(self, query_col_res: List, tbname: str, func): + col_expr = self.query_col + for i in range(col_expr.get_cardinality(tbname)): + col_val = col_expr.get_val_for_execute(tbname, i) + self.execute((col_val,)) + if not self.check_results(query_col_res): + tdLog.exit(f"check failed for {self}, query got: {query_col_res}, expect {self.get_func_res()}") + else: + tdLog.info(f"check expr: {func} with val: {col_val} got result: {query_col_res}, expect: {self.get_func_res()}") + +class DecimalCastFunction(DecimalFunction): + def __init__(self): + super().__init__("cast({0} as {1})", DecimalCastFunction.execute_cast, "cast") + + def should_skip_for_decimal(self, cols: list)->bool: + return False + + def check_results(self, query_col_res: List) -> bool: + return False + + def generate_res_type(self)->DataType: + self.query_col = self.params_[0] + self.res_type_ = self.params_[1] + return self.res_type_ + + def check(self, res: list, tbname: str): + calc_res = [] + params = [] + for i in range(self.query_col.get_cardinality(tbname)): + val = self.query_col.get_val_for_execute(tbname, i) + params.append(val) + try: + calc_val = self.execute(val) + except OverflowError as e: + tdLog.info(f"execute {self} overflow for param: {val}") + calc_res = [] + break + calc_res.append(calc_val) + if len(calc_res) != len(res): + tdLog.exit(f"check result for {self} failed len got: {len(res)}, expect: {len(calc_res)}") + if len(calc_res) == 0: + return True + for v, calc_v, p in zip(res, calc_res, params): + query_v = self.execute_cast(v) + if isinstance(calc_v, float): + eq = math.isclose(query_v, calc_v, rel_tol=1e-7) + elif isinstance(calc_v, numpy.float32): + eq = math.isclose(query_v, calc_v, abs_tol=1e-6, rel_tol=1e-6) + elif isinstance(p, float) or isinstance(p, str): + eq = math.isclose(query_v, calc_v, rel_tol=1e-7) + else: + eq = query_v == calc_v + if not eq: + tdLog.exit(f"check result for {self} failed with param: {p} query got: {v}, expect: {calc_v}") + return True + + def execute_cast(self, val): + if val is None or val == 'NULL': + return None + if self.res_type_.type == TypeEnum.BOOL: + return Decimal(val) != 0 + elif self.res_type_.type == TypeEnum.TINYINT: + dec = Decimal(val).quantize(Decimal("1"), ROUND_HALF_UP) + return int(dec) & 0xFF + elif self.res_type_.type == TypeEnum.SMALLINT: + dec = Decimal(val).quantize(Decimal("1"), ROUND_HALF_UP) + return int(dec) & 0xFFFF + elif self.res_type_.type == TypeEnum.INT: + dec = Decimal(val).quantize(Decimal("1"), ROUND_HALF_UP) + return int(dec) & 0xFFFFFFFF + elif self.res_type_.type == TypeEnum.BIGINT or self.res_type_.type == TypeEnum.TIMESTAMP: + dec = Decimal(val).quantize(Decimal("1"), ROUND_HALF_UP) + return int(dec) & 0xFFFFFFFFFFFFFFFF + elif self.res_type_.type == TypeEnum.FLOAT: + return numpy.float32(val) + elif self.res_type_.type == TypeEnum.DOUBLE: + return float(val) + elif self.res_type_.type == TypeEnum.VARCHAR or self.res_type_.type == TypeEnum.NCHAR: + if Decimal(val) == 0: + return "0" + return str(val)[0:self.res_type_.length] + elif self.res_type_.type == TypeEnum.UTINYINT: + dec = Decimal(val).quantize(Decimal("1"), ROUND_HALF_UP) + return int(dec) & 0xFF + elif self.res_type_.type == TypeEnum.USMALLINT: + dec = Decimal(val).quantize(Decimal("1"), ROUND_HALF_UP) + return int(dec) & 0xFFFF + elif self.res_type_.type == TypeEnum.UINT: + dec = Decimal(val).quantize(Decimal("1"), ROUND_HALF_UP) + return int(dec) & 0xFFFFFFFF + elif self.res_type_.type == TypeEnum.UBIGINT: + dec = Decimal(val).quantize(Decimal("1"), ROUND_HALF_UP) + return int(dec) & 0xFFFFFFFFFFFFFFFF + elif self.res_type_.is_decimal_type(): + max: Decimal = Decimal( + "9" * (self.res_type_.prec() - self.res_type_.scale()) + + "." + + "9" * self.res_type_.scale() + ) + if max < get_decimal(val, self.res_type_.scale()): + raise OverflowError() + try: + return get_decimal(val, self.res_type_.scale()) + except Exception as e: + tdLog.exit(f"failed to cast {val} to {self.res_type_}, {e}") + else: + raise Exception(f"cast unsupported type {self.res_type_.type}") + +class DecimalAggFunction(DecimalFunction): + def __init__(self, format, executor, name: str): + super().__init__(format, executor, name) + + def is_agg_func(self, op: str)-> bool: + return True + + def should_skip_for_decimal(self, cols: list)-> bool: + col: Column = cols[0] + if col.type_.is_decimal_type(): + return False + return True + + def check_results(self, query_col_res): + if len(query_col_res) == 0: + tdLog.info(f"query got no output: {self}, py calc: {self.get_func_res()}") + return True + else: + return self.get_func_res() == Decimal(query_col_res[0]) + +class DecimalLastRowFunction(DecimalAggFunction): + def __init__(self): + super().__init__("last_row({0})", DecimalLastRowFunction.execute_last_row, "last_row") + self.res_ = None + def get_func_res(self): + decimal_type:DecimalType = self.query_col.type_ + return decimal_type.aggregator.last + def generate_res_type(self): + self.res_type_ = self.query_col.type_ + def execute_last_row(self, params): + if params[0] is not None: + self.res_ = Decimal(params[0]) + +class DecimalCacheLastRowFunction(DecimalAggFunction): + def __init__(self): + super().__init__("_cache_last_row({0})", DecimalCacheLastRowFunction.execute_cache_last_row, "_cache_last_row") + def get_func_res(self): + return 1 + def generate_res_type(self): + self.res_type_ = self.query_col.type_ + def execute_cache_last_row(self, params): + return 1 + +class DecimalCacheLastFunction(DecimalAggFunction): + pass + +class DecimalFirstFunction(DecimalAggFunction): + def __init__(self): + super().__init__("first({0})", DecimalFirstFunction.execute_first, "first") + self.res_ = None + def get_func_res(self): + decimal_type: DecimalType = self.query_col.type_ + return decimal_type.aggregator.first + def generate_res_type(self): + self.res_type_ = self.query_col.type_ + def execute_first(self, params): + pass + +class DecimalLastFunction(DecimalAggFunction): + def __init__(self): + super().__init__("last({0})", DecimalLastFunction.execute_last, "last") + self.res_ = None + def get_func_res(self): + decimal_type:DecimalType = self.query_col.type_ + return decimal_type.aggregator.last + def generate_res_type(self): + self.res_type_ = self.query_col.type_ + def execute_last(self, params): + pass + +class DecimalHyperloglogFunction(DecimalAggFunction): + pass + +class DecimalSampleFunction(DecimalAggFunction): + pass + +class DecimalTailFunction(DecimalAggFunction): + pass + +class DecimalUniqueFunction(DecimalAggFunction): + pass + +class DecimalModeFunction(DecimalAggFunction): + pass + +class DecimalCountFunction(DecimalAggFunction): + def __init__(self): + super().__init__("count({0})", DecimalCountFunction.execute_count, "count") + def get_func_res(self): + decimal_type: DecimalType = self.query_col.type_ + return decimal_type.aggregator.count - decimal_type.aggregator.null_num + def generate_res_type(self): + self.res_type_ = DataType(TypeEnum.BIGINT, 8, 0) + + def execute_count(self, params): + return 1 + +class DecimalMinFunction(DecimalAggFunction): + def __init__(self): + super().__init__("min({0})", DecimalMinFunction.execute_min, "min") + self.min_: Decimal = None + + def get_func_res(self) -> Decimal: + decimal_type: DecimalType = self.query_col.type_ + return decimal_type.aggregator.min + return self.min_ + + def generate_res_type(self) -> DataType: + self.res_type_ = self.query_col.type_ + + def execute_min(self, params): + if params[0] is None: + return + if self.min_ is None: + self.min_ = Decimal(params[0]) + else: + self.min_ = min(self.min_, Decimal(params[0])) + return self.min_ + +class DecimalMaxFunction(DecimalAggFunction): + def __init__(self): + super().__init__("max({0})", DecimalMaxFunction.execute_max, "max") + self.max_: Decimal = None + + def get_func_res(self) -> Decimal: + decimal_type: DecimalType = self.query_col.type_ + return decimal_type.aggregator.max + + def generate_res_type(self) -> DataType: + self.res_type_ = self.query_col.type_ + + def execute_max(self, params): + if params[0] is None: + return + if self.max_ is None: + self.max_ = Decimal(params[0]) + else: + self.max_ = max(self.max_, Decimal(params[0])) + return self.max_ + +class DecimalSumFunction(DecimalAggFunction): + def __init__(self): + super().__init__("sum({0})", DecimalSumFunction.execute_sum, "sum") + self.sum_:Decimal = None + def get_func_res(self) -> Decimal: + decimal_type: DecimalType = self.query_col.type_ + return decimal_type.aggregator.sum + return self.sum_ + def generate_res_type(self) -> DataType: + self.res_type_ = self.query_col.type_ + self.res_type_.set_prec(DecimalType.DECIMAL_MAX_PRECISION) + def execute_sum(self, params): + if params[0] is None: + return + if self.sum_ is None: + self.sum_ = Decimal(params[0]) + else: + self.sum_ += Decimal(params[0]) + return self.sum_ + +class DecimalAvgFunction(DecimalAggFunction): + def __init__(self): + super().__init__("avg({0})", DecimalAvgFunction.execute_avg, "avg") + self.count_: Decimal = 0 + self.sum_: Decimal = None + def get_func_res(self) -> Decimal: + decimal_type: DecimalType = self.query_col.type_ + return get_decimal( + decimal_type.aggregator.sum + / (decimal_type.aggregator.count - decimal_type.aggregator.null_num), + self.res_type_.scale(), + ) + def generate_res_type(self) -> DataType: + sum_type = self.query_col.type_ + sum_type.set_prec(DecimalType.DECIMAL_MAX_PRECISION) + count_type = DataType(TypeEnum.BIGINT, 8, 0) + self.res_type_ = DecimalBinaryOperator.calc_decimal_prec_scale(sum_type, count_type, "/") + def execute_avg(self, params): + if params[0] is None: + return + if self.sum_ is None: + self.sum_ = Decimal(params[0]) + else: + self.sum_ += Decimal(params[0]) + self.count_ += 1 + return self.get_func_res() + +class DecimalBinaryOperator(DecimalColumnExpr): + def __init__(self, format, executor, op: str): + super().__init__(format, executor) + self.op_ = op + self.left_type_: DataType = None + self.right_type_: DataType = None + + def __str__(self): + return super().__str__() + + def generate(self, format_params) -> str: + return super().generate(format_params) + + def should_skip_for_decimal(self, cols: list): + left_col = cols[0] + right_col = cols[1] + if not left_col.type_.is_decimal_type() and not right_col.type_.is_decimal_type(): + return True + if self.op_ != "%": + return False + ## TODO wjm why skip decimal % float/double? it's wrong now. + left_is_real = left_col.type_.is_real_type() or left_col.type_.is_varchar_type() + right_is_real = right_col.type_.is_real_type() or right_col.type_.is_varchar_type() + if left_is_real or right_is_real: + return True + return False + + @staticmethod + def check_null(params): + if params[0] is None or params[1] is None: + return True + else: + return False + + @staticmethod + def is_compare_op(op: str)-> bool: + return op in ["==", "!=", ">", "<", ">=", "<="] + + @staticmethod + def calc_decimal_prec_scale(left: DataType, right: DataType, op: str) -> DecimalType: + left_prec = 0 + left_scale = 0 + right_prec = 0 + right_scale = 0 + if not left.is_decimal_type(): + left_prec = TypeEnum.get_type_prec(left.type) + else: + left_prec = left.prec() + left_scale = left.scale() + if not right.is_decimal_type(): + right_prec = TypeEnum.get_type_prec(right.type) + else: + right_prec = right.prec() + right_scale = right.scale() + + out_prec = 0 + out_scale = 0 + if op in ['+', '-']: + out_scale = max(left_scale, right_scale) + out_prec = max(left_prec - left_scale, right_prec - right_scale) + out_scale + 1 + elif op == '*': + out_scale = left_scale + right_scale + out_prec = left_prec + right_prec + 1 + elif op == '/': + out_scale = max(left_scale + right_prec + 1, 6) + out_prec = left_prec - left_scale + right_scale + out_scale + elif op == '%': + out_scale = max(left_scale, right_scale) + out_prec = min(left_prec - left_scale, right_prec - right_scale) + out_scale + else: + raise Exception(f"unknown op for binary operators: {op}") + + if out_prec > 38: + min_scale = min(6, out_scale) + delta = out_prec - 38 + out_prec = 38 + out_scale = max(min_scale, out_scale - delta) + return DecimalType(TypeEnum.DECIMAL, out_prec, out_scale) + + def generate_res_type(self): + left_type = self.params_[0].type_ + right_type = self.params_[1].type_ + self.left_type_ = left_type + self.right_type_ = right_type + if DecimalBinaryOperator.is_compare_op(self.op_): + self.res_type_ = DataType(TypeEnum.BOOL) + else: + ret_double_types = [TypeEnum.VARCHAR, TypeEnum.BINARY, TypeEnum.DOUBLE, TypeEnum.FLOAT, TypeEnum.NCHAR] + if left_type.type in ret_double_types or right_type.type in ret_double_types: + self.res_type_ = DataType(TypeEnum.DOUBLE) + else: + self.res_type_ = DecimalBinaryOperator.calc_decimal_prec_scale(left_type, right_type, self.op_) + + def get_input_types(self)-> list: + return [self.left_type_, self.right_type_] + + def get_convert_type(self, params): + ret_float = False + if isinstance(params[0], float) or isinstance(params[1], float): + ret_float = True + left = params[0] + right = params[1] + if isinstance(params[0], str): + ret_float = True + if isinstance(params[1], str): + ret_float = True + return (left, right), ret_float + + def execute_plus(self, params): + if DecimalBinaryOperator.check_null(params): + return 'NULL' + (left, right), ret_float = self.get_convert_type(params) + if self.res_type_.type == TypeEnum.DOUBLE: + return float(left) + float(right) + else: + return self.convert_to_res_type(Decimal(left) + Decimal(right)) + + def execute_minus(self, params): + if DecimalBinaryOperator.check_null(params): + return 'NULL' + (left, right), ret_float = self.get_convert_type(params) + if self.res_type_.type == TypeEnum.DOUBLE: + return float(left) - float(right) + else: + return self.convert_to_res_type(Decimal(left) - Decimal(right)) + + def execute_mul(self, params): + if DecimalBinaryOperator.check_null(params): + return 'NULL' + (left, right), ret_float = self.get_convert_type(params) + if self.res_type_.type == TypeEnum.DOUBLE: + return float(left) * float(right) + else: + return self.convert_to_res_type(Decimal(left) * Decimal(right)) + + def execute_div(self, params): + if DecimalBinaryOperator.check_null(params): + return 'NULL' + (left, right), _ = self.get_convert_type(params) + if self.res_type_.type == TypeEnum.DOUBLE: + if right == 0: + return 'NULL' + return float(left) / float(right) + else: + return self.convert_to_res_type(Decimal(left) / Decimal(right)) + + def execute_mod(self, params): + if DecimalBinaryOperator.check_null(params): + return 'NULL' + (left, right), _ = self.get_convert_type(params) + if self.res_type_.type == TypeEnum.DOUBLE: + return self.convert_to_res_type(Decimal(left) % Decimal(right)) + else: + return self.convert_to_res_type(Decimal(left) % Decimal(right)) + + def execute_eq(self, params): + if DecimalBinaryOperator.check_null(params): + return False + (left, right), convert_float = self.get_convert_type(params) + if convert_float: + return float(left) == float(right) + else: + return Decimal(left) == Decimal(right) + + def execute_eq_filtering(self, params): + if self.execute_eq(params): + return True + + def execute_ne(self, params): + if DecimalBinaryOperator.check_null(params): + return False + (left, right), convert_float = self.get_convert_type(params) + if convert_float: + return float(left) != float(right) + else: + return Decimal(left) != Decimal(right) + + def execute_gt(self, params): + if DecimalBinaryOperator.check_null(params): + return False + (left, right), convert_float = self.get_convert_type(params) + if convert_float: + return float(left) > float(right) + else: + return Decimal(left) > Decimal(right) + + def execute_lt(self, params): + if DecimalBinaryOperator.check_null(params): + return False + (left, right), convert_float = self.get_convert_type(params) + if convert_float: + return float(left) < float(right) + else: + return Decimal(left) < Decimal(right) + + def execute_ge(self, params): + if DecimalBinaryOperator.check_null(params): + return False + (left, right), convert_float = self.get_convert_type(params) + if convert_float: + return float(left) >= float(right) + else: + return Decimal(left) >= Decimal(right) + + def execute_le(self, params): + if DecimalBinaryOperator.check_null(params): + return False + (left, right), convert_float = self.get_convert_type(params) + if convert_float: + return float(left) <= float(right) + else: + return Decimal(left) <= Decimal(right) + + @staticmethod + def get_all_binary_ops() -> List[DecimalColumnExpr]: + return [ + DecimalBinaryOperator(" {0} + {1} ", DecimalBinaryOperator.execute_plus, "+"), + DecimalBinaryOperator(" {0} - {1} ", DecimalBinaryOperator.execute_minus, "-"), + DecimalBinaryOperator(" {0} * {1} ", DecimalBinaryOperator.execute_mul, "*"), + DecimalBinaryOperator(" {0} / {1} ", DecimalBinaryOperator.execute_div, "/"), + DecimalBinaryOperator(" {0} % {1} ", DecimalBinaryOperator.execute_mod, "%"), + DecimalBinaryOperator(" {0} == {1} ", DecimalBinaryOperator.execute_eq, "=="), + DecimalBinaryOperator(" {0} != {1} ", DecimalBinaryOperator.execute_ne, "!="), + DecimalBinaryOperator(" {0} > {1} ", DecimalBinaryOperator.execute_gt, ">"), + DecimalBinaryOperator(" {0} < {1} ", DecimalBinaryOperator.execute_lt, "<"), + DecimalBinaryOperator(" {0} >= {1} ", DecimalBinaryOperator.execute_ge, ">="), + DecimalBinaryOperator(" {0} <= {1} ", DecimalBinaryOperator.execute_le, "<="), + ] + + @staticmethod + def get_all_filtering_binary_compare_ops() -> List[DecimalColumnExpr]: + return [ + DecimalBinaryOperator(" {0} == {1} ", DecimalBinaryOperator.execute_eq, "=="), + DecimalBinaryOperator(" {0} != {1} ", DecimalBinaryOperator.execute_ne, "!="), + DecimalBinaryOperator(" {0} > {1} ", DecimalBinaryOperator.execute_gt, ">"), + DecimalBinaryOperator(" {0} < {1} ", DecimalBinaryOperator.execute_lt, "<"), + DecimalBinaryOperator(" {0} >= {1} ", DecimalBinaryOperator.execute_ge, ">="), + DecimalBinaryOperator(" {0} <= {1} ", DecimalBinaryOperator.execute_le, "<="), + ] + + def execute(self, params): + return super().execute(params) + +class DecimalUnaryOperator(DecimalColumnExpr): + def __init__(self, format, executor, op: str): + super().__init__(format, executor) + self.op_ = op + self.col_type_: DataType = None + + def should_skip_for_decimal(self, cols: list): + col:Column = cols[0] + if not col.type_.is_decimal_type(): + return True + return False + + @staticmethod + def get_all_unary_ops() -> List[DecimalColumnExpr]: + return [ + DecimalUnaryOperator(" -{0} ", DecimalUnaryOperator.execute_minus, "-"), + ] + + def get_input_types(self)-> list: + return [self.col_type_] + + def generate_res_type(self): + self.res_type_ = self.col_type_ = self.params_[0].type_ + + def execute_minus(self, params) -> Decimal: + if params[0] is None: + return 'NULL' + return -Decimal(params[0]) + +class DecimalBinaryOperatorIn(DecimalBinaryOperator): + def __init__(self, op: str): + super().__init__(op) + + def execute(self, left, right): + if self.op_.lower()() == "in": + return left in right + if self.op_.lower() == "not in": + return left not in right + + +class TDTestCase: + updatecfgDict = { + "asynclog": 0, + "ttlUnit": 1, + "ttlPushInterval": 5, + "ratioOfVnodeStreamThrea": 4, + "debugFlag": 143, + } + + def __init__(self): + self.vgroups = 4 + self.ctbNum = 10 + self.rowsPerTbl = 10000 + self.duraion = "1h" + self.norm_tb_columns = [] + self.tags = [] + self.stable_name = "meters" + self.norm_table_name = "nt" + self.col_prefix = "c" + self.c_table_prefix = "t" + self.tag_name_prefix = "t" + self.db_name = "test" + self.c_table_num = 10 + self.no_decimal_col_tb_name = "tt" + self.stb_columns = [] + self.stream_name = "stream1" + self.stream_out_stb = "stream_out_stb" + self.tsma_name = "tsma1" + self.query_test_round = 10000 + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def check_desc_for_one_ctb( + self, ctbPrefix: str, columns: List[Column], tags: List[Column] = [] + ): + ctb_idx = randrange(self.c_table_num) + return self.check_desc(f"{ctbPrefix}{ctb_idx}", columns, tags) + + def check_desc(self, tbname: str, columns: List[Column], tags: List[Column] = []): + sql = f"desc {self.db_name}.{tbname}" + tdSql.query(sql, queryTimes=1) + results = tdSql.queryResult + for i, col in enumerate(columns): + if col.type_.type == TypeEnum.DECIMAL: + if results[i + 1][1] != col.type_.__str__(): + tdLog.info(str(results)) + tdLog.exit( + f"check desc failed for table: {tbname} column {results[i+1][0]} type is {results[i+1][1]}, expect {col.type_}" + ) + if results[i + 1][4] != DecimalType.default_encode(): + tdLog.exit( + f"check desc failed for table: {tbname} column {results[i+1][0]} encode is {results[i+1][5]}, expect {DecimalType.default_encode()}" + ) + if results[i + 1][5] != DecimalType.default_compression(): + tdLog.exit( + f"check desc failed for table: {tbname} column {results[i+1][0]} compression is {results[i+1][4]}, expect {DecimalType.default_compression()}" + ) + if tbname == self.stable_name: + self.check_desc_for_one_ctb(self.c_table_prefix, columns, tags) + + def check_show_create_table( + self, tbname: str, cols: List[Column], tags: List[Column] = [] + ): + sql = f"show create table {self.db_name}.{tbname}" + tdSql.query(sql, queryTimes=1) + create_table_sql = tdSql.queryResult[0][1] + decimal_idx = 0 + results = re.findall(r"DECIMAL\((\d+),(\d+)\)", create_table_sql) + for i, col in enumerate(cols): + if ( + col.type_.type == TypeEnum.DECIMAL + or col.type_.type == TypeEnum.DECIMAL64 + ): + result_type = DecimalType( + col.type_.type, + int(results[decimal_idx][0]), + int(results[decimal_idx][1]), + ) + if result_type != col.type_: + tdLog.exit( + f"check show create table failed for: {tbname} column {i} type is {result_type}, expect {col.type}" + ) + decimal_idx += 1 + + def test_add_drop_columns_with_decimal(self, tbname: str, columns: List[Column]): + is_stb = tbname == self.stable_name + ## alter table add column + create_c99_sql = ( + f"alter table {self.db_name}.{tbname} add column c99 decimal(37, 19)" + ) + columns.append(Column(DecimalType(TypeEnum.DECIMAL, 37, 19))) + tdSql.execute(create_c99_sql, queryTimes=1, show=True) + self.check_desc(tbname, columns) + ## alter table add column with compression + create_c100_sql = f'ALTER TABLE {self.db_name}.{tbname} ADD COLUMN c100 decimal(36, 18) COMPRESS "zstd"' + tdSql.execute(create_c100_sql, queryTimes=1, show=True) + columns.append(Column(DecimalType(TypeEnum.DECIMAL, 36, 18))) + self.check_desc(tbname, columns) + + ## drop non decimal column + drop_c6_sql = f"alter table {self.db_name}.{tbname} drop column c6" + tdSql.execute(drop_c6_sql, queryTimes=1, show=True) + c6 = columns.pop(5) + self.check_desc(tbname, columns) + ## drop decimal column and not last column + drop_c99_sql = f"alter table {self.db_name}.{tbname} drop column c99" + tdSql.execute(drop_c99_sql, queryTimes=1, show=True) + c99 = columns.pop(len(columns) - 2) + self.check_desc(tbname, columns) + ## drop decimal column and last column + drop_c100_sql = f"alter table {self.db_name}.{tbname} drop column c100" + tdSql.execute(drop_c100_sql, queryTimes=1, show=True) + c100 = columns.pop(len(columns) - 1) + self.check_desc(tbname, columns) + + ## create decimal back + tdSql.execute(create_c99_sql, queryTimes=1, show=True) + tdSql.execute(create_c100_sql, queryTimes=1, show=True) + columns.append(c99) + columns.append(c100) + self.check_desc(tbname, columns) + + def test_decimal_column_ddl(self): + ## create decimal type table, normal/super table, decimal64/decimal128 + tdLog.printNoPrefix("-------- test create decimal column") + self.norm_tb_columns: List[Column] = [ + Column(DecimalType(TypeEnum.DECIMAL, 10, 2)), + Column(DecimalType(TypeEnum.DECIMAL, 20, 4)), + Column(DecimalType(TypeEnum.DECIMAL, 30, 8)), + Column(DecimalType(TypeEnum.DECIMAL, 38, 10)), + Column(DataType(TypeEnum.TINYINT)), + Column(DataType(TypeEnum.INT)), + Column(DataType(TypeEnum.BIGINT)), + Column(DataType(TypeEnum.DOUBLE)), + Column(DataType(TypeEnum.FLOAT)), + Column(DataType(TypeEnum.VARCHAR, 255)), + ] + self.tags: List[Column] = [ + Column(DataType(TypeEnum.INT)), + Column(DataType(TypeEnum.VARCHAR, 255)), + ] + self.stb_columns: List[Column] = [ + Column(DecimalType(TypeEnum.DECIMAL, 10, 2)), + Column(DecimalType(TypeEnum.DECIMAL, 20, 4)), + Column(DecimalType(TypeEnum.DECIMAL, 30, 8)), + Column(DecimalType(TypeEnum.DECIMAL, 38, 10)), + Column(DataType(TypeEnum.TINYINT)), + Column(DataType(TypeEnum.INT)), + Column(DataType(TypeEnum.BIGINT)), + Column(DataType(TypeEnum.DOUBLE)), + Column(DataType(TypeEnum.FLOAT)), + Column(DataType(TypeEnum.VARCHAR, 255)), + ] + DecimalColumnTableCreater( + tdSql, + self.db_name, + self.stable_name, + self.stb_columns, + self.tags, + col_prefix=self.col_prefix, + tag_prefix=self.tag_name_prefix, + ).create() + self.check_show_create_table("meters", self.stb_columns, self.tags) + + DecimalColumnTableCreater( + tdSql, self.db_name, self.norm_table_name, self.norm_tb_columns + ).create() + self.check_desc(self.norm_table_name, self.norm_tb_columns) + self.check_show_create_table(self.norm_table_name, self.norm_tb_columns) + + ## TODO add more values for all rows + tag_values = ["1", "t1"] + DecimalColumnTableCreater( + tdSql, self.db_name, self.stable_name, self.stb_columns + ).create_child_table( + self.c_table_prefix, self.c_table_num, self.tags, tag_values + ) + self.check_desc("meters", self.stb_columns, self.tags) + self.check_desc("t1", self.stb_columns, self.tags) + + ## invalid precision/scale + invalid_precision_scale = [ + ("decimal(-1, 2)", syntax_error), + ("decimal(39, 2)", invalid_column), + ("decimal(10, -1)", syntax_error), + ("decimal(10, 39)", invalid_column), + ("decimal(10, 2.5)", syntax_error), + ("decimal(10.5, 2)", syntax_error), + ("decimal(10.5, 2.5)", syntax_error), + ("decimal(0, 2)", invalid_column), + ("decimal(0)", invalid_column), + ("decimal", syntax_error), + ("decimal()", syntax_error), + ] + for i in invalid_precision_scale: + sql = f"create table {self.db_name}.invalid_decimal_precision_scale (ts timestamp, c1 {i[0]})" + tdSql.error(sql, i[1]) + + ## can't create decimal tag + sql = ( + "create stable %s.invalid_decimal_tag (ts timestamp) tags (t1 decimal(10, 2))" + % (self.db_name) + ) + tdSql.error(sql, invalid_column) + + ## alter table add/drop column + self.test_add_drop_columns_with_decimal( + self.norm_table_name, self.norm_tb_columns + ) + self.test_add_drop_columns_with_decimal(self.stable_name, self.stb_columns) + + ## drop index from stb + ### These ops will override the previous stbobjs and meta entries, so test it + + ## TODO test encode and compress for decimal type + sql = f'ALTER TABLE {self.db_name}.{self.norm_table_name} ADD COLUMN c101 decimal(37, 19) ENCODE "simple8b" COMPRESS "zstd"' + tdSql.error(sql, invalid_encode_param) + sql = f'ALTER TABLE {self.db_name}.{self.norm_table_name} ADD COLUMN c101 decimal(37, 19) ENCODE "delta-i" COMPRESS "zstd"' + tdSql.error(sql, invalid_encode_param) + sql = f'ALTER TABLE {self.db_name}.{self.norm_table_name} ADD COLUMN c101 decimal(37, 19) ENCODE "delta-d" COMPRESS "zstd"' + tdSql.error(sql, invalid_encode_param) + sql = f'ALTER TABLE {self.db_name}.{self.norm_table_name} ADD COLUMN c101 decimal(37, 19) ENCODE "bit-packing" COMPRESS "zstd"' + tdSql.error(sql, invalid_encode_param) + + def test_insert_decimal_values(self): + self.log_test("start to insert decimal values") + for i in range(self.c_table_num): + TableInserter( + tdSql, + self.db_name, + f"{self.c_table_prefix}{i}", + self.stb_columns, + self.tags, + ).insert(tb_insert_rows, 1537146000000, 500) + + for i in range(self.c_table_num): + TableDataValidator( + self.stb_columns, self.c_table_prefix + str(i), self.db_name, i + ).validate() + + TableInserter( + tdSql, self.db_name, self.norm_table_name, self.norm_tb_columns + ).insert(tb_insert_rows, 1537146000000, 500, flush_database=True) + TableDataValidator( + self.norm_tb_columns, self.norm_table_name, self.db_name + ).validate() + + tdSql.execute("flush database %s" % (self.db_name), queryTimes=1) + for i in range(self.c_table_num): + TableDataValidator( + self.stb_columns, self.c_table_prefix + str(i), self.db_name, i + ).validate() + TableDataValidator( + self.norm_tb_columns, self.norm_table_name, self.db_name + ).validate() + + ## insert null/None for decimal type + + ## insert with column format + + def no_decimal_table_test(self): + columns = [ + Column(DataType(TypeEnum.TINYINT)), + Column(DataType(TypeEnum.INT)), + Column(DataType(TypeEnum.BIGINT)), + Column(DataType(TypeEnum.DOUBLE)), + Column(DataType(TypeEnum.FLOAT)), + Column(DataType(TypeEnum.VARCHAR, 255)), + ] + DecimalColumnTableCreater( + tdSql, self.db_name, self.no_decimal_col_tb_name, columns, [] + ).create() + TableInserter(tdSql, self.db_name, self.no_decimal_col_tb_name, columns).insert( + 10000, 1537146000000, 500, flush_database=True + ) + ## TODO wjm test non support decimal version upgrade to decimal support version, and add decimal column + + ## Test metaentry compatibility problem for decimal type + ## How to test it? + ## Create table with no decimal type, the metaentries should not have extschma, and add decimal column, the metaentries should have extschema for all columns. + sql = f"ALTER TABLE {self.db_name}.{self.no_decimal_col_tb_name} ADD COLUMN c200 decimal(37, 19)" + tdSql.execute(sql, queryTimes=1) ## now meta entry has ext schemas + columns.append(Column(DecimalType(TypeEnum.DECIMAL, 37, 19))) + self.check_desc(self.no_decimal_col_tb_name, columns) + + ## After drop this only decimal column, the metaentries should not have extschema for all columns. + sql = ( + f"ALTER TABLE {self.db_name}.{self.no_decimal_col_tb_name} DROP COLUMN c200" + ) + tdSql.execute(sql, queryTimes=1) ## now meta entry has no ext schemas + columns.pop(len(columns) - 1) + self.check_desc(self.no_decimal_col_tb_name, columns) + sql = f"ALTER TABLE {self.db_name}.{self.no_decimal_col_tb_name} ADD COLUMN c200 int" + tdSql.execute(sql, queryTimes=1) ## meta entry has no ext schemas + columns.append(Column(DataType(TypeEnum.INT))) + self.check_desc(self.no_decimal_col_tb_name, columns) + + self.test_add_drop_columns_with_decimal(self.no_decimal_col_tb_name, columns) + + def test_decimal_ddl(self): + self.log_test("test_decimal_ddl") + tdSql.execute("create database test cachemodel 'both'", queryTimes=1) + self.test_decimal_column_ddl() + + def test_decimal_and_stream(self): + self.log_test("test_decimal_and_stream") + create_stream = f"CREATE STREAM {self.stream_name} FILL_HISTORY 1 INTO {self.db_name}.{self.stream_out_stb} AS SELECT _wstart, count(c1), avg(c2), sum(c3) FROM {self.db_name}.{self.stable_name} INTERVAL(10s)" + tdSql.execute(create_stream, queryTimes=1, show=True) + self.wait_query_result( + f"select count(*) from {self.db_name}.{self.stream_out_stb}", [(50,)], 30 + ) + ## test combine functions + create_stream = "CREATE STREAM stream2 trigger at_once watermark 100s INTO test.stream_out_stb_2 AS SELECT _wstart, count(*), min(c2), max(c2), last(c1), last(c3), avg(c2), sum(c3), min(c1), max(c1), avg(c1) FROM test.nt session(ts, 5s)" + tdSql.execute(create_stream, queryTimes=1, show=True) + cols_vals = [] + ts = datetime.now() + rows = [] + row = [] + for col in self.norm_tb_columns: + v = col.generate_value(self.norm_table_name) + cols_vals.append(v) + row.append(v) + rows.append(row) + row = [] + sql = f"insert into test.nt values('{ts}', {' ,'.join(cols_vals)})" + tdSql.execute(sql, queryTimes=1, show=True) + ts = ts + timedelta(seconds=6) + cols_vals = [] + for col in self.norm_tb_columns: + v = col.generate_value(self.norm_table_name) + cols_vals.append(v) + row.append(v) + rows.append(row) + row = [] + sql = f"insert into test.nt values('{ts}', {' ,'.join(cols_vals)})" + tdSql.execute(sql, queryTimes=1, show=True) + ts = ts - timedelta(seconds=4) + ## waiting for the last two windows been calculated + time.sleep(10) + cols_vals = [] + for col in self.norm_tb_columns: + v = col.generate_value(self.norm_table_name) + cols_vals.append(v) + row.append(v) + rows.append(row) + sql = f"insert into test.nt values('{ts}', {' ,'.join(cols_vals)})" + tdSql.execute(sql, queryTimes=1, show=True) + self.wait_query_result("select `count(*)` from test.stream_out_stb_2", [(3,)], 10) + res = TaosShell().query("select * from test.stream_out_stb_2") + if len(res) != 12: ## groupid + tdLog.exit(f"expect 12 columns but got: {len(res)}") + c1 = self.norm_tb_columns[0] + c2 = self.norm_tb_columns[1] + c3 = self.norm_tb_columns[2] + c1_vals = [get_decimal(v, c1.type_.scale()) for v in [row[0] for row in rows]] + c2_vals = [get_decimal(v, c2.type_.scale()) for v in [row[1] for row in rows]] + c3_vals = [get_decimal(v, c3.type_.scale()) for v in [row[2] for row in rows]] + min_c2 = Decimal(res[2][0]) + if min_c2 != min(c2_vals): + tdLog.exit(f"expect min(c2) = {min(c2_vals)} got: {min_c2}") + max_c2 = Decimal(res[3][0]) + if max_c2 != max(c2_vals): + tdLog.exit(f"expect max(c2) = {max(c2_vals)} got: {max_c2}") + + + def test_decimal_and_tsma(self): + self.log_test("test_decimal_and_tsma") + create_tsma = f"CREATE TSMA {self.tsma_name} ON {self.db_name}.{self.stable_name} FUNCTION(count(c1), min(c2), max(c3), avg(C3)) INTERVAL(1m)" + tdSql.execute(create_tsma, queryTimes=1, show=True) + self.wait_query_result( + f"select count(*) from {self.db_name}.{self.tsma_name}_tsma_res_stb_", + [(9 * self.c_table_num,)], + 30, + ) + + def test_decimal_and_view(self): + self.log_test("test_decimal_and_view") + c1 = self.norm_tb_columns[0] + create_view_sql = f'create view {self.db_name}.view1 as select {c1} as c1, cast({c1} as decimal(38, 10)) as c2 from {self.db_name}.{self.norm_table_name}' + tdSql.execute(create_view_sql) + res = TaosShell().query(f'select c1 from {self.db_name}.view1') + if len(res[0]) != c1.get_cardinality(self.norm_table_name): + tdLog.exit(f"query from view1 got rows: {len(res)} expect: {c1.get_cardinality(self.norm_table_name)}") + for i in range(len(res[0])): + v_query = res[0][i] + v_insert = c1.get_val_for_execute(self.norm_table_name, i) + if (v_insert is None and v_query == 'NULL') or Decimal(v_query) == v_insert: + continue + else: + tdLog.exit(f"query from view got different results: {v_query}, expect: {v_insert}") + #self.check_desc("view1", [c1, Column(DecimalType(TypeEnum.DECIMAL, 38, 10))]) + + def log_test(self, name: str): + tdLog.info(f"{datetime.now()} start to test {name}") + + def run(self): + self.test_decimal_ddl() + #self.no_decimal_table_test() + self.test_insert_decimal_values() + self.test_query_decimal() + self.test_decimal_and_tsma() + self.test_decimal_and_view() + self.test_decimal_and_stream() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + def wait_query_result(self, sql: str, expect_result, times): + for i in range(times): + tdLog.info(f"wait query result for {sql}, times: {i}") + tdSql.query(sql, queryTimes=1) + results = tdSql.queryResult + if results != expect_result: + time.sleep(1) + continue + return True + tdLog.exit( + f"wait query result timeout for {sql} failed after {times} time, expect {expect_result}, but got {results}" + ) + + def wait_query_at_least_rows(self, sql: str, rows, wait_times): + for i in range(wait_times): + tdLog.info(f"wait query rows at least for {sql}, times: {i}") + tdSql.query(sql, queryTimes=1, show=True) + results = tdSql.queryResult + if len(results) < rows: + time.sleep(1) + continue + return True + tdLog.exit( + f"wait query rows at least for {sql} failed after {wait_times} times, expect at least {rows} rows, but got {len(results)} rows" + ) + + def check_decimal_binary_expr_with_col_results( + self, dbname, tbname, tb_cols: List[Column], exprs: List[DecimalColumnExpr] + ): + if not binary_op_with_col_test: + return + for expr in exprs: + for col in tb_cols: + if col.name_ == '': + continue + for col2 in tb_cols: + if col2.name_ =='': + continue + if expr.should_skip_for_decimal([col, col2]): + continue + select_expr = expr.generate((col, col2)) + sql = f"select {select_expr} from {dbname}.{tbname}" + res = TaosShell().query(sql) + if len(res) > 0: + expr.check(res[0], tbname) + else: + tdLog.info(f"sql: {sql} got no output") + + def check_decimal_binary_expr_with_const_col_results_for_one_expr( + self, + dbname, + tbname, + tb_cols: List[Column], + expr: DecimalColumnExpr, + get_constant_cols_func, + ): + constant_cols = get_constant_cols_func() + for col in tb_cols: + if col.name_ == '': + continue + left_is_decimal = col.type_.is_decimal_type() + for const_col in constant_cols: + right_is_decimal = const_col.type_.is_decimal_type() + if expr.should_skip_for_decimal([col, const_col]): + continue + const_col.generate_value() + select_expr = expr.generate((const_col, col)) + sql = f"select {select_expr} from {dbname}.{tbname}" + shell = TaosShell() + res = shell.query(sql) + if len(res) > 0: + expr.check(res[0], tbname) + select_expr = expr.generate((col, const_col)) + sql = f"select {select_expr} from {dbname}.{tbname}" + res = shell.query(sql) + if len(res) > 0: + expr.check(res[0], tbname) + else: + tdLog.info(f"sql: {sql} got no output") + + def check_decimal_binary_expr_with_const_col_results( + self, + dbname, + tbname, + tb_cols: List[Column], + get_constant_cols_func, + get_exprs_func, + ): + exprs: List[DecimalColumnExpr] = get_exprs_func() + if not binary_op_with_const_test: + return + ts: list[threading.Thread] = [] + for expr in exprs: + t = self.run_in_thread2( + self.check_decimal_binary_expr_with_const_col_results_for_one_expr, + (dbname, tbname, tb_cols, expr, get_constant_cols_func), + ) + ts.append(t) + for t in ts: + t.join() + + def check_decimal_unary_expr_results(self, dbname, tbname, tb_cols: List[Column], exprs: List[DecimalColumnExpr]): + if not unary_op_test: + return + for expr in exprs: + for col in tb_cols: + if col.name_ == '': + continue + if expr.should_skip_for_decimal([col]): + continue + select_expr = expr.generate([col]) + sql = f"select {select_expr} from {dbname}.{tbname}" + res = TaosShell().query(sql) + if len(res) > 0: + expr.check(res[0], tbname) + else: + tdLog.info(f"sql: {sql} got no output") + + def run_in_thread(self, times, func, params) -> threading.Thread: + threads: List[threading.Thread] = [] + for i in range(times): + t = threading.Thread(target=func, args=params) + t.start() + threads.append(t) + for t in threads: + t.join() + + def run_in_thread2(self, func, params) -> threading.Thread: + t = threading.Thread(target=func, args=params) + t.start() + return t + + ## test others unsupported types operator with decimal + def test_decimal_unsupported_types(self): + unsupported_type = [ + TypeEnum.JSON, + TypeEnum.GEOMETRY, + TypeEnum.VARBINARY, + ] + all_type_columns: List[Column] = Column.get_all_type_columns([TypeEnum.JSON]) + tbname = "test_decimal_unsupported_types" + tag_cols = [Column(DataType(TypeEnum.JSON))] + tb_creater = DecimalColumnTableCreater( tdSql, self.db_name, tbname, all_type_columns, tag_cols) + tb_creater.create() + tb_creater.create_child_table(tbname, 10, tag_cols, ['{"k1": "v1"}']) + for i in range(10): + TableInserter(tdSql, self.db_name, f'{tbname}{i}', all_type_columns, tag_cols).insert(100, 1537146000000, 500, flush_database=True) + for col in all_type_columns: + ## only test decimal cols + if not col.type_.is_decimal_type(): + continue + for unsupported_col in all_type_columns: + ## only test unsupported cols + if not unsupported_col.type_.type in unsupported_type: + continue + for binary_op in DecimalBinaryOperator.get_all_binary_ops(): + select_expr = binary_op.generate((col, unsupported_col)) + sql = f"select {select_expr} from {self.db_name}.{tbname}" + select_expr_reverse = binary_op.generate((unsupported_col, col)) + sql_reverse = ( + f"select {select_expr_reverse} from {self.db_name}.{tbname}" + ) + tdLog.info( + f"select expr: {select_expr} with type: {col.type_} and {unsupported_col.type_} should err" + ) + err = tdSql.error(sql) + if tdSql.errno != invalid_operation and tdSql.errno != scalar_convert_err: + tdLog.exit(f"expected err not occured for sql: {sql}, expect: {invalid_operation} or {scalar_convert_err}, but got {tdSql.errno}") + + tdLog.info( + f"select expr: {select_expr} with type: {unsupported_col.type_} and {col.type_} should err" + ) + err = tdSql.error(sql_reverse) + if tdSql.errno != invalid_operation and tdSql.errno != scalar_convert_err: + tdLog.exit(f"expected err not occured for sql: {sql}, expect: {invalid_operation} or {scalar_convert_err}, but got {tdSql.errno}") + + def test_decimal_operators(self): + self.log_test("start to test decimal operators") + self.test_decimal_unsupported_types() + ## tables: meters, nt + ## columns: c1, c2, c3, c4, c5, c7, c8, c9, c10, c99, c100 + binary_operators = DecimalBinaryOperator.get_all_binary_ops() + + ## decimal operator with constants of all other types + self.run_in_thread( + operator_test_round, + self.check_decimal_binary_expr_with_const_col_results, + ( + self.db_name, + self.norm_table_name, + self.norm_tb_columns, + Column.get_decimal_oper_const_cols, + DecimalBinaryOperator.get_all_binary_ops, + ), + ) + + ## test decimal column op decimal column + for i in range(operator_test_round): + self.check_decimal_binary_expr_with_col_results( + self.db_name, self.norm_table_name, self.norm_tb_columns, binary_operators) + + unary_operators = DecimalUnaryOperator.get_all_unary_ops() + self.check_decimal_unary_expr_results( + self.db_name, + self.norm_table_name, + self.norm_tb_columns, + unary_operators,) + + + def test_decimal_last_first_func(self): + pass + + def check_decimal_where_with_binary_expr_with_const_col_results( + self, + dbname, + tbname, + tb_cols: List[Column], + constant_cols: List[Column], + exprs: List[DecimalColumnExpr], + ): + if not binary_op_in_where_test: + return + for expr in exprs: + tdLog.info(f"start to test decimal where filtering with const cols for expr: {expr.format_}") + for col in tb_cols: + if col.name_ == '': + continue + for const_col in constant_cols: + if expr.should_skip_for_decimal([col, const_col]): + continue + const_col.generate_value() + select_expr = expr.generate((const_col, col)) + if const_col.type_.is_decimal_type(): + expr.query_col = col + else: + expr.query_col = col + sql = f"select {expr.query_col} from {dbname}.{tbname} where {select_expr}" + res = TaosShell().query(sql) + ##TODO wjm no need to check len(res) for filtering test, cause we need to check for every row in the table to check if the filtering is working + if len(res) > 0: + expr.check_for_filtering(res[0], tbname) + select_expr = expr.generate((col, const_col)) + sql = f"select {expr.query_col} from {dbname}.{tbname} where {select_expr}" + res = TaosShell().query(sql) + if len(res) > 0: + expr.check_for_filtering(res[0], tbname) + else: + tdLog.info(f"sql: {sql} got no output") + + def check_decimal_where_with_binary_expr_with_col_results( + self, dbname, tbname, tb_cols: List[Column], exprs: List[DecimalColumnExpr] + ): + if not binary_op_in_where_test: + return + for expr in exprs: + tdLog.info(f"start to test decimal where filtering with cols for expr{expr.format_}") + for col in tb_cols: + if col.name_ == '': + continue + for col2 in tb_cols: + if col2.name_ == '': + continue + if expr.should_skip_for_decimal([col, col2]): + continue + select_expr = expr.generate((col, col2)) + if col.type_.is_decimal_type(): + expr.query_col = col + else: + expr.query_col = col2 + sql = f"select {expr.query_col} from {dbname}.{tbname} where {select_expr}" + res = TaosShell().query(sql) + if len(res) > 0: + expr.check_for_filtering(res[0], tbname) + else: + tdLog.info(f"sql: {sql} got no output") + select_expr = expr.generate((col2, col)) + sql = f"select {expr.query_col} from {dbname}.{tbname} where {select_expr}" + res = TaosShell().query(sql) + if len(res) > 0: + expr.check_for_filtering(res[0], tbname) + else: + tdLog.info(f"sql: {sql} got no output") + + def test_query_decimal_where_clause(self): + tdLog.info("start to test decimal where filtering") + binary_compare_ops = DecimalBinaryOperator.get_all_filtering_binary_compare_ops() + const_cols = Column.get_decimal_oper_const_cols() + for i in range(operator_test_round): + self.check_decimal_where_with_binary_expr_with_const_col_results( + self.db_name, + self.norm_table_name, + self.norm_tb_columns, + const_cols, + binary_compare_ops, + ) + + for i in range(operator_test_round): + self.check_decimal_where_with_binary_expr_with_col_results( + self.db_name, + self.norm_table_name, + self.norm_tb_columns, + binary_compare_ops) + + ## TODO wjm + ## 3. (dec op const col) op const col + ## 4. (dec op dec) op const col + ## 5. (dec op const col) op dec + ## 6. (dec op dec) op dec + + def test_query_with_order_by_for_tb(self, tbname: str, cols: list): + for col in cols: + if col.type_.is_decimal_type() and col.name_ != '': + self.test_query_with_order_by(col, tbname) + + def test_query_with_order_by(self, order_col: Column, tbname): + sql = f"select {order_col} from {self.db_name}.{tbname} order by {order_col} asc" + query_res = TaosShell().query(sql)[0] + calculated_ordered_res = order_col.get_ordered_result(tbname, True) + for v_from_query, v_from_calc in zip(query_res, calculated_ordered_res): + if v_from_calc is None: + if v_from_query != 'NULL': + tdLog.exit(f"query result: {v_from_query} not equal to calculated result: NULL") + elif Decimal(v_from_query) != v_from_calc: + tdLog.exit(f"query result: {v_from_query} not equal to calculated result: {v_from_calc}") + + + def test_query_decimal_order_clause(self): + self.log_test("start to test decimal order by") + self.test_query_with_order_by_for_tb(self.norm_table_name, self.norm_tb_columns) + self.test_query_with_order_by_for_tb(self.stable_name, self.stb_columns) + + def test_query_decimal_group_by_decimal(self, tbname: str, cols: list): + for col in cols: + if col.type_.is_decimal_type() and col.name_ != '': + sql = f"select count(*) from {self.db_name}.{tbname} group by {col}" + query_res = TaosShell().query(sql)[0] + calculated_grouped_res = col.get_group_num(tbname) + if len(query_res) != calculated_grouped_res: + tdLog.exit(f"query result: {len(query_res)} not equal to calculated result: {calculated_grouped_res}") + + def test_query_decimal_group_by_clause(self): + self.log_test("start to test decimal group by") + self.test_query_decimal_group_by_decimal(self.norm_table_name, self.norm_tb_columns) + self.test_query_decimal_group_by_decimal(self.stable_name, self.stb_columns) + + def test_query_decimal_group_by_with_having(self, tbname, cols: list): + for col in cols: + if col.type_.is_decimal_type() and col.name_ != '': + sql = f"select count(*) from {self.db_name}.{tbname} group by {col} having {col} is not null" + query_res = TaosShell().query(sql)[0] + calculated_grouped_res = col.get_group_num(tbname, ignore_null=True) + if len(query_res) != calculated_grouped_res: + tdLog.exit(f"query result: {len(query_res)} not equal to calculated result: {calculated_grouped_res}") + + def test_query_decimal_having_clause(self): + self.log_test("start to test decimal having") + self.test_query_decimal_group_by_with_having(self.norm_table_name, self.norm_tb_columns) + self.test_query_decimal_group_by_with_having(self.stable_name, self.stb_columns) + + def test_query_decimal_interval_fill(self): + pass + + def test_query_decimal_partition_by(self): + pass + + def test_query_decimal_case_when(self): + self.log_test("start to test decimal case when") + sql = "select case when cast(1 as decimal(10, 4)) >= 1 then cast(88888888.88 as decimal(10,2)) else cast(3.333 as decimal(10,3)) end" + res = TaosShell().query(sql)[0] + if res[0] != "88888888.88": + tdLog.exit(f"query result for sql: {sql}: {res[0]} not equal to expected result: 88888888.88") + sql = "select case when cast(1 as decimal(10, 4)) > 1 then cast(88888888.88 as decimal(10,2)) else cast(3.333 as decimal(10,3)) end" + res = TaosShell().query(sql)[0] + if res[0] != "3.33": + tdLog.exit(f"query result for sql: {sql}: {res[0]} not equal to expected result: 3.33") + + sql = "select case when cast(1 as decimal(10, 4)) > 1 then cast(88888888.88 as decimal(10,2)) else 1.23 end" + res = TaosShell().query(sql)[0] + if float(res[0]) != float(1.23): + tdLog.exit(f"query result for sql: {sql}: {res[0]} not equal to expected result: 1.23") + + sql = "select case when cast(1 as decimal(10, 4)) >= 1 then cast(88888888.88 as decimal(10,2)) else 1.23 end" + res = TaosShell().query(sql)[0] + if float(res[0]) != float(88888888.88): + tdLog.exit(f"query result for sql: {sql}: {res[0]} not equal to expected result: 88888888.88") + + sql = "select case when cast(1 as decimal(10, 4)) >= 1 then cast(88888888.88 as decimal(10,2)) else '1.23' end" + res = TaosShell().query(sql)[0] + if float(res[0]) != 88888888.88: + tdLog.exit(f"query result for sql: {sql}: {res[0]} not equal to expected result: 88888888.88") + + sql = "select case when cast(1 as decimal(10, 4)) > 1 then cast(88888888.88 as decimal(10,2)) else '1.23' end" + res = TaosShell().query(sql)[0] + if float(res[0]) != 1.23: + tdLog.exit(f"query result for sql: {sql}: {res[0]} not equal to expected result: 88888888.88") + + sql = "select case when cast(1 as decimal(10, 4)) > 1 then cast(88888888.88 as decimal(10,2)) else 'abcd' end" + res = TaosShell().query(sql)[0] + if float(res[0]) != 0: + tdLog.exit(f"query result for sql: {sql}: {res[0]} not equal to expected result: 0") + + def test_decimal_agg_funcs(self, dbname, tbname, tb_cols: List[Column], get_agg_funcs_func): + agg_funcs: List[DecimalFunction] = get_agg_funcs_func() + for func in agg_funcs: + for col in tb_cols: + if col.name_ == '' or func.should_skip_for_decimal([col]): + continue + func.query_col = col + select_expr = func.generate([col]) + sql = f"select {select_expr} from {dbname}.{tbname}" + res = TaosShell().query(sql) + if len(res) > 0: + res = res[0] + func.check_for_agg_func(res, tbname, func) + + def test_decimal_cast_func(self, dbname, tbname, tb_cols: List[Column]): + for col in tb_cols: + if col.name_ == '': + continue + to_types: list[DataType] = DecimalCastTypeGenerator(col.type_).generate(cast_func_test_round) + for t in to_types: + cast_func = DecimalCastFunction() + expr = cast_func.generate([col, t]) + sql = f"select {expr} from {dbname}.{tbname}" + res = TaosShell().query(sql) + if len(res) > 0: + res = res[0] + cast_func.check(res, tbname) + + def test_decimal_functions(self): + self.log_test("start to test decimal functions") + if not test_decimal_funcs: + return + self.test_decimal_agg_funcs( + self.db_name, + self.norm_table_name, + self.norm_tb_columns, + DecimalFunction.get_decimal_agg_funcs, + ) + ##self.test_decimal_agg_funcs( self.db_name, self.stable_name, self.stb_columns, DecimalFunction.get_decimal_agg_funcs) + self.test_decimal_cast_func(self.db_name, self.norm_table_name, self.norm_tb_columns) + + def test_query_decimal(self): + self.log_test("start to test decimal query") + if not decimal_test_query: + return + #self.test_decimal_operators() + self.test_query_decimal_where_clause() + self.test_decimal_functions() + self.test_query_decimal_order_clause() + self.test_query_decimal_case_when() + self.test_query_decimal_group_by_clause() + self.test_query_decimal_having_clause() + + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/raw_block_interface_test.py b/tests/system-test/7-tmq/raw_block_interface_test.py index 1c9798421d..141c31326a 100644 --- a/tests/system-test/7-tmq/raw_block_interface_test.py +++ b/tests/system-test/7-tmq/raw_block_interface_test.py @@ -22,11 +22,12 @@ class TDTestCase: def checkData(self): tdSql.execute('use db_raw') - tdSql.query("select * from d1") + tdSql.query("select ts, current, voltage, phase from d1") tdSql.checkRows(1) tdSql.checkData(0, 1, 120) + #tdSql.checkData(0, 4, 2.32) ## currently py conector does not support decimal - tdSql.query("select * from d2") + tdSql.query("select ts, current, voltage, phase from d2") tdSql.checkRows(1) tdSql.checkData(0, 1, None) diff --git a/tests/system-test/7-tmq/tmq_c_test.py b/tests/system-test/7-tmq/tmq_c_test.py index 0552a700dd..c04af3e3e7 100644 --- a/tests/system-test/7-tmq/tmq_c_test.py +++ b/tests/system-test/7-tmq/tmq_c_test.py @@ -23,6 +23,13 @@ class TDTestCase: tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file + def checkData(self): + tdSql.execute('use db_taosx') + tdSql.query("select * from ct0") + tdSql.checkRows(2) + tdSql.checkData(0, 4, 23.23) + + return def run(self): buildPath = tdCom.getBuildPath() cmdStr = '%s/build/bin/tmq_write_raw_test'%(buildPath) @@ -41,6 +48,7 @@ class TDTestCase: tdLog.info(cmdStr) os.system(cmdStr) + self.checkData() return def stop(self): diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 93a478dda8..10c3d351d4 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -473,6 +473,9 @@ void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, i shellFormatTimestamp(buf, sizeof(buf), *(int64_t *)val, precision); taosFprintfFile(pFile, "%s%s%s", quotationStr, buf, quotationStr); break; + case TSDB_DATA_TYPE_DECIMAL64: + case TSDB_DATA_TYPE_DECIMAL: + taosFprintfFile(pFile, "%s", val); default: break; } @@ -712,6 +715,9 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t shellFormatTimestamp(buf, sizeof(buf), taosGetInt64Aligned((int64_t *)val), precision); printf("%s", buf); break; + case TSDB_DATA_TYPE_DECIMAL: + case TSDB_DATA_TYPE_DECIMAL64: + printf("%*s", width, val); default: break; } @@ -899,7 +905,10 @@ int32_t shellCalcColWidth(TAOS_FIELD *field, int32_t precision) { } else { return TMAX(23, width); // '2020-01-01 00:00:00.000' } - + case TSDB_DATA_TYPE_DECIMAL64: + return TMAX(width, 20); + case TSDB_DATA_TYPE_DECIMAL: + return TMAX(width, 40); default: ASSERT(false); } diff --git a/utils/test/c/tmq_write_raw_test.c b/utils/test/c/tmq_write_raw_test.c index e8d4c6abae..721d65e499 100644 --- a/utils/test/c/tmq_write_raw_test.c +++ b/utils/test/c/tmq_write_raw_test.c @@ -38,10 +38,46 @@ static TAOS* use_db() { return pConn; } +static void checkData() { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT(pConn != NULL); + TAOS_RES* pRes = taos_query(pConn, "select * from db_taosx.ct0"); + ASSERT (taos_errno(pRes) == 0); + TAOS_ROW row = NULL; + while ((row = taos_fetch_row(pRes))) { + int numFields = taos_num_fields(pRes); + TAOS_FIELD* fields = taos_fetch_fields(pRes); + + for (int i = 0; i < numFields; ++i) { + if (IS_DECIMAL_TYPE(fields[i].type)) { + ASSERT(strcmp(row[i], "23.230") == 0 || strcmp(row[i], "43.530") == 0); + } + } + } + taos_free_result(pRes); + taos_close(pConn); +} + static void msg_process(TAOS_RES* msg) { printf("-----------topic-------------: %s\n", tmq_get_topic_name(msg)); printf("db: %s\n", tmq_get_db_name(msg)); printf("vg: %d\n", tmq_get_vgroup_id(msg)); + if (strcmp(tmq_get_db_name(msg), "db_query") == 0){ + TAOS_ROW row = NULL; + while ((row = taos_fetch_row(msg))) { + + int numFields = taos_num_fields(msg); + TAOS_FIELD *fields = taos_fetch_fields(msg); + + for (int i = 0; i < numFields; ++i) { + if (IS_DECIMAL_TYPE(fields[i].type)) { + ASSERT(strcmp(row[i], "3.122") == 0); + } + } + } + return; + } + TAOS* pConn = use_db(); if (tmq_get_res_type(msg) == TMQ_RES_TABLE_META || tmq_get_res_type(msg) == TMQ_RES_METADATA) { char* result = tmq_get_json_meta(msg); @@ -62,7 +98,7 @@ static void msg_process(TAOS_RES* msg) { int buildDatabase(TAOS* pConn, TAOS_RES* pRes) { pRes = taos_query(pConn, - "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " + "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16), c4 decimal(10,3)) tags(t1 int, t3 " "nchar(8), t4 bool)"); if (taos_errno(pRes) != 0) { printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); @@ -77,7 +113,7 @@ int buildDatabase(TAOS* pConn, TAOS_RES* pRes) { } taos_free_result(pRes); - pRes = taos_query(pConn, "insert into ct0 using st1 tags(1000, \"ttt\", true) values(1626006833400, 1, 2, 'a')"); + pRes = taos_query(pConn, "insert into ct0 using st1 tags(1000, \"ttt\", true) values(1626006833400, 1, 2, 'a', 3.2)"); if (taos_errno(pRes) != 0) { printf("failed to insert into ct0, reason:%s\n", taos_errstr(pRes)); return -1; @@ -98,7 +134,7 @@ int buildDatabase(TAOS* pConn, TAOS_RES* pRes) { } taos_free_result(pRes); - pRes = taos_query(pConn, "insert into ct1 using st1(t1) tags(2000) values(1626006833600, 3, 4, 'b')"); + pRes = taos_query(pConn, "insert into ct1 using st1(t1) tags(2000) values(1626006833600, 3, 4, 'b', 4.32)"); if (taos_errno(pRes) != 0) { printf("failed to insert into ct1, reason:%s\n", taos_errstr(pRes)); return -1; @@ -112,14 +148,14 @@ int buildDatabase(TAOS* pConn, TAOS_RES* pRes) { } taos_free_result(pRes); - pRes = taos_query(pConn, "insert into ct0 using st1 tags(1000, \"ttt\", true) values(1626006833400, 1, 2, 'a')"); + pRes = taos_query(pConn, "insert into ct0 using st1 tags(1000, \"ttt\", true) values(1626006833400, 1, 2, 'a', 23.23)"); if (taos_errno(pRes) != 0) { printf("failed to insert into ct0, reason:%s\n", taos_errstr(pRes)); return -1; } taos_free_result(pRes); - pRes = taos_query(pConn, "insert into ct1 using st1(t1) tags(2000) values(1626006833600, 3, 4, 'b')"); + pRes = taos_query(pConn, "insert into ct1 using st1(t1) tags(2000) values(1626006833600, 3, 4, 'b', 43.53)"); if (taos_errno(pRes) != 0) { printf("failed to insert into ct1, reason:%s\n", taos_errstr(pRes)); return -1; @@ -128,8 +164,8 @@ int buildDatabase(TAOS* pConn, TAOS_RES* pRes) { pRes = taos_query( pConn, - "insert into ct3 using st1(t1) tags(3000) values(1626006833600, 5, 6, 'c') ct1 using st1(t1) tags(2000) values(1626006833601, 2, 3, 'sds') (1626006833602, 4, 5, " - "'ddd') ct0 using st1 tags(1000, \"ttt\", true) values(1626006833603, 4, 3, 'hwj') ct1 using st1(t1) tags(2000) values(now+5s, 23, 32, 's21ds')"); + "insert into ct3 using st1(t1) tags(3000) values(1626006833600, 5, 6, 'c', 43.53) ct1 using st1(t1) tags(2000) values(1626006833601, 2, 3, 'sds', 43.53) (1626006833602, 4, 5, " + "'ddd', 43.53) ct0 using st1 tags(1000, \"ttt\", true) values(1626006833603, 4, 3, 'hwj', 43.53) ct1 using st1(t1) tags(2000) values(now+5s, 23, 32, 's21ds', 43.53)"); if (taos_errno(pRes) != 0) { printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); return -1; @@ -196,6 +232,43 @@ int32_t init_env() { buildDatabase(pConn, pRes); + pRes = taos_query(pConn, "drop topic if exists topic_query"); + if (taos_errno(pRes) != 0) { + printf("error in drop topic, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "drop database if exists db_query"); + if (taos_errno(pRes) != 0) { + printf("error in drop db_taosx, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create database if not exists db_query vgroups 1 wal_retention_period 3600"); + if (taos_errno(pRes) != 0) { + printf("error in create db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + // create and insert another stable + pRes = taos_query(pConn, + "create stable if not exists db_query.st11 (ts timestamp, c1 int, c2 float, c3 binary(16), c4 decimal(9,3)) tags(t1 int, t3 " + "nchar(8), t4 bool)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into db_query.ct10 using db_query.st11 tags(1000, \"ttt\", true) values(1626006833400, 1, 2, 'a', 3.122)"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct0, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); taos_close(pConn); return 0; } @@ -215,6 +288,13 @@ int32_t create_topic() { } taos_free_result(pRes); + pRes = taos_query(pConn, "create topic topic_query as select * from db_query.st11"); + if (taos_errno(pRes) != 0) { + printf("failed to create topic topic_query, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + taos_close(pConn); return 0; } @@ -254,6 +334,7 @@ tmq_t* build_consumer() { tmq_list_t* build_topic_list() { tmq_list_t* topic_list = tmq_list_new(); tmq_list_append(topic_list, "topic_db"); + tmq_list_append(topic_list, "topic_query"); return topic_list; } @@ -294,4 +375,5 @@ int main(int argc, char* argv[]) { tmq_list_t* topic_list = build_topic_list(); basic_consume_loop(tmq, topic_list); tmq_list_destroy(topic_list); + checkData(); } diff --git a/utils/test/c/write_raw_block_test.c b/utils/test/c/write_raw_block_test.c index ae4a606e6e..590e9143fc 100644 --- a/utils/test/c/write_raw_block_test.c +++ b/utils/test/c/write_raw_block_test.c @@ -62,12 +62,12 @@ void init_env() { action("use db_raw"); action( - "CREATE STABLE `meters` (`ts` TIMESTAMP, `current` INT, `voltage` INT, `phase` FLOAT) TAGS (`groupid` INT, " + "CREATE STABLE `meters` (`ts` TIMESTAMP, `current` INT, `voltage` INT, `phase` FLOAT, data decimal(4,2)) TAGS (`groupid` INT, " "`location` VARCHAR(16))"); action("create table d0 using meters tags(1, 'San Francisco')"); action("create table d1 using meters tags(2, 'San Francisco')"); action("create table d2 using meters tags(3, 'San Francisco')"); - action("insert into d0 (ts, current) values (now, 120)"); + action("insert into d0 (ts, current, data) values (now, 120, 2.32)"); action("create table ntba(ts timestamp, addr binary(32))"); action("create table ntbb(ts timestamp, addr binary(8))");