From eccbae4e527c61ebc24113eb6b054626ad0dc615 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Tue, 24 Dec 2024 17:52:52 +0800 Subject: [PATCH] support insert and query decimal type --- include/libs/decimal/decimal.h | 14 ++ include/libs/nodes/querynodes.h | 1 + include/util/tdef.h | 2 +- source/client/CMakeLists.txt | 4 +- source/client/inc/clientInt.h | 4 +- source/client/inc/clientStmt.h | 2 +- source/client/src/clientImpl.c | 45 +++++- source/client/src/clientMain.c | 2 +- source/client/src/clientStmt.c | 5 +- source/client/src/clientStmt2.c | 3 +- source/client/src/clientTmq.c | 3 +- source/dnode/vnode/src/meta/metaTable2.c | 2 +- source/libs/decimal/src/decimal.c | 179 ++++++++++++++++++++--- source/libs/nodes/src/nodesUtilFuncs.c | 1 + source/libs/parser/inc/parInt.h | 2 +- source/libs/parser/src/parInsertSql.c | 1 + source/libs/parser/src/parTranslater.c | 17 ++- source/libs/parser/src/parser.c | 2 +- tests/system-test/2-query/decimal.py | 11 +- tools/shell/src/shellEngine.c | 8 +- 20 files changed, 259 insertions(+), 49 deletions(-) diff --git a/include/libs/decimal/decimal.h b/include/libs/decimal/decimal.h index 0fbf39deb9..970c4ea66a 100644 --- a/include/libs/decimal/decimal.h +++ b/include/libs/decimal/decimal.h @@ -44,6 +44,8 @@ int32_t decimal128FromStr(const char* str, int32_t len, uint8_t* precision, uint int32_t decimal64ToDataVal(const Decimal64* dec, SValue* pVal); int32_t decimal128ToDataVal(const Decimal128* dec, SValue* pVal); +int32_t decimalToStr(DecimalWord* pDec, int8_t precision, int8_t scale, char* pBuf, int32_t bufLen); + typedef struct DecimalVar DecimalVar; typedef struct SDecimalVarOps { @@ -52,12 +54,24 @@ typedef struct SDecimalVarOps { int32_t (*multiply)(DecimalVar* pLeft, const DecimalVar* pRight); } SDecimalVarOps; +typedef struct SWideInteger { + int32_t wordNum; + DecimalWord *words; +} SWideInteger; + +// TODO wjm rename it typedef struct SWideIntegerOps { uint8_t wordNum; + int32_t (*abs)(DecimalWord* pInt); int32_t (*add)(DecimalWord* pLeft, DecimalWord* pRight, uint8_t rightWordNum); int32_t (*subtract)(DecimalWord* pLeft, DecimalWord* pRight, uint8_t rightWordNum); int32_t (*multiply)(DecimalWord* pLeft, DecimalWord* pRight, uint8_t rightWordNum); int32_t (*divide)(DecimalWord* pLeft, DecimalWord* pRight, uint8_t rightWordNum); + int32_t (*mod)(DecimalWord* pLeft, DecimalWord* pRight, uint8_t rightWordNum); + bool (*lt)(DecimalWord* pLeft, DecimalWord* pRight, uint8_t rightWordNum); + bool (*gt)(DecimalWord* pLeft, DecimalWord* pRight, uint8_t rightWordNum); + bool (*eq)(DecimalWord* pLeft, DecimalWord* pRight, uint8_t rightWordNum); + int32_t (*toStr)(DecimalWord* pInt, char* pBuf, int32_t bufLen); } SWideIntegerOps; #ifdef __cplusplus diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 7191b4ef4a..404ade04d9 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -647,6 +647,7 @@ typedef struct SQuery { SArray* pPlaceholderValues; SNode* pPrepareRoot; bool stableQuery; + SExtSchema* pResExtSchema; } SQuery; void nodesWalkSelectStmtImpl(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext); diff --git a/include/util/tdef.h b/include/util/tdef.h index 6c386a1fc4..1145285a24 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -693,7 +693,7 @@ typedef enum { #define TSDB_DECIMAL_MIN_SCALE 0 #define TSDB_DECIMAL_MAX_SCALE TSDB_DECIMAL_MAX_PRECISION -typedef uint64_t DecimalWord; +typedef int64_t DecimalWord; #ifdef __cplusplus } diff --git a/source/client/CMakeLists.txt b/source/client/CMakeLists.txt index 2113aa7921..cff60667c3 100644 --- a/source/client/CMakeLists.txt +++ b/source/client/CMakeLists.txt @@ -24,7 +24,7 @@ target_include_directories( target_link_libraries( ${TAOS_LIB} INTERFACE api - PRIVATE os util common transport monitor nodes parser command planner catalog scheduler function qcom geometry + PRIVATE os util common transport monitor nodes parser command planner catalog scheduler function qcom geometry decimal ) if(TD_WINDOWS) @@ -63,7 +63,7 @@ target_include_directories( target_link_libraries( ${TAOS_LIB_STATIC} INTERFACE api - PRIVATE os util common transport monitor nodes parser command planner catalog scheduler function qcom geometry + PRIVATE os util common transport monitor nodes parser command planner catalog scheduler function qcom geometry decimal ) if(${BUILD_TEST}) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index ed37b45bd7..4128c64566 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -205,7 +205,7 @@ typedef struct SReqResultInfo { SExecResult execRes; const char* pRspMsg; const char* pData; - TAOS_FIELD* fields; // todo, column names are not needed. + TAOS_FIELD_E* fields; // todo, column names are not needed. TAOS_FIELD* userFields; // the fields info that return to user uint32_t numOfCols; int32_t* length; @@ -316,7 +316,7 @@ void doSetOneRowPtr(SReqResultInfo* pResultInfo); void setResPrecision(SReqResultInfo* pResInfo, int32_t precision); int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4); int32_t setResultDataPtr(SReqResultInfo* pResultInfo, bool convertUcs4); -int32_t setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols); +int32_t setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols, const SExtSchema* pExtSchema); void doFreeReqResultInfo(SReqResultInfo* pResInfo); int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName, SArray** pReq); void syncCatalogFn(SMetaData* pResult, void* param, int32_t code); diff --git a/source/client/inc/clientStmt.h b/source/client/inc/clientStmt.h index ec61b2ff57..5800e415e2 100644 --- a/source/client/inc/clientStmt.h +++ b/source/client/inc/clientStmt.h @@ -48,7 +48,7 @@ typedef struct SStmtTableCache { } SStmtTableCache; typedef struct SStmtQueryResInfo { - TAOS_FIELD *fields; + TAOS_FIELD_E *fields; TAOS_FIELD *userFields; uint32_t numOfCols; int32_t precision; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 6c0ddbb672..43d2b5bc5f 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -28,6 +28,8 @@ #include "tref.h" #include "tsched.h" #include "tversion.h" +#include "decimal.h" + static int32_t initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet); static int32_t buildConnectMsg(SRequestObj* pRequest, SMsgSendInfo** pMsgSendInfo); @@ -313,7 +315,7 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtC code = qParseSql(&cxt, pQuery); if (TSDB_CODE_SUCCESS == code) { if ((*pQuery)->haveResultSet) { - code = setResSchemaInfo(&pRequest->body.resInfo, (*pQuery)->pResSchema, (*pQuery)->numOfResCols); + code = setResSchemaInfo(&pRequest->body.resInfo, (*pQuery)->pResSchema, (*pQuery)->numOfResCols, (*pQuery)->pResExtSchema); setResPrecision(&pRequest->body.resInfo, (*pQuery)->precision); } } @@ -515,7 +517,7 @@ int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArra return qCreateQueryPlan(&cxt, pPlan, pNodeList); } -int32_t setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols) { +int32_t setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols, const SExtSchema* pExtSchema) { if (pResInfo == NULL || pSchema == NULL || numOfCols <= 0) { tscError("invalid paras, pResInfo == NULL || pSchema == NULL || numOfCols <= 0"); return TSDB_CODE_INVALID_PARA; @@ -528,7 +530,7 @@ int32_t setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32 if (pResInfo->userFields != NULL) { taosMemoryFree(pResInfo->userFields); } - pResInfo->fields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD)); + pResInfo->fields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD_E)); if (NULL == pResInfo->fields) return terrno; pResInfo->userFields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD)); if (NULL == pResInfo->userFields) { @@ -552,6 +554,8 @@ int32_t setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32 pResInfo->userFields[i].bytes -= VARSTR_HEADER_SIZE; } else if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR || pSchema[i].type == TSDB_DATA_TYPE_JSON) { pResInfo->userFields[i].bytes = (pResInfo->userFields[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + } else if (IS_DECIMAL_TYPE(pSchema[i].type) && pExtSchema) { + decimalFromTypeMod(pExtSchema[i].typeMod, &pResInfo->fields[i].precision, &pResInfo->fields[i].scale); } tstrncpy(pResInfo->fields[i].name, pSchema[i].name, tListLen(pResInfo->fields[i].name)); @@ -2134,6 +2138,37 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t* colLength) { return TSDB_CODE_SUCCESS; } +static int32_t convertDecimalType(SReqResultInfo* pResultInfo) { + for (int32_t i = 0; i < pResultInfo->numOfCols; ++i) { + TAOS_FIELD_E* pField = pResultInfo->fields + i; + int32_t type = pField->type; + int32_t bufLen = 0; + char* p = NULL; + if (!IS_DECIMAL_TYPE(type) || !pResultInfo->pCol[i].pData) { + continue; + } else { + bufLen = 64; + p = taosMemoryRealloc(pResultInfo->convertBuf[i], bufLen * pResultInfo->numOfRows); + pField->bytes = bufLen; + } + if (!p) return terrno; + pResultInfo->convertBuf[i] = p; + + for (int32_t j = 0; j < pResultInfo->numOfRows; ++j) { + int32_t code = decimalToStr((DecimalWord*)(pResultInfo->pCol[i].pData + j * tDataTypes[type].bytes), pField->precision, pField->scale, p, bufLen); + p += bufLen; + if (TSDB_CODE_SUCCESS != code) { + return code; + } + } + // TODO wjm handle NULL??? + // TODO wjm use vardatalen??? + pResultInfo->pCol[i].pData = pResultInfo->convertBuf[i]; + pResultInfo->row[i] = pResultInfo->pCol[i].pData; + } + return 0; +} + int32_t getVersion1BlockMetaSize(const char* p, int32_t numOfCols) { return sizeof(int32_t) + sizeof(int32_t) + sizeof(int32_t) * 3 + sizeof(uint64_t) + numOfCols * (sizeof(int8_t) + sizeof(int32_t)); @@ -2472,7 +2507,9 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, bool convertUcs4) { if (convertUcs4) { code = doConvertUCS4(pResultInfo, colLength); } - + if (TSDB_CODE_SUCCESS == code) { + code = convertDecimalType(pResultInfo); + } return code; } diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 82c509cf0a..a9659a0dce 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -1260,7 +1260,7 @@ void handleQueryAnslyseRes(SSqlCallbackWrapper *pWrapper, SMetaData *pResultMeta } if (pQuery->haveResultSet) { - code = setResSchemaInfo(&pRequest->body.resInfo, pQuery->pResSchema, pQuery->numOfResCols); + code = setResSchemaInfo(&pRequest->body.resInfo, pQuery->pResSchema, pQuery->numOfResCols, pQuery->pResExtSchema); setResPrecision(&pRequest->body.resInfo, pQuery->precision); } } diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 14815b009a..5b277e6877 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -213,7 +213,7 @@ int32_t stmtBackupQueryFields(STscStmt* pStmt) { int32_t stmtRestoreQueryFields(STscStmt* pStmt) { SStmtQueryResInfo* pRes = &pStmt->sql.queryRes; - int32_t size = pRes->numOfCols * sizeof(TAOS_FIELD); + int32_t size = pRes->numOfCols * sizeof(TAOS_FIELD_E); pStmt->exec.pRequest->body.resInfo.numOfCols = pRes->numOfCols; pStmt->exec.pRequest->body.resInfo.precision = pRes->precision; @@ -1270,8 +1270,9 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) { if (pStmt->sql.pQuery->haveResultSet) { STMT_ERR_RET(setResSchemaInfo(&pStmt->exec.pRequest->body.resInfo, pStmt->sql.pQuery->pResSchema, - pStmt->sql.pQuery->numOfResCols)); + pStmt->sql.pQuery->numOfResCols, pStmt->sql.pQuery->pResExtSchema)); taosMemoryFreeClear(pStmt->sql.pQuery->pResSchema); + taosMemoryFreeClear(pStmt->sql.pQuery->pResExtSchema); setResPrecision(&pStmt->exec.pRequest->body.resInfo, pStmt->sql.pQuery->precision); } diff --git a/source/client/src/clientStmt2.c b/source/client/src/clientStmt2.c index e47f9dd2d8..a697ecfd84 100644 --- a/source/client/src/clientStmt2.c +++ b/source/client/src/clientStmt2.c @@ -1385,8 +1385,9 @@ int stmtBindBatch2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* bind, int32_t colIdx) { if (pStmt->sql.pQuery->haveResultSet) { STMT_ERR_RET(setResSchemaInfo(&pStmt->exec.pRequest->body.resInfo, pStmt->sql.pQuery->pResSchema, - pStmt->sql.pQuery->numOfResCols)); + pStmt->sql.pQuery->numOfResCols, pStmt->sql.pQuery->pResExtSchema)); taosMemoryFreeClear(pStmt->sql.pQuery->pResSchema); + taosMemoryFreeClear(pStmt->sql.pQuery->pResExtSchema); setResPrecision(&pStmt->exec.pRequest->body.resInfo, pStmt->sql.pQuery->precision); } diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index d19bd9e009..86b111abd4 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -3012,7 +3012,8 @@ int32_t tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4, SReqResultInfo** pRes doFreeReqResultInfo(&pRspObj->resInfo); SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(data->blockSchema, pRspObj->resIter); if (pSW) { - TAOS_CHECK_RETURN(setResSchemaInfo(&pRspObj->resInfo, pSW->pSchema, pSW->nCols)); + // TODO wjm tmq support ext schema + TAOS_CHECK_RETURN(setResSchemaInfo(&pRspObj->resInfo, pSW->pSchema, pSW->nCols, NULL)); } } diff --git a/source/dnode/vnode/src/meta/metaTable2.c b/source/dnode/vnode/src/meta/metaTable2.c index 7e0743d3d5..a4701ec2a8 100644 --- a/source/dnode/vnode/src/meta/metaTable2.c +++ b/source/dnode/vnode/src/meta/metaTable2.c @@ -1824,4 +1824,4 @@ int32_t metaDropMultipleTables(SMeta *pMeta, int64_t version, SArray *uidArray) } } return code; -} \ No newline at end of file +} diff --git a/source/libs/decimal/src/decimal.c b/source/libs/decimal/src/decimal.c index 234ab70d1b..2d8b5b4f01 100644 --- a/source/libs/decimal/src/decimal.c +++ b/source/libs/decimal/src/decimal.c @@ -22,8 +22,25 @@ typedef enum DecimalInternalType { DECIMAL_128 = 1, } DecimalInternalType; +#define DECIMAL_GET_INTERNAL_TYPE(precision) ((precision) > TSDB_DECIMAL64_MAX_SCALE ? DECIMAL_128 : DECIMAL_64) +#define DECIMAL_GET_WORD_NUM(decimalInternalType) ((decimalInternalType) == DECIMAL_64 ? DECIMAL_WORD_NUM(Decimal64) : DECIMAL_WORD_NUM(Decimal128)) +#define DecimalMax Decimal128 + +static int32_t int64Abs(DecimalWord* pInt); +static int32_t int64Add(DecimalWord* pLeft, DecimalWord* pRight, uint8_t rightWordNum); +static int32_t int64Subtract(DecimalWord* pLeft, DecimalWord* pRight, uint8_t rightWordNum); +static int32_t int64Multiply(DecimalWord* pLeft, DecimalWord* pRight, uint8_t rightWordNum); +static int32_t int64divide(DecimalWord* pLeft, DecimalWord* pRight, uint8_t rightWordNum); +static int32_t int64Mod(DecimalWord* pLeft, DecimalWord* pRight, uint8_t rightWordNum); +static bool int64Lt(DecimalWord* pLeft, DecimalWord* pRight, uint8_t rightWordNum); +static bool int64Gt(DecimalWord* pLeft, DecimalWord* pRight, uint8_t rightWordNum); +static bool int64Eq(DecimalWord* pLeft, DecimalWord* pRight, uint8_t rightWordNum); +static int32_t int64ToStr(DecimalWord* pInt, char* pBuf, int32_t bufLen); + +typedef __int128_t Int128; + SWideIntegerOps wideIntegerOps [2] = { - {DECIMAL_WORD_NUM(Decimal64), 0, 0, 0}, + {DECIMAL_WORD_NUM(Decimal64), int64Abs, int64Add, int64Subtract, int64Multiply, int64divide, int64Mod, int64Lt, int64Gt, int64Eq, int64ToStr}, {DECIMAL_WORD_NUM(Decimal128), 0, 0, 0}}; SDecimalVarOps decimalVarOps[2] = { @@ -40,7 +57,7 @@ struct DecimalVar { DecimalWord* words; }; -uint8_t maxPrecision(DecimalInternalType type) { +static uint8_t maxPrecision(DecimalInternalType type) { switch (type) { case DECIMAL_64: return TSDB_DECIMAL64_MAX_PRECISION; @@ -65,18 +82,20 @@ void decimalFromTypeMod(STypeMod typeMod, uint8_t* precision, uint8_t* scale) { *scale = (uint8_t)(typeMod & 0xFF); } -#define DECIMAL_INTERNAL_TYPE(TYPE) DECIMAL_WORD_NUM(TYPE) - 1 - int32_t decimal64FromStr(const char* str, int32_t len, uint8_t* precision, uint8_t* scale, Decimal64* result) { int32_t code = 0; - DecimalVar var = {.type = DECIMAL_INTERNAL_TYPE(Decimal64), .words = result->words}; + DecimalVar var = {.type = DECIMAL_64, .words = result->words}; code = decimalVarFromStr(str, len, &var); + *precision = var.precision; + *scale = var.scale; return code; } int32_t decimal128FromStr(const char* str, int32_t len, uint8_t* precision, uint8_t* scale, Decimal128* result) { int32_t code = 0; - DecimalVar var = {.type = DECIMAL_INTERNAL_TYPE(Decimal128), .words = result->words}; + DecimalVar var = {.type = DECIMAL_128, .words = result->words}; + *precision = var.precision; + *scale = var.scale; return code; } @@ -86,6 +105,7 @@ static int32_t decimalVarFromStr(const char* str, int32_t len, DecimalVar* resul result->scale = 0; bool leadingZeroes = true, afterPoint = false; uint32_t places = 0; + result->sign = 1; if (len == 0) return TSDB_CODE_INVALID_DATA_FMT; SWideIntegerOps ops = @@ -97,7 +117,6 @@ static int32_t decimalVarFromStr(const char* str, int32_t len, DecimalVar* resul case '-': result->sign = -1; case '+': - result->sign = 1; pos++; default: break; @@ -151,22 +170,11 @@ static int32_t decimalVarFromStr(const char* str, int32_t len, DecimalVar* resul default: break; } - ++pos; } - DecimalWord sign = result->sign; - ops.multiply(result->words, &sign, 1); - return code; -} - -int32_t varMultiply(DecimalVar* pLeft, const DecimalVar* pRight) { - int32_t code = 0; - - return code; -} - -int32_t varAdd(DecimalVar* pLeft, const DecimalVar* pRight) { - int32_t code = 0; - + if (result->sign < 0) { + DecimalWord sign = (DecimalWord)result->sign; + ops.multiply(result->words, &sign, 1); + } return code; } @@ -181,3 +189,130 @@ int32_t decimal128ToDataVal(const Decimal128* dec, SValue* pVal) { valueSetDatum(pVal, TSDB_DATA_TYPE_DECIMAL, dec->words, DECIMAL_WORD_NUM(Decimal128) * sizeof(DecimalWord)); return TSDB_CODE_SUCCESS; } + +static int32_t int64Abs(DecimalWord* pInt) { + *pInt = TABS(*pInt); + return 0; +} + +static int32_t int64Add(DecimalWord* pLeft, DecimalWord* pRight, uint8_t rightWordNum) { + int32_t code = 0; + Int128 sum = *pLeft; + sum += *pRight; + *pLeft = (DecimalWord)sum; + return code; +} + +static int32_t int64Subtract(DecimalWord* pLeft, DecimalWord* pRight, uint8_t rightWordNum) { + int32_t code = 0; + Int128 res = *pLeft; + res -= *pRight; + *pLeft = (DecimalWord)res; + return code; +} + +static int32_t int64Multiply(DecimalWord* pLeft, DecimalWord* pRight, uint8_t rightWordNum) { + Int128 res = *pLeft; + res *= *pRight; + *pLeft = (DecimalWord)res; + return 0; +} + +static int32_t int64divide(DecimalWord* pLeft, DecimalWord* pRight, uint8_t rightWordNum) { + Int128 res = *pLeft; + res /= *pRight; + *pLeft = (DecimalWord)res; + return 0; +} + +static int32_t int64Mod(DecimalWord* pLeft, DecimalWord* pRight, uint8_t rightWordNum) { + Int128 res = *pLeft; + res %= *pRight; + *pLeft = (DecimalWord)res; + return 0; +} + +static bool int64Lt(DecimalWord* pLeft, DecimalWord* pRight, uint8_t rightWordNum) { + return *pLeft < *pRight; +} +static bool int64Gt(DecimalWord* pLeft, DecimalWord* pRight, uint8_t rightWordNum) { + return *pLeft > *pRight; +} +static bool int64Eq(DecimalWord* pLeft, DecimalWord* pRight, uint8_t rightWordNum) { + return *pLeft == *pRight; +} +static int32_t int64ToStr(DecimalWord* pInt, char *pBuf, int32_t bufLen) { + return snprintf(pBuf, bufLen, "%"PRId64, *pInt); +} + +static int64_t SCALE_MULTIPLIER_64[19] = {1LL, + 10LL, + 100LL, + 1000LL, + 10000LL, + 100000LL, + 1000000LL, + 10000000LL, + 100000000LL, + 1000000000LL, + 10000000000LL, + 100000000000LL, + 1000000000000LL, + 10000000000000LL, + 100000000000000LL, + 1000000000000000LL, + 10000000000000000LL, + 100000000000000000LL, + 1000000000000000000LL}; + +static int32_t decimalGetWhole(const DecimalWord* pDec, DecimalInternalType type, int8_t scale, DecimalWord* pWhole) { + SWideIntegerOps ops = wideIntegerOps[type]; + if (type == DECIMAL_64) { + pWhole[0] = *pDec; + DecimalWord scaleMul = SCALE_MULTIPLIER_64[scale]; + int32_t code = ops.divide(pWhole, &scaleMul, 1); + if (TSDB_CODE_SUCCESS != 0) { + // TODO wjm + } + ops.abs(pWhole); + } else { + memcpy(pWhole, pDec, DECIMAL_GET_WORD_NUM(type) * sizeof(DecimalWord)); + // TODO wjm + //ops.divide(pWhole->words, ) + } + return 0; +} + +static int32_t decimalGetFrac(const DecimalWord* pDec, DecimalInternalType type, int8_t scale, DecimalWord* pFrac) { + SWideIntegerOps ops = wideIntegerOps[type]; + if (type == DECIMAL_64) { + pFrac[0] = *pDec; + DecimalWord scaleMul = SCALE_MULTIPLIER_64[scale]; + int32_t code = ops.mod(pFrac, &scaleMul, 1); + ops.abs(pFrac); + } else { + + } + return 0; +} + +int32_t decimalToStr(DecimalWord* pDec, int8_t precision, int8_t scale, char* pBuf, int32_t bufLen) { + DecimalInternalType iType = DECIMAL_GET_INTERNAL_TYPE(precision); + int32_t wordNum = DECIMAL_GET_WORD_NUM(iType); + SWideIntegerOps ops = wideIntegerOps[iType]; + DecimalMax whole = {0}, frac = {0}; + DecimalWord zero = 0; + int32_t pos = 0; + + if (ops.lt(pDec, &zero, 1)) { + pos = sprintf(pBuf, "-"); + } + int32_t code = decimalGetWhole(pDec, iType, scale, whole.words); + if (!ops.eq(whole.words, &zero, 1)) { + pos += ops.toStr(whole.words, pBuf + pos, bufLen - pos); + } + pos += snprintf(pBuf + pos, bufLen - pos, "."); + code = decimalGetFrac(pDec, iType, scale, frac.words); + ops.toStr(frac.words, pBuf + pos, bufLen - pos); + return 0; +} diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 9b7d9e53bf..59f7813d63 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -1640,6 +1640,7 @@ void nodesDestroyNode(SNode* pNode) { taosMemoryFreeClear(pQuery->pCmdMsg->pMsg); taosMemoryFreeClear(pQuery->pCmdMsg); } + taosMemoryFreeClear(pQuery->pResExtSchema); taosArrayDestroy(pQuery->pDbList); taosArrayDestroy(pQuery->pTableList); taosArrayDestroy(pQuery->pTargetTableList); diff --git a/source/libs/parser/inc/parInt.h b/source/libs/parser/inc/parInt.h index 5999ada70f..c8cd062fed 100644 --- a/source/libs/parser/inc/parInt.h +++ b/source/libs/parser/inc/parInt.h @@ -36,7 +36,7 @@ int32_t parse(SParseContext* pParseCxt, SQuery** pQuery); int32_t collectMetaKey(SParseContext* pParseCxt, SQuery* pQuery, SParseMetaCache* pMetaCache); int32_t authenticate(SParseContext* pParseCxt, SQuery* pQuery, SParseMetaCache* pMetaCache); int32_t translate(SParseContext* pParseCxt, SQuery* pQuery, SParseMetaCache* pMetaCache); -int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema); +int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema, SExtSchema** pExtSchema); int32_t calculateConstant(SParseContext* pParseCxt, SQuery* pQuery); int32_t translatePostCreateStream(SParseContext* pParseCxt, SQuery* pQuery, SSDataBlock* pBlock); int32_t translatePostCreateSmaIndex(SParseContext* pParseCxt, SQuery* pQuery, SSDataBlock* pBlock); diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 9f43c5182f..abea6e7af2 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1809,6 +1809,7 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, if (TSDB_CODE_SUCCESS != code) { return code; } + // TODO wjm // precision check // scale auto fit code = decimal64ToDataVal(&dec, &pVal->value); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index cac26d78c3..429fb36320 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -14130,12 +14130,17 @@ static int32_t translateSubquery(STranslateContext* pCxt, SNode* pNode) { return code; } -static int32_t extractQueryResultSchema(const SNodeList* pProjections, int32_t* numOfCols, SSchema** pSchema) { +static int32_t extractQueryResultSchema(const SNodeList* pProjections, int32_t* numOfCols, SSchema** pSchema, SExtSchema** ppExtSchemas) { *numOfCols = LIST_LENGTH(pProjections); *pSchema = taosMemoryCalloc((*numOfCols), sizeof(SSchema)); if (NULL == (*pSchema)) { return terrno; } + if (ppExtSchemas) *ppExtSchemas = taosMemoryCalloc(*numOfCols, sizeof(SExtSchema)); + if (!ppExtSchemas) { + taosMemoryFreeClear(*pSchema); + return terrno; + } SNode* pNode; int32_t index = 0; @@ -14147,6 +14152,9 @@ static int32_t extractQueryResultSchema(const SNodeList* pProjections, int32_t* } else { (*pSchema)[index].type = pExpr->resType.type; (*pSchema)[index].bytes = pExpr->resType.bytes; + if (ppExtSchemas) { + (*ppExtSchemas)[index].typeMod = calcTypeMod(&pExpr->resType); + } } (*pSchema)[index].colId = index + 1; if ('\0' != pExpr->userAlias[0]) { @@ -14321,7 +14329,7 @@ static int32_t extractCompactDbResultSchema(int32_t* numOfCols, SSchema** pSchem return TSDB_CODE_SUCCESS; } -int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema) { +int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema, SExtSchema** ppExtSchemas) { if (NULL == pRoot) { return TSDB_CODE_SUCCESS; } @@ -14329,7 +14337,7 @@ int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pS switch (nodeType(pRoot)) { case QUERY_NODE_SELECT_STMT: case QUERY_NODE_SET_OPERATOR: - return extractQueryResultSchema(getProjectList(pRoot), numOfCols, pSchema); + return extractQueryResultSchema(getProjectList(pRoot), numOfCols, pSchema, ppExtSchemas); case QUERY_NODE_EXPLAIN_STMT: return extractExplainResultSchema(numOfCols, pSchema); case QUERY_NODE_DESCRIBE_STMT: { @@ -17804,7 +17812,8 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) { if (pQuery->haveResultSet) { taosMemoryFreeClear(pQuery->pResSchema); - if (TSDB_CODE_SUCCESS != extractResultSchema(pQuery->pRoot, &pQuery->numOfResCols, &pQuery->pResSchema)) { + taosMemoryFreeClear(pQuery->pResExtSchema); + if (TSDB_CODE_SUCCESS != extractResultSchema(pQuery->pRoot, &pQuery->numOfResCols, &pQuery->pResSchema, &pQuery->pResExtSchema)) { return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 6d2822f038..cdf2ecb52a 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -398,7 +398,7 @@ void qDestroyParseContext(SParseContext* pCxt) { void qDestroyQuery(SQuery* pQueryNode) { nodesDestroyNode((SNode*)pQueryNode); } int32_t qExtractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema) { - return extractResultSchema(pRoot, numOfCols, pSchema); + return extractResultSchema(pRoot, numOfCols, pSchema, NULL); } int32_t qSetSTableIdForRsma(SNode* pStmt, int64_t uid) { diff --git a/tests/system-test/2-query/decimal.py b/tests/system-test/2-query/decimal.py index 161aaf7336..b6f5764772 100644 --- a/tests/system-test/2-query/decimal.py +++ b/tests/system-test/2-query/decimal.py @@ -150,6 +150,7 @@ class DataType: return f"'{val}'" else: return val + ## TODO generate NULL, None def generate_value(self) -> str: if self.type == TypeEnum.BOOL: return str(secrets.randbelow(2)) @@ -164,7 +165,7 @@ class DataType: if self.type == TypeEnum.FLOAT or self.type == TypeEnum.DOUBLE: return str(random.random()) if self.type == TypeEnum.VARCHAR or self.type == TypeEnum.NCHAR or self.type == TypeEnum.VARBINARY: - return f"'{secrets.token_urlsafe(random.randint(0, self.length-10))}'" + return f"'{secrets.token_urlsafe(random.randint(0, self.length))[0:random.randint(0, self.length)]}'" if self.type == TypeEnum.TIMESTAMP: return str(secrets.randbelow(9223372036854775808)) if self.type == TypeEnum.UTINYINT: @@ -240,9 +241,11 @@ class TableInserter: if i != rows - 1: sql += ", " if len(sql) > 1000: + tdLog.debug(f"insert into with sql{sql}") self.conn.execute(sql, queryTimes=1) sql = pre_insert if len(sql) > len(pre_insert): + tdLog.debug(f"insert into with sql{sql}") self.conn.execute(sql, queryTimes=1) class TDTestCase: @@ -406,9 +409,9 @@ class TDTestCase: tdLog.printNoPrefix("-------- test create decimal column") self.columns = [ DataType(TypeEnum.DECIMAL, type_mod=DataType.get_decimal_type_mod(DecimalType(10, 2))), - DataType(TypeEnum.DECIMAL, type_mod=DataType.get_decimal_type_mod(DecimalType(20, 2))), - DataType(TypeEnum.DECIMAL, type_mod=DataType.get_decimal_type_mod(DecimalType(30, 2))), - DataType(TypeEnum.DECIMAL, type_mod=DataType.get_decimal_type_mod(DecimalType(38, 2))), + #DataType(TypeEnum.DECIMAL, type_mod=DataType.get_decimal_type_mod(DecimalType(20, 2))), + #DataType(TypeEnum.DECIMAL, type_mod=DataType.get_decimal_type_mod(DecimalType(30, 2))), + #DataType(TypeEnum.DECIMAL, type_mod=DataType.get_decimal_type_mod(DecimalType(38, 2))), DataType(TypeEnum.TINYINT), DataType(TypeEnum.INT), DataType(TypeEnum.BIGINT), diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 99b1c204a2..ffd8e92ecb 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -708,6 +708,9 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t shellFormatTimestamp(buf, sizeof(buf), *(int64_t *)val, precision); printf("%s", buf); break; + case TSDB_DATA_TYPE_DECIMAL: + case TSDB_DATA_TYPE_DECIMAL64: + printf("%*s", width, val); default: break; } @@ -895,7 +898,10 @@ int32_t shellCalcColWidth(TAOS_FIELD *field, int32_t precision) { } else { return TMAX(23, width); // '2020-01-01 00:00:00.000' } - + case TSDB_DATA_TYPE_DECIMAL64: + return TMAX(width, 20); + case TSDB_DATA_TYPE_DECIMAL: + return TMAX(width, 40); default: ASSERT(false); }