From 82c10655233892e86d0a30b828cda27fd8e51fa0 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Fri, 13 Dec 2024 15:53:57 +0800 Subject: [PATCH] decimal: create table --- include/client/taos.h | 3 +- include/common/tmsg.h | 15 ++++++ include/common/ttypes.h | 6 ++- include/libs/executor/storageapi.h | 1 + include/util/tdef.h | 13 ++++- source/common/src/msg/tmsg.c | 56 ++++++++++++++++++++++ source/common/src/tcol.c | 8 +++- source/common/src/ttypes.c | 19 +++++--- source/dnode/mnode/impl/inc/mndDef.h | 2 + source/dnode/mnode/impl/src/mndStb.c | 40 ++++++++++++++++ source/dnode/vnode/src/meta/metaEntry.c | 60 ++++++++++++++++++++++++ source/dnode/vnode/src/meta/metaTable2.c | 1 + source/libs/CMakeLists.txt | 1 + source/libs/parser/CMakeLists.txt | 2 +- source/libs/parser/inc/parAst.h | 1 + source/libs/parser/inc/parUtil.h | 1 + source/libs/parser/inc/sql.y | 6 +-- source/libs/parser/src/parAstCreater.c | 9 ++++ source/libs/parser/src/parTokenizer.c | 1 + source/libs/parser/src/parTranslater.c | 23 ++++++++- source/libs/parser/src/parUtil.c | 8 ++++ tests/system-test/2-query/tsma.py | 1 + 22 files changed, 260 insertions(+), 17 deletions(-) diff --git a/include/client/taos.h b/include/client/taos.h index 433779f811..5e0bce6115 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -52,7 +52,8 @@ typedef void TAOS_SUB; #define TSDB_DATA_TYPE_MEDIUMBLOB 19 #define TSDB_DATA_TYPE_BINARY TSDB_DATA_TYPE_VARCHAR // string #define TSDB_DATA_TYPE_GEOMETRY 20 // geometry -#define TSDB_DATA_TYPE_MAX 21 +#define TSDB_DATA_TYPE_DECIMAL64 21 // decimal64 +#define TSDB_DATA_TYPE_MAX 22 typedef enum { TSDB_OPTION_LOCALE, diff --git a/include/common/tmsg.h b/include/common/tmsg.h index a068f19175..f9e9323a39 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -515,6 +515,7 @@ typedef struct SFieldWithOptions { int8_t flags; int32_t bytes; uint32_t compress; + STypeMod typeMod; } SFieldWithOptions; typedef struct SRetention { @@ -587,6 +588,7 @@ STSRow* tGetSubmitBlkNext(SSubmitBlkIter* pIter); // for debug int32_t tPrintFixedSchemaSubmitReq(SSubmitReq* pReq, STSchema* pSchema); +// TODO wjm resolve compatibility problem struct SSchema { int8_t type; int8_t flags; @@ -660,6 +662,7 @@ void tFreeSSubmitRsp(SSubmitRsp* pRsp); #define COL_SET_NULL ((int8_t)0x10) #define COL_SET_VAL ((int8_t)0x20) #define COL_IS_SYSINFO ((int8_t)0x40) +#define COL_HAS_TYPE_MOD ((int8_t)0x80) #define COL_IS_SET(FLG) (((FLG) & (COL_SET_VAL | COL_SET_NULL)) != 0) #define COL_CLR_SET(FLG) ((FLG) &= (~(COL_SET_VAL | COL_SET_NULL))) @@ -678,6 +681,13 @@ void tFreeSSubmitRsp(SSubmitRsp* pRsp); (s)->flags &= (~COL_IDX_ON); \ } while (0) +#define SSCHEMA_SET_TYPE_MOD(s) \ + do { \ + (s)->flags |= COL_HAS_TYPE_MOD; \ + } while (0) + +#define HAS_TYPE_MOD(s) (((s)->flags & COL_HAS_TYPE_MOD)) + #define SSCHMEA_TYPE(s) ((s)->type) #define SSCHMEA_FLAGS(s) ((s)->flags) #define SSCHMEA_COLID(s) ((s)->colId) @@ -694,6 +704,10 @@ typedef struct { char tsSlowLogExceptDb[TSDB_DB_NAME_LEN]; } SMonitorParas; +typedef struct { + STypeMod typeMod; +} SExtSchema; + typedef struct { int32_t nCols; int32_t version; @@ -3241,6 +3255,7 @@ typedef struct SVCreateStbReq { int8_t source; int8_t colCmpred; SColCmprWrapper colCmpr; + SExtSchema* pExtSchema; } SVCreateStbReq; int tEncodeSVCreateStbReq(SEncoder* pCoder, const SVCreateStbReq* pReq); diff --git a/include/common/ttypes.h b/include/common/ttypes.h index 95fe14e572..84862f3689 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -265,8 +265,9 @@ typedef struct { #define IS_INTEGER_TYPE(_t) ((IS_SIGNED_NUMERIC_TYPE(_t)) || (IS_UNSIGNED_NUMERIC_TYPE(_t))) #define IS_TIMESTAMP_TYPE(_t) ((_t) == TSDB_DATA_TYPE_TIMESTAMP) #define IS_BOOLEAN_TYPE(_t) ((_t) == TSDB_DATA_TYPE_BOOL) +#define IS_DECIMAL_TYPE(_t) ((_t) == TSDB_DATA_TYPE_DECIMAL || (_t) == TSDB_DATA_TYPE_DECIMAL64) -#define IS_NUMERIC_TYPE(_t) ((IS_SIGNED_NUMERIC_TYPE(_t)) || (IS_UNSIGNED_NUMERIC_TYPE(_t)) || (IS_FLOAT_TYPE(_t))) +#define IS_NUMERIC_TYPE(_t) ((IS_SIGNED_NUMERIC_TYPE(_t)) || (IS_UNSIGNED_NUMERIC_TYPE(_t)) || (IS_FLOAT_TYPE(_t)) || (IS_DECIMAL_TYPE(_t))) #define IS_MATHABLE_TYPE(_t) \ (IS_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL) || (_t) == (TSDB_DATA_TYPE_TIMESTAMP)) @@ -371,6 +372,9 @@ void assignVal(char *val, const char *src, int32_t len, int32_t type); void *getDataMin(int32_t type, void *value); void *getDataMax(int32_t type, void *value); +#define STypeMod int32_t +uint8_t getDecimalType(uint8_t precision); + #ifdef __cplusplus } #endif diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 72dec77905..53bf97a824 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -83,6 +83,7 @@ typedef struct SMetaEntry { uint8_t* pBuf; SColCmprWrapper colCmpr; // col compress alg + SExtSchema* pExtSchema; } SMetaEntry; typedef struct SMetaReader { diff --git a/include/util/tdef.h b/include/util/tdef.h index fea057dca8..fb9a1a3974 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -33,7 +33,7 @@ extern "C" { #define TD_VER_MAX UINT64_MAX // TODO: use the real max version from query handle // Bytes for each type. -extern const int32_t TYPE_BYTES[21]; +extern const int32_t TYPE_BYTES[22]; #define CHAR_BYTES sizeof(char) #define SHORT_BYTES sizeof(int16_t) @@ -46,6 +46,9 @@ extern const int32_t TYPE_BYTES[21]; #define TSDB_KEYSIZE sizeof(TSKEY) #define TSDB_NCHAR_SIZE sizeof(TdUcs4) +#define DECIMAL64_BYTES 8 +#define DECIMAL128_BYTES 16 + // NULL definition #define TSDB_DATA_BOOL_NULL 0x02 #define TSDB_DATA_TINYINT_NULL 0x80 @@ -678,6 +681,14 @@ typedef enum { #define MIN_RESERVE_MEM_SIZE 1024 // MB +// Decimal +#define TSDB_DECIMAL_MIN_PRECISION 1 +#define TSDB_DECIMAL_MAX_PRECISION 38 +#define TSDB_DECIMAL_MAX_SCALE TSDB_DECIMAL_MAX_PRECISION + +#define TSDB_DECIMAL64_MAX_PRECISION 18 +#define TSDB_DECIMAL64_MAX_SCALE TSDB_DECIMAL64_MAX_PRECISION + #ifdef __cplusplus } #endif diff --git a/source/common/src/msg/tmsg.c b/source/common/src/msg/tmsg.c index 77c115339c..1cddd64f97 100644 --- a/source/common/src/msg/tmsg.c +++ b/source/common/src/msg/tmsg.c @@ -673,6 +673,7 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq TAOS_CHECK_EXIT(tEncodeI32(&encoder, pField->bytes)); TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pField->name)); TAOS_CHECK_EXIT(tEncodeU32(&encoder, pField->compress)); + TAOS_CHECK_EXIT(tEncodeI32(&encoder, pField->typeMod)); } for (int32_t i = 0; i < pReq->numOfTags; ++i) { @@ -759,6 +760,7 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR TAOS_CHECK_EXIT(tDecodeI32(&decoder, &field.bytes)); TAOS_CHECK_EXIT(tDecodeCStrTo(&decoder, field.name)); TAOS_CHECK_EXIT(tDecodeU32(&decoder, &field.compress)); + TAOS_CHECK_EXIT(tDecodeI32(&decoder, &field.typeMod)); if (taosArrayPush(pReq->pColumns, &field) == NULL) { TAOS_CHECK_EXIT(terrno); } @@ -903,6 +905,7 @@ int32_t tSerializeSMAlterStbReq(void *buf, int32_t bufLen, SMAlterStbReq *pReq) TAOS_CHECK_EXIT(tEncodeI32(&encoder, pField->bytes)); TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pField->name)); TAOS_CHECK_EXIT(tEncodeU32(&encoder, pField->compress)); + TAOS_CHECK_EXIT(tEncodeI32(&encoder, pField->typeMod)); } else { SField *pField = taosArrayGet(pReq->pFields, i); @@ -955,6 +958,7 @@ int32_t tDeserializeSMAlterStbReq(void *buf, int32_t bufLen, SMAlterStbReq *pReq TAOS_CHECK_EXIT(tDecodeI32(&decoder, &field.bytes)); TAOS_CHECK_EXIT(tDecodeCStrTo(&decoder, field.name)); TAOS_CHECK_EXIT(tDecodeU32(&decoder, &field.compress)); + TAOS_CHECK_EXIT(tDecodeI32(&decoder, &field.typeMod)); if (taosArrayPush(pReq->pFields, &field) == NULL) { TAOS_CHECK_EXIT(terrno); } @@ -10430,6 +10434,44 @@ _exit: return code; } +static int32_t tEncodeSExtSchema(SEncoder* pCoder, const SExtSchema* pExtSchema) { + int32_t code = 0, lino; + TAOS_CHECK_EXIT(tEncodeI32v(pCoder, pExtSchema->typeMod)); + +_exit: + return code; +} + +int32_t tDecodeSExtSchema(SDecoder* pCoder, SExtSchema* pExtSchema) { + int32_t code = 0, lino; + TAOS_CHECK_EXIT(tDecodeI32v(pCoder, &pExtSchema->typeMod)); + +_exit: + return code; +} + +static int32_t tEncodeSExtSchemas(SEncoder* pCoder, const SExtSchema* pExtSchemas, int32_t nCol) { + int32_t code = 0, lino; + for (int32_t i = 0; i < nCol; ++i) { + TAOS_CHECK_EXIT(tEncodeSExtSchema(pCoder, pExtSchemas + i)); + } + +_exit: + return code; +} + +static int32_t tDecodeSExtSchemas(SDecoder* pCoder, SExtSchema** ppExtSchema, int32_t nCol) { + int32_t code = 0, lino; + *ppExtSchema = tDecoderMalloc(pCoder, sizeof(SExtSchema) * nCol); + if (!*ppExtSchema) TAOS_CHECK_EXIT(terrno); + for (int32_t i = 0; i < nCol; ++i) { + TAOS_CHECK_EXIT(tDecodeSExtSchema(pCoder, (*ppExtSchema) + i)); + } + +_exit: + return code; +} + int tEncodeSVCreateStbReq(SEncoder *pCoder, const SVCreateStbReq *pReq) { int32_t code = 0; int32_t lino; @@ -10453,6 +10495,12 @@ int tEncodeSVCreateStbReq(SEncoder *pCoder, const SVCreateStbReq *pReq) { TAOS_CHECK_EXIT(tEncodeI8(pCoder, pReq->colCmpred)); TAOS_CHECK_EXIT(tEncodeSColCmprWrapper(pCoder, &pReq->colCmpr)); + if (pReq->pExtSchema) { + TAOS_CHECK_EXIT(tEncodeI8(pCoder, 1)); + TAOS_CHECK_EXIT(tEncodeSExtSchemas(pCoder, pReq->pExtSchema, pReq->schemaRow.nCols)); + } else { + TAOS_CHECK_EXIT(tEncodeI8(pCoder, 0)); + } tEndEncode(pCoder); _exit: @@ -10487,6 +10535,14 @@ int tDecodeSVCreateStbReq(SDecoder *pCoder, SVCreateStbReq *pReq) { if (!tDecodeIsEnd(pCoder)) { TAOS_CHECK_EXIT(tDecodeSColCmprWrapperEx(pCoder, &pReq->colCmpr)); } + + if (!tDecodeIsEnd(pCoder)) { + int8_t hasExtSchema = 0; + TAOS_CHECK_EXIT(tDecodeI8(pCoder, &hasExtSchema)); + if (hasExtSchema) { + TAOS_CHECK_EXIT(tDecodeSExtSchemas(pCoder, &pReq->pExtSchema, pReq->schemaRow.nCols)); + } + } } tEndDecode(pCoder); diff --git a/source/common/src/tcol.c b/source/common/src/tcol.c index 32e0109b2b..80a73bd5bd 100644 --- a/source/common/src/tcol.c +++ b/source/common/src/tcol.c @@ -62,8 +62,9 @@ uint8_t getDefaultEncode(uint8_t type) { return TSDB_COLVAL_ENCODE_DISABLED; case TSDB_DATA_TYPE_VARBINARY: return TSDB_COLVAL_ENCODE_DISABLED; + case TSDB_DATA_TYPE_DECIMAL64: case TSDB_DATA_TYPE_DECIMAL: - return TSDB_COLVAL_ENCODE_DELTAD; + return TSDB_COLVAL_ENCODE_SIMPLE8B; case TSDB_DATA_TYPE_BLOB: return TSDB_COLVAL_ENCODE_SIMPLE8B; case TSDB_DATA_TYPE_MEDIUMBLOB: @@ -110,8 +111,9 @@ uint16_t getDefaultCompress(uint8_t type) { return TSDB_COLVAL_COMPRESS_LZ4; case TSDB_DATA_TYPE_VARBINARY: return TSDB_COLVAL_COMPRESS_ZSTD; + case TSDB_DATA_TYPE_DECIMAL64: case TSDB_DATA_TYPE_DECIMAL: - return TSDB_COLVAL_COMPRESS_LZ4; + return TSDB_COLVAL_COMPRESS_ZSTD; case TSDB_DATA_TYPE_BLOB: return TSDB_COLVAL_COMPRESS_LZ4; case TSDB_DATA_TYPE_MEDIUMBLOB: @@ -413,6 +415,8 @@ int8_t validColEncode(uint8_t type, uint8_t l1) { return TSDB_COLVAL_ENCODE_SIMPLE8B == l1 || TSDB_COLVAL_ENCODE_XOR == l1 ? 1 : 0; } else if (type == TSDB_DATA_TYPE_GEOMETRY) { return 1; + } else if (type == TSDB_DATA_TYPE_DECIMAL64 || type == TSDB_DATA_TYPE_DECIMAL) { + return 1; } return 0; } diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index 1a0740b2b9..afbc471789 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -17,7 +17,7 @@ #include "ttypes.h" #include "tcompression.h" -const int32_t TYPE_BYTES[21] = { +const int32_t TYPE_BYTES[TSDB_DATA_TYPE_MAX] = { 2, // TSDB_DATA_TYPE_NULL CHAR_BYTES, // TSDB_DATA_TYPE_BOOL CHAR_BYTES, // TSDB_DATA_TYPE_TINYINT @@ -35,10 +35,11 @@ const int32_t TYPE_BYTES[21] = { sizeof(uint64_t), // TSDB_DATA_TYPE_UBIGINT TSDB_MAX_JSON_TAG_LEN, // TSDB_DATA_TYPE_JSON sizeof(VarDataOffsetT), // TSDB_DATA_TYPE_VARBINARY - TSDB_MAX_TAGS_LEN, // TSDB_DATA_TYPE_DECIMAL: placeholder, not implemented + DECIMAL128_BYTES, // TSDB_DATA_TYPE_DECIMAL: placeholder, not implemented TSDB_MAX_TAGS_LEN, // TSDB_DATA_TYPE_BLOB: placeholder, not implemented TSDB_MAX_TAGS_LEN, // TSDB_DATA_TYPE_MEDIUMBLOB: placeholder, not implemented sizeof(VarDataOffsetT), // TSDB_DATA_TYPE_GEOMETRY + DECIMAL64_BYTES, // TSDB_DATA_TYPE_DECIMAL64 }; tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX] = { @@ -62,11 +63,12 @@ tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX] = { {TSDB_DATA_TYPE_UBIGINT, 15, LONG_BYTES, "BIGINT UNSIGNED", 0, UINT64_MAX, tsCompressBigint, tsDecompressBigint}, {TSDB_DATA_TYPE_JSON, 4, TSDB_MAX_JSON_TAG_LEN, "JSON", 0, 0, tsCompressString, tsDecompressString}, {TSDB_DATA_TYPE_VARBINARY, 9, 1, "VARBINARY", 0, 0, tsCompressString, - tsDecompressString}, // placeholder, not implemented - {TSDB_DATA_TYPE_DECIMAL, 7, 1, "DECIMAL", 0, 0, NULL, NULL}, // placeholder, not implemented - {TSDB_DATA_TYPE_BLOB, 4, 1, "BLOB", 0, 0, NULL, NULL}, // placeholder, not implemented - {TSDB_DATA_TYPE_MEDIUMBLOB, 10, 1, "MEDIUMBLOB", 0, 0, NULL, NULL}, // placeholder, not implemented + tsDecompressString}, // placeholder, not implemented + {TSDB_DATA_TYPE_DECIMAL, 7, DECIMAL128_BYTES, "DECIMAL", 0, 0, NULL, NULL}, // placeholder, not implemented + {TSDB_DATA_TYPE_BLOB, 4, 1, "BLOB", 0, 0, NULL, NULL}, // placeholder, not implemented + {TSDB_DATA_TYPE_MEDIUMBLOB, 10, 1, "MEDIUMBLOB", 0, 0, NULL, NULL}, // placeholder, not implemented {TSDB_DATA_TYPE_GEOMETRY, 8, 1, "GEOMETRY", 0, 0, tsCompressString, tsDecompressString}, + {TSDB_DATA_TYPE_DECIMAL64, 7, DECIMAL64_BYTES, "DECIMAL", 0, 0, NULL, NULL}, }; tDataTypeCompress tDataCompress[TSDB_DATA_TYPE_MAX] = { @@ -96,6 +98,7 @@ tDataTypeCompress tDataCompress[TSDB_DATA_TYPE_MAX] = { {TSDB_DATA_TYPE_BLOB, 4, 1, "BLOB", 0, 0, NULL, NULL}, // placeholder, not implemented {TSDB_DATA_TYPE_MEDIUMBLOB, 10, 1, "MEDIUMBLOB", 0, 0, NULL, NULL}, // placeholder, not implemented {TSDB_DATA_TYPE_GEOMETRY, 8, 1, "GEOMETRY", 0, 0, tsCompressString2, tsDecompressString2}, + // TODO wjm decimal compress }; @@ -229,3 +232,7 @@ int32_t operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type) { return 0; } + +uint8_t getDecimalType(uint8_t precision) { + return precision > TSDB_DECIMAL64_MAX_PRECISION ? TSDB_DATA_TYPE_DECIMAL : TSDB_DATA_TYPE_DECIMAL64; +} diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 050c722167..058fae1db1 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -561,6 +561,7 @@ typedef struct { col_id_t colId; int32_t cmprAlg; } SCmprObj; + typedef struct { char name[TSDB_TABLE_FNAME_LEN]; char db[TSDB_DB_FNAME_LEN]; @@ -590,6 +591,7 @@ typedef struct { SRWLatch lock; int8_t source; SColCmpr* pCmpr; + SExtSchema* pExtSchemas; } SStbObj; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 05b9826ae8..f6d25a8f1e 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -117,6 +117,7 @@ void mndCleanupStb(SMnode *pMnode) {} SSdbRaw *mndStbActionEncode(SStbObj *pStb) { int32_t code = 0; int32_t lino = 0; + bool hasTypeMod = false; terrno = TSDB_CODE_OUT_OF_MEMORY; int32_t size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema) + pStb->commentLen + @@ -155,6 +156,7 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) { SDB_SET_INT16(pRaw, dataPos, pSchema->colId, _OVER) SDB_SET_INT32(pRaw, dataPos, pSchema->bytes, _OVER) SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER) + hasTypeMod = hasTypeMod || HAS_TYPE_MOD(pSchema); } for (int32_t i = 0; i < pStb->numOfTags; ++i) { @@ -190,6 +192,14 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) { SDB_SET_INT32(pRaw, dataPos, p->alg, _OVER) } } + + // TODO wjm test it, what if some cols are deleted, maybe rewrite it + if (hasTypeMod) { + for (int32_t i = 0; i < pStb->numOfColumns; ++i) { + SDB_SET_INT32(pRaw, dataPos, pStb->pExtSchemas[i].typeMod, _OVER); + } + } + SDB_SET_RESERVE(pRaw, dataPos, STB_RESERVE_SIZE, _OVER) SDB_SET_DATALEN(pRaw, dataPos, _OVER) @@ -212,6 +222,7 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_OUT_OF_MEMORY; SSdbRow *pRow = NULL; SStbObj *pStb = NULL; + bool hasExtSchemas = false; int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; @@ -264,6 +275,7 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { SDB_GET_INT16(pRaw, dataPos, &pSchema->colId, _OVER) SDB_GET_INT32(pRaw, dataPos, &pSchema->bytes, _OVER) SDB_GET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER) + hasExtSchemas = hasExtSchemas || HAS_TYPE_MOD(pSchema); } for (int32_t i = 0; i < pStb->numOfTags; ++i) { @@ -317,6 +329,16 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { } } + // type mod + if (hasExtSchemas) { + pStb->pExtSchemas = taosMemoryCalloc(pStb->numOfColumns, sizeof(SExtSchema)); + if (!pStb->pExtSchemas) goto _OVER; + for (int32_t i = 0; i < pStb->numOfColumns; ++i) { + SSchema *pSchema = &pStb->pColumns[i]; + SDB_GET_INT32(pRaw, dataPos, &pStb->pExtSchemas[i].typeMod, _OVER) + } + } + SDB_GET_RESERVE(pRaw, dataPos, STB_RESERVE_SIZE, _OVER) terrno = 0; @@ -557,6 +579,7 @@ void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int3 } } } + req.pExtSchema = pStb->pExtSchemas; // only reference to it. // get length int32_t ret = 0; tEncodeSize(tEncodeSVCreateStbReq, &req, contLen, ret); @@ -855,6 +878,7 @@ static SSchema *mndFindStbColumns(const SStbObj *pStb, const char *colName) { int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreate, SDbObj *pDb) { int32_t code = 0; + bool hasTypeMods = false; memcpy(pDst->name, pCreate->name, TSDB_TABLE_FNAME_LEN); memcpy(pDst->db, pDb->name, TSDB_DB_FNAME_LEN); pDst->createdTime = taosGetTimestampMs(); @@ -930,6 +954,7 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat memcpy(pSchema->name, pField->name, TSDB_COL_NAME_LEN); pSchema->colId = pDst->nextColId; pDst->nextColId++; + hasTypeMods = hasTypeMods || pField->typeMod != 0; } for (int32_t i = 0; i < pDst->numOfTags; ++i) { @@ -954,6 +979,18 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat pColCmpr->id = pSchema->colId; pColCmpr->alg = pField->compress; } + + if (hasTypeMods) { + pDst->pExtSchemas = taosMemoryCalloc(pDst->numOfColumns, sizeof(SExtSchema)); + if (!pDst->pExtSchemas) { + code = terrno; + TAOS_RETURN(code); + } + for (int32_t i = 0; i < pDst->numOfColumns; ++i) { + SFieldWithOptions * pField = taosArrayGet(pCreate->pColumns, i); + pDst->pExtSchemas[i].typeMod = pField->typeMod; + } + } TAOS_RETURN(code); } static int32_t mndGenIdxNameForFirstTag(char *fullname, char *dbname, char *stbname, char *tagname) { @@ -1247,6 +1284,7 @@ static int32_t mndBuildStbFromAlter(SStbObj *pStb, SStbObj *pDst, SMCreateStbReq p->alg = pField->compress; } } + // TODO wjm alter table with deicmal table pDst->tagVer = createReq->tagVer; pDst->colVer = createReq->colVer; return TSDB_CODE_SUCCESS; @@ -3634,6 +3672,8 @@ static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB colTypeLen += tsnprintf(varDataVal(colTypeStr) + colTypeLen, sizeof(colTypeStr) - colTypeLen - VARSTR_HEADER_SIZE, "(%d)", (int32_t)((pStb->pColumns[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); + } else if (IS_DECIMAL_TYPE(colType)) { + //colTypeLen += sprintf(varDataVal(colTypeStr) + colTypeLen, "(%d,%d)", pStb->pColumns[i].precision, pStb->pColumns[i].scale); } varDataSetLen(colTypeStr, colTypeLen); RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (char *)colTypeStr, false), pStb, &lino, _OVER); diff --git a/source/dnode/vnode/src/meta/metaEntry.c b/source/dnode/vnode/src/meta/metaEntry.c index a234ea257c..474875aa36 100644 --- a/source/dnode/vnode/src/meta/metaEntry.c +++ b/source/dnode/vnode/src/meta/metaEntry.c @@ -15,6 +15,61 @@ #include "meta.h" +static bool schemasHasTypeMod(const SSchema *pSchema, int32_t nCols) { + for (int32_t i = 0; i < nCols; i++) { + if (HAS_TYPE_MOD(pSchema + i)) { + return true; + } + } + return false; +} + +static int32_t metaEncodeExtSchema(SEncoder* pCoder, const SMetaEntry* pME) { + if (pME->pExtSchema) { + const SSchemaWrapper *pSchWrapper = NULL; + bool hasTypeMods = false; + if (pME->type == TSDB_SUPER_TABLE) { + pSchWrapper = &pME->stbEntry.schemaRow; + } else if (pME->type == TSDB_NORMAL_TABLE) { + pSchWrapper = &pME->ntbEntry.schemaRow; + } else { + return 0; + } + hasTypeMods = schemasHasTypeMod(pSchWrapper->pSchema, pSchWrapper->nCols); + + for (int32_t i = 0; i < pSchWrapper->nCols && hasTypeMods; ++i) { + TAOS_CHECK_RETURN(tEncodeI32v(pCoder, pME->pExtSchema[i].typeMod)); + } + } + return 0; +} + +static int32_t metaDecodeExtSchemas(SDecoder* pDecoder, SMetaEntry* pME) { + bool hasExtSchema = false; + SSchemaWrapper* pSchWrapper = NULL; + if (pME->type == TSDB_SUPER_TABLE) { + pSchWrapper = &pME->stbEntry.schemaRow; + } else if (pME->type == TSDB_NORMAL_TABLE) { + pSchWrapper = &pME->ntbEntry.schemaRow; + } else { + return 0; + } + + hasExtSchema = schemasHasTypeMod(pSchWrapper->pSchema, pSchWrapper->nCols); + if (hasExtSchema && pSchWrapper->nCols > 0) { + pME->pExtSchema = (SExtSchema*)tDecoderMalloc(pDecoder, sizeof(SExtSchema) * pSchWrapper->nCols); + if (pME->pExtSchema == NULL) { + return terrno; + } + + for (int32_t i = 0; i < pSchWrapper->nCols && hasExtSchema; i++) { + TAOS_CHECK_RETURN(tDecodeI32v(pDecoder, &pME->pExtSchema[i].typeMod)); + } + } + + return 0; +} + int meteEncodeColCmprEntry(SEncoder *pCoder, const SMetaEntry *pME) { const SColCmprWrapper *pw = &pME->colCmpr; TAOS_CHECK_RETURN(tEncodeI32v(pCoder, pw->nCols)); @@ -129,6 +184,7 @@ int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) { return TSDB_CODE_INVALID_PARA; } TAOS_CHECK_RETURN(meteEncodeColCmprEntry(pCoder, pME)); + TAOS_CHECK_RETURN(metaEncodeExtSchema(pCoder, pME)); } tEndEncode(pCoder); @@ -208,8 +264,12 @@ int metaDecodeEntryImpl(SDecoder *pCoder, SMetaEntry *pME, bool headerOnly) { } TABLE_SET_COL_COMPRESSED(pME->flags); } + if (!tDecodeIsEnd(pCoder)) { + TAOS_CHECK_RETURN(metaDecodeExtSchemas(pCoder, pME)); + } } + tEndDecode(pCoder); return 0; } diff --git a/source/dnode/vnode/src/meta/metaTable2.c b/source/dnode/vnode/src/meta/metaTable2.c index abab15ff58..eb4e25c9e0 100644 --- a/source/dnode/vnode/src/meta/metaTable2.c +++ b/source/dnode/vnode/src/meta/metaTable2.c @@ -189,6 +189,7 @@ int32_t metaCreateSuperTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq TABLE_SET_COL_COMPRESSED(entry.flags); entry.colCmpr = pReq->colCmpr; } + entry.pExtSchema = pReq->pExtSchema; code = metaHandleEntry2(pMeta, &entry); if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index 033582f2c0..8ee417ac57 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -25,3 +25,4 @@ add_subdirectory(geometry) add_subdirectory(command) add_subdirectory(azure) add_subdirectory(tcs) +add_subdirectory(decimal) diff --git a/source/libs/parser/CMakeLists.txt b/source/libs/parser/CMakeLists.txt index 088cdc4368..50896a2715 100644 --- a/source/libs/parser/CMakeLists.txt +++ b/source/libs/parser/CMakeLists.txt @@ -33,7 +33,7 @@ target_include_directories( target_link_libraries( parser - PRIVATE os util nodes catalog function scalar geometry transport qcom + PRIVATE os util nodes catalog function scalar geometry transport qcom decimal ) if(${BUILD_TEST}) diff --git a/source/libs/parser/inc/parAst.h b/source/libs/parser/inc/parAst.h index cb0a8f971e..7939599a5e 100644 --- a/source/libs/parser/inc/parAst.h +++ b/source/libs/parser/inc/parAst.h @@ -191,6 +191,7 @@ SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* SDataType createDataType(uint8_t type); SDataType createVarLenDataType(uint8_t type, const SToken* pLen); +SDataType createDecimalDataType(uint8_t type, const SToken* pPrecisionToken, const SToken* pScaleToken); SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt); SNode* createAlterDatabaseOptions(SAstCreateContext* pCxt); diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index ede31ec2a5..584ae16252 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -186,6 +186,7 @@ int32_t getTsmaFromCache(SParseMetaCache* pMetaCache, const SName* pTsmaName, ST * @retval val range between [INT64_MIN, INT64_MAX] */ int64_t int64SafeSub(int64_t a, int64_t b); +STypeMod calcTypeMod(const SDataType* pType); #ifdef __cplusplus } diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 7f36e3a1dd..f7c6b65daa 100755 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -504,9 +504,9 @@ type_name(A) ::= MEDIUMBLOB. type_name(A) ::= BLOB. { A = createDataType(TSDB_DATA_TYPE_BLOB); } type_name(A) ::= VARBINARY NK_LP NK_INTEGER(B) NK_RP. { A = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &B); } type_name(A) ::= GEOMETRY NK_LP NK_INTEGER(B) NK_RP. { A = createVarLenDataType(TSDB_DATA_TYPE_GEOMETRY, &B); } -type_name(A) ::= DECIMAL. { A = createDataType(TSDB_DATA_TYPE_DECIMAL); } -type_name(A) ::= DECIMAL NK_LP NK_INTEGER NK_RP. { A = createDataType(TSDB_DATA_TYPE_DECIMAL); } -type_name(A) ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP. { A = createDataType(TSDB_DATA_TYPE_DECIMAL); } +// type_name(A) ::= DECIMAL. { A = createDataType(TSDB_DATA_TYPE_DECIMAL); } +type_name(A) ::= DECIMAL NK_LP NK_INTEGER(B) NK_RP. { A = createDecimalDataType(TSDB_DATA_TYPE_DECIMAL, &B, NULL); } +type_name(A) ::= DECIMAL NK_LP NK_INTEGER(B) NK_COMMA NK_INTEGER(C) NK_RP. { A = createDecimalDataType(TSDB_DATA_TYPE_DECIMAL, &B, &C); } %type type_name_default_len { SDataType } %destructor type_name_default_len { } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 20f2ab81e0..4538176adc 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -2487,6 +2487,15 @@ SDataType createVarLenDataType(uint8_t type, const SToken* pLen) { return dt; } +SDataType createDecimalDataType(uint8_t type, const SToken* pPrecisionToken, const SToken* pScaleToken) { + SDataType dt = {0}; + dt.precision = taosStr2UInt8(pPrecisionToken->z, NULL, 10); + dt.scale = pScaleToken ? taosStr2Int32(pScaleToken->z, NULL, 10) : 0; + dt.type = getDecimalType(dt.precision); + dt.bytes = tDataTypes[dt.type].bytes; + return dt; +} + SNode* createCreateTableStmt(SAstCreateContext* pCxt, bool ignoreExists, SNode* pRealTable, SNodeList* pCols, SNodeList* pTags, SNode* pOptions) { CHECK_PARSER_STATUS(pCxt); diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index 215804da15..1acbeadcfc 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -89,6 +89,7 @@ static SKeyword keywordTable[] = { {"DATABASE", TK_DATABASE}, {"DATABASES", TK_DATABASES}, {"DBS", TK_DBS}, + {"DECIMAL", TK_DECIMAL}, {"DELETE", TK_DELETE}, {"DELETE_MARK", TK_DELETE_MARK}, {"DESC", TK_DESC}, diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 182c5965ad..723a0ff1db 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -9272,7 +9272,7 @@ static int32_t columnDefNodeToField(SNodeList* pList, SArray** pArray, bool calB } else { field.bytes = pCol->dataType.bytes; } - + field.typeMod = calcTypeMod(&pCol->dataType); tstrncpy(field.name, pCol->colName, TSDB_COL_NAME_LEN); if (pCol->pOptions) { setColEncode(&field.compress, columnEncodeVal(((SColumnOptions*)pCol->pOptions)->encode)); @@ -9285,6 +9285,9 @@ static int32_t columnDefNodeToField(SNodeList* pList, SArray** pArray, bool calB if (pCol->pOptions && ((SColumnOptions*)pCol->pOptions)->bPrimaryKey) { field.flags |= COL_IS_KEY; } + if (field.typeMod > 0) { + field.flags |= COL_HAS_TYPE_MOD; + } if (NULL == taosArrayPush(*pArray, &field)) { code = terrno; break; @@ -9401,6 +9404,17 @@ static int32_t checkTableRollupOption(STranslateContext* pCxt, SNodeList* pFuncs return TSDB_CODE_SUCCESS; } +static int32_t checkDecimalDataType(STranslateContext* pCxt, SDataType pDataType) { + if (IS_DECIMAL_TYPE(pDataType.type)) { + if (pDataType.precision < TSDB_DECIMAL_MIN_PRECISION || pDataType.precision > TSDB_DECIMAL_MAX_PRECISION || + pDataType.precision < pDataType.scale) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, + "Invalid column type: %s, invalid precision or scale"); + } + } + return TSDB_CODE_SUCCESS; +} + static int32_t checkTableTagsSchema(STranslateContext* pCxt, SHashObj* pHash, SNodeList* pTags) { int32_t ntags = LIST_LENGTH(pTags); if (0 == ntags) { @@ -9437,6 +9451,7 @@ static int32_t checkTableTagsSchema(STranslateContext* pCxt, SHashObj* pHash, SN } else { break; } + // TODO wjm can't create tag with decimal type } if (TSDB_CODE_SUCCESS == code && tagsSize > TSDB_MAX_TAGS_LEN) { @@ -9500,7 +9515,7 @@ static int32_t checkTableColsSchema(STranslateContext* pCxt, SHashObj* pHash, in } } - if (TSDB_CODE_SUCCESS == code && isAggrRollup && 0 != colIndex) { + if (TSDB_CODE_SUCCESS == code && isAggrRollup && 0 != colIndex) { // TODO wjm, test is agg rollup if (pCol->dataType.type != TSDB_DATA_TYPE_FLOAT && pCol->dataType.type != TSDB_DATA_TYPE_DOUBLE) { code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, @@ -9508,6 +9523,10 @@ static int32_t checkTableColsSchema(STranslateContext* pCxt, SHashObj* pHash, in } } + if (TSDB_CODE_SUCCESS == code && IS_DECIMAL_TYPE(pCol->dataType.type)) { + code = checkDecimalDataType(pCxt, pCol->dataType); + } + if (TSDB_CODE_SUCCESS == code) { code = taosHashPut(pHash, pCol->colName, len, &pCol, POINTER_BYTES); } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index a5758a17c3..5952c2371a 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -18,6 +18,7 @@ #include "querynodes.h" #include "tarray.h" #include "tlog.h" +#include "decimal.h" #define USER_AUTH_KEY_MAX_LEN TSDB_USER_LEN + TSDB_TABLE_FNAME_LEN + 2 @@ -1503,3 +1504,10 @@ int64_t int64SafeSub(int64_t a, int64_t b) { } return res; } + +STypeMod calcTypeMod(const SDataType* pType) { + if (IS_DECIMAL_TYPE(pType->type)) { + return decimalCalcTypeMod(pType); + } + return 0; +} diff --git a/tests/system-test/2-query/tsma.py b/tests/system-test/2-query/tsma.py index 1c688d568c..0a1e94c932 100644 --- a/tests/system-test/2-query/tsma.py +++ b/tests/system-test/2-query/tsma.py @@ -1227,6 +1227,7 @@ class TDTestCase: def run(self): self.init_data() + time.sleep(9999999) self.test_ddl() self.test_query_with_tsma() # bug to fix