diff --git a/include/common/tcol.h b/include/common/tcol.h index 32fbe33290..e4a53cb05b 100644 --- a/include/common/tcol.h +++ b/include/common/tcol.h @@ -13,14 +13,14 @@ * along with this program. If not, see . */ #include "taosdef.h" - #ifndef _TD_TCOL_H_ #define _TD_TCOL_H_ #define TSDB_COLUMN_ENCODE_UNKNOWN "Unknown" -#define TSDB_COLUMN_ENCODE_SIMPLE8B "simple8b" -#define TSDB_COLUMN_ENCODE_XOR "xor" -#define TSDB_COLUMN_ENCODE_RLE "rle" +#define TSDB_COLUMN_ENCODE_SIMPLE8B "Simple8b" +#define TSDB_COLUMN_ENCODE_XOR "DeltaI" +#define TSDB_COLUMN_ENCODE_RLE "Bit-Packing" +#define TSDB_COLUMN_ENCODE_DELTAD "DeltaD" #define TSDB_COLUMN_ENCODE_DISABLED "disabled" #define TSDB_COLUMN_COMPRESS_UNKNOWN "Unknown" @@ -40,6 +40,8 @@ #define TSDB_COLVAL_ENCODE_SIMPLE8B 1 #define TSDB_COLVAL_ENCODE_XOR 2 #define TSDB_COLVAL_ENCODE_RLE 3 +#define TSDB_COLVAL_ENCODE_DELTAD 4 + #define TSDB_COLVAL_ENCODE_DISABLED 0xff #define TSDB_COLVAL_COMPRESS_NOCHANGE 0 @@ -59,7 +61,7 @@ #define TSDB_CL_COMMENT_LEN 1025 #define TSDB_CL_COMPRESS_OPTION_LEN 12 -extern const char* supportedEncode[4]; +extern const char* supportedEncode[5]; extern const char* supportedCompress[6]; extern const char* supportedLevel[3]; diff --git a/source/common/src/tcol.c b/source/common/src/tcol.c index 61243b69d9..a6aceedd53 100644 --- a/source/common/src/tcol.c +++ b/source/common/src/tcol.c @@ -16,8 +16,8 @@ #include "tcol.h" #include "tutil.h" -const char* supportedEncode[4] = {TSDB_COLUMN_ENCODE_SIMPLE8B, TSDB_COLUMN_ENCODE_XOR, TSDB_COLUMN_ENCODE_RLE, - TSDB_COLUMN_ENCODE_DISABLED}; +const char* supportedEncode[5] = {TSDB_COLUMN_ENCODE_SIMPLE8B, TSDB_COLUMN_ENCODE_XOR, TSDB_COLUMN_ENCODE_RLE, + TSDB_COLUMN_ENCODE_DELTAD, TSDB_COLUMN_ENCODE_DISABLED}; const char* supportedCompress[6] = {TSDB_COLUMN_COMPRESS_LZ4, TSDB_COLUMN_COMPRESS_TSZ, TSDB_COLUMN_COMPRESS_XZ, TSDB_COLUMN_COMPRESS_ZLIB, @@ -33,13 +33,17 @@ uint8_t getDefaultEncode(uint8_t type) { switch (type) { case TSDB_DATA_TYPE_NULL: case TSDB_DATA_TYPE_BOOL: + return TSDB_COLVAL_ENCODE_RLE; case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_BIGINT: + return TSDB_COLVAL_ENCODE_SIMPLE8B; case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_DOUBLE: + return TSDB_COLVAL_ENCODE_DELTAD; case TSDB_DATA_TYPE_VARCHAR: // TSDB_DATA_TYPE_BINARY + return case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_UTINYINT: @@ -108,6 +112,9 @@ const char* columnEncodeStr(uint8_t type) { case TSDB_COLVAL_ENCODE_RLE: encode = TSDB_COLUMN_ENCODE_RLE; break; + case TSDB_COLVAL_ENCODE_DELTAD: + encode = TSDB_COLUMN_ENCODE_DELTAD; + break; case TSDB_COLVAL_ENCODE_DISABLED: encode = TSDB_COLUMN_ENCODE_DISABLED; break; @@ -187,6 +194,8 @@ uint8_t columnEncodeVal(const char* encode) { e = TSDB_COLVAL_ENCODE_XOR; } else if (0 == strcmp(encode, TSDB_COLUMN_ENCODE_RLE)) { e = TSDB_COLVAL_ENCODE_RLE; + } else if (0 == strcmp(encode, TSDB_COLUMN_ENCODE_DELTAD)) { + e = TSDB_COLVAL_ENCODE_DELTAD; } else if (0 == strcmp(encode, TSDB_COLUMN_ENCODE_DISABLED)) { e = TSDB_COLVAL_ENCODE_DISABLED; } else { @@ -298,3 +307,22 @@ void setColCompressByOption(uint32_t* compress, uint8_t encode, uint16_t compres } bool useCompress(uint8_t tableType) { return TSDB_SUPER_TABLE == tableType || TSDB_NORMAL_TABLE == tableType; } + +int8_t validColCompressOption(uint8_t type, uint8_t encode, uint8_t compress, uint8_t level) { + if (level < TSDB_COLVAL_LEVEL_HIGH || level > TSDB_COLVAL_LEVEL_LOW) { + return 0; + } + + if (type == TSDB_DATA_TYPE_BOOL) { + } else if (type >= TSDB_DATA_TYPE_TINYINT && type <= TSDB_DATA_TYPE_BIGINT) { + } else if (type >= TSDB_DATA_TYPE_FLOAT && type <= TSDB_DATA_TYPE_DOUBLE) { + if (compress == TSDB_COLVAL_COMPRESS_TSZ) { + return 1; + } + } else if (type == TSDB_DATA_TYPE_VARCHAR && type == TSDB_DATA_TYPE_NCHAR) { + } else if (type == TSDB_DATA_TYPE_TIMESTAMP) { + } else if (type >= TSDB_DATA_TYPE_USMALLINT || type <= TSDB_DATA_TYPE_UBIGINT) { + } else if (type == TSDB_DATA_TYPE_JSON || type == TSDB_DATA_TYPE_VARBINARY) { + } + return 0; +} diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 5cbcd65bc3..0135904822 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -266,12 +266,12 @@ int32_t l2DecompressImpl_xz(const char *const input, const int32_t compressedSiz return -1; } -TCompressL1FnSet compressL1Dict[] = {{"unknown", NULL, tsCompressUnknow2, tsDecompressUnknow2}, - {"timestamp", NULL, tsCompressTimestampImp2, tsDecompressTimestampImp2}, - {"int", NULL, tsCompressINTImp2, tsDecompressINTImp2}, - {"double", NULL, tsCompressDoubleImp2, tsDecompressDoubleImp2}, - {"bool", NULL, tsCompressBoolImp2, tsDecompressBoolImp2}, - {NULL, NULL, NULL}}; +TCompressL1FnSet compressL1Dict[] = {{"PLAIN", NULL, tsCompressUnknow2, tsDecompressUnknow2}, + {"SIMPLE-8B", NULL, tsCompressINTImp2, tsDecompressINTImp2}, + {"DELTAI", NULL, tsCompressTimestampImp2, tsDecompressTimestampImp2}, + {"BIT-PACKING", NULL, tsCompressBoolImp2, tsDecompressBoolImp2}, + {"DELTAD", NULL, tsCompressDoubleImp2, tsDecompressDoubleImp2}}; + TCompressL2FnSet compressL2Dict[] = { {"unknown", l2ComressInitImpl_disabled, l2CompressImpl_disabled, l2DecompressImpl_disabled}, {"lz4", l2ComressInitImpl_lz4, l2CompressImpl_lz4, l2DecompressImpl_lz4}, @@ -2940,13 +2940,13 @@ int32_t tsFindCompressAlg(int8_t dataType, uint8_t compress, TCompressL1FnSet *l return 0; } -typedef struct { - int8_t dtype; - SArray *l1Set; - SArray *l2Set; -} TCompressCompatible; +// typedef struct { +// int8_t dtype; +// SArray *l1Set; +// SArray *l2Set; +// } TCompressCompatible; -SHashObj *algSet = NULL; +// SHashObj *algSet = NULL; // int32_t tsCompressSetInit() { // algSet = taosHashInit(24, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); @@ -2968,38 +2968,38 @@ SHashObj *algSet = NULL; // } // return 0; // } -int32_t tsCompressSetDestroy() { - void *p = taosHashIterate(algSet, NULL); - while (p) { - TCompressCompatible *v = p; - taosArrayDestroy(v->l1Set); - taosArrayDestroy(v->l2Set); +// int32_t tsCompressSetDestroy() { +// void *p = taosHashIterate(algSet, NULL); +// while (p) { +// TCompressCompatible *v = p; +// taosArrayDestroy(v->l1Set); +// taosArrayDestroy(v->l2Set); - taosHashIterate(algSet, p); - } - return 0; -} +// taosHashIterate(algSet, p); +// } +// return 0; +// } -int32_t tsValidCompressAlgByDataTypes(int8_t type, int8_t compress) { - // compress alg - int8_t l1 = COMPRESS_L1_TYPE_U8(compress); - int8_t l2 = COMPRESS_L2_TYPE_U8(compress); - int8_t lvl = COMPRESS_L2_TYPE_LEVEL_U8(compress); +// int32_t tsValidCompressAlgByDataTypes(int8_t type, int8_t compress) { +// // compress alg +// int8_t l1 = COMPRESS_L1_TYPE_U8(compress); +// int8_t l2 = COMPRESS_L2_TYPE_U8(compress); +// int8_t lvl = COMPRESS_L2_TYPE_LEVEL_U8(compress); - TCompressCompatible *p = taosHashGet(algSet, &type, sizeof(type)); - if (p == NULL) return -1; +// TCompressCompatible *p = taosHashGet(algSet, &type, sizeof(type)); +// if (p == NULL) return -1; - if (p->dtype != type) return -1; +// if (p->dtype != type) return -1; - if (taosArraySearch(p->l1Set, &l1, compareInt8Val, 0) == NULL) { - return -1; - } +// if (taosArraySearch(p->l1Set, &l1, compareInt8Val, 0) == NULL) { +// return -1; +// } - if (taosArraySearch(p->l2Set, &l2, compareInt8Val, 0) == NULL) { - return -1; - } - return 0; -} +// if (taosArraySearch(p->l2Set, &l2, compareInt8Val, 0) == NULL) { +// return -1; +// } +// return 0; +// } int32_t tcompressDebug(uint32_t cmprAlg, uint8_t *l1Alg, uint8_t *l2Alg, uint8_t *level) { DEFINE_VAR(cmprAlg) @@ -3038,3 +3038,9 @@ int8_t tUpdateCompress(uint32_t oldCmpr, uint32_t newCmpr, uint8_t l2Disabled, u return 0; } +// int32_t validCompress(int8_t type, uint8_t encode, uint8_t compress, uint8_t level) { +// if (type == TSDB_DATA_TYPE_BOOL) { + +// } else +// return 0; +// }