xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

refactor code

This commit is contained in:
yihaoDeng 2024-03-28 01:44:04 +00:00
parent 6d19283d1f
commit 94e2aa2ed5
3 changed files with 81 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
include/common/tcol.h
View File

@ -13,14 +13,14 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "taosdef.h" #include "taosdef.h"
#ifndef _TD_TCOL_H_ #ifndef _TD_TCOL_H_
#define _TD_TCOL_H_ #define _TD_TCOL_H_
#define TSDB_COLUMN_ENCODE_UNKNOWN "Unknown" #define TSDB_COLUMN_ENCODE_UNKNOWN "Unknown"
#define TSDB_COLUMN_ENCODE_SIMPLE8B "simple8b" #define TSDB_COLUMN_ENCODE_SIMPLE8B "Simple8b"
#define TSDB_COLUMN_ENCODE_XOR "xor" #define TSDB_COLUMN_ENCODE_XOR "DeltaI"
#define TSDB_COLUMN_ENCODE_RLE "rle" #define TSDB_COLUMN_ENCODE_RLE "Bit-Packing"
#define TSDB_COLUMN_ENCODE_DELTAD "DeltaD"
#define TSDB_COLUMN_ENCODE_DISABLED "disabled" #define TSDB_COLUMN_ENCODE_DISABLED "disabled"
#define TSDB_COLUMN_COMPRESS_UNKNOWN "Unknown" #define TSDB_COLUMN_COMPRESS_UNKNOWN "Unknown"
@ -40,6 +40,8 @@
#define TSDB_COLVAL_ENCODE_SIMPLE8B 1 #define TSDB_COLVAL_ENCODE_SIMPLE8B 1
#define TSDB_COLVAL_ENCODE_XOR 2 #define TSDB_COLVAL_ENCODE_XOR 2
#define TSDB_COLVAL_ENCODE_RLE 3 #define TSDB_COLVAL_ENCODE_RLE 3
#define TSDB_COLVAL_ENCODE_DELTAD 4
#define TSDB_COLVAL_ENCODE_DISABLED 0xff #define TSDB_COLVAL_ENCODE_DISABLED 0xff
#define TSDB_COLVAL_COMPRESS_NOCHANGE 0 #define TSDB_COLVAL_COMPRESS_NOCHANGE 0
@ -59,7 +61,7 @@
#define TSDB_CL_COMMENT_LEN 1025 #define TSDB_CL_COMMENT_LEN 1025
#define TSDB_CL_COMPRESS_OPTION_LEN 12 #define TSDB_CL_COMPRESS_OPTION_LEN 12
extern const char* supportedEncode[4]; extern const char* supportedEncode[5];
extern const char* supportedCompress[6]; extern const char* supportedCompress[6];
extern const char* supportedLevel[3]; extern const char* supportedLevel[3];

32
source/common/src/tcol.c
View File

@ -16,8 +16,8 @@
#include "tcol.h" #include "tcol.h"
#include "tutil.h" #include "tutil.h"
const char* supportedEncode[4] = {TSDB_COLUMN_ENCODE_SIMPLE8B, TSDB_COLUMN_ENCODE_XOR, TSDB_COLUMN_ENCODE_RLE, const char* supportedEncode[5] = {TSDB_COLUMN_ENCODE_SIMPLE8B, TSDB_COLUMN_ENCODE_XOR, TSDB_COLUMN_ENCODE_RLE,
TSDB_COLUMN_ENCODE_DISABLED}; TSDB_COLUMN_ENCODE_DELTAD, TSDB_COLUMN_ENCODE_DISABLED};
const char* supportedCompress[6] = {TSDB_COLUMN_COMPRESS_LZ4, TSDB_COLUMN_COMPRESS_TSZ, const char* supportedCompress[6] = {TSDB_COLUMN_COMPRESS_LZ4, TSDB_COLUMN_COMPRESS_TSZ,
TSDB_COLUMN_COMPRESS_XZ, TSDB_COLUMN_COMPRESS_ZLIB, TSDB_COLUMN_COMPRESS_XZ, TSDB_COLUMN_COMPRESS_ZLIB,
@ -33,13 +33,17 @@ uint8_t getDefaultEncode(uint8_t type) {
switch (type) { switch (type) {
case TSDB_DATA_TYPE_NULL: case TSDB_DATA_TYPE_NULL:
case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_BOOL:
return TSDB_COLVAL_ENCODE_RLE;
case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_SMALLINT:
case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_INT:
case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_BIGINT:
return TSDB_COLVAL_ENCODE_SIMPLE8B;
case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_FLOAT:
case TSDB_DATA_TYPE_DOUBLE: case TSDB_DATA_TYPE_DOUBLE:
return TSDB_COLVAL_ENCODE_DELTAD;
case TSDB_DATA_TYPE_VARCHAR: // TSDB_DATA_TYPE_BINARY case TSDB_DATA_TYPE_VARCHAR: // TSDB_DATA_TYPE_BINARY
return
case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_TIMESTAMP:
case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_UTINYINT: case TSDB_DATA_TYPE_UTINYINT:
@ -108,6 +112,9 @@ const char* columnEncodeStr(uint8_t type) {
case TSDB_COLVAL_ENCODE_RLE: case TSDB_COLVAL_ENCODE_RLE:
encode = TSDB_COLUMN_ENCODE_RLE; encode = TSDB_COLUMN_ENCODE_RLE;
break; break;
case TSDB_COLVAL_ENCODE_DELTAD:
encode = TSDB_COLUMN_ENCODE_DELTAD;
break;
case TSDB_COLVAL_ENCODE_DISABLED: case TSDB_COLVAL_ENCODE_DISABLED:
encode = TSDB_COLUMN_ENCODE_DISABLED; encode = TSDB_COLUMN_ENCODE_DISABLED;
break; break;
@ -187,6 +194,8 @@ uint8_t columnEncodeVal(const char* encode) {
e = TSDB_COLVAL_ENCODE_XOR; e = TSDB_COLVAL_ENCODE_XOR;
} else if (0 == strcmp(encode, TSDB_COLUMN_ENCODE_RLE)) { } else if (0 == strcmp(encode, TSDB_COLUMN_ENCODE_RLE)) {
e = TSDB_COLVAL_ENCODE_RLE; e = TSDB_COLVAL_ENCODE_RLE;
} else if (0 == strcmp(encode, TSDB_COLUMN_ENCODE_DELTAD)) {
e = TSDB_COLVAL_ENCODE_DELTAD;
} else if (0 == strcmp(encode, TSDB_COLUMN_ENCODE_DISABLED)) { } else if (0 == strcmp(encode, TSDB_COLUMN_ENCODE_DISABLED)) {
e = TSDB_COLVAL_ENCODE_DISABLED; e = TSDB_COLVAL_ENCODE_DISABLED;
} else { } else {
@ -298,3 +307,22 @@ void setColCompressByOption(uint32_t* compress, uint8_t encode, uint16_t compres
} }
bool useCompress(uint8_t tableType) { return TSDB_SUPER_TABLE == tableType || TSDB_NORMAL_TABLE == tableType; } bool useCompress(uint8_t tableType) { return TSDB_SUPER_TABLE == tableType || TSDB_NORMAL_TABLE == tableType; }
int8_t validColCompressOption(uint8_t type, uint8_t encode, uint8_t compress, uint8_t level) {
if (level < TSDB_COLVAL_LEVEL_HIGH || level > TSDB_COLVAL_LEVEL_LOW) {
return 0;
}
if (type == TSDB_DATA_TYPE_BOOL) {
} else if (type >= TSDB_DATA_TYPE_TINYINT && type <= TSDB_DATA_TYPE_BIGINT) {
} else if (type >= TSDB_DATA_TYPE_FLOAT && type <= TSDB_DATA_TYPE_DOUBLE) {
if (compress == TSDB_COLVAL_COMPRESS_TSZ) {
return 1;
}
} else if (type == TSDB_DATA_TYPE_VARCHAR && type == TSDB_DATA_TYPE_NCHAR) {
} else if (type == TSDB_DATA_TYPE_TIMESTAMP) {
} else if (type >= TSDB_DATA_TYPE_USMALLINT || type <= TSDB_DATA_TYPE_UBIGINT) {
} else if (type == TSDB_DATA_TYPE_JSON || type == TSDB_DATA_TYPE_VARBINARY) {
}
return 0;
}

82
source/util/src/tcompression.c
View File

@ -266,12 +266,12 @@ int32_t l2DecompressImpl_xz(const char *const input, const int32_t compressedSiz
return -1; return -1;
} }
TCompressL1FnSet compressL1Dict[] = {{"unknown", NULL, tsCompressUnknow2, tsDecompressUnknow2}, TCompressL1FnSet compressL1Dict[] = {{"PLAIN", NULL, tsCompressUnknow2, tsDecompressUnknow2},
{"timestamp", NULL, tsCompressTimestampImp2, tsDecompressTimestampImp2}, {"SIMPLE-8B", NULL, tsCompressINTImp2, tsDecompressINTImp2},
{"int", NULL, tsCompressINTImp2, tsDecompressINTImp2}, {"DELTAI", NULL, tsCompressTimestampImp2, tsDecompressTimestampImp2},
{"double", NULL, tsCompressDoubleImp2, tsDecompressDoubleImp2}, {"BIT-PACKING", NULL, tsCompressBoolImp2, tsDecompressBoolImp2},
{"bool", NULL, tsCompressBoolImp2, tsDecompressBoolImp2}, {"DELTAD", NULL, tsCompressDoubleImp2, tsDecompressDoubleImp2}};
{NULL, NULL, NULL}};
TCompressL2FnSet compressL2Dict[] = { TCompressL2FnSet compressL2Dict[] = {
{"unknown", l2ComressInitImpl_disabled, l2CompressImpl_disabled, l2DecompressImpl_disabled}, {"unknown", l2ComressInitImpl_disabled, l2CompressImpl_disabled, l2DecompressImpl_disabled},
{"lz4", l2ComressInitImpl_lz4, l2CompressImpl_lz4, l2DecompressImpl_lz4}, {"lz4", l2ComressInitImpl_lz4, l2CompressImpl_lz4, l2DecompressImpl_lz4},
@ -2940,13 +2940,13 @@ int32_t tsFindCompressAlg(int8_t dataType, uint8_t compress, TCompressL1FnSet *l
return 0; return 0;
} }
typedef struct { // typedef struct {
int8_t dtype; // int8_t dtype;
SArray *l1Set; // SArray *l1Set;
SArray *l2Set; // SArray *l2Set;
} TCompressCompatible; // } TCompressCompatible;
SHashObj *algSet = NULL; // SHashObj *algSet = NULL;
// int32_t tsCompressSetInit() { // int32_t tsCompressSetInit() {
// algSet = taosHashInit(24, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); // algSet = taosHashInit(24, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK);
@ -2968,38 +2968,38 @@ SHashObj *algSet = NULL;
// } // }
// return 0; // return 0;
// } // }
int32_t tsCompressSetDestroy() { // int32_t tsCompressSetDestroy() {
void *p = taosHashIterate(algSet, NULL); // void *p = taosHashIterate(algSet, NULL);
while (p) { // while (p) {
TCompressCompatible *v = p; // TCompressCompatible *v = p;
taosArrayDestroy(v->l1Set); // taosArrayDestroy(v->l1Set);
taosArrayDestroy(v->l2Set); // taosArrayDestroy(v->l2Set);
taosHashIterate(algSet, p); // taosHashIterate(algSet, p);
} // }
return 0; // return 0;
} // }
int32_t tsValidCompressAlgByDataTypes(int8_t type, int8_t compress) { // int32_t tsValidCompressAlgByDataTypes(int8_t type, int8_t compress) {
// compress alg // // compress alg
int8_t l1 = COMPRESS_L1_TYPE_U8(compress); // int8_t l1 = COMPRESS_L1_TYPE_U8(compress);
int8_t l2 = COMPRESS_L2_TYPE_U8(compress); // int8_t l2 = COMPRESS_L2_TYPE_U8(compress);
int8_t lvl = COMPRESS_L2_TYPE_LEVEL_U8(compress); // int8_t lvl = COMPRESS_L2_TYPE_LEVEL_U8(compress);
TCompressCompatible *p = taosHashGet(algSet, &type, sizeof(type)); // TCompressCompatible *p = taosHashGet(algSet, &type, sizeof(type));
if (p == NULL) return -1; // if (p == NULL) return -1;
if (p->dtype != type) return -1; // if (p->dtype != type) return -1;
if (taosArraySearch(p->l1Set, &l1, compareInt8Val, 0) == NULL) { // if (taosArraySearch(p->l1Set, &l1, compareInt8Val, 0) == NULL) {
return -1; // return -1;
} // }
if (taosArraySearch(p->l2Set, &l2, compareInt8Val, 0) == NULL) { // if (taosArraySearch(p->l2Set, &l2, compareInt8Val, 0) == NULL) {
return -1; // return -1;
} // }
return 0; // return 0;
} // }
int32_t tcompressDebug(uint32_t cmprAlg, uint8_t *l1Alg, uint8_t *l2Alg, uint8_t *level) { int32_t tcompressDebug(uint32_t cmprAlg, uint8_t *l1Alg, uint8_t *l2Alg, uint8_t *level) {
DEFINE_VAR(cmprAlg) DEFINE_VAR(cmprAlg)
@ -3038,3 +3038,9 @@ int8_t tUpdateCompress(uint32_t oldCmpr, uint32_t newCmpr, uint8_t l2Disabled, u
return 0; return 0;
} }
// int32_t validCompress(int8_t type, uint8_t encode, uint8_t compress, uint8_t level) {
// if (type == TSDB_DATA_TYPE_BOOL) {
// } else
// return 0;
// }