xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

refactor code

This commit is contained in:
yihaoDeng 2024-03-28 11:56:44 +00:00
parent 94e2aa2ed5
commit ee81a43f81
5 changed files with 163 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
include/common/tcol.h
View File

@ -17,10 +17,10 @@
#define _TD_TCOL_H_ #define _TD_TCOL_H_
#define TSDB_COLUMN_ENCODE_UNKNOWN "Unknown" #define TSDB_COLUMN_ENCODE_UNKNOWN "Unknown"
#define TSDB_COLUMN_ENCODE_SIMPLE8B "Simple8b" #define TSDB_COLUMN_ENCODE_SIMPLE8B "simple8b"
#define TSDB_COLUMN_ENCODE_XOR "DeltaI" #define TSDB_COLUMN_ENCODE_XOR "delta-i"
#define TSDB_COLUMN_ENCODE_RLE "Bit-Packing" #define TSDB_COLUMN_ENCODE_RLE "bit-packing"
#define TSDB_COLUMN_ENCODE_DELTAD "DeltaD" #define TSDB_COLUMN_ENCODE_DELTAD "delta-d"
#define TSDB_COLUMN_ENCODE_DISABLED "disabled" #define TSDB_COLUMN_ENCODE_DISABLED "disabled"
#define TSDB_COLUMN_COMPRESS_UNKNOWN "Unknown" #define TSDB_COLUMN_COMPRESS_UNKNOWN "Unknown"
@ -41,7 +41,6 @@
#define TSDB_COLVAL_ENCODE_XOR 2 #define TSDB_COLVAL_ENCODE_XOR 2
#define TSDB_COLVAL_ENCODE_RLE 3 #define TSDB_COLVAL_ENCODE_RLE 3
#define TSDB_COLVAL_ENCODE_DELTAD 4 #define TSDB_COLVAL_ENCODE_DELTAD 4
#define TSDB_COLVAL_ENCODE_DISABLED 0xff #define TSDB_COLVAL_ENCODE_DISABLED 0xff
#define TSDB_COLVAL_COMPRESS_NOCHANGE 0 #define TSDB_COLVAL_COMPRESS_NOCHANGE 0
@ -53,9 +52,9 @@
#define TSDB_COLVAL_COMPRESS_DISABLED 0xff #define TSDB_COLVAL_COMPRESS_DISABLED 0xff
#define TSDB_COLVAL_LEVEL_NOCHANGE 0 #define TSDB_COLVAL_LEVEL_NOCHANGE 0
#define TSDB_COLVAL_LEVEL_HIGH 1 #define TSDB_COLVAL_LEVEL_LOW 1
#define TSDB_COLVAL_LEVEL_MEDIUM 2 #define TSDB_COLVAL_LEVEL_MEDIUM 2
#define TSDB_COLVAL_LEVEL_LOW 3 #define TSDB_COLVAL_LEVEL_HIGH 3
#define TSDB_COLVAL_LEVEL_DISABLED 0xff #define TSDB_COLVAL_LEVEL_DISABLED 0xff
#define TSDB_CL_COMMENT_LEN 1025 #define TSDB_CL_COMMENT_LEN 1025
@ -90,6 +89,9 @@ bool checkColumnLevelOrSetDefault(uint8_t type, char level[TSDB_CL_COMPRESS_OPTI
void setColEncode(uint32_t* compress, uint8_t encode); void setColEncode(uint32_t* compress, uint8_t encode);
void setColCompress(uint32_t* compress, uint16_t compressType); void setColCompress(uint32_t* compress, uint16_t compressType);
void setColLevel(uint32_t* compress, uint8_t level); void setColLevel(uint32_t* compress, uint8_t level);
void setColCompressByOption(uint32_t* compress, uint8_t encode, uint16_t compressType, uint8_t level); int8_t setColCompressByOption(uint8_t type, uint32_t* compress, uint8_t encode, uint16_t compressType, uint8_t level);
int8_t validColCompressLevel(uint8_t type, uint8_t level);
int8_t validColCompress(uint8_t type, uint8_t l2);
int8_t validColEncode(uint8_t type, uint8_t l1);
#endif /*_TD_TCOL_H_*/ #endif /*_TD_TCOL_H_*/

1
include/util/tcompression.h
View File

@ -260,6 +260,7 @@ typedef enum L1Compress {
L1_SIMPLE_8B, L1_SIMPLE_8B,
L1_XOR, L1_XOR,
L1_RLE, L1_RLE,
L1_DELTAD,
L1_DISABLED = 0xFF, L1_DISABLED = 0xFF,
} EL1CompressFuncType; } EL1CompressFuncType;

76
source/common/src/tcol.c
View File

@ -43,17 +43,27 @@ uint8_t getDefaultEncode(uint8_t type) {
case TSDB_DATA_TYPE_DOUBLE: case TSDB_DATA_TYPE_DOUBLE:
return TSDB_COLVAL_ENCODE_DELTAD; return TSDB_COLVAL_ENCODE_DELTAD;
case TSDB_DATA_TYPE_VARCHAR: // TSDB_DATA_TYPE_BINARY case TSDB_DATA_TYPE_VARCHAR: // TSDB_DATA_TYPE_BINARY
return return TSDB_COLVAL_ENCODE_SIMPLE8B;
case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_TIMESTAMP:
return TSDB_COLVAL_ENCODE_XOR;
case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_NCHAR:
return TSDB_COLVAL_ENCODE_SIMPLE8B;
case TSDB_DATA_TYPE_UTINYINT: case TSDB_DATA_TYPE_UTINYINT:
return TSDB_COLVAL_ENCODE_SIMPLE8B;
case TSDB_DATA_TYPE_USMALLINT: case TSDB_DATA_TYPE_USMALLINT:
return TSDB_COLVAL_ENCODE_SIMPLE8B;
case TSDB_DATA_TYPE_UINT: case TSDB_DATA_TYPE_UINT:
return TSDB_COLVAL_ENCODE_SIMPLE8B;
case TSDB_DATA_TYPE_UBIGINT: case TSDB_DATA_TYPE_UBIGINT:
return TSDB_COLVAL_ENCODE_SIMPLE8B;
case TSDB_DATA_TYPE_JSON: case TSDB_DATA_TYPE_JSON:
return TSDB_COLVAL_ENCODE_SIMPLE8B;
case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_VARBINARY:
return TSDB_COLVAL_ENCODE_SIMPLE8B;
case TSDB_DATA_TYPE_DECIMAL: case TSDB_DATA_TYPE_DECIMAL:
return TSDB_COLVAL_ENCODE_DELTAD;
case TSDB_DATA_TYPE_BLOB: case TSDB_DATA_TYPE_BLOB:
return TSDB_COLVAL_ENCODE_SIMPLE8B;
case TSDB_DATA_TYPE_MEDIUMBLOB: case TSDB_DATA_TYPE_MEDIUMBLOB:
case TSDB_DATA_TYPE_GEOMETRY: case TSDB_DATA_TYPE_GEOMETRY:
case TSDB_DATA_TYPE_MAX: case TSDB_DATA_TYPE_MAX:
@ -90,7 +100,6 @@ uint16_t getDefaultCompress(uint8_t type) {
case TSDB_DATA_TYPE_GEOMETRY: case TSDB_DATA_TYPE_GEOMETRY:
case TSDB_DATA_TYPE_MAX: case TSDB_DATA_TYPE_MAX:
return TSDB_COLVAL_COMPRESS_LZ4; return TSDB_COLVAL_COMPRESS_LZ4;
default: default:
return TSDB_COLVAL_COMPRESS_LZ4; return TSDB_COLVAL_COMPRESS_LZ4;
} }
@ -294,34 +303,75 @@ void setColLevel(uint32_t* compress, uint8_t level) {
return; return;
} }
void setColCompressByOption(uint32_t* compress, uint8_t encode, uint16_t compressType, uint8_t level) { int8_t setColCompressByOption(uint8_t type, uint32_t* compress, uint8_t encode, uint16_t compressType, uint8_t level) {
if (!validColEncode(type, encode)) return 0;
setColEncode(compress, encode); setColEncode(compress, encode);
if (compressType == TSDB_COLVAL_COMPRESS_DISABLED) { if (compressType == TSDB_COLVAL_COMPRESS_DISABLED) {
setColCompress(compress, compressType); setColCompress(compress, compressType);
setColLevel(compress, TSDB_COLVAL_LEVEL_DISABLED); setColLevel(compress, TSDB_COLVAL_LEVEL_DISABLED);
} else { } else {
if (!validColCompress(type, compressType)) return 0;
setColCompress(compress, compressType); setColCompress(compress, compressType);
if (!validColCompressLevel(type, level)) return 0;
setColLevel(compress, level); setColLevel(compress, level);
} }
return; return 1;
} }
bool useCompress(uint8_t tableType) { return TSDB_SUPER_TABLE == tableType || TSDB_NORMAL_TABLE == tableType; } bool useCompress(uint8_t tableType) { return TSDB_SUPER_TABLE == tableType || TSDB_NORMAL_TABLE == tableType; }
int8_t validColCompressOption(uint8_t type, uint8_t encode, uint8_t compress, uint8_t level) { int8_t validColCompressLevel(uint8_t type, uint8_t level) {
if (level < TSDB_COLVAL_LEVEL_HIGH || level > TSDB_COLVAL_LEVEL_LOW) { if (level < TSDB_COLVAL_LEVEL_LOW || level > TSDB_COLVAL_LEVEL_HIGH) {
return 0; return 0;
} }
if (type == TSDB_DATA_TYPE_BOOL) {
} else if (type >= TSDB_DATA_TYPE_TINYINT && type <= TSDB_DATA_TYPE_BIGINT) {
} else if (type >= TSDB_DATA_TYPE_FLOAT && type <= TSDB_DATA_TYPE_DOUBLE) {
if (compress == TSDB_COLVAL_COMPRESS_TSZ) {
return 1; return 1;
} }
} else if (type == TSDB_DATA_TYPE_VARCHAR && type == TSDB_DATA_TYPE_NCHAR) { int8_t validColCompress(uint8_t type, uint8_t l2) {
// if (type == TSDB_DATA_TYPE_DOUBLE || type == TSDB_DATA_TYPE_FLOAT) {
// if (l2 != TSDB_COLVAL_COMPRESS_TSZ) {
// return 0;
// }
// }
if (l2 <= TSDB_COLVAL_COMPRESS_NOCHANGE || l2 >= TSDB_COLVAL_COMPRESS_DISABLED) {
return TSDB_COLVAL_COMPRESS_DISABLED == l2 ? 1 : 0;
}
return 1;
}
//
// | --------type----------|----- supported encode ----|
// |tinyint/smallint/int/bigint/utinyint/usmallinit/uint/ubiginint| simple8b |
// | timestamp/bigint/ubigint | delta-i |
// | bool | bit-packing |
// | flout/double | delta-d |
//
int8_t validColEncode(uint8_t type, uint8_t l1) {
if (type == TSDB_DATA_TYPE_BOOL) {
return TSDB_COLVAL_ENCODE_RLE == l1 ? 1 : 0;
} else if (type >= TSDB_DATA_TYPE_TINYINT && type <= TSDB_DATA_TYPE_INT) {
return TSDB_COLVAL_ENCODE_SIMPLE8B == l1 ? 1 : 0;
} else if (type == TSDB_DATA_TYPE_BIGINT) {
return TSDB_COLVAL_ENCODE_SIMPLE8B == l1 || TSDB_COLVAL_ENCODE_XOR == l1 ? 1 : 0;
} else if (type >= TSDB_DATA_TYPE_FLOAT && type <= TSDB_DATA_TYPE_DOUBLE) {
return TSDB_COLVAL_ENCODE_DELTAD == l1 ? 1 : 0;
} else if ((type == TSDB_DATA_TYPE_VARCHAR && type == TSDB_DATA_TYPE_NCHAR) || type == TSDB_DATA_TYPE_JSON ||
type == TSDB_DATA_TYPE_VARBINARY) {
if (l1 >= TSDB_COLVAL_ENCODE_NOCHANGE || l1 <= TSDB_COLVAL_ENCODE_DELTAD) {
return 1;
} else if (l1 == TSDB_COLVAL_ENCODE_DISABLED) {
return 1;
} else {
return 0;
}
} else if (type == TSDB_DATA_TYPE_TIMESTAMP) { } else if (type == TSDB_DATA_TYPE_TIMESTAMP) {
} else if (type >= TSDB_DATA_TYPE_USMALLINT || type <= TSDB_DATA_TYPE_UBIGINT) { return TSDB_COLVAL_ENCODE_XOR == l1 ? 1 : 0;
} else if (type >= TSDB_DATA_TYPE_USMALLINT || type <= TSDB_DATA_TYPE_UINT) {
return TSDB_COLVAL_ENCODE_SIMPLE8B == l1 ? 1 : 0;
} else if (type == TSDB_DATA_TYPE_UBIGINT) {
return TSDB_COLVAL_ENCODE_SIMPLE8B == l1 || TSDB_COLVAL_ENCODE_XOR == l1 ? 1 : 0;
} else if (type == TSDB_DATA_TYPE_JSON || type == TSDB_DATA_TYPE_VARBINARY) { } else if (type == TSDB_DATA_TYPE_JSON || type == TSDB_DATA_TYPE_VARBINARY) {
} }
return 0; return 0;

28
source/libs/parser/src/parTranslater.c
View File

@ -5769,9 +5769,11 @@ static int32_t checkColumnOptions(SNodeList* pList) {
FOREACH(pNode, pList) { FOREACH(pNode, pList) {
SColumnDefNode* pCol = (SColumnDefNode*)pNode; SColumnDefNode* pCol = (SColumnDefNode*)pNode;
if (!pCol->pOptions) return TSDB_CODE_TSC_ENCODE_PARAM_NULL; if (!pCol->pOptions) return TSDB_CODE_TSC_ENCODE_PARAM_NULL;
if (!checkColumnEncodeOrSetDefault(pCol->type, pCol->pOptions->encode)) return TSDB_CODE_TSC_ENCODE_PARAM_ERROR; if (!checkColumnEncodeOrSetDefault(pCol->dataType.type, pCol->pOptions->encode))
if (!checkColumnCompressOrSetDefault(pCol->type, pCol->pOptions->compress)) return TSDB_CODE_TSC_ENCODE_PARAM_ERROR; return TSDB_CODE_TSC_ENCODE_PARAM_ERROR;
if (!checkColumnLevelOrSetDefault(pCol->type, pCol->pOptions->compressLevel)) if (!checkColumnCompressOrSetDefault(pCol->dataType.type, pCol->pOptions->compress))
return TSDB_CODE_TSC_ENCODE_PARAM_ERROR;
if (!checkColumnLevelOrSetDefault(pCol->dataType.type, pCol->pOptions->compressLevel))
return TSDB_CODE_TSC_ENCODE_PARAM_ERROR; return TSDB_CODE_TSC_ENCODE_PARAM_ERROR;
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -6614,9 +6616,10 @@ static int32_t buildAlterSuperTableReq(STranslateContext* pCxt, SAlterTableStmt*
if (!checkColumnEncode(pStmt->pColOptions->encode)) return TSDB_CODE_TSC_ENCODE_PARAM_ERROR; if (!checkColumnEncode(pStmt->pColOptions->encode)) return TSDB_CODE_TSC_ENCODE_PARAM_ERROR;
if (!checkColumnCompress(pStmt->pColOptions->compress)) return TSDB_CODE_TSC_ENCODE_PARAM_ERROR; if (!checkColumnCompress(pStmt->pColOptions->compress)) return TSDB_CODE_TSC_ENCODE_PARAM_ERROR;
if (!checkColumnLevel(pStmt->pColOptions->compressLevel)) return TSDB_CODE_TSC_ENCODE_PARAM_ERROR; if (!checkColumnLevel(pStmt->pColOptions->compressLevel)) return TSDB_CODE_TSC_ENCODE_PARAM_ERROR;
setColCompressByOption((uint32_t*)&field.bytes, columnEncodeVal(pStmt->pColOptions->encode), int8_t valid = setColCompressByOption(
columnCompressVal(pStmt->pColOptions->compress), pStmt->dataType.type, (uint32_t*)&field.bytes, columnEncodeVal(pStmt->pColOptions->encode),
columnLevelVal(pStmt->pColOptions->compressLevel)); columnCompressVal(pStmt->pColOptions->compress), columnLevelVal(pStmt->pColOptions->compressLevel));
if (!valid) return TSDB_CODE_TSC_ENCODE_PARAM_ERROR;
taosArrayPush(pAlterReq->pFields, &field); taosArrayPush(pAlterReq->pFields, &field);
break; break;
} }
@ -9912,12 +9915,14 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt*
tInitDefaultSColCmprWrapperByCols(&req.colCmpr, req.ntb.schemaRow.nCols); tInitDefaultSColCmprWrapperByCols(&req.colCmpr, req.ntb.schemaRow.nCols);
FOREACH(pCol, pStmt->pCols) { FOREACH(pCol, pStmt->pCols) {
SColumnDefNode* pColDef = (SColumnDefNode*)pCol; SColumnDefNode* pColDef = (SColumnDefNode*)pCol;
toSchema(pColDef, index + 1, req.ntb.schemaRow.pSchema + index); SSchema* pScheam = req.ntb.schemaRow.pSchema + index;
toSchema(pColDef, index + 1, pScheam);
if (pColDef->pOptions) { if (pColDef->pOptions) {
req.colCmpr.pColCmpr[index].id = index + 1; req.colCmpr.pColCmpr[index].id = index + 1;
setColCompressByOption(&req.colCmpr.pColCmpr[index].alg, columnEncodeVal(pColDef->pOptions->encode), int8_t valid = setColCompressByOption(
columnCompressVal(pColDef->pOptions->compress), pScheam->type, &req.colCmpr.pColCmpr[index].alg, columnEncodeVal(pColDef->pOptions->encode),
columnLevelVal(pColDef->pOptions->compressLevel)); columnCompressVal(pColDef->pOptions->compress), columnLevelVal(pColDef->pOptions->compressLevel));
if (!valid) return TSDB_CODE_TSC_ENCODE_PARAM_ERROR;
} }
++index; ++index;
} }
@ -10743,9 +10748,10 @@ static int buildAlterTableColumnCompress(STranslateContext* pCxt, SAlterTableStm
if (!checkColumnEncode(pStmt->pColOptions->encode)) return TSDB_CODE_TSC_ENCODE_PARAM_ERROR; if (!checkColumnEncode(pStmt->pColOptions->encode)) return TSDB_CODE_TSC_ENCODE_PARAM_ERROR;
if (!checkColumnCompress(pStmt->pColOptions->compress)) return TSDB_CODE_TSC_ENCODE_PARAM_ERROR; if (!checkColumnCompress(pStmt->pColOptions->compress)) return TSDB_CODE_TSC_ENCODE_PARAM_ERROR;
if (!checkColumnLevel(pStmt->pColOptions->compressLevel)) return TSDB_CODE_TSC_ENCODE_PARAM_ERROR; if (!checkColumnLevel(pStmt->pColOptions->compressLevel)) return TSDB_CODE_TSC_ENCODE_PARAM_ERROR;
setColCompressByOption(&pReq->compress, columnEncodeVal(pStmt->pColOptions->encode), int8_t valid = setColCompressByOption(pSchema->type, &pReq->compress, columnEncodeVal(pStmt->pColOptions->encode),
columnCompressVal(pStmt->pColOptions->compress), columnCompressVal(pStmt->pColOptions->compress),
columnLevelVal(pStmt->pColOptions->compressLevel)); columnLevelVal(pStmt->pColOptions->compressLevel));
if (!valid) return TSDB_CODE_TSC_ENCODE_PARAM_ERROR;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }

146
source/util/src/tcompression.c
View File

@ -63,8 +63,8 @@
#include "td_sz.h" #include "td_sz.h"
#endif #endif
int32_t tsCompressUnknow2(const char *const input, const int32_t nelements, char *const output, const char type); int32_t tsCompressPlain2(const char *const input, const int32_t nelements, char *const output, const char type);
int32_t tsDecompressUnknow2(const char *const input, const int32_t nelements, char *const output, const char type); int32_t tsDecompressPlain2(const char *const input, const int32_t nelements, char *const output, const char type);
// delta // delta
int32_t tsCompressTimestampImp2(const char *const input, const int32_t nelements, char *const output, const char type); int32_t tsCompressTimestampImp2(const char *const input, const int32_t nelements, char *const output, const char type);
@ -266,7 +266,7 @@ int32_t l2DecompressImpl_xz(const char *const input, const int32_t compressedSiz
return -1; return -1;
} }
TCompressL1FnSet compressL1Dict[] = {{"PLAIN", NULL, tsCompressUnknow2, tsDecompressUnknow2}, TCompressL1FnSet compressL1Dict[] = {{"PLAIN", NULL, tsCompressPlain2, tsDecompressPlain2},
{"SIMPLE-8B", NULL, tsCompressINTImp2, tsDecompressINTImp2}, {"SIMPLE-8B", NULL, tsCompressINTImp2, tsDecompressINTImp2},
{"DELTAI", NULL, tsCompressTimestampImp2, tsDecompressTimestampImp2}, {"DELTAI", NULL, tsCompressTimestampImp2, tsDecompressTimestampImp2},
{"BIT-PACKING", NULL, tsCompressBoolImp2, tsDecompressBoolImp2}, {"BIT-PACKING", NULL, tsCompressBoolImp2, tsDecompressBoolImp2},
@ -858,13 +858,13 @@ int32_t tsDecompressTimestampImp(const char *const input, const int32_t nelement
return nelements * longBytes; return nelements * longBytes;
} }
int32_t tsCompressUnknow2(const char *const input, const int32_t nelements, char *const output, const char type) { int32_t tsCompressPlain2(const char *const input, const int32_t nelements, char *const output, const char type) {
int32_t bytes = tDataTypes[type].type * nelements; int32_t bytes = tDataTypes[type].type * nelements;
output[0] = 0; output[0] = 0;
memcpy(output + 1, input, bytes); memcpy(output + 1, input, bytes);
return bytes + 1; return bytes + 1;
} }
int32_t tsDecompressUnknow2(const char *const input, const int32_t nelements, char *const output, const char type) { int32_t tsDecompressPlain2(const char *const input, const int32_t nelements, char *const output, const char type) {
int32_t bytes = tDataTypes[type].bytes * nelements; int32_t bytes = tDataTypes[type].bytes * nelements;
memcpy(output, input + 1, bytes); memcpy(output, input + 1, bytes);
return bytes; return bytes;
@ -2787,8 +2787,8 @@ int32_t tsCompressBool2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_
int32_t nBuf) { int32_t nBuf) {
uint32_t tCmprAlg = 0; uint32_t tCmprAlg = 0;
DEFINE_VAR(cmprAlg) DEFINE_VAR(cmprAlg)
if (l1 != 4) { if (l1 != L1_RLE) {
SET_COMPRESS(4, l2, lvl, tCmprAlg); SET_COMPRESS(L1_RLE, l2, lvl, tCmprAlg);
} }
FUNC_COMPRESS_IMPL(pIn, nIn, nEle, pOut, nOut, tCmprAlg, pBuf, nBuf, TSDB_DATA_TYPE_BOOL, 1); FUNC_COMPRESS_IMPL(pIn, nIn, nEle, pOut, nOut, tCmprAlg, pBuf, nBuf, TSDB_DATA_TYPE_BOOL, 1);
} }
@ -2797,8 +2797,8 @@ int32_t tsDecompressBool2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int3
int32_t nBuf) { int32_t nBuf) {
uint32_t tCmprAlg = 0; uint32_t tCmprAlg = 0;
DEFINE_VAR(cmprAlg) DEFINE_VAR(cmprAlg)
if (l1 != 4) { if (l1 != L1_RLE) {
SET_COMPRESS(4, l2, lvl, tCmprAlg); SET_COMPRESS(L1_RLE, l2, lvl, tCmprAlg);
} }
FUNC_COMPRESS_IMPL(pIn, nIn, nEle, pOut, nOut, tCmprAlg, pBuf, nBuf, TSDB_DATA_TYPE_BOOL, 0); FUNC_COMPRESS_IMPL(pIn, nIn, nEle, pOut, nOut, tCmprAlg, pBuf, nBuf, TSDB_DATA_TYPE_BOOL, 0);
} }
@ -2808,8 +2808,8 @@ int32_t tsCompressTinyint2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int
int32_t nBuf) { int32_t nBuf) {
uint32_t tCmprAlg = 0; uint32_t tCmprAlg = 0;
DEFINE_VAR(cmprAlg) DEFINE_VAR(cmprAlg)
if (l1 != L1_XOR) { if (l1 != L1_SIMPLE_8B) {
SET_COMPRESS(L1_XOR, l2, lvl, tCmprAlg); SET_COMPRESS(L1_SIMPLE_8B, l2, lvl, tCmprAlg);
} }
FUNC_COMPRESS_IMPL(pIn, nIn, nEle, pOut, nOut, tCmprAlg, pBuf, nBuf, TSDB_DATA_TYPE_TINYINT, 1); FUNC_COMPRESS_IMPL(pIn, nIn, nEle, pOut, nOut, tCmprAlg, pBuf, nBuf, TSDB_DATA_TYPE_TINYINT, 1);
} }
@ -2818,8 +2818,8 @@ int32_t tsDecompressTinyint2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, i
void *pBuf, int32_t nBuf) { void *pBuf, int32_t nBuf) {
uint32_t tCmprAlg = 0; uint32_t tCmprAlg = 0;
DEFINE_VAR(cmprAlg) DEFINE_VAR(cmprAlg)
if (l1 != L1_XOR) { if (l1 != L1_SIMPLE_8B) {
SET_COMPRESS(L1_XOR, l2, lvl, tCmprAlg); SET_COMPRESS(L1_SIMPLE_8B, l2, lvl, tCmprAlg);
} }
FUNC_COMPRESS_IMPL(pIn, nIn, nEle, pOut, nOut, tCmprAlg, pBuf, nBuf, TSDB_DATA_TYPE_TINYINT, 0); FUNC_COMPRESS_IMPL(pIn, nIn, nEle, pOut, nOut, tCmprAlg, pBuf, nBuf, TSDB_DATA_TYPE_TINYINT, 0);
} }
@ -2829,8 +2829,8 @@ int32_t tsCompressSmallint2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, in
void *pBuf, int32_t nBuf) { void *pBuf, int32_t nBuf) {
uint32_t tCmprAlg = 0; uint32_t tCmprAlg = 0;
DEFINE_VAR(cmprAlg) DEFINE_VAR(cmprAlg)
if (l1 != L1_XOR) { if (l1 != L1_SIMPLE_8B) {
SET_COMPRESS(L1_XOR, l2, lvl, tCmprAlg); SET_COMPRESS(L1_SIMPLE_8B, l2, lvl, tCmprAlg);
} }
FUNC_COMPRESS_IMPL(pIn, nIn, nEle, pOut, nOut, tCmprAlg, pBuf, nBuf, TSDB_DATA_TYPE_SMALLINT, 1); FUNC_COMPRESS_IMPL(pIn, nIn, nEle, pOut, nOut, tCmprAlg, pBuf, nBuf, TSDB_DATA_TYPE_SMALLINT, 1);
} }
@ -2839,8 +2839,8 @@ int32_t tsDecompressSmallint2(void *pIn, int32_t nIn, int32_t nEle, void *pOut,
void *pBuf, int32_t nBuf) { void *pBuf, int32_t nBuf) {
uint32_t tCmprAlg = 0; uint32_t tCmprAlg = 0;
DEFINE_VAR(cmprAlg) DEFINE_VAR(cmprAlg)
if (l1 != L1_XOR) { if (l1 != L1_SIMPLE_8B) {
SET_COMPRESS(L1_XOR, l2, lvl, tCmprAlg); SET_COMPRESS(L1_SIMPLE_8B, l2, lvl, tCmprAlg);
} }
FUNC_COMPRESS_IMPL(pIn, nIn, nEle, pOut, nOut, tCmprAlg, pBuf, nBuf, TSDB_DATA_TYPE_SMALLINT, 0); FUNC_COMPRESS_IMPL(pIn, nIn, nEle, pOut, nOut, tCmprAlg, pBuf, nBuf, TSDB_DATA_TYPE_SMALLINT, 0);
} }
@ -2851,8 +2851,8 @@ int32_t tsCompressInt2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t
uint32_t tCmprAlg = 0; uint32_t tCmprAlg = 0;
{ {
DEFINE_VAR(cmprAlg) DEFINE_VAR(cmprAlg)
if (l1 != L1_XOR) { if (l1 != L1_SIMPLE_8B) {
SET_COMPRESS(L1_XOR, l2, lvl, tCmprAlg); SET_COMPRESS(L1_SIMPLE_8B, l2, lvl, tCmprAlg);
} }
} }
FUNC_COMPRESS_IMPL(pIn, nIn, nEle, pOut, nOut, tCmprAlg, pBuf, nBuf, TSDB_DATA_TYPE_INT, 1); FUNC_COMPRESS_IMPL(pIn, nIn, nEle, pOut, nOut, tCmprAlg, pBuf, nBuf, TSDB_DATA_TYPE_INT, 1);
@ -2863,8 +2863,8 @@ int32_t tsDecompressInt2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32
uint32_t tCmprAlg = 0; uint32_t tCmprAlg = 0;
{ {
DEFINE_VAR(cmprAlg) DEFINE_VAR(cmprAlg)
if (l1 != L1_XOR) { if (l1 != L1_SIMPLE_8B) {
SET_COMPRESS(L1_XOR, l2, lvl, tCmprAlg); SET_COMPRESS(L1_SIMPLE_8B, l2, lvl, tCmprAlg);
} }
} }
FUNC_COMPRESS_IMPL(pIn, nIn, nEle, pOut, nOut, tCmprAlg, pBuf, nBuf, TSDB_DATA_TYPE_INT, 0); FUNC_COMPRESS_IMPL(pIn, nIn, nEle, pOut, nOut, tCmprAlg, pBuf, nBuf, TSDB_DATA_TYPE_INT, 0);
@ -2881,64 +2881,65 @@ int32_t tsDecompressBigint2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, in
FUNC_COMPRESS_IMPL(pIn, nIn, nEle, pOut, nOut, cmprAlg, pBuf, nBuf, TSDB_DATA_TYPE_BIGINT, 0); FUNC_COMPRESS_IMPL(pIn, nIn, nEle, pOut, nOut, cmprAlg, pBuf, nBuf, TSDB_DATA_TYPE_BIGINT, 0);
} }
int32_t tsFindCompressAlg(int8_t dataType, uint8_t compress, TCompressL1FnSet *l1Fn, TCompressL2FnSet *l2Fn); // int32_t tsFindCompressAlg(int8_t dataType, uint8_t compress, TCompressL1FnSet *l1Fn, TCompressL2FnSet *l2Fn);
int32_t tsCompressImpl(int8_t type, void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, // int32_t tsCompressImpl(int8_t type, void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg,
void *pBuf, int32_t nBuf) { // void *pBuf, int32_t nBuf) {
TCompressL1FnSet fn1; // TCompressL1FnSet fn1;
TCompressL2FnSet fn2; // TCompressL2FnSet fn2;
if (tsFindCompressAlg(type, cmprAlg, &fn1, &fn2)) return -1; // if (tsFindCompressAlg(type, cmprAlg, &fn1, &fn2)) return -1;
int32_t len = 0; // int32_t len = 0;
uint8_t l1 = COMPRESS_L1_TYPE_U8(cmprAlg); // uint8_t l1 = COMPRESS_L1_TYPE_U8(cmprAlg);
uint8_t l2 = COMPRESS_L2_TYPE_U8(cmprAlg); // uint8_t l2 = COMPRESS_L2_TYPE_U8(cmprAlg);
uint8_t lvl = COMPRESS_L2_TYPE_LEVEL_U8(cmprAlg); // uint8_t lvl = COMPRESS_L2_TYPE_LEVEL_U8(cmprAlg);
if (l2 == L2_DISABLED) { // if (l2 == L2_DISABLED) {
len = fn1.comprFn(pIn, nEle, pOut, type); // len = fn1.comprFn(pIn, nEle, pOut, type);
} else { // } else {
len = fn1.comprFn(pIn, nEle, pBuf, type); // len = fn1.comprFn(pIn, nEle, pBuf, type);
len = fn2.comprFn(pBuf, len, pOut, nOut, type, lvl); // len = fn2.comprFn(pBuf, len, pOut, nOut, type, lvl);
} // }
return len; // return len;
} // }
int32_t tsDecompressImpl(int8_t type, void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, // int32_t tsDecompressImpl(int8_t type, void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t
void *pBuf, int32_t nBuf) { // cmprAlg,
TCompressL1FnSet fn1; // void *pBuf, int32_t nBuf) {
TCompressL2FnSet fn2; // TCompressL1FnSet fn1;
// TCompressL2FnSet fn2;
if (tsFindCompressAlg(type, cmprAlg, &fn1, &fn2) != 0) return -1; // if (tsFindCompressAlg(type, cmprAlg, &fn1, &fn2) != 0) return -1;
int8_t l1 = COMPRESS_L1_TYPE_U8(cmprAlg); // uint8_t l1 = COMPRESS_L1_TYPE_U8(cmprAlg);
int8_t l2 = COMPRESS_L2_TYPE_U8(cmprAlg); // uint8_t l2 = COMPRESS_L2_TYPE_U8(cmprAlg);
int8_t lvl = COMPRESS_L2_TYPE_LEVEL_U8(cmprAlg); // uint8_t lvl = COMPRESS_L2_TYPE_LEVEL_U8(cmprAlg);
int32_t len = 0; // uint32_t len = 0;
if (l2 == L2_DISABLED) { // if (l2 == L2_DISABLED) {
len = fn1.decomprFn(pIn, nEle, pOut, type); // len = fn1.decomprFn(pIn, nEle, pOut, type);
} else { // } else {
len = fn2.decomprFn(pIn, nIn, pBuf, nBuf, type); // len = fn2.decomprFn(pIn, nIn, pBuf, nBuf, type);
if (len < 0) return -1; // if (len < 0) return -1;
len = fn1.decomprFn(pBuf, nEle, pOut, type); // len = fn1.decomprFn(pBuf, nEle, pOut, type);
} // }
return len; // return len;
} // }
int32_t tsFindCompressAlg(int8_t dataType, uint8_t compress, TCompressL1FnSet *l1Fn, TCompressL2FnSet *l2Fn) { // int32_t tsFindCompressAlg(int8_t dataType, uint8_t compress, TCompressL1FnSet *l1Fn, TCompressL2FnSet *l2Fn) {
int8_t l1 = COMPRESS_L1_TYPE_U8(compress); // uint8_t l1 = COMPRESS_L1_TYPE_U8(compress);
int8_t l2 = COMPRESS_L2_TYPE_U8(compress); // uint8_t l2 = COMPRESS_L2_TYPE_U8(compress);
int8_t lvl = COMPRESS_L2_TYPE_LEVEL_U8(compress); // uint8_t lvl = COMPRESS_L2_TYPE_LEVEL_U8(compress);
static int32_t l1Sz = sizeof(compressL1Dict) / sizeof(compressL1Dict[0]); // static int32_t l1Sz = sizeof(compressL1Dict) / sizeof(compressL1Dict[0]);
if (l1 >= l1Sz) return -1; // if (l1 >= l1Sz) return -1;
static int32_t l2Sz = sizeof(compressL2Dict) / sizeof(compressL2Dict[0]); // static int32_t l2Sz = sizeof(compressL2Dict) / sizeof(compressL2Dict[0]);
if (l2 >= l2Sz) return -1; // if (l2 >= l2Sz) return -1;
*l1Fn = compressL1Dict[l1]; // *l1Fn = compressL1Dict[l1];
*l2Fn = compressL2Dict[l2]; // *l2Fn = compressL2Dict[l2];
return 0; // return 0;
} // }
// typedef struct { // typedef struct {
// int8_t dtype; // int8_t dtype;
@ -3035,12 +3036,5 @@ int8_t tUpdateCompress(uint32_t oldCmpr, uint32_t newCmpr, uint8_t l2Disabled, u
SET_COMPRESS(ol1, ol2, nlvl, *dst); SET_COMPRESS(ol1, ol2, nlvl, *dst);
return 1; return 1;
} }
return 0; return 0;
} }
// int32_t validCompress(int8_t type, uint8_t encode, uint8_t compress, uint8_t level) {
// if (type == TSDB_DATA_TYPE_BOOL) {
// } else
// return 0;
// }