refactor code

This commit is contained in:
yihaoDeng 2024-03-25 09:49:28 +00:00
parent bba1b8d88a
commit 7aaef32304
3 changed files with 79 additions and 5 deletions

View File

@ -6617,6 +6617,7 @@ static int32_t buildAlterSuperTableReq(STranslateContext* pCxt, SAlterTableStmt*
setColCompressByOption((uint32_t*)&field.bytes, columnEncodeVal(pStmt->pColOptions->encode), setColCompressByOption((uint32_t*)&field.bytes, columnEncodeVal(pStmt->pColOptions->encode),
columnCompressVal(pStmt->pColOptions->compress), columnCompressVal(pStmt->pColOptions->compress),
columnLevelVal(pStmt->pColOptions->compressLevel)); columnLevelVal(pStmt->pColOptions->compressLevel));
taosArrayPush(pAlterReq->pFields, &field);
break; break;
} }
default: default:

View File

@ -2700,7 +2700,8 @@ int32_t tsDecompressTimestamp2(void *pIn, int32_t nIn, int32_t nEle, void *pOut,
// Float ===================================================== // Float =====================================================
int32_t tsCompressFloat2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg, void *pBuf, int32_t tsCompressFloat2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg, void *pBuf,
int32_t nBuf) { int32_t nBuf) {
if (lossyFloat) { DEFINE_VAR(cmprAlg)
if (lvl != 0 && lossyFloat) {
return tsCompressFloatLossyImp(pIn, nEle, pOut); return tsCompressFloatLossyImp(pIn, nEle, pOut);
} }
FUNC_COMPRESS_IMPL(pIn, nIn, nEle, pOut, nOut, cmprAlg, pBuf, nBuf, TSDB_DATA_TYPE_FLOAT, 1); FUNC_COMPRESS_IMPL(pIn, nIn, nEle, pOut, nOut, cmprAlg, pBuf, nBuf, TSDB_DATA_TYPE_FLOAT, 1);
@ -2708,7 +2709,8 @@ int32_t tsCompressFloat2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32
int32_t tsDecompressFloat2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg, void *pBuf, int32_t tsDecompressFloat2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg, void *pBuf,
int32_t nBuf) { int32_t nBuf) {
if (HEAD_ALGO(((uint8_t *)pIn)[0]) == ALGO_SZ_LOSSY) { DEFINE_VAR(cmprAlg)
if (lvl != 0 && HEAD_ALGO(((uint8_t *)pIn)[0]) == ALGO_SZ_LOSSY) {
return tsDecompressFloatLossyImp(pIn, nIn, nEle, pOut); return tsDecompressFloatLossyImp(pIn, nIn, nEle, pOut);
} }
FUNC_COMPRESS_IMPL(pIn, nIn, nEle, pOut, nOut, cmprAlg, pBuf, nBuf, TSDB_DATA_TYPE_FLOAT, 0); FUNC_COMPRESS_IMPL(pIn, nIn, nEle, pOut, nOut, cmprAlg, pBuf, nBuf, TSDB_DATA_TYPE_FLOAT, 0);
@ -2717,7 +2719,8 @@ int32_t tsDecompressFloat2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int
// Double ===================================================== // Double =====================================================
int32_t tsCompressDouble2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg, void *pBuf, int32_t tsCompressDouble2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg, void *pBuf,
int32_t nBuf) { int32_t nBuf) {
if (lossyDouble) { DEFINE_VAR(cmprAlg)
if (lvl != 0 && lossyDouble) {
// lossy mode // lossy mode
return tsCompressDoubleLossyImp(pIn, nEle, pOut); return tsCompressDoubleLossyImp(pIn, nEle, pOut);
} }
@ -2726,7 +2729,8 @@ int32_t tsCompressDouble2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int3
int32_t tsDecompressDouble2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg, int32_t tsDecompressDouble2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg,
void *pBuf, int32_t nBuf) { void *pBuf, int32_t nBuf) {
if (HEAD_ALGO(((uint8_t *)pIn)[0]) == ALGO_SZ_LOSSY) { DEFINE_VAR(cmprAlg)
if (lvl != 0 && HEAD_ALGO(((uint8_t *)pIn)[0]) == ALGO_SZ_LOSSY) {
// decompress lossy // decompress lossy
return tsDecompressDoubleLossyImp(pIn, nIn, nEle, pOut); return tsDecompressDoubleLossyImp(pIn, nIn, nEle, pOut);
} }

View File

@ -201,6 +201,44 @@ void* genCompressData(int32_t type, int32_t num, bool order) {
} }
return pBuf; return pBuf;
} }
void* genCompressData_float(int32_t type, int32_t num, bool order) {
tDataTypeDescriptor desc = tDataTypes[type];
int32_t cnt = num * (desc.bytes);
char* pBuf = (char*)taosMemoryCalloc(1, cnt);
char* p = pBuf;
uint32_t v = taosGetTimestampMs();
int64_t iniVal = 0;
for (int32_t i = 0; i < num; i++) {
int64_t d = taosRandR(&v);
if (type == TSDB_DATA_TYPE_FLOAT) {
float f = d * 1.0 / 3;
fillDataByData(p, &f, desc.bytes);
} else if (type == TSDB_DATA_TYPE_DOUBLE) {
double f = d * 1.0 / 3;
fillDataByData(p, &f, desc.bytes);
}
// if (type == TSDB_DATA_TYPE_BOOL) {
// int8_t val = d % 2;
// fillDataByData(p, &val, desc.bytes);
// } else if (type == TSDB_DATA_TYPE_TINYINT) {
// int8_t val = d % INT8_MAX;
// fillDataByData(p, &val, desc.bytes);
// } else if (type == TSDB_DATA_TYPE_SMALLINT) {
// int16_t val = d % INT8_MAX;
// fillDataByData(p, &val, desc.bytes);
// } else if (type == TSDB_DATA_TYPE_INT) {
// int32_t val = d % INT8_MAX;
// fillDataByData(p, &val, desc.bytes);
// } else if (type == TSDB_DATA_TYPE_BIGINT) {
// int64_t val = d % INT8_MAX;
// fillDataByData(p, &val, desc.bytes);
// }
p += desc.bytes;
}
return pBuf;
}
TEST(utilTest, compressAlg) { TEST(utilTest, compressAlg) {
int32_t num = 4096; int32_t num = 4096;
int64_t* pList = static_cast<int64_t*>(taosMemoryCalloc(num, sizeof(int64_t))); int64_t* pList = static_cast<int64_t*>(taosMemoryCalloc(num, sizeof(int64_t)));
@ -328,6 +366,7 @@ TEST(utilTest, compressAlg) {
printf("-------------"); printf("-------------");
} }
} }
// bool
for (int8_t type = 1; type <= 1; type++) { for (int8_t type = 1; type <= 1; type++) {
printf("------summary, type: %s-------\n", tDataTypes[type].name); printf("------summary, type: %s-------\n", tDataTypes[type].name);
char* p = (char*)genCompressData(type, num, 0); char* p = (char*)genCompressData(type, num, 0);
@ -346,4 +385,34 @@ TEST(utilTest, compressAlg) {
taosMemoryFree(p); taosMemoryFree(p);
printf("-------------"); printf("-------------");
} }
} // float/double
float fPresion = 1E-8;
double dPresion = 1E-16;
uint32_t maxRange = 500; // max quantization intervals
uint32_t curRange = 100; // current quantization intervals
bool ifAdtFse = false; // ADT-FSE algorithom or original huffman algorithom
char compressor[32] = "ZSTD_COMPRESSOR"; // ZSTD_COMPRESSOR or GZIP_COMPRESSOR
tsCompressInit((char*)"float|double", fPresion, dPresion, maxRange, curRange, ifAdtFse, compressor);
for (int8_t type = 6; type <= 7; type++) {
printf("------summary, type: %s-------\n", tDataTypes[type].name);
char* p = (char*)genCompressData_float(type, num, 0);
for (int8_t i = 1; i <= 3; i++) {
uint32_t cmprAlg = 0;
setColCompress(&cmprAlg, i);
setColEncode(&cmprAlg, 3);
setColLevel(&cmprAlg, 1);
compressImplTestByAlg(p, type, num, cmprAlg);
}
{
// uint32_t cmprAlg = 0;
// setColCompress(&cmprAlg, 4);
// setColEncode(&cmprAlg, 3);
// compressImplTestByAlg(p, type, num, cmprAlg);
// }
// taosMemoryFree(p);
// printf("-------------");
}
taosMemoryFree(p);
}
}