xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

add sz config to taos.cfg

This commit is contained in:
tickduan 2021-07-07 20:39:49 +08:00
parent f204a0d350
commit ed84111382
18 changed files with 300 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
deps/SZ/sz/include/conf.h vendored
View File

@ -16,6 +16,11 @@ extern "C" {
#include <stdio.h>
//
// set default value
//
void setDefaulParams(sz_exedata* exedata, sz_params* params);
//conf.c
void updateQuantizationInfo(int quant_intervals);
int SZ_ReadConf(const char* sz_cfgFile);

3
deps/SZ/sz/include/pub.h vendored
View File

@ -33,7 +33,8 @@ typedef struct sz_params
int szMode; //* 0 (best speed) or 1 (better compression with Zstd/Gzip) or 3 temporal-dimension based compression
int gzipMode; //* four options: Z_NO_COMPRESSION, or Z_BEST_SPEED, Z_BEST_COMPRESSION, Z_DEFAULT_COMPRESSION
int errorBoundMode; //4bits (0.5byte), //SZ_ABS, REL, ABS_AND_REL, or ABS_OR_REL, PSNR, or PW_REL, PSNR
double absErrBound; //absolute error bound
double absErrBound; //absolute error bound for float
double absErrBoundDouble; // for double
double relBoundRatio; //value range based relative error bound ratio
double psnr; //PSNR
double normErr;

2
deps/SZ/sz/include/sz.h vendored
View File

@ -161,11 +161,9 @@ extern int dataEndianType; //*endian type of the data read from disk
extern int sysEndianType; //*sysEndianType is actually set automatically.
extern sz_params *confparams_cpr;
extern sz_params *confparams_dec;
extern sz_exedata *exe_params;
void SZ_Finalize();
int SZ_Init(const char *configFilePath);
int SZ_Init_Params(sz_params *params);

12
deps/SZ/sz/include/td_sz.h vendored
View File

@ -3,12 +3,19 @@
#define _TD_SZ_H
#include "defines.h"
#ifdef __cplusplus
extern "C" {
#endif
void cost_start();
double cost_end(const char* tag);
//
// Init success return 1 else 0
//
void tdszInit(double fPrecision, double dPrecision, unsigned int maxIntervals, unsigned int intervals, const char* compressor);
//
// compress interface to tdengine return value is count of output with bytes
//
@ -19,6 +26,11 @@ int tdszCompress(int type, const char * input, const int nelements, const char *
//
int tdszDecompress(int type, const char * input, int compressedSize, const int nelements, const char * output);
//
// tdszExit
//
void tdszExit();
#ifdef __cplusplus
}
#endif

117
deps/SZ/sz/src/conf.c vendored
View File

@ -12,25 +12,67 @@
#include "sz.h"
#include "iniparser.h"
#include "Huffman.h"
#include "pastri.h"
/*-------------------------------------------------------------------------*/
/**
@brief It reads the configuration given in the configuration file.
@return integer 1 if successfull.
//
// set default value
//
void setDefaulParams(sz_exedata* exedata, sz_params* params)
{
// sz_params
if(params)
{
// first important
params->errorBoundMode = SZ_ABS;
params->absErrBound = 1E-8;
params->absErrBoundDouble = 1E-16;
params->max_quant_intervals = 800;
params->quantization_intervals = 500;
params->losslessCompressor = ZSTD_COMPRESSOR; //other option: GZIP_COMPRESSOR;
This function reads the configuration given in the SZ configuration
file and sets other required parameters.
// second important
params->sol_ID = SZ;
params->maxRangeRadius = params->max_quant_intervals/2;
params->predThreshold = 0.99;
params->sampleDistance = 100;
params->szMode = SZ_BEST_COMPRESSION;
if(params->losslessCompressor==ZSTD_COMPRESSOR)
params->gzipMode = 3; //fast mode
else
params->gzipMode = 1; //high speed mode
**/
// other
params->psnr = 90;
params->relBoundRatio = 1E-8;
params->accelerate_pw_rel_compression = 1;
params->pw_relBoundRatio = 1E-3;
params->segment_size = 36;
params->pwr_type = SZ_PWR_MIN_TYPE;
params->snapshotCmprStep = 5;
params->withRegression = SZ_WITH_LINEAR_REGRESSION;
params->randomAccess = 0; //0: no random access , 1: support random access
params->protectValueRange = 0;
params->plus_bits = 3;
}
// sz_exedata
if(exedata)
{
exedata->optQuantMode = 1;
exedata->SZ_SIZE_TYPE = 4;
if(params)
{
exedata->intvCapacity = params->maxRangeRadius*2;
exedata->intvRadius = params->maxRangeRadius;
}
else
{
exedata->intvCapacity = 500;
exedata->intvRadius = 200;
}
}
}
/*struct node_t *pool;
node *qqq;
node *qq;
int n_nodes = 0, qend;
unsigned long **code;
unsigned char *cout;
int n_inode;*/
unsigned int roundUpToPowerOf2(unsigned int base)
{
@ -74,8 +116,10 @@ double computeABSErrBoundFromNORM_ERR(double normErr, size_t nbEle)
int SZ_ReadConf(const char* sz_cfgFile) {
// Check access to SZ configuration file and load dictionary
//record the setting in confparams_cpr
confparams_cpr = (sz_params*)malloc(sizeof(sz_params));
exe_params = (sz_exedata*)malloc(sizeof(sz_exedata));
if(confparams_cpr == NULL)
confparams_cpr = (sz_params*)malloc(sizeof(sz_params));
if(exe_params == NULL)
exe_params = (sz_exedata*)malloc(sizeof(sz_exedata));
int x = 1;
char sol_name[256];
@ -92,49 +136,12 @@ int SZ_ReadConf(const char* sz_cfgFile) {
else //=0
sysEndianType = BIG_ENDIAN_SYSTEM;
confparams_cpr->plus_bits = 3;
// default option
if(sz_cfgFile == NULL || access(sz_cfgFile, F_OK) != 0)
{
dataEndianType = LITTLE_ENDIAN_DATA;
confparams_cpr->sol_ID = SZ;
confparams_cpr->max_quant_intervals = 800;
confparams_cpr->maxRangeRadius = confparams_cpr->max_quant_intervals/2;
exe_params->intvCapacity = confparams_cpr->maxRangeRadius*2;
exe_params->intvRadius = confparams_cpr->maxRangeRadius;
confparams_cpr->quantization_intervals = 500;
exe_params->optQuantMode = 1;
confparams_cpr->predThreshold = 0.99;
confparams_cpr->sampleDistance = 100;
confparams_cpr->szMode = SZ_BEST_COMPRESSION;
confparams_cpr->losslessCompressor = ZSTD_COMPRESSOR; //other option: GZIP_COMPRESSOR;
if(confparams_cpr->losslessCompressor==ZSTD_COMPRESSOR)
confparams_cpr->gzipMode = 3; //fast mode
else
confparams_cpr->gzipMode = 1; //high speed mode
confparams_cpr->errorBoundMode = SZ_ABS;
confparams_cpr->psnr = 90;
confparams_cpr->absErrBound = 1E-8;
confparams_cpr->relBoundRatio = 1E-8;
confparams_cpr->accelerate_pw_rel_compression = 1;
confparams_cpr->pw_relBoundRatio = 1E-3;
confparams_cpr->segment_size = 36;
confparams_cpr->pwr_type = SZ_PWR_MIN_TYPE;
confparams_cpr->snapshotCmprStep = 5;
confparams_cpr->withRegression = SZ_WITH_LINEAR_REGRESSION;
confparams_cpr->randomAccess = 0; //0: no random access , 1: support random access
confparams_cpr->protectValueRange = 0;
setDefaulParams(exe_params, confparams_cpr);
return SZ_SUCCESS;
}

2
deps/SZ/sz/src/dataCompression.c vendored
View File

@ -95,7 +95,7 @@ double getRealPrecision_double(double valueRangeSize, int errBoundMode, double a
int state = SZ_SUCCESS;
double precision = 0;
if(errBoundMode==SZ_ABS||errBoundMode==ABS_OR_PW_REL||errBoundMode==ABS_AND_PW_REL)
precision = absErrBound*0.00000001;
precision = absErrBound;
else if(errBoundMode==REL||errBoundMode==REL_OR_PW_REL||errBoundMode==REL_AND_PW_REL)
precision = relBoundRatio*valueRangeSize;
else if(errBoundMode==ABS_AND_REL)

11
deps/SZ/sz/src/sz.c vendored
View File

@ -31,13 +31,8 @@ int sysEndianType = LITTLE_ENDIAN_SYSTEM ; //*sysEndianType is actually set aut
//the confparams should be separate between compression and decopmression, in case of mutual-affection when calling compression/decompression alternatively
sz_params *confparams_cpr = NULL; //used for compression
sz_params *confparams_dec = NULL; //used for decompression
sz_exedata *exe_params = NULL;
int SZ_Init(const char *configFilePath)
{
// check CPU EndianType
@ -152,14 +147,8 @@ size_t SZ_decompress(int dataType, unsigned char *bytes, size_t byteLength, size
return outSize;
}
void SZ_Finalize()
{
if(confparams_dec!=NULL)
{
free(confparams_dec);
confparams_dec = NULL;
}
if(confparams_cpr!=NULL)
{
free(confparams_cpr);

8
deps/SZ/sz/src/sz_double.c vendored
View File

@ -325,18 +325,18 @@ int SZ_compress_args_double(double *oriData, size_t r1, unsigned char* newByteDa
if(params->errorBoundMode==PSNR)
{
params->errorBoundMode = SZ_ABS;
realPrecision = params->absErrBound = computeABSErrBoundFromPSNR(params->psnr, (double)params->predThreshold, valueRangeSize);
realPrecision = params->absErrBoundDouble = computeABSErrBoundFromPSNR(params->psnr, (double)params->predThreshold, valueRangeSize);
}
else if(params->errorBoundMode==NORM) //norm error = sqrt(sum((xi-xi_)^2))
{
params->errorBoundMode = SZ_ABS;
realPrecision = params->absErrBound = computeABSErrBoundFromNORM_ERR(params->normErr, dataLength);
realPrecision = params->absErrBoundDouble = computeABSErrBoundFromNORM_ERR(params->normErr, dataLength);
//printf("realPrecision=%lf\n", realPrecision);
}
else
{
realPrecision = getRealPrecision_double(valueRangeSize, params->errorBoundMode, params->absErrBound, params->relBoundRatio, &status);
params->absErrBound = realPrecision;
realPrecision = getRealPrecision_double(valueRangeSize, params->errorBoundMode, params->absErrBoundDouble, params->relBoundRatio, &status);
params->absErrBoundDouble = realPrecision;
}
if(valueRangeSize <= realPrecision)
{

44
deps/SZ/sz/src/td_sz.c vendored
View File

@ -5,7 +5,34 @@
#include "td_sz.h"
#include "sz.h"
#include "conf.h"
//
// Init success return 1 else 0
//
void tdszInit(double fPrecision, double dPrecision, unsigned int maxIntervals, unsigned int intervals, const char* compressor)
{
// need malloc
if(confparams_cpr == NULL)
confparams_cpr = (sz_params*)malloc(sizeof(sz_params));
if(exe_params == NULL)
exe_params = (sz_exedata*)malloc(sizeof(sz_exedata));
// set default
setDefaulParams(exe_params, confparams_cpr);
// overwrite with args
confparams_cpr->absErrBound = fPrecision;
confparams_cpr->absErrBoundDouble = dPrecision;
confparams_cpr->max_quant_intervals = maxIntervals;
confparams_cpr->quantization_intervals = intervals;
if(strcmp(compressor, "GZIP_COMPRESSOR")==0)
confparams_cpr->losslessCompressor = GZIP_COMPRESSOR;
else if(strcmp(compressor, "ZSTD_COMPRESSOR")==0)
confparams_cpr->losslessCompressor = ZSTD_COMPRESSOR;
return ;
}
//
@ -28,3 +55,20 @@ int tdszDecompress(int type, const char * input, int compressedSize, const int n
size_t outSize = SZ_decompress(type, (void*)input, compressedSize, (size_t)nelements, (unsigned char*)output);
return (int)outSize;
}
//
// tdszExit
//
void tdszExit()
{
if(confparams_cpr!=NULL)
{
free(confparams_cpr);
confparams_cpr = NULL;
}
if(exe_params!=NULL)
{
free(exe_params);
exe_params = NULL;
}
}

8
src/common/inc/tglobal.h
View File

@ -205,6 +205,14 @@ extern int32_t wDebugFlag;
extern int32_t cqDebugFlag;
extern int32_t debugFlag;
// lossy
extern char lossyColumns[];
extern double fPrecision;
extern double dPrecision;
extern uint32_t maxIntervals;
extern uint32_t intervals;
extern char Compressor[];
typedef struct {
char dir[TSDB_FILENAME_LEN];
int level;

74
src/common/src/tglobal.c
View File

@ -244,6 +244,18 @@ int32_t tsdbDebugFlag = 131;
int32_t cqDebugFlag = 131;
int32_t fsDebugFlag = 135;
//
// lossy compress 6
//
char lossyColumns[32] = ""; // "float|double" means all float and double columns can be lossy compressed. set empty can close lossy compress.
// below option can take effect when tsLossyColumns not empty
double fPrecision = 1E-8; // float column precision
double dPrecision = 1E-16; // double column precision
uint32_t maxIntervals = 800; // max intervals
uint32_t intervals = 500; // intervals
char Compressor[32] = "ZSTD_COMPRESSOR"; // ZSTD_COMPRESSOR or GZIP_COMPRESSOR
int32_t (*monStartSystemFp)() = NULL;
void (*monStopSystemFp)() = NULL;
void (*monExecuteSQLFp)(char *sql) = NULL;
@ -1517,6 +1529,68 @@ static void doInitGlobalConfig(void) {
cfg.ptrLength = tListLen(tsTempDir);
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
// lossy compress
cfg.option = "lossyColumns";
cfg.ptr = lossyColumns;
cfg.valType = TAOS_CFG_VTYPE_STRING;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = 0;
cfg.maxValue = 0;
cfg.ptrLength = tListLen(lossyColumns);
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "Compressor";
cfg.ptr = Compressor;
cfg.valType = TAOS_CFG_VTYPE_STRING;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = 0;
cfg.maxValue = 0;
cfg.ptrLength = tListLen(Compressor);
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "fPrecision";
cfg.ptr = &fPrecision;
cfg.valType = TAOS_CFG_VTYPE_DOUBLE;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = -MAXFLOAT;
cfg.maxValue = MAXFLOAT;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "dPrecision";
cfg.ptr = &dPrecision;
cfg.valType = TAOS_CFG_VTYPE_DOUBLE;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = -MAXFLOAT;
cfg.maxValue = MAXFLOAT;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "maxIntervals";
cfg.ptr = &maxIntervals;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = 0;
cfg.maxValue = 65536;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "intervals";
cfg.ptr = &intervals;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = 0;
cfg.maxValue = 65536;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
}
void taosInitGlobalCfg() {

13
src/dnode/src/dnodeMain.c
View File

@ -41,6 +41,7 @@
#include "dnodeTelemetry.h"
#include "module.h"
#include "mnode.h"
#include "tscompression.h"
#if !defined(_MODULE) || !defined(_TD_LINUX)
int32_t moduleStart() { return 0; }
@ -234,6 +235,10 @@ static void dnodeCheckDataDirOpenned(char *dir) {
}
static int32_t dnodeInitStorage() {
// compress module init
tsCompressInit();
// storage module init
if (tsDiskCfgNum == 1 && dnodeCreateDir(tsDataDir) < 0) {
dError("failed to create dir: %s, reason: %s", tsDataDir, strerror(errno));
return -1;
@ -309,7 +314,13 @@ static int32_t dnodeInitStorage() {
return 0;
}
static void dnodeCleanupStorage() { tfsDestroy(); }
static void dnodeCleanupStorage() {
// storage destroy
tfsDestroy();
// compress destroy
tsCompressExit();
}
bool dnodeIsFirstDeploy() {
return strcmp(tsFirst, tsLocalEp) == 0;

15
src/kit/taospack/taospack.c
View File

@ -142,7 +142,6 @@ double check_same_double(double* ft1, double* ft2, int count){
//
// test compress and decompress
//
extern bool gOpenLossy;
bool DoDouble(double* doubles, int cnt, int algorithm) {
// compress
@ -677,7 +676,8 @@ void test_same_double(int algo){
}
extern bool lossyFloat;
extern bool lossyDouble;
//
// ----------------- main ----------------------
@ -685,8 +685,9 @@ void test_same_double(int algo){
int main(int argc, char *argv[]) {
printf("welcome to use taospack tools v1.3\n");
gOpenLossy = false;
tsLossyInit();
tsCompressInit();
lossyFloat = lossyDouble = true;
//
//tsCompressExit();
//return 1;
@ -697,11 +698,11 @@ int main(int argc, char *argv[]) {
// t
if(strcmp(argv[1], "-tone") == 0 || strcmp(argv[1], "-t") == 0 ) {
algo = ONE_STAGE_COMP;
gOpenLossy = true;
lossyFloat = lossyDouble = true;
}
if(strcmp(argv[1], "-tw") == 0) {
algo = TWO_STAGE_COMP;
gOpenLossy = false;
lossyFloat = lossyDouble = false;
}
if(strcmp(argv[1], "-sf") == 0) {
@ -737,7 +738,7 @@ int main(int argc, char *argv[]) {
unitTestFloat();
}
//memTest();
tsCompressExit();
return 0;
}

5
src/tsdb/src/tsdbMain.c
View File

@ -16,6 +16,7 @@
// no test file errors here
#include "tsdbint.h"
#include "tscompression.h"
#include "tsdbLog.h"
#define IS_VALID_PRECISION(precision) \
(((precision) >= TSDB_TIME_PRECISION_MILLI) && ((precision) <= TSDB_TIME_PRECISION_NANO))
@ -67,10 +68,6 @@ STsdbRepo *tsdbOpenRepo(STsdbCfg *pCfg, STsdbAppH *pAppH) {
terrno = TSDB_CODE_SUCCESS;
// Compress Init
//if(pCfg->compressLossy)
tsLossyInit();
// Check and set default configurations
if (tsdbCheckAndSetDefaultCfg(&config) < 0) {
tsdbError("vgId:%d failed to open TSDB repository since %s", config.tsdbId, tstrerror(terrno));

5
src/util/inc/tconfig.h
View File

@ -20,7 +20,7 @@
extern "C" {
#endif
#define TSDB_CFG_MAX_NUM 110
#define TSDB_CFG_MAX_NUM 116 // 110 + 6 with lossy option
#define TSDB_CFG_PRINT_LEN 23
#define TSDB_CFG_OPTION_LEN 24
#define TSDB_CFG_VALUE_LEN 41
@ -32,6 +32,8 @@ extern "C" {
#define TSDB_CFG_CTYPE_B_OPTION 16U // can be configured by taos_options function
#define TSDB_CFG_CTYPE_B_NOT_PRINT 32U // such as password
#define MAXFLOAT 0x1.fffffep+127f
enum {
TAOS_CFG_CSTATUS_NONE, // not configured
TAOS_CFG_CSTATUS_DEFAULT, // use system default value
@ -50,6 +52,7 @@ enum {
TAOS_CFG_VTYPE_IPSTR,
TAOS_CFG_VTYPE_DIRECTORY,
TAOS_CFG_VTYPE_DATA_DIRCTORY,
TAOS_CFG_VTYPE_DOUBLE,
};
enum {

24
src/util/inc/tscompression.h
View File

@ -23,7 +23,6 @@ extern "C" {
#include "taosdef.h"
#include "tutil.h"
#define COMP_OVERFLOW_BYTES 2
#define BITS_PER_BYTE 8
// Masks
@ -51,9 +50,6 @@ extern "C" {
#define HEAD_MODE(x) x%2
#define HEAD_ALGO(x) x/2
extern bool gOpenLossy;
extern int tsCompressINTImp(const char *const input, const int nelements, char *const output, const char type);
extern int tsDecompressINTImp(const char *const input, const int nelements, char *const output, const char type);
extern int tsCompressBoolImp(const char *const input, const int nelements, char *const output);
@ -67,13 +63,17 @@ extern int tsDecompressDoubleImp(const char *const input, const int nelements, c
extern int tsCompressFloatImp(const char *const input, const int nelements, char *const output);
extern int tsDecompressFloatImp(const char *const input, const int nelements, char *const output);
// lossy
int tsCompressFloatLossyImp(const char * input, const int nelements, char *const output);
int tsDecompressFloatLossyImp(const char * input, int compressedSize, const int nelements, char *const output);
int tsCompressDoubleLossyImp(const char * input, const int nelements, char *const output);
int tsDecompressDoubleLossyImp(const char * input, int compressedSize, const int nelements, char *const output);
extern int tsCompressFloatLossyImp(const char * input, const int nelements, char *const output);
extern int tsDecompressFloatLossyImp(const char * input, int compressedSize, const int nelements, char *const output);
extern int tsCompressDoubleLossyImp(const char * input, const int nelements, char *const output);
extern int tsDecompressDoubleLossyImp(const char * input, int compressedSize, const int nelements, char *const output);
// init
bool tsLossyInit();
extern bool lossyFloat;
extern bool lossyDouble;
// init call
int tsCompressInit();
// exit call
void tsCompressExit();
void cost_start();
double cost_end(const char* tag);
@ -223,7 +223,7 @@ static FORCE_INLINE int tsDecompressString(const char *const input, int compress
static FORCE_INLINE int tsCompressFloat(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
char algorithm, char *const buffer, int bufferSize) {
// lossy mode
if(gOpenLossy) {
if(lossyFloat) {
return tsCompressFloatLossyImp(input, nelements, output);
// lossless mode
} else {
@ -269,7 +269,7 @@ static FORCE_INLINE int tsDecompressFloat(const char *const input, int compresse
static FORCE_INLINE int tsCompressDouble(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
char algorithm, char *const buffer, int bufferSize) {
if(gOpenLossy){
if(lossyDouble){
// lossy mode
return tsCompressDoubleLossyImp(input, nelements, output);
} else {

44
src/util/src/tcompression.c
View File

@ -53,6 +53,8 @@
#include "taosdef.h"
#include "tscompression.h"
#include "tulog.h"
#include "tglobal.h"
static const int TEST_NUMBER = 1;
#define is_bigendian() ((*(char *)&TEST_NUMBER) == 0)
@ -62,7 +64,33 @@ static const int TEST_NUMBER = 1;
#define ZIGZAG_ENCODE(T, v) ((u##T)((v) >> (sizeof(T) * 8 - 1))) ^ (((u##T)(v)) << 1) // zigzag encode
#define ZIGZAG_DECODE(T, v) ((v) >> 1) ^ -((T)((v)&1)) // zigzag decode
bool gOpenLossy = true;
bool lossyFloat = false;
bool lossyDouble = false;
// init call
int tsCompressInit(){
// config
if(lossyColumns[0] == 0){
lossyFloat = false;
lossyDouble = false;
return 0;
}
lossyFloat = strstr(lossyColumns, "float") != NULL;
lossyDouble = strstr(lossyColumns, "double") != NULL;
if(lossyFloat == false && lossyDouble == false)
return 0;
tdszInit(fPrecision, dPrecision, maxIntervals, intervals, Compressor);
uInfo("lossy compression is opened. columns = %s \n", lossyColumns);
return 1;
}
// exit call
void tsCompressExit(){
tdszExit();
}
/*
* Compress Integer (Simple8B).
@ -891,20 +919,6 @@ int tsDecompressFloatImp(const char *const input, const int nelements, char *con
return nelements * FLOAT_BYTES;
}
//
// ----------- global init and exit resource ------
//
int SZ_Init(const char *configFilePath); //declare deps/sz/include/sz.h
bool gLossyInited = false;
bool tsLossyInit() {
// init compress init
if(!gLossyInited){
gLossyInited = true;
SZ_Init("./sz.config");
}
return true;
}
//
// ---------- float double lossy -----------

21
src/util/src/tconfig.c
View File

@ -61,6 +61,24 @@ static void taosReadFloatConfig(SGlobalCfg *cfg, char *input_value) {
}
}
static void taosReadDoubleConfig(SGlobalCfg *cfg, char *input_value) {
double value = atof(input_value);
double *option = (double *)cfg->ptr;
if (value < cfg->minValue || value > cfg->maxValue) {
uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%f",
cfg->option, input_value, cfg->minValue, cfg->maxValue, *option);
} else {
if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) {
*option = value;
cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE;
} else {
uWarn("config option:%s, input value:%s, is configured by %s, use %f", cfg->option, input_value,
tsCfgStatusStr[cfg->cfgStatus], *option);
}
}
}
static void taosReadInt32Config(SGlobalCfg *cfg, char *input_value) {
int32_t value = atoi(input_value);
int32_t *option = (int32_t *)cfg->ptr;
@ -262,6 +280,9 @@ static void taosReadConfigOption(const char *option, char *value, char *value2,
case TAOS_CFG_VTYPE_FLOAT:
taosReadFloatConfig(cfg, value);
break;
case TAOS_CFG_VTYPE_DOUBLE:
taosReadDoubleConfig(cfg, value);
break;
case TAOS_CFG_VTYPE_STRING:
taosReadStringConfig(cfg, value);
break;