add sz config to taos.cfg

deps/SZ/sz/include/conf.h

@@ -16,6 +16,11 @@ extern "C" {
 
 #include <stdio.h>
 
+//
+// set default value
+//
+void setDefaulParams(sz_exedata* exedata, sz_params* params);
+
 //conf.c
 void updateQuantizationInfo(int quant_intervals);
 int SZ_ReadConf(const char* sz_cfgFile);

deps/SZ/sz/include/pub.h

@@ -33,7 +33,8 @@ typedef struct sz_params
 	int szMode; //* 0 (best speed) or 1 (better compression with Zstd/Gzip) or 3 temporal-dimension based compression
 	int gzipMode; //* four options: Z_NO_COMPRESSION, or Z_BEST_SPEED, Z_BEST_COMPRESSION, Z_DEFAULT_COMPRESSION
 	int  errorBoundMode; //4bits (0.5byte), //SZ_ABS, REL, ABS_AND_REL, or ABS_OR_REL, PSNR, or PW_REL, PSNR
-	double absErrBound; //absolute error bound
+	double absErrBound; //absolute error bound for float
+	double absErrBoundDouble; // for double
 	double relBoundRatio; //value range based relative error bound ratio
 	double psnr; //PSNR
 	double normErr;

deps/SZ/sz/include/sz.h

@@ -161,11 +161,9 @@ extern int dataEndianType; //*endian type of the data read from disk
 extern int sysEndianType; //*sysEndianType is actually set automatically.
 
 extern sz_params *confparams_cpr;
-extern sz_params *confparams_dec;
 extern sz_exedata *exe_params;
 
 
-
 void SZ_Finalize();
 int SZ_Init(const char *configFilePath);
 int SZ_Init_Params(sz_params *params);

deps/SZ/sz/include/td_sz.h

@@ -3,12 +3,19 @@
 #define _TD_SZ_H
 #include "defines.h"
 
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 void cost_start();
 double cost_end(const char* tag);
+
+//
+// Init  success return 1 else 0
+//
+void tdszInit(double fPrecision, double dPrecision, unsigned int maxIntervals, unsigned int intervals, const char* compressor);
+
 //
 // compress interface to tdengine return value is count of output with bytes
 //
@@ -19,6 +26,11 @@ int tdszCompress(int type, const char * input, const int nelements, const char *
 //
 int tdszDecompress(int type, const char * input, int compressedSize, const int nelements, const char * output);
 
+//
+//  tdszExit
+//
+void tdszExit();
+
 #ifdef __cplusplus
 }
 #endif

deps/SZ/sz/src/conf.c

@@ -12,25 +12,67 @@
 #include "sz.h"
 #include "iniparser.h"
 #include "Huffman.h"
-#include "pastri.h"
 
-/*-------------------------------------------------------------------------*/
+//
-/**
+// set default value
-    @brief      It reads the configuration given in the configuration file.
+//
-    @return     integer         1 if successfull.
+void setDefaulParams(sz_exedata* exedata, sz_params* params)
+{
+	// sz_params
+	if(params)
+	{
+		// first important
+		params->errorBoundMode = SZ_ABS;
+		params->absErrBound    = 1E-8;
+		params->absErrBoundDouble = 1E-16;
+		params->max_quant_intervals = 800;
+		params->quantization_intervals = 500;
+		params->losslessCompressor = ZSTD_COMPRESSOR; //other option: GZIP_COMPRESSOR;
 
-    This function reads the configuration given in the SZ configuration
+        // second important
-    file and sets other required parameters.
+		params->sol_ID = SZ;		
+		params->maxRangeRadius = params->max_quant_intervals/2;		
+		params->predThreshold = 0.99;
+		params->sampleDistance = 100;
+		params->szMode = SZ_BEST_COMPRESSION;
+		if(params->losslessCompressor==ZSTD_COMPRESSOR)
+			params->gzipMode = 3; //fast mode
+		else
+			params->gzipMode = 1; //high speed mode
 
- **/
+        // other
+		params->psnr = 90;
+		params->relBoundRatio = 1E-8;
+		params->accelerate_pw_rel_compression = 1;
+		params->pw_relBoundRatio = 1E-3;
+		params->segment_size = 36;
+		params->pwr_type = SZ_PWR_MIN_TYPE;
+		params->snapshotCmprStep = 5;
+		params->withRegression = SZ_WITH_LINEAR_REGRESSION;
+		params->randomAccess = 0; //0: no random access , 1: support random access
+		params->protectValueRange = 0;		
+		params->plus_bits = 3;
+	}
+
+    // sz_exedata
+	if(exedata)
+	{
+		exedata->optQuantMode = 1;
+		exedata->SZ_SIZE_TYPE = 4;
+		if(params)
+		{
+			exedata->intvCapacity = params->maxRangeRadius*2;
+			exedata->intvRadius   = params->maxRangeRadius;
+		}
+		else
+		{
+			exedata->intvCapacity = 500;
+			exedata->intvRadius   = 200;
+		}
+	}
+	
+}
 
-/*struct node_t *pool;
-node *qqq;
-node *qq;
-int n_nodes = 0, qend;
-unsigned long **code;
-unsigned char *cout;
-int n_inode;*/ 
  
 unsigned int roundUpToPowerOf2(unsigned int base)
 {
@@ -74,8 +116,10 @@ double computeABSErrBoundFromNORM_ERR(double normErr, size_t nbEle)
 int SZ_ReadConf(const char* sz_cfgFile) {
     // Check access to SZ configuration file and load dictionary
     //record the setting in confparams_cpr
-    confparams_cpr = (sz_params*)malloc(sizeof(sz_params));    
+	if(confparams_cpr == NULL)
-    exe_params = (sz_exedata*)malloc(sizeof(sz_exedata));
+	   confparams_cpr = (sz_params*)malloc(sizeof(sz_params));    
+	if(exe_params == NULL)
+       exe_params = (sz_exedata*)malloc(sizeof(sz_exedata));
     
     int x = 1;
     char sol_name[256];
@@ -92,49 +136,12 @@ int SZ_ReadConf(const char* sz_cfgFile) {
 	else //=0
 		sysEndianType = BIG_ENDIAN_SYSTEM;
     
-    confparams_cpr->plus_bits = 3;
     
 	// default option
     if(sz_cfgFile == NULL || access(sz_cfgFile, F_OK) != 0)
     {
 		dataEndianType = LITTLE_ENDIAN_DATA;
-		confparams_cpr->sol_ID = SZ;
+		setDefaulParams(exe_params, confparams_cpr);
-		confparams_cpr->max_quant_intervals = 800;
-		confparams_cpr->maxRangeRadius = confparams_cpr->max_quant_intervals/2;
-				
-		exe_params->intvCapacity = confparams_cpr->maxRangeRadius*2;
-		exe_params->intvRadius = confparams_cpr->maxRangeRadius;
-		
-		confparams_cpr->quantization_intervals = 500;
-		exe_params->optQuantMode = 1;
-		confparams_cpr->predThreshold = 0.99;
-		confparams_cpr->sampleDistance = 100;
-		
-		confparams_cpr->szMode = SZ_BEST_COMPRESSION;
-		confparams_cpr->losslessCompressor = ZSTD_COMPRESSOR; //other option: GZIP_COMPRESSOR;
-		if(confparams_cpr->losslessCompressor==ZSTD_COMPRESSOR)
-			confparams_cpr->gzipMode = 3; //fast mode
-		else
-			confparams_cpr->gzipMode = 1; //high speed mode
-		
-		confparams_cpr->errorBoundMode = SZ_ABS;
-		confparams_cpr->psnr = 90;
-		confparams_cpr->absErrBound = 1E-8;
-		confparams_cpr->relBoundRatio = 1E-8;
-		confparams_cpr->accelerate_pw_rel_compression = 1;
-		
-		confparams_cpr->pw_relBoundRatio = 1E-3;
-		confparams_cpr->segment_size = 36;
-		
-		confparams_cpr->pwr_type = SZ_PWR_MIN_TYPE;
-		
-		confparams_cpr->snapshotCmprStep = 5;
-		
-		confparams_cpr->withRegression = SZ_WITH_LINEAR_REGRESSION;
-	
-		confparams_cpr->randomAccess = 0; //0: no random access , 1: support random access
-	
-		confparams_cpr->protectValueRange = 0;
 	
 		return SZ_SUCCESS;
 	}

deps/SZ/sz/src/dataCompression.c

@@ -95,7 +95,7 @@ double getRealPrecision_double(double valueRangeSize, int errBoundMode, double a
 	int state = SZ_SUCCESS;
 	double precision = 0;
 	if(errBoundMode==SZ_ABS||errBoundMode==ABS_OR_PW_REL||errBoundMode==ABS_AND_PW_REL)
-		precision = absErrBound*0.00000001; 
+		precision = absErrBound; 
 	else if(errBoundMode==REL||errBoundMode==REL_OR_PW_REL||errBoundMode==REL_AND_PW_REL)
 		precision = relBoundRatio*valueRangeSize;
 	else if(errBoundMode==ABS_AND_REL)

deps/SZ/sz/src/sz.c

@@ -31,13 +31,8 @@ int sysEndianType  = LITTLE_ENDIAN_SYSTEM ; //*sysEndianType is actually set aut
 
 //the confparams should be separate between compression and decopmression, in case of mutual-affection when calling compression/decompression alternatively
 sz_params *confparams_cpr = NULL; //used for compression
-sz_params *confparams_dec = NULL; //used for decompression 
-
 sz_exedata *exe_params = NULL;
 
-
-
-
 int SZ_Init(const char *configFilePath)
 {
 	// check CPU EndianType
@@ -152,14 +147,8 @@ size_t SZ_decompress(int dataType, unsigned char *bytes, size_t byteLength, size
     return outSize;
 }
 
-
 void SZ_Finalize()
 {
-	if(confparams_dec!=NULL)
-	{
-		free(confparams_dec);
-		confparams_dec = NULL;
-	}
 	if(confparams_cpr!=NULL)
 	{
 		free(confparams_cpr);

deps/SZ/sz/src/sz_double.c

@@ -325,18 +325,18 @@ int SZ_compress_args_double(double *oriData, size_t r1, unsigned char* newByteDa
 	if(params->errorBoundMode==PSNR)
 	{
 		params->errorBoundMode = SZ_ABS;
-		realPrecision = params->absErrBound = computeABSErrBoundFromPSNR(params->psnr, (double)params->predThreshold, valueRangeSize);
+		realPrecision = params->absErrBoundDouble = computeABSErrBoundFromPSNR(params->psnr, (double)params->predThreshold, valueRangeSize);
 	}
 	else if(params->errorBoundMode==NORM) //norm error = sqrt(sum((xi-xi_)^2))
 	{
 		params->errorBoundMode = SZ_ABS;
-		realPrecision = params->absErrBound = computeABSErrBoundFromNORM_ERR(params->normErr, dataLength);
+		realPrecision = params->absErrBoundDouble = computeABSErrBoundFromNORM_ERR(params->normErr, dataLength);
 		//printf("realPrecision=%lf\n", realPrecision);				
 	}	
 	else
 	{
-		realPrecision = getRealPrecision_double(valueRangeSize, params->errorBoundMode, params->absErrBound, params->relBoundRatio, &status);
+		realPrecision = getRealPrecision_double(valueRangeSize, params->errorBoundMode, params->absErrBoundDouble, params->relBoundRatio, &status);
-		params->absErrBound = realPrecision;
+		params->absErrBoundDouble = realPrecision;
 	}	
 	if(valueRangeSize <= realPrecision)
 	{		

deps/SZ/sz/src/td_sz.c

@@ -5,7 +5,34 @@
 
 #include "td_sz.h"
 #include "sz.h"
+#include "conf.h"
 
+//
+// Init  success return 1 else 0
+//
+void tdszInit(double fPrecision, double dPrecision, unsigned int maxIntervals, unsigned int intervals, const char* compressor)
+{
+	// need malloc
+	if(confparams_cpr == NULL)
+	   confparams_cpr = (sz_params*)malloc(sizeof(sz_params));    
+	if(exe_params == NULL)
+       exe_params = (sz_exedata*)malloc(sizeof(sz_exedata));
+	 
+	// set default
+	setDefaulParams(exe_params, confparams_cpr);
+
+    // overwrite with args
+	confparams_cpr->absErrBound = fPrecision;
+	confparams_cpr->absErrBoundDouble = dPrecision;
+	confparams_cpr->max_quant_intervals = maxIntervals;
+	confparams_cpr->quantization_intervals = intervals;
+	if(strcmp(compressor, "GZIP_COMPRESSOR")==0)
+		confparams_cpr->losslessCompressor = GZIP_COMPRESSOR;
+	else if(strcmp(compressor, "ZSTD_COMPRESSOR")==0)
+		confparams_cpr->losslessCompressor = ZSTD_COMPRESSOR;
+	
+	return ;
+}
 
 
 //
@@ -28,3 +55,20 @@ int tdszDecompress(int type, const char * input, int compressedSize, const int n
 	size_t outSize = SZ_decompress(type, (void*)input, compressedSize, (size_t)nelements, (unsigned char*)output);
     return (int)outSize;
 }
+
+//
+//  tdszExit
+//
+void tdszExit()
+{
+	if(confparams_cpr!=NULL)
+	{
+		free(confparams_cpr);
+		confparams_cpr = NULL;
+	}	
+	if(exe_params!=NULL)
+	{
+		free(exe_params);
+		exe_params = NULL;
+	}
+}

src/common/inc/tglobal.h

@@ -205,6 +205,14 @@ extern int32_t wDebugFlag;
 extern int32_t cqDebugFlag;
 extern int32_t debugFlag;
 
+// lossy 
+extern char lossyColumns[];
+extern double fPrecision;
+extern double dPrecision;
+extern uint32_t maxIntervals;
+extern uint32_t intervals;
+extern char Compressor[];
+
 typedef struct {
   char dir[TSDB_FILENAME_LEN];
   int  level;

src/common/src/tglobal.c

@@ -244,6 +244,18 @@ int32_t tsdbDebugFlag = 131;
 int32_t cqDebugFlag = 131;
 int32_t fsDebugFlag = 135;
 
+//
+// lossy compress 6
+//
+char lossyColumns[32] = "";  // "float|double" means all float and double columns can be lossy compressed.  set empty can close lossy compress.
+// below option can take effect when tsLossyColumns not empty 
+double   fPrecision   = 1E-8;   // float column precision
+double   dPrecision   = 1E-16;  // double column precision
+uint32_t maxIntervals = 800;    // max intervals
+uint32_t intervals    = 500;    // intervals
+char     Compressor[32] = "ZSTD_COMPRESSOR"; // ZSTD_COMPRESSOR or GZIP_COMPRESSOR 
+
+
 int32_t (*monStartSystemFp)() = NULL;
 void (*monStopSystemFp)() = NULL;
 void (*monExecuteSQLFp)(char *sql) = NULL;
@@ -1517,6 +1529,68 @@ static void doInitGlobalConfig(void) {
   cfg.ptrLength = tListLen(tsTempDir);
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
+
+  // lossy compress
+  cfg.option = "lossyColumns";
+  cfg.ptr = lossyColumns;
+  cfg.valType = TAOS_CFG_VTYPE_STRING;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
+  cfg.minValue = 0;
+  cfg.maxValue = 0;
+  cfg.ptrLength = tListLen(lossyColumns);
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
+  cfg.option = "Compressor";
+  cfg.ptr = Compressor;
+  cfg.valType = TAOS_CFG_VTYPE_STRING;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
+  cfg.minValue = 0;
+  cfg.maxValue = 0;
+  cfg.ptrLength = tListLen(Compressor);
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
+  cfg.option = "fPrecision";
+  cfg.ptr = &fPrecision;
+  cfg.valType = TAOS_CFG_VTYPE_DOUBLE;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
+  cfg.minValue = -MAXFLOAT;
+  cfg.maxValue = MAXFLOAT;
+  cfg.ptrLength = 0;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
+  cfg.option = "dPrecision";
+  cfg.ptr = &dPrecision;
+  cfg.valType = TAOS_CFG_VTYPE_DOUBLE;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
+  cfg.minValue = -MAXFLOAT;
+  cfg.maxValue = MAXFLOAT;
+  cfg.ptrLength = 0;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
+  cfg.option = "maxIntervals";
+  cfg.ptr = &maxIntervals;
+  cfg.valType = TAOS_CFG_VTYPE_INT32;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
+  cfg.minValue = 0;
+  cfg.maxValue = 65536;
+  cfg.ptrLength = 0;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
+  cfg.option = "intervals";
+  cfg.ptr = &intervals;
+  cfg.valType = TAOS_CFG_VTYPE_INT32;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
+  cfg.minValue = 0;
+  cfg.maxValue = 65536;
+  cfg.ptrLength = 0;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
 }
 
 void taosInitGlobalCfg() {

src/dnode/src/dnodeMain.c

@@ -41,6 +41,7 @@
 #include "dnodeTelemetry.h"
 #include "module.h"
 #include "mnode.h"
+#include "tscompression.h"
 
 #if !defined(_MODULE) || !defined(_TD_LINUX)
 int32_t moduleStart() { return 0; }
@@ -234,6 +235,10 @@ static void dnodeCheckDataDirOpenned(char *dir) {
 }
 
 static int32_t dnodeInitStorage() {
+  // compress module init
+  tsCompressInit();
+  
+  // storage module init
   if (tsDiskCfgNum == 1 && dnodeCreateDir(tsDataDir) < 0) {
     dError("failed to create dir: %s, reason: %s", tsDataDir, strerror(errno));
     return -1;
@@ -309,7 +314,13 @@ static int32_t dnodeInitStorage() {
   return 0;
 }
 
-static void dnodeCleanupStorage() { tfsDestroy(); }
+static void dnodeCleanupStorage() {
+  // storage destroy
+  tfsDestroy(); 
+  
+  // compress destroy
+  tsCompressExit();
+}
 
 bool  dnodeIsFirstDeploy() {
   return strcmp(tsFirst, tsLocalEp) == 0;

src/kit/taospack/taospack.c

@@ -142,7 +142,6 @@ double check_same_double(double* ft1, double* ft2, int count){
 //
 // test compress and decompress
 //
-extern bool gOpenLossy;
 
 bool DoDouble(double* doubles, int cnt, int algorithm) {
   // compress
@@ -677,7 +676,8 @@ void test_same_double(int algo){
 
 }
 
-
+extern bool lossyFloat;
+extern bool lossyDouble;
 
 //
 //   -----------------  main ----------------------
@@ -685,8 +685,9 @@ void test_same_double(int algo){
 int main(int argc, char *argv[]) {
   printf("welcome to use taospack tools v1.3\n");
  
-  gOpenLossy = false;
+  tsCompressInit();
-  tsLossyInit();
+  lossyFloat = lossyDouble = true;
+ 
   //
   //tsCompressExit();
   //return 1; 
@@ -697,11 +698,11 @@ int main(int argc, char *argv[]) {
     // t
     if(strcmp(argv[1], "-tone") == 0 || strcmp(argv[1], "-t") == 0 ) {
         algo = ONE_STAGE_COMP;
-        gOpenLossy = true;
+        lossyFloat = lossyDouble = true;
     }
     if(strcmp(argv[1], "-tw") == 0) {
         algo = TWO_STAGE_COMP;
-        gOpenLossy = false;
+        lossyFloat = lossyDouble = false;
     }
 
     if(strcmp(argv[1], "-sf") == 0) {
@@ -737,7 +738,7 @@ int main(int argc, char *argv[]) {
     unitTestFloat();
   }
 
-  //memTest();
+  tsCompressExit();
   return 0;
 }
 

src/tsdb/src/tsdbMain.c

@@ -16,6 +16,7 @@
 // no test file errors here
 #include "tsdbint.h"
 #include "tscompression.h"
+#include "tsdbLog.h"
 
 #define IS_VALID_PRECISION(precision) \
   (((precision) >= TSDB_TIME_PRECISION_MILLI) && ((precision) <= TSDB_TIME_PRECISION_NANO))
@@ -67,10 +68,6 @@ STsdbRepo *tsdbOpenRepo(STsdbCfg *pCfg, STsdbAppH *pAppH) {
 
   terrno = TSDB_CODE_SUCCESS;
 
-  // Compress Init
-  //if(pCfg->compressLossy)
-  tsLossyInit();
-
   // Check and set default configurations
   if (tsdbCheckAndSetDefaultCfg(&config) < 0) {
     tsdbError("vgId:%d failed to open TSDB repository since %s", config.tsdbId, tstrerror(terrno));

src/util/inc/tconfig.h

@@ -20,7 +20,7 @@
 extern "C" {
 #endif
 
-#define TSDB_CFG_MAX_NUM    110
+#define TSDB_CFG_MAX_NUM    116  // 110 + 6 with lossy option
 #define TSDB_CFG_PRINT_LEN  23
 #define TSDB_CFG_OPTION_LEN 24
 #define TSDB_CFG_VALUE_LEN  41
@@ -32,6 +32,8 @@ extern "C" {
 #define TSDB_CFG_CTYPE_B_OPTION    16U  // can be configured by taos_options function
 #define TSDB_CFG_CTYPE_B_NOT_PRINT 32U  // such as password
 
+#define MAXFLOAT    0x1.fffffep+127f
+
 enum {
   TAOS_CFG_CSTATUS_NONE,     // not configured
   TAOS_CFG_CSTATUS_DEFAULT,  // use system default value
@@ -50,6 +52,7 @@ enum {
   TAOS_CFG_VTYPE_IPSTR,
   TAOS_CFG_VTYPE_DIRECTORY,
   TAOS_CFG_VTYPE_DATA_DIRCTORY,
+  TAOS_CFG_VTYPE_DOUBLE,
 };
 
 enum {

src/util/inc/tscompression.h

@@ -23,7 +23,6 @@ extern "C" {
 #include "taosdef.h"
 #include "tutil.h"
 
-
 #define COMP_OVERFLOW_BYTES 2
 #define BITS_PER_BYTE 8
 // Masks
@@ -51,9 +50,6 @@ extern "C" {
 #define HEAD_MODE(x)  x%2
 #define HEAD_ALGO(x)  x/2
 
-
-extern bool gOpenLossy;
-
 extern int tsCompressINTImp(const char *const input, const int nelements, char *const output, const char type);
 extern int tsDecompressINTImp(const char *const input, const int nelements, char *const output, const char type);
 extern int tsCompressBoolImp(const char *const input, const int nelements, char *const output);
@@ -67,13 +63,17 @@ extern int tsDecompressDoubleImp(const char *const input, const int nelements, c
 extern int tsCompressFloatImp(const char *const input, const int nelements, char *const output);
 extern int tsDecompressFloatImp(const char *const input, const int nelements, char *const output);
 // lossy
-int tsCompressFloatLossyImp(const char * input, const int nelements, char *const output);
+extern int tsCompressFloatLossyImp(const char * input, const int nelements, char *const output);
-int tsDecompressFloatLossyImp(const char * input, int compressedSize, const int nelements, char *const output);
+extern int tsDecompressFloatLossyImp(const char * input, int compressedSize, const int nelements, char *const output);
-int tsCompressDoubleLossyImp(const char * input, const int nelements, char *const output);
+extern int tsCompressDoubleLossyImp(const char * input, const int nelements, char *const output);
-int tsDecompressDoubleLossyImp(const char * input, int compressedSize, const int nelements, char *const output);
+extern int tsDecompressDoubleLossyImp(const char * input, int compressedSize, const int nelements, char *const output);
 
-// init
+extern bool lossyFloat;
-bool tsLossyInit();
+extern bool lossyDouble;
+// init call
+int tsCompressInit();
+// exit call
+void tsCompressExit();
 
 void cost_start();
 double cost_end(const char* tag);
@@ -223,7 +223,7 @@ static FORCE_INLINE int tsDecompressString(const char *const input, int compress
 static FORCE_INLINE int tsCompressFloat(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
                     char algorithm, char *const buffer, int bufferSize) {
   // lossy mode
-  if(gOpenLossy) {
+  if(lossyFloat) {
     return tsCompressFloatLossyImp(input, nelements, output);
   // lossless mode  
   } else {
@@ -269,7 +269,7 @@ static FORCE_INLINE int tsDecompressFloat(const char *const input, int compresse
 
 static FORCE_INLINE int tsCompressDouble(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
                      char algorithm, char *const buffer, int bufferSize) {
-  if(gOpenLossy){
+  if(lossyDouble){
     // lossy mode
     return tsCompressDoubleLossyImp(input, nelements, output);
   } else {

src/util/src/tcompression.c

@@ -53,6 +53,8 @@
 #include "taosdef.h"
 #include "tscompression.h"
 #include "tulog.h"
+#include "tglobal.h"
+
 
 static const int TEST_NUMBER = 1;
 #define is_bigendian() ((*(char *)&TEST_NUMBER) == 0)
@@ -62,7 +64,33 @@ static const int TEST_NUMBER = 1;
 #define ZIGZAG_ENCODE(T, v) ((u##T)((v) >> (sizeof(T) * 8 - 1))) ^ (((u##T)(v)) << 1)  // zigzag encode
 #define ZIGZAG_DECODE(T, v) ((v) >> 1) ^ -((T)((v)&1))                                 // zigzag decode
 
-bool  gOpenLossy = true;
+bool lossyFloat  = false;
+bool lossyDouble = false;
+
+// init call
+int tsCompressInit(){
+  // config 
+  if(lossyColumns[0] == 0){
+    lossyFloat = false;
+    lossyDouble = false;
+    return 0;
+  }
+
+  lossyFloat  = strstr(lossyColumns, "float") != NULL;
+  lossyDouble = strstr(lossyColumns, "double") != NULL;
+
+  if(lossyFloat == false && lossyDouble == false)
+        return 0;
+  
+  tdszInit(fPrecision, dPrecision, maxIntervals, intervals, Compressor);
+  uInfo("lossy compression is opened. columns = %s \n", lossyColumns);
+  return 1;
+}
+// exit call
+void tsCompressExit(){
+   tdszExit();
+}
+
 
 /*
  * Compress Integer (Simple8B).
@@ -891,20 +919,6 @@ int tsDecompressFloatImp(const char *const input, const int nelements, char *con
   return nelements * FLOAT_BYTES;
 }
 
-//
-// ----------- global init and exit resource ------
-//
-int SZ_Init(const char *configFilePath);  //declare deps/sz/include/sz.h
-
-bool gLossyInited = false;
-bool tsLossyInit() {
-  // init compress init
-  if(!gLossyInited){
-    gLossyInited = true;
-    SZ_Init("./sz.config");
-  }
-  return true;
-}
 
 //
 //   ----------  float double lossy  -----------

src/util/src/tconfig.c

@@ -61,6 +61,24 @@ static void taosReadFloatConfig(SGlobalCfg *cfg, char *input_value) {
   }
 }
 
+static void taosReadDoubleConfig(SGlobalCfg *cfg, char *input_value) {
+  double  value = atof(input_value);
+  double *option = (double *)cfg->ptr;
+  if (value < cfg->minValue || value > cfg->maxValue) {
+    uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%f",
+           cfg->option, input_value, cfg->minValue, cfg->maxValue, *option);
+  } else {
+    if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) {
+      *option = value;
+      cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE;
+    } else {
+      uWarn("config option:%s, input value:%s, is configured by %s, use %f", cfg->option, input_value,
+            tsCfgStatusStr[cfg->cfgStatus], *option);
+    }
+  }
+}
+
+
 static void taosReadInt32Config(SGlobalCfg *cfg, char *input_value) {
   int32_t  value = atoi(input_value);
   int32_t *option = (int32_t *)cfg->ptr;
@@ -262,6 +280,9 @@ static void taosReadConfigOption(const char *option, char *value, char *value2,
       case TAOS_CFG_VTYPE_FLOAT:
         taosReadFloatConfig(cfg, value);
         break;
+      case TAOS_CFG_VTYPE_DOUBLE:
+        taosReadDoubleConfig(cfg, value);
+        break;
       case TAOS_CFG_VTYPE_STRING:
         taosReadStringConfig(cfg, value);
         break;