From 3e24cb15ce3d349e905aa5fcf90c2ca057bf5d12 Mon Sep 17 00:00:00 2001 From: tickduan <417921451@qq.com> Date: Tue, 6 Jul 2021 09:23:08 +0800 Subject: [PATCH] remove pwr_ reduce to 41s --- deps/SZ/sz/include/CompressElement.h | 1 - deps/SZ/sz/include/TightDataPointStorageD.h | 23 +- deps/SZ/sz/include/TightDataPointStorageF.h | 23 +- deps/SZ/sz/include/dataCompression.h | 24 - deps/SZ/sz/include/defines.h | 5 +- deps/SZ/sz/include/sz.h | 6 - deps/SZ/sz/include/sz_double.h | 4 - deps/SZ/sz/include/sz_double_pwr.h | 40 -- deps/SZ/sz/include/sz_float.h | 6 - deps/SZ/sz/include/sz_float_pwr.h | 43 -- deps/SZ/sz/include/szd_double.h | 2 - deps/SZ/sz/include/szd_double_pwr.h | 28 - deps/SZ/sz/include/szd_double_ts.h | 25 - deps/SZ/sz/include/szd_float.h | 2 - deps/SZ/sz/include/szd_float_pwr.h | 30 - deps/SZ/sz/src/ByteToolkit.c | 6 +- deps/SZ/sz/src/CompressElement.c | 25 - deps/SZ/sz/src/TightDataPointStorageD.c | 431 +---------- deps/SZ/sz/src/TightDataPointStorageF.c | 417 ++--------- deps/SZ/sz/src/dataCompression.c | 298 -------- deps/SZ/sz/src/sz.c | 19 +- deps/SZ/sz/src/sz_double.c | 271 +------ deps/SZ/sz/src/sz_double_pwr.c | 752 -------------------- deps/SZ/sz/src/sz_double_ts.c | 191 ----- deps/SZ/sz/src/sz_float.c | 258 +------ deps/SZ/sz/src/sz_float_pwr.c | 736 ------------------- deps/SZ/sz/src/szd_double.c | 139 +--- deps/SZ/sz/src/szd_double_pwr.c | 371 ---------- deps/SZ/sz/src/szd_float.c | 185 +---- deps/SZ/sz/src/szd_float_pwr.c | 372 ---------- src/kit/taospack/taospack.c | 26 +- 31 files changed, 214 insertions(+), 4545 deletions(-) delete mode 100644 deps/SZ/sz/include/sz_double_pwr.h delete mode 100644 deps/SZ/sz/include/sz_float_pwr.h delete mode 100644 deps/SZ/sz/include/szd_double_pwr.h delete mode 100644 deps/SZ/sz/include/szd_double_ts.h delete mode 100644 deps/SZ/sz/include/szd_float_pwr.h delete mode 100644 deps/SZ/sz/src/sz_double_pwr.c delete mode 100644 deps/SZ/sz/src/sz_double_ts.c delete mode 100644 deps/SZ/sz/src/sz_float_pwr.c delete mode 100644 deps/SZ/sz/src/szd_double_pwr.c delete mode 100644 deps/SZ/sz/src/szd_float_pwr.c diff --git a/deps/SZ/sz/include/CompressElement.h b/deps/SZ/sz/include/CompressElement.h index e85cb62ad2..e396284436 100644 --- a/deps/SZ/sz/include/CompressElement.h +++ b/deps/SZ/sz/include/CompressElement.h @@ -45,7 +45,6 @@ typedef struct LossyCompressionElement int residualMidBits; } LossyCompressionElement; -char* decompressGroupIDArray(unsigned char* bytes, size_t dataLength); short computeGroupNum_float(float value); short computeGroupNum_double(double value); diff --git a/deps/SZ/sz/include/TightDataPointStorageD.h b/deps/SZ/sz/include/TightDataPointStorageD.h index b6b01aca80..a32a89aaab 100644 --- a/deps/SZ/sz/include/TightDataPointStorageD.h +++ b/deps/SZ/sz/include/TightDataPointStorageD.h @@ -33,10 +33,7 @@ typedef struct TightDataPointStorageD size_t exactDataNum; double reservedValue; - - unsigned char* rtypeArray; - size_t rtypeArray_size; - + unsigned char* typeArray; //its size is dataSeriesLength/4 (or xxx/4+1) size_t typeArray_size; @@ -54,9 +51,6 @@ typedef struct TightDataPointStorageD unsigned char isLossless; //a mark to denote whether it's lossless compression (1 is yes, 0 is no) size_t segment_size; - - unsigned char* pwrErrBoundBytes; - int pwrErrBoundBytes_size; unsigned char* raBytes; size_t raBytes_size; @@ -75,22 +69,11 @@ void new_TightDataPointStorageD(TightDataPointStorageD **self, unsigned char* leadNumIntArray, //leadNumIntArray contains readable numbers.... unsigned char* resiMidBits, size_t resiMidBits_size, unsigned char resiBitLength, - double realPrecision, double medianValue, char reqLength, unsigned int intervals, - unsigned char* pwrErrBoundBytes, size_t pwrErrBoundBytes_size, unsigned char radExpo); - -void new_TightDataPointStorageD2(TightDataPointStorageD **self, - size_t dataSeriesLength, size_t exactDataNum, - int* type, unsigned char* exactMidBytes, size_t exactMidBytes_size, - unsigned char* leadNumIntArray, //leadNumIntArray contains readable numbers.... - unsigned char* resiMidBits, size_t resiMidBits_size, - unsigned char* resiBitLength, size_t resiBitLengthSize, - double realPrecision, double medianValue, char reqLength, unsigned int intervals, - unsigned char* pwrErrBoundBytes, size_t pwrErrBoundBytes_size, unsigned char radExpo); + double realPrecision, double medianValue, char reqLength, + unsigned int intervals, unsigned char radExpo); void convertTDPStoBytes_double(TightDataPointStorageD* tdps, unsigned char* bytes, unsigned char* dsLengthBytes, unsigned char sameByte); -void convertTDPStoBytes_double_reserve(TightDataPointStorageD* tdps, unsigned char* bytes, unsigned char* dsLengthBytes, unsigned char sameByte); bool convertTDPStoFlatBytes_double(TightDataPointStorageD *tdps, unsigned char* bytes, size_t *size); -void convertTDPStoFlatBytes_double_args(TightDataPointStorageD *tdps, unsigned char* bytes, size_t *size); void free_TightDataPointStorageD(TightDataPointStorageD *tdps); void free_TightDataPointStorageD2(TightDataPointStorageD *tdps); diff --git a/deps/SZ/sz/include/TightDataPointStorageF.h b/deps/SZ/sz/include/TightDataPointStorageF.h index 536be706e7..d4f0ea1edc 100644 --- a/deps/SZ/sz/include/TightDataPointStorageF.h +++ b/deps/SZ/sz/include/TightDataPointStorageF.h @@ -33,13 +33,10 @@ typedef struct TightDataPointStorageF size_t exactDataNum; float reservedValue; - unsigned char* rtypeArray; - size_t rtypeArray_size; - float minLogValue; unsigned char* typeArray; //its size is dataSeriesLength/4 (or xxx/4+1) - size_t typeArray_size; + size_t typeArray_size; unsigned char* leadNumArray; //its size is exactDataNum/4 (or exactDataNum/4+1) size_t leadNumArray_size; @@ -56,8 +53,6 @@ typedef struct TightDataPointStorageF size_t segment_size; - unsigned char* pwrErrBoundBytes; - int pwrErrBoundBytes_size; unsigned char* raBytes; size_t raBytes_size; @@ -77,24 +72,10 @@ void new_TightDataPointStorageF(TightDataPointStorageF **self, unsigned char* resiMidBits, size_t resiMidBits_size, unsigned char resiBitLength, double realPrecision, float medianValue, char reqLength, unsigned int intervals, - unsigned char* pwrErrBoundBytes, size_t pwrErrBoundBytes_size, unsigned char radExpo); - -/** - * This function is designed for first-version of the point-wise relative error bound (developed by Sheng Di for TPDS18 paper) - * - * */ -void new_TightDataPointStorageF2(TightDataPointStorageF **self, - size_t dataSeriesLength, size_t exactDataNum, - int* type, unsigned char* exactMidBytes, size_t exactMidBytes_size, - unsigned char* leadNumIntArray, //leadNumIntArray contains readable numbers.... - unsigned char* resiMidBits, size_t resiMidBits_size, - unsigned char* resiBitLength, size_t resiBitLengthSize, - double realPrecision, float medianValue, char reqLength, unsigned int intervals, - unsigned char* pwrErrBoundBytes, size_t pwrErrBoundBytes_size, unsigned char radExpo); + unsigned char radExpo); void convertTDPStoBytes_float(TightDataPointStorageF* tdps, unsigned char* bytes, unsigned char* dsLengthBytes, unsigned char sameByte); bool convertTDPStoFlatBytes_float(TightDataPointStorageF *tdps, unsigned char* bytes, size_t *size); -void convertTDPStoFlatBytes_float_args(TightDataPointStorageF *tdps, unsigned char* bytes, size_t *size); void free_TightDataPointStorageF(TightDataPointStorageF *tdps); void free_TightDataPointStorageF2(TightDataPointStorageF *tdps); diff --git a/deps/SZ/sz/include/dataCompression.h b/deps/SZ/sz/include/dataCompression.h index f30696c9c0..3e15555acc 100644 --- a/deps/SZ/sz/include/dataCompression.h +++ b/deps/SZ/sz/include/dataCompression.h @@ -30,21 +30,9 @@ extern "C" { //dataCompression.c -int computeByteSizePerIntValue(long valueRangeSize); -long computeRangeSize_int(void* oriData, int dataType, size_t size, int64_t* valueRangeSize); double computeRangeSize_double(double* oriData, size_t size, double* valueRangeSize, double* medianValue); float computeRangeSize_float(float* oriData, size_t size, float* valueRangeSize, float* medianValue); -float computeRangeSize_float_MSST19(float* oriData, size_t size, float* valueRangeSize, float* medianValue, unsigned char * signs, bool* positive, float* nearZero); -double computeRangeSize_double_MSST19(double* oriData, size_t size, double* valueRangeSize, double* medianValue, unsigned char * signs, bool* positive, double* nearZero); -double computeRangeSize_double_subblock(double* oriData, double* valueRangeSize, double* medianValue, -size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, -size_t s5, size_t s4, size_t s3, size_t s2, size_t s1, -size_t e5, size_t e4, size_t e3, size_t e2, size_t e1); -float computeRangeSize_float_subblock(float* oriData, float* valueRangeSize, float* medianValue, -size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, -size_t s5, size_t s4, size_t s3, size_t s2, size_t s1, -size_t e5, size_t e4, size_t e3, size_t e2, size_t e1); double min_d(double a, double b); double max_d(double a, double b); float min_f(float a, float b); @@ -56,22 +44,10 @@ void symTransform_8bytes(unsigned char data[8]); void symTransform_2bytes(unsigned char data[2]); void symTransform_4bytes(unsigned char data[4]); -void compressInt8Value(int8_t tgtValue, int8_t minValue, int byteSize, unsigned char* bytes); -void compressInt16Value(int16_t tgtValue, int16_t minValue, int byteSize, unsigned char* bytes); -void compressInt32Value(int32_t tgtValue, int32_t minValue, int byteSize, unsigned char* bytes); -void compressInt64Value(int64_t tgtValue, int64_t minValue, int byteSize, unsigned char* bytes); - -void compressUInt8Value(uint8_t tgtValue, uint8_t minValue, int byteSize, unsigned char* bytes); -void compressUInt16Value(uint16_t tgtValue, uint16_t minValue, int byteSize, unsigned char* bytes); -void compressUInt32Value(uint32_t tgtValue, uint32_t minValue, int byteSize, unsigned char* bytes); -void compressUInt64Value(uint64_t tgtValue, uint64_t minValue, int byteSize, unsigned char* bytes); - void compressSingleFloatValue(FloatValueCompressElement *vce, float tgtValue, float precision, float medianValue, int reqLength, int reqBytesLength, int resiBitsLength); -void compressSingleFloatValue_MSST19(FloatValueCompressElement *vce, float tgtValue, float precision, int reqLength, int reqBytesLength, int resiBitsLength); void compressSingleDoubleValue(DoubleValueCompressElement *vce, double tgtValue, double precision, double medianValue, int reqLength, int reqBytesLength, int resiBitsLength); -void compressSingleDoubleValue_MSST19(DoubleValueCompressElement *vce, double tgtValue, double precision, int reqLength, int reqBytesLength, int resiBitsLength); int compIdenticalLeadingBytesCount_double(unsigned char* preBytes, unsigned char* curBytes); int compIdenticalLeadingBytesCount_float(unsigned char* preBytes, unsigned char* curBytes); diff --git a/deps/SZ/sz/include/defines.h b/deps/SZ/sz/include/defines.h index 009ec6f636..ed12bfd8cb 100644 --- a/deps/SZ/sz/include/defines.h +++ b/deps/SZ/sz/include/defines.h @@ -10,8 +10,8 @@ #ifndef _SZ_DEFINES_H #define _SZ_DEFINES_H -#define SZ_VERNUM 0x0200 -#define SZ_VER_MAJOR 1 // curretn version +// define data format version +#define DATA_FROMAT_VER1 1 // curretn version #define PASTRI 103 @@ -88,6 +88,7 @@ #define SZ_BERR -6 //bound-mode error (should be only SZ_ABS, REL, ABS_AND_REL, ABS_OR_REL, or PW_REL) #define SZ_LITTER_ELEMENT -7 #define SZ_ALGORITHM_ERR -8 +#define SZ_FORMAT_ERR -9 #define SZ_MAINTAIN_VAR_DATA 0 #define SZ_DESTROY_WHOLE_VARSET 1 diff --git a/deps/SZ/sz/include/sz.h b/deps/SZ/sz/include/sz.h index 96cf212a29..2b7a86fd07 100644 --- a/deps/SZ/sz/include/sz.h +++ b/deps/SZ/sz/include/sz.h @@ -19,7 +19,6 @@ #include "DynamicByteArray.h" #include "DynamicIntArray.h" #include "VarSet.h" -#include "Huffman.h" #include "TightDataPointStorageD.h" #include "TightDataPointStorageF.h" #include "conf.h" @@ -30,8 +29,6 @@ #include "sz_double.h" #include "szd_float.h" #include "szd_double.h" -#include "sz_float_pwr.h" -#include "sz_double_pwr.h" #include "sz_opencl.h" #include "callZlib.h" #include "rw.h" @@ -185,9 +182,6 @@ extern sz_tsc_metadata *sz_tsc; extern pastri_params pastri_par; #endif -//sz.h -HuffmanTree* SZ_Reset(); - void SZ_Finalize(); int SZ_Init(const char *configFilePath); diff --git a/deps/SZ/sz/include/sz_double.h b/deps/SZ/sz/include/sz_double.h index 930968f70f..b83632fba5 100644 --- a/deps/SZ/sz/include/sz_double.h +++ b/deps/SZ/sz/include/sz_double.h @@ -20,22 +20,18 @@ extern "C" { unsigned char* SZ_skip_compress_double(double* data, size_t dataLength, size_t* outSize); void computeReqLength_double(double realPrecision, short radExpo, int* reqLength, double* medianValue); -short computeReqLength_double_MSST19(double realPrecision); unsigned int optimize_intervals_double_1D(double *oriData, size_t dataLength, double realPrecision); unsigned int optimize_intervals_double_1D_opt(double *oriData, size_t dataLength, double realPrecision); -unsigned int optimize_intervals_double_1D_opt_MSST19(double *oriData, size_t dataLength, double realPrecision); TightDataPointStorageD* SZ_compress_double_1D_MDQ(double *oriData, size_t dataLength, double realPrecision, double valueRangeSize, double medianValue_d); void SZ_compress_args_double_StoreOriData(double* oriData, size_t dataLength, unsigned char* newByteData, size_t *outSize); bool SZ_compress_args_double_NoCkRngeNoGzip_1D( unsigned char* newByteData, double *oriData, size_t dataLength, double realPrecision, size_t *outSize, double valueRangeSize, double medianValue_d); -TightDataPointStorageD* SZ_compress_double_1D_MDQ_MSST19(double *oriData, size_t dataLength, double realPrecision, double valueRangeSize, double medianValue_f); - void SZ_compress_args_double_withinRange(unsigned char* newByteData, double *oriData, size_t dataLength, size_t *outSize); diff --git a/deps/SZ/sz/include/sz_double_pwr.h b/deps/SZ/sz/include/sz_double_pwr.h deleted file mode 100644 index 93d53c7c29..0000000000 --- a/deps/SZ/sz/include/sz_double_pwr.h +++ /dev/null @@ -1,40 +0,0 @@ -/** - * @file sz_double.h - * @author Sheng Di - * @date July, 2017 - * @brief Header file for the sz_double.c. - * (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -#ifndef _SZ_Double_PWR_H -#define _SZ_Double_PWR_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include - -void compute_segment_precisions_double_1D(double *oriData, size_t dataLength, double* pwrErrBound, unsigned char* pwrErrBoundBytes, double globalPrecision); -unsigned int optimize_intervals_double_1D_pwr(double *oriData, size_t dataLength, double* pwrErrBound); -void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr(unsigned char* newByteData, double *oriData, double globalPrecision, size_t dataLength, size_t *outSize, double min, double max); -void createRangeGroups_double(double** posGroups, double** negGroups, int** posFlags, int** negFlags); -void compressGroupIDArray_double(char* groupID, TightDataPointStorageD* tdps); -TightDataPointStorageD* SZ_compress_double_1D_MDQ_pwrGroup(double* oriData, size_t dataLength, int errBoundMode, -double absErrBound, double relBoundRatio, double pwrErrRatio, double valueRangeSize, double medianValue_f); -void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwrgroup(unsigned char* newByteData, double *oriData, -size_t dataLength, double absErrBound, double relBoundRatio, double pwrErrRatio, double valueRangeSize, double medianValue_f, size_t *outSize); - -void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log(unsigned char* newByteData, double *oriData, double globalPrecision, size_t dataLength, size_t *outSize, double min, double max); - -void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log_MSST19(unsigned char* newByteData, double *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, double valueRangeSize, double medianValue_f, - unsigned char* signs, bool* positive, double min, double max, double nearZero); - -#ifdef __cplusplus -} -#endif - -#endif /* ----- #ifndef _SZ_Double_PWR_H ----- */ - diff --git a/deps/SZ/sz/include/sz_float.h b/deps/SZ/sz/include/sz_float.h index 3298f863ef..80962d5f12 100644 --- a/deps/SZ/sz/include/sz_float.h +++ b/deps/SZ/sz/include/sz_float.h @@ -17,15 +17,12 @@ extern "C" { unsigned char* SZ_skip_compress_float(float* data, size_t dataLength, size_t* outSize); void computeReqLength_float(double realPrecision, short radExpo, int* reqLength, float* medianValue); -short computeReqLength_float_MSST19(double realPrecision); unsigned int optimize_intervals_float_1D(float *oriData, size_t dataLength, double realPrecision); unsigned int optimize_intervals_and_compute_dense_position_float_1D(float *oriData, size_t dataLength, double realPrecision, float * dense_pos); unsigned int optimize_intervals_float_1D_opt(float *oriData, size_t dataLength, double realPrecision); -unsigned int optimize_intervals_float_1D_opt_MSST19(float *oriData, size_t dataLength, double realPrecision); - TightDataPointStorageF* SZ_compress_float_1D_MDQ(float *oriData, size_t dataLength, float realPrecision, float valueRangeSize, float medianValue_f); @@ -40,9 +37,6 @@ void SZ_blocked_regression(float * block_ori_data, size_t dim_0, size_t dim_1, s unsigned char * SZ_compress_float_1D_MDQ_RA(float *oriData, size_t r1, double realPrecision, size_t * comp_size); -TightDataPointStorageF* SZ_compress_float_1D_MDQ_MSST19(float *oriData, -size_t dataLength, double realPrecision, float valueRangeSize, float medianValue_f); - void SZ_compress_args_float_withinRange(unsigned char* newByteData, float *oriData, size_t dataLength, size_t *outSize); diff --git a/deps/SZ/sz/include/sz_float_pwr.h b/deps/SZ/sz/include/sz_float_pwr.h deleted file mode 100644 index 7fd75044ae..0000000000 --- a/deps/SZ/sz/include/sz_float_pwr.h +++ /dev/null @@ -1,43 +0,0 @@ -/** - * @file sz_float.h - * @author Sheng Di - * @date July, 2017 - * @brief Header file for the sz_float.c. - * (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -#ifndef _SZ_Float_PWR_H -#define _SZ_Float_PWR_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include - -void compute_segment_precisions_float_1D(float *oriData, size_t dataLength, float* pwrErrBound, unsigned char* pwrErrBoundBytes, double globalPrecision); -unsigned int optimize_intervals_float_1D_pwr(float *oriData, size_t dataLength, float* pwrErrBound); -void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr(unsigned char* newByteData, float *oriData, double globalPrecision, size_t dataLength, size_t *outSize, float min, float max); - - -void createRangeGroups_float(float** posGroups, float** negGroups, int** posFlags, int** negFlags); -void compressGroupIDArray_float(char* groupID, TightDataPointStorageF* tdps); -int* generateGroupLowerBounds(); -TightDataPointStorageF* SZ_compress_float_1D_MDQ_pwrGroup(float* oriData, size_t dataLength, int errBoundMode, -double absErrBound, double relBoundRatio, double pwrErrRatio, float valueRangeSize, float medianValue_f); - -void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwrgroup(unsigned char* newByteData, float *oriData, -size_t dataLength, double absErrBound, double relBoundRatio, double pwrErrRatio, float valueRangeSize, float medianValue_f, size_t *outSize); - -void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_pre_log(unsigned char* newByteData, float *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, float min, float max); -void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_pre_log_MSST19(unsigned char* newByteData, float *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, float valueRangeSize, float medianValue_f, - unsigned char* signs, bool* positive, float min, float max, float nearZero); - -#ifdef __cplusplus -} -#endif - -#endif /* ----- #ifndef _SZ_Float_PWR_H ----- */ - diff --git a/deps/SZ/sz/include/szd_double.h b/deps/SZ/sz/include/szd_double.h index a314e0b3fa..565324a25f 100644 --- a/deps/SZ/sz/include/szd_double.h +++ b/deps/SZ/sz/include/szd_double.h @@ -18,8 +18,6 @@ extern "C" { void decompressDataSeries_double_1D(double* data, size_t dataSeriesLength, double* hist_data, TightDataPointStorageD* tdps); -void decompressDataSeries_double_1D_MSST19(double* data, size_t dataSeriesLength, TightDataPointStorageD* tdps); - void getSnapshotData_double_1D(double* data, size_t dataSeriesLength, TightDataPointStorageD* tdps, int errBoundMode, int compressionType, double* hist_data, sz_params* pde_params); int SZ_decompress_args_double(double* newData, size_t r1, unsigned char* cmpBytes, size_t cmpSize, int compressionType, double* hist_data, sz_exedata* pde_exe, sz_params* pde_params); diff --git a/deps/SZ/sz/include/szd_double_pwr.h b/deps/SZ/sz/include/szd_double_pwr.h deleted file mode 100644 index 9dd87cf5d1..0000000000 --- a/deps/SZ/sz/include/szd_double_pwr.h +++ /dev/null @@ -1,28 +0,0 @@ -/** - * @file szd_double_pwr.h - * @author Sheng Di - * @date July, 2017 - * @brief Header file for the szd_double_pwr.c. - * (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -#ifndef _SZD_Double_PWR_H -#define _SZD_Double_PWR_H - -#ifdef __cplusplus -extern "C" { -#endif - -void decompressDataSeries_double_1D_pwr(double** data, size_t dataSeriesLength, TightDataPointStorageD* tdps); - -void decompressDataSeries_double_1D_pwrgroup(double** data, size_t dataSeriesLength, TightDataPointStorageD* tdps); -void decompressDataSeries_double_1D_pwr_pre_log(double** data, size_t dataSeriesLength, TightDataPointStorageD* tdps); - -void decompressDataSeries_double_1D_pwr_pre_log_MSST19(double** data, size_t dataSeriesLength, TightDataPointStorageD* tdps); - -#ifdef __cplusplus -} -#endif - -#endif /* ----- #ifndef _SZD_Double_PWR_H ----- */ diff --git a/deps/SZ/sz/include/szd_double_ts.h b/deps/SZ/sz/include/szd_double_ts.h deleted file mode 100644 index 6f7a768aef..0000000000 --- a/deps/SZ/sz/include/szd_double_ts.h +++ /dev/null @@ -1,25 +0,0 @@ -/** - * @file szd_double_ts.h - * @author Sheng Di - * @date July, 2017 - * @brief Header file for the szd_double_ts.c. - * (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -#ifndef _SZD_Double_TS_H -#define _SZD_Double_TS_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "TightDataPointStorageD.h" - -void decompressDataSeries_double_1D_ts(double** data, size_t dataSeriesLength, double* hist_data, TightDataPointStorageD* tdps); - -#ifdef __cplusplus -} -#endif - -#endif /* ----- #ifndef _SZD_Double_TS_H ----- */ diff --git a/deps/SZ/sz/include/szd_float.h b/deps/SZ/sz/include/szd_float.h index 224853d081..39caabff73 100644 --- a/deps/SZ/sz/include/szd_float.h +++ b/deps/SZ/sz/include/szd_float.h @@ -18,8 +18,6 @@ extern "C" { void decompressDataSeries_float_1D(float* data, size_t dataSeriesLength, float* hist_data, TightDataPointStorageF* tdps); -void decompressDataSeries_float_1D_MSST19(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps); - void getSnapshotData_float_1D(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps, int errBoundMode, int compressionType, float* hist_data, sz_params* pde_params); int SZ_decompress_args_float(float* newData, size_t r1, unsigned char* cmpBytes, size_t cmpSize, int compressionType, float* hist_data, sz_exedata* pde_exe, sz_params* pde_params); diff --git a/deps/SZ/sz/include/szd_float_pwr.h b/deps/SZ/sz/include/szd_float_pwr.h deleted file mode 100644 index 769686dc17..0000000000 --- a/deps/SZ/sz/include/szd_float_pwr.h +++ /dev/null @@ -1,30 +0,0 @@ -/** - * @file szd_float_pwr.h - * @author Sheng Di - * @date July, 2017 - * @brief Header file for the szd_float_pwr.c. - * (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -#ifndef _SZD_Float_PWR_H -#define _SZD_Float_PWR_H - -#ifdef __cplusplus -extern "C" { -#endif - -void decompressDataSeries_float_1D_pwr(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps); - -char* decompressGroupIDArray(unsigned char* bytes, size_t dataLength); -void decompressDataSeries_float_1D_pwrgroup(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps); -void decompressDataSeries_float_1D_pwr_pre_log(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps); - -void decompressDataSeries_float_1D_pwr_pre_log_MSST19(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps); - -#ifdef __cplusplus -} -#endif - -#endif /* ----- #ifndef _SZD_Float_PWR_H ----- */ - diff --git a/deps/SZ/sz/src/ByteToolkit.c b/deps/SZ/sz/src/ByteToolkit.c index 2df6736c81..24041efeab 100644 --- a/deps/SZ/sz/src/ByteToolkit.c +++ b/deps/SZ/sz/src/ByteToolkit.c @@ -906,10 +906,10 @@ void convertSZParamsToBytes(sz_params* params, unsigned char* result) //int16ToBytes_bigEndian(&result[3], tmp2); //errorBoundMode; //4bits(0.5 byte) - result[1] = params->errorBoundMode; + //result[1] = params->errorBoundMode; //data type (float, double, int8, int16, ....) //10 choices, so 4 bits - result[1] = (result[1] << 4) | (params->dataType & 0x17); + //result[1] = (result[1] << 4) | (params->dataType & 0x17); //result[5]: abs_err_bound or psnr //4 bytes //result[9]: rel_bound_ratio or pwr_err_bound//4 bytes @@ -1001,10 +1001,10 @@ void convertBytesToSZParams(unsigned char* bytes, sz_params* params, sz_exedata* //params->sampleDistance = bytesToInt16_bigEndian(&bytes[1]); //params->predThreshold = 1.0*bytesToInt16_bigEndian(&bytes[3])/10000.0; + /* params->dataType = bytes[1] & 0x07; params->errorBoundMode = (bytes[1] & 0xf0) >> 4; - /* switch(params->errorBoundMode) { case SZ_ABS: diff --git a/deps/SZ/sz/src/CompressElement.c b/deps/SZ/sz/src/CompressElement.c index ee8a179b9f..c941ded46d 100644 --- a/deps/SZ/sz/src/CompressElement.c +++ b/deps/SZ/sz/src/CompressElement.c @@ -16,31 +16,6 @@ #include #include -char* decompressGroupIDArray(unsigned char* bytes, size_t dataLength) -{ - HuffmanTree* huffmanTree = SZ_Reset(); //create a default huffman tree - int* standGroupID = (int*)malloc(dataLength*sizeof(int)); - decode_withTree(huffmanTree, bytes, dataLength, standGroupID); - SZ_ReleaseHuffman(huffmanTree); - - char* groupID = (char*)malloc(dataLength*sizeof(char)); - size_t i = 0; - int lastGroupIDValue = 0, curStandIDValue = 0, curGroupIDValue = 0; - int offset = 2*(GROUP_COUNT + 2); - - curGroupIDValue = groupID[0] = standGroupID[0] - GROUP_COUNT; - lastGroupIDValue = curGroupIDValue; - for(i=1;i #include "TightDataPointStorageD.h" #include "sz.h" +#include "defines.h" #include "Huffman.h" -//#include "rw.h" void new_TightDataPointStorageD_Empty(TightDataPointStorageD **this) { @@ -25,12 +25,6 @@ void new_TightDataPointStorageD_Empty(TightDataPointStorageD **this) (*this)->reqLength = 0; (*this)->radExpo = 0; - (*this)->rtypeArray = NULL; - (*this)->rtypeArray_size = 0; - - (*this)->typeArray = NULL; //its size is dataSeriesLength/4 (or xxx/4+1) - (*this)->typeArray_size = 0; - (*this)->leadNumArray = NULL; //its size is exactDataNum/4 (or exactDataNum/4+1) (*this)->leadNumArray_size = 0; @@ -44,8 +38,6 @@ void new_TightDataPointStorageD_Empty(TightDataPointStorageD **this) (*this)->isLossless = 0; (*this)->segment_size = 0; - (*this)->pwrErrBoundBytes = NULL; - (*this)->pwrErrBoundBytes_size = 0; (*this)->raBytes = NULL; (*this)->raBytes_size = 0; @@ -56,29 +48,25 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi { new_TightDataPointStorageD_Empty(this); size_t i, index = 0; - size_t pwrErrBoundBytes_size = 0, segmentL = 0, radExpoL = 0, pwrErrBoundBytesL = 0; unsigned char version = flatBytes[index++]; //3 unsigned char sameRByte = flatBytes[index++]; //1 - if(checkVersion(version)!=1) + + // parse data format + switch (version) { - //wrong version - printf("Wrong version: \nCompressed-data %s . program version=%d\n", version, versionNumber); - return 0; + case DATA_FROMAT_VER1: + break; + default: + printf(" error, compressed data format can not be recognised. ver=%d\n ", version); + return SZ_ABS; } int same = sameRByte & 0x01; (*this)->isLossless = (sameRByte & 0x10)>>4; - int isPW_REL = (sameRByte & 0x20)>>5; exe_params->SZ_SIZE_TYPE = ((sameRByte & 0x40)>>6)==1?8:4; //pde_params->protectValueRange = (sameRByte & 0x04)>>2; pde_params->accelerate_pw_rel_compression = (sameRByte & 0x08) >> 3; int errorBoundMode = SZ_ABS; - if(isPW_REL) - { - errorBoundMode = PW_REL; - segmentL = exe_params->SZ_SIZE_TYPE; - pwrErrBoundBytesL = 4; - } convertBytesToSZParams(&(flatBytes[index]), pde_params, pde_exe); @@ -111,9 +99,7 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi (*this)->raBytes_size = flatBytesLength - 3 - 1 - MetaDataByteLength_double - exe_params->SZ_SIZE_TYPE; (*this)->raBytes = &(flatBytes[index]); return errorBoundMode; - } - - int rtype_ = 0;//sameRByte & 0x08; //1000 + } unsigned char byteBuf[8]; @@ -123,24 +109,6 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi pde_params->maxRangeRadius = max_quant_intervals/2; - if(errorBoundMode>=PW_REL) - { - (*this)->radExpo = flatBytes[index++];//1 - radExpoL = 1; - for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++) - byteBuf[i] = flatBytes[index++]; - pde_params->segment_size = (*this)->segment_size = bytesToSize(byteBuf);// exe_params->SZ_SIZE_TYPE - - for (i = 0; i < 4; i++) - byteBuf[i] = flatBytes[index++]; - pwrErrBoundBytes_size = (*this)->pwrErrBoundBytes_size = bytesToInt_bigEndian(byteBuf);// 4 - } - else - { - pwrErrBoundBytes_size = 0; - (*this)->pwrErrBoundBytes = NULL; - } - for (i = 0; i < 4; i++) byteBuf[i] = flatBytes[index++]; (*this)->intervals = bytesToInt_bigEndian(byteBuf);// 4 @@ -151,29 +119,14 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi (*this)->reqLength = flatBytes[index++]; //1 - if(isPW_REL && pde_params->accelerate_pw_rel_compression) - { - (*this)->plus_bits = flatBytes[index++]; - (*this)->max_bits = flatBytes[index++]; - } - for (i = 0; i < 8; i++) byteBuf[i] = flatBytes[index++]; (*this)->realPrecision = bytesToDouble(byteBuf);//8 for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++) - byteBuf[i] = flatBytes[index++]; + byteBuf[i] = flatBytes[index++]; (*this)->typeArray_size = bytesToSize(byteBuf);// exe_params->SZ_SIZE_TYPE - if(rtype_!=0) - { - for(i = 0;iSZ_SIZE_TYPE;i++) - byteBuf[i] = flatBytes[index++]; - (*this)->rtypeArray_size = bytesToSize(byteBuf);//ST - } - else - (*this)->rtypeArray_size = 0; - for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++) byteBuf[i] = flatBytes[index++]; (*this)->exactDataNum = bytesToSize(byteBuf);// ST @@ -182,17 +135,6 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi byteBuf[i] = flatBytes[index++]; (*this)->exactMidBytes_size = bytesToSize(byteBuf);// ST - if (rtype_ != 0) { - if((*this)->rtypeArray_size>0) - (*this)->rtypeArray = (unsigned char*)malloc(sizeof(unsigned char)*(*this)->rtypeArray_size); - else - (*this)->rtypeArray = NULL; - - for (i = 0; i < 8; i++) - byteBuf[i] = flatBytes[index++]; - (*this)->reservedValue = bytesToDouble(byteBuf);//8 - } - size_t logicLeadNumBitsNum = (*this)->exactDataNum * 2; if (logicLeadNumBitsNum % 8 == 0) { @@ -203,43 +145,19 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi (*this)->leadNumArray_size = (logicLeadNumBitsNum >> 3) + 1; } - int minLogValueSize = 0; - if(errorBoundMode>=PW_REL) - minLogValueSize = 8; - - if ((*this)->rtypeArray != NULL) - { - // -3 -2 modify to -1 -1 - (*this)->residualMidBits_size = flatBytesLength - 1 - 1 - MetaDataByteLength_double - exe_params->SZ_SIZE_TYPE - 4 - radExpoL - segmentL - pwrErrBoundBytesL - 4 - 8 - 1 - 8 - - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - minLogValueSize - exe_params->SZ_SIZE_TYPE - 8 - (*this)->rtypeArray_size - - minLogValueSize - (*this)->typeArray_size - (*this)->leadNumArray_size - - (*this)->exactMidBytes_size - pwrErrBoundBytes_size - 1 - 1; - for (i = 0; i < (*this)->rtypeArray_size; i++) - (*this)->rtypeArray[i] = flatBytes[index++]; - } - else - { - (*this)->residualMidBits_size = flatBytesLength - 1 - 1 - MetaDataByteLength_double - exe_params->SZ_SIZE_TYPE - 4 - radExpoL - segmentL - pwrErrBoundBytesL - 4 - 8 - 1 - 8 - - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - minLogValueSize - (*this)->typeArray_size - - (*this)->leadNumArray_size - (*this)->exactMidBytes_size - pwrErrBoundBytes_size - 1 - 1; - } - - if(errorBoundMode >= PW_REL){ - (*this)->minLogValue = bytesToDouble(&flatBytes[index]); - index+=8; - } - (*this)->typeArray = &flatBytes[index]; //retrieve the number of states (i.e., stateNum) (*this)->allNodes = bytesToInt_bigEndian((*this)->typeArray); //the first 4 bytes store the stateNum (*this)->stateNum = ((*this)->allNodes+1)/2; index+=(*this)->typeArray_size; - - (*this)->pwrErrBoundBytes = &flatBytes[index]; - - index+=pwrErrBoundBytes_size; - + + + // todo need check length + (*this)->residualMidBits_size = flatBytesLength - 1 - 1 - MetaDataByteLength - exe_params->SZ_SIZE_TYPE - 4 - 4 - 4 - 1 - 8 + - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE + - (*this)->leadNumArray_size - (*this)->exactMidBytes_size - (*this)->typeArray_size; + (*this)->leadNumArray = &flatBytes[index]; index+=(*this)->leadNumArray_size; @@ -250,7 +168,6 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi (*this)->residualMidBits = &flatBytes[index]; - //index+=(*this)->residualMidBits_size; return errorBoundMode; } @@ -270,7 +187,8 @@ void new_TightDataPointStorageD(TightDataPointStorageD **this, unsigned char* resiMidBits, size_t resiMidBits_size, unsigned char resiBitLength, double realPrecision, double medianValue, char reqLength, unsigned int intervals, - unsigned char* pwrErrBoundBytes, size_t pwrErrBoundBytes_size, unsigned char radExpo) { + unsigned char radExpo) +{ //int i = 0; *this = (TightDataPointStorageD *)malloc(sizeof(TightDataPointStorageD)); (*this)->allSameData = 0; @@ -281,15 +199,9 @@ void new_TightDataPointStorageD(TightDataPointStorageD **this, (*this)->dataSeriesLength = dataSeriesLength; (*this)->exactDataNum = exactDataNum; - (*this)->rtypeArray = NULL; - (*this)->rtypeArray_size = 0; - int stateNum = 2*intervals; HuffmanTree* huffmanTree = createHuffmanTree(stateNum); - if(confparams_cpr->errorBoundMode == PW_REL && confparams_cpr->accelerate_pw_rel_compression) - (*this)->max_bits = encode_withTree_MSST19(huffmanTree, type, dataSeriesLength, &(*this)->typeArray, &(*this)->typeArray_size); - else - encode_withTree(huffmanTree, type, dataSeriesLength, &(*this)->typeArray, &(*this)->typeArray_size); + encode_withTree(huffmanTree, type, dataSeriesLength, &(*this)->typeArray, &(*this)->typeArray_size); SZ_ReleaseHuffman(huffmanTree); (*this)->exactMidBytes = exactMidBytes; @@ -302,65 +214,8 @@ void new_TightDataPointStorageD(TightDataPointStorageD **this, (*this)->intervals = intervals; (*this)->isLossless = 0; - - if(confparams_cpr->errorBoundMode>=PW_REL) - (*this)->pwrErrBoundBytes = pwrErrBoundBytes; - else - (*this)->pwrErrBoundBytes = NULL; - + (*this)->radExpo = radExpo; - - (*this)->pwrErrBoundBytes_size = pwrErrBoundBytes_size; -} - -void new_TightDataPointStorageD2(TightDataPointStorageD **this, - size_t dataSeriesLength, size_t exactDataNum, - int* type, unsigned char* exactMidBytes, size_t exactMidBytes_size, - unsigned char* leadNumIntArray, //leadNumIntArray contains readable numbers.... - unsigned char* resiMidBits, size_t resiMidBits_size, - unsigned char* resiBitLength, size_t resiBitLengthSize, - double realPrecision, double medianValue, char reqLength, unsigned int intervals, - unsigned char* pwrErrBoundBytes, size_t pwrErrBoundBytes_size, unsigned char radExpo) { - //int i = 0; - *this = (TightDataPointStorageD *)malloc(sizeof(TightDataPointStorageD)); - (*this)->allSameData = 0; - (*this)->realPrecision = realPrecision; - (*this)->medianValue = medianValue; - (*this)->reqLength = reqLength; - - (*this)->dataSeriesLength = dataSeriesLength; - (*this)->exactDataNum = exactDataNum; - - (*this)->rtypeArray = NULL; - (*this)->rtypeArray_size = 0; - - int stateNum = 2*intervals; - HuffmanTree* huffmanTree = createHuffmanTree(stateNum); - encode_withTree(huffmanTree, type, dataSeriesLength, &(*this)->typeArray, &(*this)->typeArray_size); - SZ_ReleaseHuffman(huffmanTree); - - (*this)->exactMidBytes = exactMidBytes; - (*this)->exactMidBytes_size = exactMidBytes_size; - - (*this)->leadNumArray_size = convertIntArray2ByteArray_fast_2b(leadNumIntArray, exactDataNum, &((*this)->leadNumArray)); - - //(*this)->residualMidBits = resiMidBits; - //(*this)->residualMidBits_size = resiMidBits_size; - - (*this)->residualMidBits_size = convertIntArray2ByteArray_fast_dynamic2(resiMidBits, resiBitLength, resiBitLengthSize, &((*this)->residualMidBits)); - - (*this)->intervals = intervals; - - (*this)->isLossless = 0; - - if(confparams_cpr->errorBoundMode>=PW_REL) - (*this)->pwrErrBoundBytes = pwrErrBoundBytes; - else - (*this)->pwrErrBoundBytes = NULL; - - (*this)->radExpo = radExpo; - - (*this)->pwrErrBoundBytes_size = pwrErrBoundBytes_size; } void convertTDPStoBytes_double(TightDataPointStorageD* tdps, unsigned char* bytes, unsigned char* dsLengthBytes, unsigned char sameByte) @@ -373,9 +228,6 @@ void convertTDPStoBytes_double(TightDataPointStorageD* tdps, unsigned char* byte unsigned char realPrecisionBytes[8]; unsigned char medianValueBytes[8]; - - unsigned char segment_sizeBytes[8]; - unsigned char pwrErrBoundBytes_sizeBytes[4]; unsigned char max_quant_intervals_Bytes[4]; bytes[k++] = versionNumber; @@ -390,19 +242,6 @@ void convertTDPStoBytes_double(TightDataPointStorageD* tdps, unsigned char* byte for(i = 0;i<4;i++)//4 bytes[k++] = max_quant_intervals_Bytes[i]; - if(confparams_cpr->errorBoundMode>=PW_REL) - { - bytes[k++] = tdps->radExpo; //1 byte - - sizeToBytes(segment_sizeBytes, confparams_cpr->segment_size); - for(i = 0;iSZ_SIZE_TYPE;i++)//ST - bytes[k++] = segment_sizeBytes[i]; - - intToBytes_bigEndian(pwrErrBoundBytes_sizeBytes, tdps->pwrErrBoundBytes_size); - for(i = 0;i<4;i++)//4 - bytes[k++] = pwrErrBoundBytes_sizeBytes[i]; - } - intToBytes_bigEndian(intervalsBytes, tdps->intervals); for(i = 0;i<4;i++)//4 bytes[k++] = intervalsBytes[i]; @@ -413,16 +252,10 @@ void convertTDPStoBytes_double(TightDataPointStorageD* tdps, unsigned char* byte bytes[k++] = tdps->reqLength; //1 byte - if(confparams_cpr->errorBoundMode == PW_REL && confparams_cpr->accelerate_pw_rel_compression==1) - { - bytes[k++] = tdps->plus_bits; - bytes[k++] = tdps->max_bits; - } - doubleToBytes(realPrecisionBytes, tdps->realPrecision); for (i = 0; i < 8; i++)// 8 - bytes[k++] = realPrecisionBytes[i]; - + bytes[k++] = realPrecisionBytes[i]; + sizeToBytes(typeArrayLengthBytes, tdps->typeArray_size); for(i = 0;iSZ_SIZE_TYPE;i++)//ST bytes[k++] = typeArrayLengthBytes[i]; @@ -441,15 +274,10 @@ void convertTDPStoBytes_double(TightDataPointStorageD* tdps, unsigned char* byte for(i = 0;i < 8; i++) bytes[k++] = exactMidBytesLength[i]; } - + + // copy data memcpy(&(bytes[k]), tdps->typeArray, tdps->typeArray_size); k += tdps->typeArray_size; - if(confparams_cpr->errorBoundMode>=PW_REL) - { - memcpy(&(bytes[k]), tdps->pwrErrBoundBytes, tdps->pwrErrBoundBytes_size); - k += tdps->pwrErrBoundBytes_size; - } - memcpy(&(bytes[k]), tdps->leadNumArray, tdps->leadNumArray_size); k += tdps->leadNumArray_size; memcpy(&(bytes[k]), tdps->exactMidBytes, tdps->exactMidBytes_size); @@ -462,110 +290,6 @@ void convertTDPStoBytes_double(TightDataPointStorageD* tdps, unsigned char* byte } } -void convertTDPStoBytes_double_reserve(TightDataPointStorageD* tdps, unsigned char* bytes, unsigned char* dsLengthBytes, unsigned char sameByte) -{ - size_t i, k = 0; - unsigned char intervalsBytes[4]; - unsigned char typeArrayLengthBytes[8]; - unsigned char rTypeLengthBytes[8]; - unsigned char exactLengthBytes[8]; - unsigned char exactMidBytesLength[8]; - unsigned char reservedValueBytes[8]; - unsigned char realPrecisionBytes[8]; - - unsigned char medianValueBytes[8]; - - unsigned char segment_sizeBytes[8]; - unsigned char pwrErrBoundBytes_sizeBytes[4]; - unsigned char max_quant_intervals_Bytes[4]; - - bytes[k++] = versionNumber; - bytes[k++] = sameByte; //1 - - convertSZParamsToBytes(confparams_cpr, &(bytes[k])); - k = k + MetaDataByteLength_double; - - for(i = 0;iSZ_SIZE_TYPE;i++)//ST - bytes[k++] = dsLengthBytes[i]; - - intToBytes_bigEndian(max_quant_intervals_Bytes, confparams_cpr->max_quant_intervals); - for(i = 0;i<4;i++)//4 - bytes[k++] = max_quant_intervals_Bytes[i]; - - if(confparams_cpr->errorBoundMode>=PW_REL) - { - bytes[k++] = tdps->radExpo; //1 byte - - sizeToBytes(segment_sizeBytes, confparams_cpr->segment_size); - for(i = 0;iSZ_SIZE_TYPE;i++)//4 - bytes[k++] = segment_sizeBytes[i]; - - intToBytes_bigEndian(pwrErrBoundBytes_sizeBytes, tdps->pwrErrBoundBytes_size); - for(i = 0;i<4;i++)//4 - bytes[k++] = pwrErrBoundBytes_sizeBytes[i]; - } - intToBytes_bigEndian(intervalsBytes, tdps->intervals); - for(i = 0;i<4;i++)//4 - bytes[k++] = intervalsBytes[i]; - - doubleToBytes(medianValueBytes, tdps->medianValue); - for (i = 0; i < 8; i++)// 8 - bytes[k++] = medianValueBytes[i]; - - bytes[k++] = tdps->reqLength; //1 byte - - doubleToBytes(realPrecisionBytes, tdps->realPrecision); - for (i = 0; i < 8; i++)// 8 - bytes[k++] = realPrecisionBytes[i]; - - sizeToBytes(typeArrayLengthBytes, tdps->typeArray_size); - for(i = 0;iSZ_SIZE_TYPE;i++)//ST - bytes[k++] = typeArrayLengthBytes[i]; - - sizeToBytes(rTypeLengthBytes, tdps->rtypeArray_size); - for(i = 0;iSZ_SIZE_TYPE;i++)//ST - bytes[k++] = rTypeLengthBytes[i]; - - sizeToBytes(exactLengthBytes, tdps->exactDataNum); - for(i = 0;iSZ_SIZE_TYPE;i++)//ST - bytes[k++] = exactLengthBytes[i]; - - sizeToBytes(exactMidBytesLength, tdps->exactMidBytes_size); - for(i = 0;iSZ_SIZE_TYPE;i++)//ST - bytes[k++] = exactMidBytesLength[i]; - - doubleToBytes(reservedValueBytes, tdps->reservedValue); - for (i = 0; i < 8; i++)// 8 - bytes[k++] = reservedValueBytes[i]; - - memcpy(&(bytes[k]), tdps->rtypeArray, tdps->rtypeArray_size); - k += tdps->rtypeArray_size; - - if(confparams_cpr->errorBoundMode>=PW_REL) - { - doubleToBytes(exactMidBytesLength, tdps->minLogValue); - for(i = 0;i < 8; i++) - bytes[k++] = exactMidBytesLength[i]; - } - - memcpy(&(bytes[k]), tdps->typeArray, tdps->typeArray_size); - k += tdps->typeArray_size; - if(confparams_cpr->errorBoundMode>=PW_REL) - { - memcpy(&(bytes[k]), tdps->pwrErrBoundBytes, tdps->pwrErrBoundBytes_size); - k += tdps->pwrErrBoundBytes_size; - } - memcpy(&(bytes[k]), tdps->leadNumArray, tdps->leadNumArray_size); - k += tdps->leadNumArray_size; - memcpy(&(bytes[k]), tdps->exactMidBytes, tdps->exactMidBytes_size); - k += tdps->exactMidBytes_size; - if(tdps->residualMidBits!=NULL) - { - memcpy(&(bytes[k]), tdps->residualMidBits, tdps->residualMidBits_size); - k += tdps->residualMidBits_size; - } -} - //Convert TightDataPointStorageD to bytes... bool convertTDPStoFlatBytes_double(TightDataPointStorageD *tdps, unsigned char* bytes, size_t *size) { @@ -611,27 +335,15 @@ bool convertTDPStoFlatBytes_double(TightDataPointStorageD *tdps, unsigned char* *size = totalByteLength; } - else if (tdps->rtypeArray == NULL) + else { size_t residualMidBitsLength = tdps->residualMidBits == NULL ? 0 : tdps->residualMidBits_size; - size_t segmentL = 0, radExpoL = 0, pwrBoundArrayL = 0; - - int minLogValueSize = 0; - if(confparams_cpr->errorBoundMode>=PW_REL) - { - segmentL = exe_params->SZ_SIZE_TYPE; - radExpoL = 1; - pwrBoundArrayL = 4; - minLogValueSize = 8; - } - - size_t totalByteLength = 3 + 1 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + 4 + radExpoL + segmentL + pwrBoundArrayL + 4 + 8 + 1 + 8 - + exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE - + minLogValueSize /*max absolute log value*/ - + tdps->typeArray_size + tdps->leadNumArray_size - + tdps->exactMidBytes_size + residualMidBitsLength + tdps->pwrErrBoundBytes_size; - if(confparams_cpr->errorBoundMode == PW_REL && confparams_cpr->accelerate_pw_rel_compression) - totalByteLength += (1+1); // for MSST19 + size_t totalByteLength = 1 + 1 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + 4 + 4 + 8 + 1 + 8 + + exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE + + tdps->typeArray_size + + tdps->leadNumArray_size + + tdps->exactMidBytes_size + + residualMidBitsLength; //*bytes = (unsigned char *)malloc(sizeof(unsigned char)*totalByteLength); comment by tickduan if(totalByteLength >= tdps->dataSeriesLength * sizeof(double)) @@ -642,96 +354,19 @@ bool convertTDPStoFlatBytes_double(TightDataPointStorageD *tdps, unsigned char* convertTDPStoBytes_double(tdps, bytes, dsLengthBytes, sameByte); *size = totalByteLength; } - else //the case with reserved value - { - //TODO - return false; - } return true; } -void convertTDPStoFlatBytes_double_args(TightDataPointStorageD *tdps, unsigned char* bytes, size_t *size) -{ - size_t i, k = 0; - unsigned char dsLengthBytes[8]; - - if(exe_params->SZ_SIZE_TYPE==4) - intToBytes_bigEndian(dsLengthBytes, tdps->dataSeriesLength);//4 - else - longToBytes_bigEndian(dsLengthBytes, tdps->dataSeriesLength);//8 - - unsigned char sameByte = tdps->allSameData==1?(unsigned char)1:(unsigned char)0; //0000,0001 - sameByte = sameByte | (confparams_cpr->szMode << 1); //0000,0110 - if(tdps->isLossless) - sameByte = (unsigned char) (sameByte | 0x10); // 0001,0000 - if(confparams_cpr->errorBoundMode>=PW_REL) - sameByte = (unsigned char) (sameByte | 0x20); // 0010,0000, the 5th bit - if(exe_params->SZ_SIZE_TYPE==8) - sameByte = (unsigned char) (sameByte | 0x40); //0100,0000, the 6th bit - if(confparams_cpr->errorBoundMode == PW_REL && confparams_cpr->accelerate_pw_rel_compression) - sameByte = (unsigned char) (sameByte | 0x08); //0000,1000, the 7th bit - //if(confparams_cpr->protectValueRange) - // sameByte = (unsigned char) (sameByte | 0x04); //0000,0100 - if(tdps->allSameData==1) - { - size_t totalByteLength = 1 + 1 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + tdps->exactMidBytes_size; - - bytes[k++] = versionNumber; - bytes[k++] = sameByte; - - convertSZParamsToBytes(confparams_cpr, &(bytes[k])); - k = k + MetaDataByteLength_double; - - for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++) - bytes[k++] = dsLengthBytes[i]; - for (i = 0; i < tdps->exactMidBytes_size; i++) - bytes[k++] = tdps->exactMidBytes[i]; - - *size = totalByteLength; - } - else if (tdps->rtypeArray == NULL) - { - size_t residualMidBitsLength = tdps->residualMidBits == NULL ? 0 : tdps->residualMidBits_size; - size_t segmentL = 0, radExpoL = 0, pwrBoundArrayL = 0; - if(confparams_cpr->errorBoundMode>=PW_REL) - { - segmentL = exe_params->SZ_SIZE_TYPE; - radExpoL = 1; - pwrBoundArrayL = 4; - } - - size_t totalByteLength = 3 + 1 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE+ 4 + radExpoL + segmentL + pwrBoundArrayL + 4 + 8 + 1 + 8 - + exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE - + tdps->typeArray_size + tdps->leadNumArray_size - + tdps->exactMidBytes_size + residualMidBitsLength + tdps->pwrErrBoundBytes_size; - if(confparams_cpr->errorBoundMode == PW_REL && confparams_cpr->accelerate_pw_rel_compression) - totalByteLength += (1+1); // for MSST19 - convertTDPStoBytes_double(tdps, bytes, dsLengthBytes, sameByte); - - *size = totalByteLength; - } - else //the case with reserved value - { - //TODO - } -} - void free_TightDataPointStorageD(TightDataPointStorageD *tdps) { - if(tdps->rtypeArray!=NULL) - free(tdps->rtypeArray); - if(tdps->typeArray!=NULL) - free(tdps->typeArray); if(tdps->leadNumArray!=NULL) free(tdps->leadNumArray); if(tdps->exactMidBytes!=NULL) free(tdps->exactMidBytes); if(tdps->residualMidBits!=NULL) free(tdps->residualMidBits); - if(tdps->pwrErrBoundBytes!=NULL) - free(tdps->pwrErrBoundBytes); free(tdps); } diff --git a/deps/SZ/sz/src/TightDataPointStorageF.c b/deps/SZ/sz/src/TightDataPointStorageF.c index 53eda91c22..7107891129 100644 --- a/deps/SZ/sz/src/TightDataPointStorageF.c +++ b/deps/SZ/sz/src/TightDataPointStorageF.c @@ -12,8 +12,8 @@ #include #include "TightDataPointStorageF.h" #include "sz.h" +#include "defines.h" #include "Huffman.h" -//#include "rw.h" void new_TightDataPointStorageF_Empty(TightDataPointStorageF **this) { @@ -25,12 +25,6 @@ void new_TightDataPointStorageF_Empty(TightDataPointStorageF **this) (*this)->reqLength = 0; (*this)->radExpo = 0; - (*this)->rtypeArray = NULL; - (*this)->rtypeArray_size = 0; - - (*this)->typeArray = NULL; //its size is dataSeriesLength/4 (or xxx/4+1) - (*this)->typeArray_size = 0; - (*this)->leadNumArray = NULL; //its size is exactDataNum/4 (or exactDataNum/4+1) (*this)->leadNumArray_size = 0; @@ -44,8 +38,6 @@ void new_TightDataPointStorageF_Empty(TightDataPointStorageF **this) (*this)->isLossless = 0; (*this)->segment_size = 0; - (*this)->pwrErrBoundBytes = NULL; - (*this)->pwrErrBoundBytes_size = 0; (*this)->raBytes = NULL; (*this)->raBytes_size = 0; @@ -55,47 +47,38 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi { new_TightDataPointStorageF_Empty(this); size_t i, index = 0; - size_t pwrErrBoundBytes_size = 0, segmentL = 0, radExpoL = 0, pwrErrBoundBytesL = 0; + + // + // parse tdps + // + + // 1 version(1) unsigned char version = flatBytes[index++]; //1 unsigned char sameRByte = flatBytes[index++]; //1 - if(checkVersion(version) != 1) + + // parse data format + switch (version) { - //wrong version - printf("Wrong version: \nCompressed-data version =%d , program version=%d \n", version, versionNumber); - return 0; - } - //note that 1000,0000 is reserved for regression tag. + case DATA_FROMAT_VER1: + break; + default: + printf(" error, float compressed data format can not be recognised. ver=%d\n ", version); + return SZ_ABS; + } + + // 2 same(1) //note that 1000,0000 is reserved for regression tag. int same = sameRByte & 0x01; //0000,0001 - (*this)->isLossless = (sameRByte & 0x10)>>4; //0001,0000 - int isPW_REL = (sameRByte & 0x20)>>5; //0010,0000 - exe_params->SZ_SIZE_TYPE = ((sameRByte & 0x40)>>6)==1?8:4; //0100,0000 - //pde_params->randomAccess = (sameRByte & 0x02) >> 1; - //pde_params->szMode = (sameRByte & 0x06) >> 1; //0000,0110 (in fact, this szMode could be removed because convertSZParamsToBytes will overwrite it) - - //pde_params->protectValueRange = (sameRByte & 0x04)>>2; - - pde_params->accelerate_pw_rel_compression = (sameRByte & 0x08) >> 3;//0000,1000 - + (*this)->isLossless = (sameRByte & 0x10)>>4; //0001,0000 //0010,0000 + exe_params->SZ_SIZE_TYPE = ((sameRByte & 0x40)>>6)==1?8:4; //0100,0000 int errorBoundMode = SZ_ABS; - if(isPW_REL) - { - errorBoundMode = PW_REL; - segmentL = exe_params->SZ_SIZE_TYPE; - pwrErrBoundBytesL = 4; - } - - + // 3 meta(2) convertBytesToSZParams(&(flatBytes[index]), pde_params, pde_exe); - index += MetaDataByteLength; - - int isRegression = (sameRByte >> 7) & 0x01; - + // 4 element count(4) unsigned char dsLengthBytes[8]; for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++) dsLengthBytes[i] = flatBytes[index++]; - (*this)->dataSeriesLength = bytesToSize(dsLengthBytes);// 4 or 8 - + (*this)->dataSeriesLength = bytesToSize(dsLengthBytes);// 4 or 8 if((*this)->isLossless==1) { //(*this)->exactMidBytes = flatBytes+8; @@ -110,90 +93,48 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi } else (*this)->allSameData = 0; + // regression + int isRegression = (sameRByte >> 7) & 0x01; if(isRegression == 1) { (*this)->raBytes_size = flatBytesLength - 1 - 1 - MetaDataByteLength - exe_params->SZ_SIZE_TYPE; (*this)->raBytes = &(flatBytes[index]); return errorBoundMode; } - - int rtype_ = 0;//sameRByte & 0x08; //=00001000 + // 5 quant intervals(4) unsigned char byteBuf[8]; - for (i = 0; i < 4; i++) byteBuf[i] = flatBytes[index++]; int max_quant_intervals = bytesToInt_bigEndian(byteBuf);// 4 - pde_params->maxRangeRadius = max_quant_intervals/2; - - if(errorBoundMode>=PW_REL) - { - (*this)->radExpo = flatBytes[index++];//1 - radExpoL = 1; - for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++) - byteBuf[i] = flatBytes[index++]; - pde_params->segment_size = (*this)->segment_size = bytesToSize(byteBuf);// exe_params->SZ_SIZE_TYPE - - for (i = 0; i < 4; i++) - byteBuf[i] = flatBytes[index++]; - pwrErrBoundBytes_size = (*this)->pwrErrBoundBytes_size = bytesToInt_bigEndian(byteBuf);// 4 - } - else - { - pwrErrBoundBytes_size = 0; - (*this)->pwrErrBoundBytes = NULL; - } + // 6 intervals for (i = 0; i < 4; i++) byteBuf[i] = flatBytes[index++]; (*this)->intervals = bytesToInt_bigEndian(byteBuf);// 4 - + // 7 median for (i = 0; i < 4; i++) byteBuf[i] = flatBytes[index++]; (*this)->medianValue = bytesToFloat(byteBuf); //4 - + // 8 reqLength (*this)->reqLength = flatBytes[index++]; //1 - - if(isPW_REL && pde_params->accelerate_pw_rel_compression) - { - (*this)->plus_bits = flatBytes[index++]; - (*this)->max_bits = flatBytes[index++]; - } - + // 9 realPrecision(8) for (i = 0; i < 8; i++) byteBuf[i] = flatBytes[index++]; (*this)->realPrecision = bytesToDouble(byteBuf);//8 - + // 10 typeArray_size for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++) byteBuf[i] = flatBytes[index++]; (*this)->typeArray_size = bytesToSize(byteBuf);// 4 - if(rtype_!=0) - { - for(i = 0;iSZ_SIZE_TYPE;i++) - byteBuf[i] = flatBytes[index++]; - (*this)->rtypeArray_size = bytesToSize(byteBuf);//(ST) - } - else - (*this)->rtypeArray_size = 0; - + // 11 exactNum for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++) - byteBuf[i] = flatBytes[index++]; + byteBuf[i] = flatBytes[index++]; (*this)->exactDataNum = bytesToSize(byteBuf);// ST - + // 12 mid size for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++) byteBuf[i] = flatBytes[index++]; (*this)->exactMidBytes_size = bytesToSize(byteBuf);// ST - - if (rtype_ != 0) { - if((*this)->rtypeArray_size>0) - (*this)->rtypeArray = (unsigned char*)malloc(sizeof(unsigned char)*(*this)->rtypeArray_size); - else - (*this)->rtypeArray = NULL; - - for (i = 0; i < 4; i++) - byteBuf[i] = flatBytes[index++]; - (*this)->reservedValue = bytesToFloat(byteBuf);//4 - } - // leadNum size calc from exactDataNum + + // calc leadNumArray_size size_t logicLeadNumBitsNum = (*this)->exactDataNum * 2; if (logicLeadNumBitsNum % 8 == 0) { @@ -202,57 +143,29 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi else { (*this)->leadNumArray_size = (logicLeadNumBitsNum >> 3) + 1; - } - - int minLogValueSize = 0; - if(errorBoundMode>=PW_REL) - minLogValueSize = 4; - - if ((*this)->rtypeArray != NULL) - { - // modify from -3 -1 to -1 -1 tickduan - (*this)->residualMidBits_size = flatBytesLength - 1 - 1 - MetaDataByteLength - exe_params->SZ_SIZE_TYPE - 4 - radExpoL - segmentL - pwrErrBoundBytesL - 4 - 4 - 1 - 8 - - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - minLogValueSize - exe_params->SZ_SIZE_TYPE - 4 - (*this)->rtypeArray_size - - minLogValueSize - (*this)->typeArray_size - (*this)->leadNumArray_size - - (*this)->exactMidBytes_size - pwrErrBoundBytes_size - 1 - 1; - for (i = 0; i < (*this)->rtypeArray_size; i++) - (*this)->rtypeArray[i] = flatBytes[index++]; - } - else - { - (*this)->residualMidBits_size = flatBytesLength - 1 - 1 - MetaDataByteLength - exe_params->SZ_SIZE_TYPE - 4 - radExpoL - segmentL - pwrErrBoundBytesL - 4 - 4 - 1 - 8 - - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - minLogValueSize - (*this)->typeArray_size - - (*this)->leadNumArray_size - (*this)->exactMidBytes_size - pwrErrBoundBytes_size - 1 - 1; - } - - if(errorBoundMode>=PW_REL) - { - (*this)->minLogValue = bytesToFloat(&flatBytes[index]); - index+=4; - } + } + // 13 typeArray (*this)->typeArray = &flatBytes[index]; //retrieve the number of states (i.e., stateNum) (*this)->allNodes = bytesToInt_bigEndian((*this)->typeArray); //the first 4 bytes store the stateNum (*this)->stateNum = ((*this)->allNodes+1)/2; - index+=(*this)->typeArray_size; - - (*this)->pwrErrBoundBytes = &flatBytes[index]; - - index+=pwrErrBoundBytes_size; - + + // 14 leadNumArray (*this)->leadNumArray = &flatBytes[index]; - - index+=(*this)->leadNumArray_size; - + index += (*this)->leadNumArray_size; + // 15 exactMidBytes (*this)->exactMidBytes = &flatBytes[index]; - index+=(*this)->exactMidBytes_size; - + // 16 residualMidBits (*this)->residualMidBits = &flatBytes[index]; + + // calc residualMidBits_size + (*this)->residualMidBits_size = flatBytesLength - 1 - 1 - MetaDataByteLength - exe_params->SZ_SIZE_TYPE - 4 - 4 - 4 - 1 - 8 + - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE + - (*this)->leadNumArray_size - (*this)->exactMidBytes_size - (*this)->typeArray_size; - //index+=(*this)->residualMidBits_size; return errorBoundMode; } @@ -272,7 +185,7 @@ void new_TightDataPointStorageF(TightDataPointStorageF **this, unsigned char* resiMidBits, size_t resiMidBits_size, unsigned char resiBitLength, double realPrecision, float medianValue, char reqLength, unsigned int intervals, - unsigned char* pwrErrBoundBytes, size_t pwrErrBoundBytes_size, unsigned char radExpo) { + unsigned char radExpo) { *this = (TightDataPointStorageF *)malloc(sizeof(TightDataPointStorageF)); (*this)->allSameData = 0; @@ -283,9 +196,7 @@ void new_TightDataPointStorageF(TightDataPointStorageF **this, (*this)->dataSeriesLength = dataSeriesLength; (*this)->exactDataNum = exactDataNum; - (*this)->rtypeArray = NULL; - (*this)->rtypeArray_size = 0; - + // encode type with huffMan int stateNum = 2*intervals; HuffmanTree* huffmanTree = createHuffmanTree(stateNum); if(confparams_cpr->errorBoundMode == PW_REL && confparams_cpr->accelerate_pw_rel_compression) @@ -293,7 +204,7 @@ void new_TightDataPointStorageF(TightDataPointStorageF **this, else encode_withTree(huffmanTree, type, dataSeriesLength, &(*this)->typeArray, &(*this)->typeArray_size); SZ_ReleaseHuffman(huffmanTree); - + (*this)->exactMidBytes = exactMidBytes; (*this)->exactMidBytes_size = exactMidBytes_size; @@ -305,64 +216,7 @@ void new_TightDataPointStorageF(TightDataPointStorageF **this, (*this)->isLossless = 0; - if(confparams_cpr->errorBoundMode>=PW_REL) - (*this)->pwrErrBoundBytes = pwrErrBoundBytes; - else - (*this)->pwrErrBoundBytes = NULL; - (*this)->radExpo = radExpo; - - (*this)->pwrErrBoundBytes_size = pwrErrBoundBytes_size; -} - -void new_TightDataPointStorageF2(TightDataPointStorageF **this, - size_t dataSeriesLength, size_t exactDataNum, - int* type, unsigned char* exactMidBytes, size_t exactMidBytes_size, - unsigned char* leadNumIntArray, //leadNumIntArray contains readable numbers.... - unsigned char* resiMidBits, size_t resiMidBits_size, - unsigned char* resiBitLength, size_t resiBitLengthSize, - double realPrecision, float medianValue, char reqLength, unsigned int intervals, - unsigned char* pwrErrBoundBytes, size_t pwrErrBoundBytes_size, unsigned char radExpo) { - //int i = 0; - *this = (TightDataPointStorageF *)malloc(sizeof(TightDataPointStorageF)); - (*this)->allSameData = 0; - (*this)->realPrecision = realPrecision; - (*this)->medianValue = medianValue; - (*this)->reqLength = reqLength; - - (*this)->dataSeriesLength = dataSeriesLength; - (*this)->exactDataNum = exactDataNum; - - (*this)->rtypeArray = NULL; - (*this)->rtypeArray_size = 0; - - int stateNum = 2*intervals; - HuffmanTree* huffmanTree = createHuffmanTree(stateNum); - encode_withTree(huffmanTree, type, dataSeriesLength, &(*this)->typeArray, &(*this)->typeArray_size); - SZ_ReleaseHuffman(huffmanTree); - - (*this)->exactMidBytes = exactMidBytes; - (*this)->exactMidBytes_size = exactMidBytes_size; - - (*this)->leadNumArray_size = convertIntArray2ByteArray_fast_2b(leadNumIntArray, exactDataNum, &((*this)->leadNumArray)); - - //(*this)->residualMidBits = resiMidBits; - //(*this)->residualMidBits_size = resiMidBits_size; - - (*this)->residualMidBits_size = convertIntArray2ByteArray_fast_dynamic2(resiMidBits, resiBitLength, resiBitLengthSize, &((*this)->residualMidBits)); - - (*this)->intervals = intervals; - - (*this)->isLossless = 0; - - if(confparams_cpr->errorBoundMode>=PW_REL) - (*this)->pwrErrBoundBytes = pwrErrBoundBytes; - else - (*this)->pwrErrBoundBytes = NULL; - - (*this)->radExpo = radExpo; - - (*this)->pwrErrBoundBytes_size = pwrErrBoundBytes_size; } void convertTDPStoBytes_float(TightDataPointStorageF* tdps, unsigned char* bytes, unsigned char* dsLengthBytes, unsigned char sameByte) @@ -377,7 +231,6 @@ void convertTDPStoBytes_float(TightDataPointStorageF* tdps, unsigned char* bytes unsigned char medianValueBytes[4]; unsigned char segment_sizeBytes[8]; - unsigned char pwrErrBoundBytes_sizeBytes[4]; unsigned char max_quant_intervals_Bytes[4]; // 1 version @@ -393,76 +246,43 @@ void convertTDPStoBytes_float(TightDataPointStorageF* tdps, unsigned char* bytes intToBytes_bigEndian(max_quant_intervals_Bytes, confparams_cpr->max_quant_intervals); // 5 max_quant_intervals length for(i = 0;i<4;i++)//4 - bytes[k++] = max_quant_intervals_Bytes[i]; - - if(confparams_cpr->errorBoundMode>=PW_REL) - { - // 6 range exponent - bytes[k++] = tdps->radExpo; //1 byte - // 7 segmetn size - sizeToBytes(segment_sizeBytes, confparams_cpr->segment_size); - for(i = 0;iSZ_SIZE_TYPE;i++)//ST - bytes[k++] = segment_sizeBytes[i]; - // 8 pwrErrBoundBytes_size - intToBytes_bigEndian(pwrErrBoundBytes_sizeBytes, tdps->pwrErrBoundBytes_size); - for(i = 0;i<4;i++)//4 - bytes[k++] = pwrErrBoundBytes_sizeBytes[i]; - } - - // 9 intervals + bytes[k++] = max_quant_intervals_Bytes[i]; + // 6 intervals intToBytes_bigEndian(intervalsBytes, tdps->intervals); for(i = 0;i<4;i++)//4 - bytes[k++] = intervalsBytes[i]; - - // 10 median + bytes[k++] = intervalsBytes[i]; + // 7 median floatToBytes(medianValueBytes, tdps->medianValue); for (i = 0; i < 4; i++)// 4 bytes[k++] = medianValueBytes[i]; - // 11 reqLength + // 8 reqLength bytes[k++] = tdps->reqLength; //1 byte - // 12 plus max - if(confparams_cpr->errorBoundMode == PW_REL && confparams_cpr->accelerate_pw_rel_compression) - { - bytes[k++] = tdps->plus_bits; - bytes[k++] = tdps->max_bits; - } - // 13 realPrecision + // 9 realPrecision doubleToBytes(realPrecisionBytes, tdps->realPrecision); for (i = 0; i < 8; i++)// 8 - bytes[k++] = realPrecisionBytes[i]; - // 14 typeArray size + bytes[k++] = realPrecisionBytes[i]; + // 10 typeArray size sizeToBytes(typeArrayLengthBytes, tdps->typeArray_size); for(i = 0;iSZ_SIZE_TYPE;i++)//ST - bytes[k++] = typeArrayLengthBytes[i]; - // 15 exactDataNum leadNum calc by this , so not save leadNum + bytes[k++] = typeArrayLengthBytes[i]; + // 11 exactDataNum leadNum calc by this , so not save leadNum sizeToBytes(exactLengthBytes, tdps->exactDataNum); for(i = 0;iSZ_SIZE_TYPE;i++)//ST bytes[k++] = exactLengthBytes[i]; - // 16 Mid Length + // 12 Mid size sizeToBytes(exactMidBytesLength, tdps->exactMidBytes_size); for(i = 0;iSZ_SIZE_TYPE;i++)//ST bytes[k++] = exactMidBytesLength[i]; - if(confparams_cpr->errorBoundMode>=PW_REL) - { - floatToBytes(exactMidBytesLength, tdps->minLogValue); - for(i=0;i<4;i++) - bytes[k++] = exactMidBytesLength[i]; - } - // 17 type data + // 13 typeArray memcpy(&(bytes[k]), tdps->typeArray, tdps->typeArray_size); - k += tdps->typeArray_size; - if(confparams_cpr->errorBoundMode>=PW_REL) - { - memcpy(&(bytes[k]), tdps->pwrErrBoundBytes, tdps->pwrErrBoundBytes_size); - k += tdps->pwrErrBoundBytes_size; - } - //18 leadNum data + k += tdps->typeArray_size; + // 14 leadNumArray_size memcpy(&(bytes[k]), tdps->leadNumArray, tdps->leadNumArray_size); k += tdps->leadNumArray_size; - // 19 mid data + // 15 mid data memcpy(&(bytes[k]), tdps->exactMidBytes, tdps->exactMidBytes_size); - k += tdps->exactMidBytes_size; - // 20 residual + k += tdps->exactMidBytes_size; + // 16 residualMidBits if(tdps->residualMidBits!=NULL) { memcpy(&(bytes[k]), tdps->residualMidBits, tdps->residualMidBits_size); @@ -524,29 +344,20 @@ bool convertTDPStoFlatBytes_float(TightDataPointStorageF *tdps, unsigned char* b *size = totalByteLength; } - else if (tdps->rtypeArray == NULL) + else { // // not same format // size_t residualMidBitsLength = tdps->residualMidBits == NULL ? 0 : tdps->residualMidBits_size; - size_t segmentL = 0, radExpoL = 0, pwrBoundArrayL = 0; - int minLogValueSize = 0; - if(confparams_cpr->errorBoundMode>=PW_REL) - { - segmentL = exe_params->SZ_SIZE_TYPE; - radExpoL = 1; - pwrBoundArrayL = 4; - minLogValueSize = 4; - } // version(1) + samebyte(1) - size_t totalByteLength = 1 + 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 4 + radExpoL + segmentL + pwrBoundArrayL + 4 + 4 + 1 + 8 - + exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE + minLogValueSize - + tdps->typeArray_size + tdps->leadNumArray_size - + tdps->exactMidBytes_size + residualMidBitsLength + tdps->pwrErrBoundBytes_size; - if(confparams_cpr->errorBoundMode == PW_REL && confparams_cpr->accelerate_pw_rel_compression) - totalByteLength += (1+1); // for MSST19 + size_t totalByteLength = 1 + 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 4 + 4 + 4 + 1 + 8 + + exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE + + tdps->typeArray_size + + tdps->leadNumArray_size + + tdps->exactMidBytes_size + + residualMidBitsLength; //*bytes = (unsigned char *)malloc(sizeof(unsigned char)*totalByteLength); // comment by tickduan if(totalByteLength >= tdps->dataSeriesLength * sizeof(float)) @@ -559,99 +370,21 @@ bool convertTDPStoFlatBytes_float(TightDataPointStorageF *tdps, unsigned char* b *size = totalByteLength; return true; } - else //the case with reserved value - { - //TODO - *size = 0; - return false; - } return true; } -void convertTDPStoFlatBytes_float_args(TightDataPointStorageF *tdps, unsigned char* bytes, size_t *size) -{ - size_t i, k = 0; - unsigned char dsLengthBytes[8]; - - if(exe_params->SZ_SIZE_TYPE==4) - intToBytes_bigEndian(dsLengthBytes, tdps->dataSeriesLength);//4 - else - longToBytes_bigEndian(dsLengthBytes, tdps->dataSeriesLength);//8 - - unsigned char sameByte = tdps->allSameData==1?(unsigned char)1:(unsigned char)0; - sameByte = sameByte | (confparams_cpr->szMode << 1); - if(tdps->isLossless) - sameByte = (unsigned char) (sameByte | 0x10); - if(confparams_cpr->errorBoundMode>=PW_REL) - sameByte = (unsigned char) (sameByte | 0x20); // 00100000, the 5th bit - if(exe_params->SZ_SIZE_TYPE==8) - sameByte = (unsigned char) (sameByte | 0x40); // 01000000, the 6th bit - if(confparams_cpr->errorBoundMode == PW_REL && confparams_cpr->accelerate_pw_rel_compression) - sameByte = (unsigned char) (sameByte | 0x08); - - if(tdps->allSameData==1) - { - size_t totalByteLength = 1 + 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + tdps->exactMidBytes_size; - //*bytes = (unsigned char *)malloc(sizeof(unsigned char)*totalByteLength); - - bytes[k++] = versionNumber; - bytes[k++] = sameByte; - - convertSZParamsToBytes(confparams_cpr, &(bytes[k])); - k = k + MetaDataByteLength; - - for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++) - bytes[k++] = dsLengthBytes[i]; - for (i = 0; i < tdps->exactMidBytes_size; i++) - bytes[k++] = tdps->exactMidBytes[i]; - - *size = totalByteLength; - } - else if (tdps->rtypeArray == NULL) - { - size_t residualMidBitsLength = tdps->residualMidBits == NULL ? 0 : tdps->residualMidBits_size; - size_t segmentL = 0, radExpoL = 0, pwrBoundArrayL = 0; - if(confparams_cpr->errorBoundMode>=PW_REL) - { - segmentL = exe_params->SZ_SIZE_TYPE; - radExpoL = 1; - pwrBoundArrayL = 4; - } - - size_t totalByteLength = 1 + 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 4 + radExpoL + segmentL + pwrBoundArrayL + 4 + 4 + 1 + 8 - + exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE - + tdps->typeArray_size + tdps->leadNumArray_size - + tdps->exactMidBytes_size + residualMidBitsLength + tdps->pwrErrBoundBytes_size; - if(confparams_cpr->errorBoundMode == PW_REL && confparams_cpr->accelerate_pw_rel_compression) - totalByteLength += (1+1); // for MSST19 - convertTDPStoBytes_float(tdps, bytes, dsLengthBytes, sameByte); - - *size = totalByteLength; - } - else //the case with reserved value - { - //TODO - } -} - /** * to free the memory used in the compression * */ void free_TightDataPointStorageF(TightDataPointStorageF *tdps) { - if(tdps->rtypeArray!=NULL) - free(tdps->rtypeArray); - if(tdps->typeArray!=NULL) - free(tdps->typeArray); if(tdps->leadNumArray!=NULL) free(tdps->leadNumArray); if(tdps->exactMidBytes!=NULL) free(tdps->exactMidBytes); if(tdps->residualMidBits!=NULL) free(tdps->residualMidBits); - if(tdps->pwrErrBoundBytes!=NULL) - free(tdps->pwrErrBoundBytes); free(tdps); } diff --git a/deps/SZ/sz/src/dataCompression.c b/deps/SZ/sz/src/dataCompression.c index 355f3ac28e..237a3315c1 100644 --- a/deps/SZ/sz/src/dataCompression.c +++ b/deps/SZ/sz/src/dataCompression.c @@ -18,83 +18,7 @@ #include "CompressElement.h" #include "dataCompression.h" -int computeByteSizePerIntValue(long valueRangeSize) -{ - if(valueRangeSize<=256) - return 1; - else if(valueRangeSize<=65536) - return 2; - else if(valueRangeSize<=4294967296) //2^32 - return 4; - else - return 8; -} -long computeRangeSize_int(void* oriData, int dataType, size_t size, int64_t* valueRangeSize) -{ - size_t i = 0; - long max = 0, min = 0; - - if(dataType==SZ_UINT8) - { - unsigned char* data = (unsigned char*)oriData; - unsigned char data_; - min = data[0], max = min; - computeMinMax(data); - } - else if(dataType == SZ_INT8) - { - char* data = (char*)oriData; - char data_; - min = data[0], max = min; - computeMinMax(data); - } - else if(dataType == SZ_UINT16) - { - unsigned short* data = (unsigned short*)oriData; - unsigned short data_; - min = data[0], max = min; - computeMinMax(data); - } - else if(dataType == SZ_INT16) - { - short* data = (short*)oriData; - short data_; - min = data[0], max = min; - computeMinMax(data); - } - else if(dataType == SZ_UINT32) - { - unsigned int* data = (unsigned int*)oriData; - unsigned int data_; - min = data[0], max = min; - computeMinMax(data); - } - else if(dataType == SZ_INT32) - { - int* data = (int*)oriData; - int data_; - min = data[0], max = min; - computeMinMax(data); - } - else if(dataType == SZ_UINT64) - { - unsigned long* data = (unsigned long*)oriData; - unsigned long data_; - min = data[0], max = min; - computeMinMax(data); - } - else if(dataType == SZ_INT64) - { - long* data = (long *)oriData; - long data_; - min = data[0], max = min; - computeMinMax(data); - } - - *valueRangeSize = max - min; - return min; -} float computeRangeSize_float(float* oriData, size_t size, float* valueRangeSize, float* medianValue) { @@ -115,34 +39,6 @@ float computeRangeSize_float(float* oriData, size_t size, float* valueRangeSize, return min; } -float computeRangeSize_float_MSST19(float* oriData, size_t size, float* valueRangeSize, float* medianValue, unsigned char * signs, bool* positive, float* nearZero) -{ - size_t i = 0; - float min = oriData[0]; - float max = min; - *nearZero = min; - - for(i=1;idata) - min = data; - else if(maxdata) - min = data; - else if(maxdata) - min = data; - else if(maxdata) - min = data; - else if(maxresiBitsLength = resiBitsLength; } -void compressSingleFloatValue_MSST19(FloatValueCompressElement *vce, float tgtValue, float precision, int reqLength, int reqBytesLength, int resiBitsLength) -{ - float normValue = tgtValue; - - lfloat lfBuf; - lfBuf.value = normValue; - - int ignBytesLength = 32 - reqLength; - if(ignBytesLength<0) - ignBytesLength = 0; - - int tmp_int = lfBuf.ivalue; - intToBytes_bigEndian(vce->curBytes, tmp_int); - - lfBuf.ivalue = (lfBuf.ivalue >> ignBytesLength) << ignBytesLength; - - //float tmpValue = lfBuf.value; - - vce->data = lfBuf.value; - vce->curValue = tmp_int; - vce->reqBytesLength = reqBytesLength; - vce->resiBitsLength = resiBitsLength; -} - -void compressSingleDoubleValue_MSST19(DoubleValueCompressElement *vce, double tgtValue, double precision, int reqLength, int reqBytesLength, int resiBitsLength) -{ - ldouble lfBuf; - lfBuf.value = tgtValue; - - int ignBytesLength = 64 - reqLength; - if(ignBytesLength<0) - ignBytesLength = 0; - - long tmp_long = lfBuf.lvalue; - longToBytes_bigEndian(vce->curBytes, tmp_long); - - lfBuf.lvalue = (lfBuf.lvalue >> ignBytesLength) << ignBytesLength; - - //float tmpValue = lfBuf.value; - - vce->data = lfBuf.value; - vce->curValue = tmp_long; - vce->reqBytesLength = reqBytesLength; - vce->resiBitsLength = resiBitsLength; -} - void compressSingleDoubleValue(DoubleValueCompressElement *vce, double tgtValue, double precision, double medianValue, int reqLength, int reqBytesLength, int resiBitsLength) { diff --git a/deps/SZ/sz/src/sz.c b/deps/SZ/sz/src/sz.c index 31032021ac..f4ff348aa5 100644 --- a/deps/SZ/sz/src/sz.c +++ b/deps/SZ/sz/src/sz.c @@ -19,13 +19,12 @@ #include "TightDataPointStorageF.h" #include "zlib.h" #include "rw.h" -#include "Huffman.h" #include "conf.h" #include "utility.h" //#include "CurveFillingCompressStorage.h" -unsigned char versionNumber = SZ_VER_MAJOR; +unsigned char versionNumber = DATA_FROMAT_VER1; int SZ_SIZE_TYPE_DEFUALT = 4; int dataEndianType = LITTLE_ENDIAN_DATA; //*endian type of the data read from disk @@ -48,10 +47,7 @@ sz_tsc_metadata *sz_tsc = NULL; pastri_params pastri_par; #endif -HuffmanTree* SZ_Reset() -{ - return createDefaultHuffmanTree(); -} + int SZ_Init(const char *configFilePath) { @@ -191,7 +187,15 @@ void SZ_Finalize() } } - +void modulePath(char *buf, int size) +{ + char path[1024]; + sprintf(path, "/proc/%d/exe", getpid()); + readlink(path, buf, size); + char* pos = strrchr(buf, '/'); + if(pos) + pos[1]=0; +} struct timeval startTime; @@ -200,6 +204,7 @@ struct timeval costStart; /*only used for recording the cost*/ double totalCost = 0; + void cost_start() { totalCost = 0; diff --git a/deps/SZ/sz/src/sz_double.c b/deps/SZ/sz/src/sz_double.c index ab8b82c9b8..e7a932b30e 100644 --- a/deps/SZ/sz/src/sz_double.c +++ b/deps/SZ/sz/src/sz_double.c @@ -19,12 +19,9 @@ #include "DynamicIntArray.h" #include "TightDataPointStorageD.h" #include "sz_double.h" -#include "sz_double_pwr.h" #include "szd_double.h" -#include "szd_double_pwr.h" #include "zlib.h" #include "rw.h" -#include "sz_double_ts.h" #include "utility.h" #include "CacheTable.h" #include "MultiLevelCacheTableWideInterval.h" @@ -51,11 +48,6 @@ inline void computeReqLength_double(double realPrecision, short radExpo, int* re } } -inline short computeReqLength_double_MSST19(double realPrecision) -{ - short reqExpo = getPrecisionReqLength_double(realPrecision); - return 12-reqExpo; -} unsigned int optimize_intervals_double_1D(double *oriData, size_t dataLength, double realPrecision) { @@ -117,9 +109,8 @@ size_t dataLength, double realPrecision, double valueRangeSize, double medianVal short radExpo = getExponent_double(valueRangeSize/2); computeReqLength_double(realPrecision, radExpo, &reqLength, &medianValue); - - int* type = (int*) malloc(dataLength*sizeof(int)); + int* type = (int*) malloc(dataLength*sizeof(int)); double* spaceFillingValue = oriData; // DynamicIntArray *exactLeadNumArray; @@ -141,8 +132,8 @@ size_t dataLength, double realPrecision, double valueRangeSize, double medianVal DoubleValueCompressElement *vce = (DoubleValueCompressElement*)malloc(sizeof(DoubleValueCompressElement)); LossyCompressionElement *lce = (LossyCompressionElement*)malloc(sizeof(LossyCompressionElement)); - //add the first data - type[0] = 0; + //add the first data + type[0] = 0; compressSingleDoubleValue(vce, spaceFillingValue[0], realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength); updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce); memcpy(preDataBytes,vce->curBytes,8); @@ -166,7 +157,7 @@ size_t dataLength, double realPrecision, double valueRangeSize, double medianVal double interval = 2*realPrecision; double recip_realPrecision = 1/realPrecision; - for(i=2;icurBytes, preDataBytes, reqBytesLength, resiBitsLength, lce); memcpy(preDataBytes,vce->curBytes,8); @@ -199,7 +190,6 @@ size_t dataLength, double realPrecision, double valueRangeSize, double medianVal //listAdd_double(last3CmprsData, vce->data); pred = vce->data; - }//end of for @@ -212,7 +202,7 @@ size_t dataLength, double realPrecision, double valueRangeSize, double medianVal exactLeadNumArray->array, resiBitArray->array, resiBitArray->size, resiBitsLength, - realPrecision, medianValue, (char)reqLength, quantization_intervals, NULL, 0, 0); + realPrecision, medianValue, (char)reqLength, quantization_intervals, 0); // printf("exactDataNum=%d, expSegmentsInBytes_size=%d, exactMidByteArray->size=%d\n", // exactDataNum, expSegmentsInBytes_size, exactMidByteArray->size); @@ -283,171 +273,16 @@ size_t dataLength, double realPrecision, size_t *outSize, double valueRangeSize, return true; } -/*MSST19*/ -TightDataPointStorageD* SZ_compress_double_1D_MDQ_MSST19(double *oriData, -size_t dataLength, double realPrecision, double valueRangeSize, double medianValue_f) -{ - - //struct ClockPoint clockPointBuild; - //TimeDurationStart("build", &clockPointBuild); - unsigned int quantization_intervals; - if(exe_params->optQuantMode==1) - quantization_intervals = optimize_intervals_double_1D_opt_MSST19(oriData, dataLength, realPrecision); - else - quantization_intervals = exe_params->intvCapacity; - //updateQuantizationInfo(quantization_intervals); - int intvRadius = quantization_intervals/2; - - double* precisionTable = (double*)malloc(sizeof(double) * quantization_intervals); - double inv = 2.0-pow(2, -(confparams_cpr->plus_bits)); - for(int i=0; iplus_bits); - - size_t i; - int reqLength; - double medianValue = medianValue_f; - //double medianInverse = 1 / medianValue_f; - //short radExpo = getExponent_double(realPrecision); - - reqLength = computeReqLength_double_MSST19(realPrecision); - - int* type = (int*) malloc(dataLength*sizeof(int)); - - double* spaceFillingValue = oriData; // - - DynamicIntArray *exactLeadNumArray; - new_DIA(&exactLeadNumArray, dataLength/2/8); - - DynamicByteArray *exactMidByteArray; - new_DBA(&exactMidByteArray, dataLength/2); - - DynamicIntArray *resiBitArray; - new_DIA(&resiBitArray, DynArrayInitLen); - - unsigned char preDataBytes[8]; - intToBytes_bigEndian(preDataBytes, 0); - - int reqBytesLength = reqLength/8; - int resiBitsLength = reqLength%8; - double last3CmprsData[3] = {0}; - - //size_t miss=0, hit=0; - - DoubleValueCompressElement *vce = (DoubleValueCompressElement*)malloc(sizeof(DoubleValueCompressElement)); - LossyCompressionElement *lce = (LossyCompressionElement*)malloc(sizeof(LossyCompressionElement)); - - //add the first data - type[0] = 0; - compressSingleDoubleValue_MSST19(vce, spaceFillingValue[0], realPrecision, reqLength, reqBytesLength, resiBitsLength); - updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce); - memcpy(preDataBytes,vce->curBytes,8); - addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce); - listAdd_double(last3CmprsData, vce->data); - //miss++; - - //add the second data - type[1] = 0; - compressSingleDoubleValue_MSST19(vce, spaceFillingValue[1], realPrecision, reqLength, reqBytesLength, resiBitsLength); - updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce); - memcpy(preDataBytes,vce->curBytes,8); - addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce); - listAdd_double(last3CmprsData, vce->data); - //miss++; - - int state; - //double checkRadius; - double curData; - double pred = vce->data; - - double predRelErrRatio; - - const uint64_t top = levelTable.topIndex, base = levelTable.baseIndex; - const uint64_t range = top - base; - const int bits = levelTable.bits; - uint64_t* const buffer = (uint64_t*)&predRelErrRatio; - const int shift = 52-bits; - uint64_t expoIndex, mantiIndex; - uint16_t* tables[range+1]; - for(int i=0; i<=range; i++){ - tables[i] = levelTable.subTables[i].table; - } - - for(i=2;i> 52) - base; - if(expoIndex <= range){ - mantiIndex = (*buffer & 0x000fffffffffffff) >> shift; - state = tables[expoIndex][mantiIndex]; - }else{ - state = 0; - } - - if(state) - { - type[i] = state; - pred *= precisionTable[state]; - //hit++; - continue; - } - - //unpredictable data processing - type[i] = 0; - compressSingleDoubleValue_MSST19(vce, curData, realPrecision, reqLength, reqBytesLength, resiBitsLength); - updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce); - memcpy(preDataBytes,vce->curBytes,8); - addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce); - pred = vce->data; - //miss++; - - }//end of for - -// printf("miss:%d, hit:%d\n", miss, hit); - - size_t exactDataNum = exactLeadNumArray->size; - - TightDataPointStorageD* tdps; - - new_TightDataPointStorageD(&tdps, dataLength, exactDataNum, - type, exactMidByteArray->array, exactMidByteArray->size, - exactLeadNumArray->array, - resiBitArray->array, resiBitArray->size, - resiBitsLength, - realPrecision, medianValue, (char)reqLength, quantization_intervals, NULL, 0, 0); - tdps->plus_bits = confparams_cpr->plus_bits; - - //free memory - free_DIA(exactLeadNumArray); - free_DIA(resiBitArray); - free(type); - free(vce); - free(lce); - free(exactMidByteArray); //exactMidByteArray->array has been released in free_TightDataPointStorageF(tdps); - free(precisionTable); - freeTopLevelTableWideInterval(&levelTable); - return tdps; -} - void SZ_compress_args_double_withinRange(unsigned char* newByteData, double *oriData, size_t dataLength, size_t *outSize) { TightDataPointStorageD* tdps = (TightDataPointStorageD*) malloc(sizeof(TightDataPointStorageD)); - tdps->rtypeArray = NULL; - tdps->typeArray = NULL; tdps->leadNumArray = NULL; tdps->residualMidBits = NULL; tdps->allSameData = 1; tdps->dataSeriesLength = dataLength; tdps->exactMidBytes = (unsigned char*)malloc(sizeof(unsigned char)*8); - tdps->pwrErrBoundBytes = NULL; tdps->isLossless = 0; double value = oriData[0]; doubleToBytes(tdps->exactMidBytes, value); @@ -472,12 +307,8 @@ int SZ_compress_args_double(double *oriData, size_t r1, unsigned char* newByteDa double valueRangeSize = 0, medianValue = 0; - unsigned char * signs = NULL; bool positive = true; double nearZero = 0.0; - double min = 0; - if(params->pw_relBoundRatio < 0.000009999) - params->accelerate_pw_rel_compression = 0; // check at least elements count if(dataLength <= MIN_NUM_OF_ELEMENTS) @@ -486,14 +317,8 @@ int SZ_compress_args_double(double *oriData, size_t r1, unsigned char* newByteDa return SZ_LITTER_ELEMENT; } - if(params->errorBoundMode == PW_REL && params->accelerate_pw_rel_compression == 1) - { - signs = (unsigned char *) malloc(dataLength); - memset(signs, 0, dataLength); - min = computeRangeSize_double_MSST19(oriData, dataLength, &valueRangeSize, &medianValue, signs, &positive, &nearZero); - } - else - min = computeRangeSize_double(oriData, dataLength, &valueRangeSize, &medianValue); + + double min = computeRangeSize_double(oriData, dataLength, &valueRangeSize, &medianValue); double max = min+valueRangeSize; params->dmin = min; params->dmax = max; @@ -517,9 +342,7 @@ int SZ_compress_args_double(double *oriData, size_t r1, unsigned char* newByteDa params->absErrBound = realPrecision; } if(valueRangeSize <= realPrecision) - { - if(params->errorBoundMode>=PW_REL && params->accelerate_pw_rel_compression == 1) - free(signs); + { SZ_compress_args_double_withinRange(newByteData, oriData, dataLength, outSize); } else @@ -532,25 +355,15 @@ int SZ_compress_args_double(double *oriData, size_t r1, unsigned char* newByteDa tmpByteData = (unsigned char*)malloc(r1*sizeof(double)*1.2); } - if(params->errorBoundMode>=PW_REL) + if(!SZ_compress_args_double_NoCkRngeNoGzip_1D(tmpByteData, oriData, r1, realPrecision, &tmpOutSize, valueRangeSize, medianValue)) { - if(params->accelerate_pw_rel_compression && params->maxRangeRadius <= 32768) - SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log_MSST19(tmpByteData, oriData, params->pw_relBoundRatio, r1, &tmpOutSize, valueRangeSize, medianValue, signs, &positive, min, max, nearZero); - else - SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log(tmpByteData, oriData, params->pw_relBoundRatio, r1, &tmpOutSize, min, max); - } - else - { - if(!SZ_compress_args_double_NoCkRngeNoGzip_1D(tmpByteData, oriData, r1, realPrecision, &tmpOutSize, valueRangeSize, medianValue)) - { - if(twoStage) - free(tmpByteData); + if(twoStage) + free(tmpByteData); - return SZ_ALGORITHM_ERR; - } - //if(tmpOutSize>=dataLength*sizeof(double) + 3 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + 1) - // SZ_compress_args_double_StoreOriData(oriData, dataLength, tmpByteData, &tmpOutSize); + return SZ_ALGORITHM_ERR; } + //if(tmpOutSize>=dataLength*sizeof(double) + 3 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + 1) + // SZ_compress_args_double_StoreOriData(oriData, dataLength, tmpByteData, &tmpOutSize); // //Call Gzip to do the further compression. @@ -569,58 +382,6 @@ int SZ_compress_args_double(double *oriData, size_t r1, unsigned char* newByteDa return status; } - -unsigned int optimize_intervals_double_1D_opt_MSST19(double *oriData, size_t dataLength, double realPrecision) -{ - size_t i = 0, radiusIndex; - double pred_value = 0; - double pred_err; - size_t *intervals = (size_t*)malloc(confparams_cpr->maxRangeRadius*sizeof(size_t)); - memset(intervals, 0, confparams_cpr->maxRangeRadius*sizeof(size_t)); - size_t totalSampleSize = 0;//dataLength/confparams_cpr->sampleDistance; - - double * data_pos = oriData + 2; - double divider = log2(1+realPrecision)*2; - int tempIndex = 0; - while(data_pos - oriData < dataLength){ - if(*data_pos == 0){ - data_pos += confparams_cpr->sampleDistance; - continue; - } - tempIndex++; - totalSampleSize++; - pred_value = data_pos[-1]; - pred_err = fabs((double)*data_pos / pred_value); - radiusIndex = (unsigned long)fabs(log2(pred_err)/divider+0.5); - if(radiusIndex>=confparams_cpr->maxRangeRadius) - radiusIndex = confparams_cpr->maxRangeRadius - 1; - intervals[radiusIndex]++; - - data_pos += confparams_cpr->sampleDistance; - } - //compute the appropriate number - size_t targetCount = totalSampleSize*confparams_cpr->predThreshold; - size_t sum = 0; - for(i=0;imaxRangeRadius;i++) - { - sum += intervals[i]; - if(sum>targetCount) - break; - } - if(i>=confparams_cpr->maxRangeRadius) - i = confparams_cpr->maxRangeRadius-1; - - unsigned int accIntervals = 2*(i+1); - unsigned int powerOf2 = roundUpToPowerOf2(accIntervals); - - if(powerOf2<64) - powerOf2 = 64; - - free(intervals); - return powerOf2; -} - - unsigned int optimize_intervals_double_1D_opt(double *oriData, size_t dataLength, double realPrecision) { size_t i = 0, radiusIndex; diff --git a/deps/SZ/sz/src/sz_double_pwr.c b/deps/SZ/sz/src/sz_double_pwr.c deleted file mode 100644 index 00259dbd04..0000000000 --- a/deps/SZ/sz/src/sz_double_pwr.c +++ /dev/null @@ -1,752 +0,0 @@ -/** - * @file sz_double_pwr.c - * @author Sheng Di, Dingwen Tao, Xin Liang, Xiangyu Zou, Tao Lu, Wen Xia, Xuan Wang, Weizhe Zhang - * @date Aug, 2016 - * @brief SZ_Init, Compression and Decompression functions - * This file contains the compression/decompression functions related to point-wise relative errors - * (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - - -#include -#include -#include -#include -#include -#include "sz.h" -#include "CompressElement.h" -#include "DynamicByteArray.h" -#include "TightDataPointStorageD.h" -#include "sz_double.h" -#include "sz_double_pwr.h" -#include "zlib.h" -#include "rw.h" -#include "utility.h" - -void compute_segment_precisions_double_1D(double *oriData, size_t dataLength, double* pwrErrBound, unsigned char* pwrErrBoundBytes, double globalPrecision) -{ - size_t i = 0, j = 0, k = 0; - double realPrecision = oriData[0]!=0?fabs(confparams_cpr->pw_relBoundRatio*oriData[0]):confparams_cpr->pw_relBoundRatio; - double approxPrecision; - unsigned char realPrecBytes[8]; - double curPrecision; - double curValue; - double sum = 0; - for(i=0;isegment_size==0&&i>0) - { - //get two first bytes of the realPrecision - if(confparams_cpr->pwr_type==SZ_PWR_AVG_TYPE) - { - realPrecision = sum/confparams_cpr->segment_size; - sum = 0; - } - realPrecision *= confparams_cpr->pw_relBoundRatio; - if(confparams_cpr->errorBoundMode==ABS_AND_PW_REL||confparams_cpr->errorBoundMode==REL_AND_PW_REL) - realPrecision = realPrecisionerrorBoundMode==ABS_OR_PW_REL||confparams_cpr->errorBoundMode==REL_OR_PW_REL) - realPrecision = realPrecisionpwr_type) - { - case SZ_PWR_MIN_TYPE: - if(realPrecision>curPrecision) - realPrecision = curPrecision; - break; - case SZ_PWR_AVG_TYPE: - sum += curPrecision; - break; - case SZ_PWR_MAX_TYPE: - if(realPrecisionpwr_type==SZ_PWR_AVG_TYPE) - { - int size = dataLength%confparams_cpr->segment_size==0?confparams_cpr->segment_size:dataLength%confparams_cpr->segment_size; - realPrecision = sum/size; - } - if(confparams_cpr->errorBoundMode==ABS_AND_PW_REL||confparams_cpr->errorBoundMode==REL_AND_PW_REL) - realPrecision = realPrecisionerrorBoundMode==ABS_OR_PW_REL||confparams_cpr->errorBoundMode==REL_OR_PW_REL) - realPrecision = realPrecisionmaxRangeRadius*sizeof(int)); - memset(intervals, 0, confparams_cpr->maxRangeRadius*sizeof(int)); - int totalSampleSize = dataLength/confparams_cpr->sampleDistance; - for(i=2;isegment_size==0) - realPrecision = pwrErrBound[j++]; - if(i%confparams_cpr->sampleDistance==0) - { - //pred_value = 2*oriData[i-1] - oriData[i-2]; - pred_value = oriData[i-1]; - pred_err = fabs(pred_value - oriData[i]); - radiusIndex = (unsigned long)((pred_err/realPrecision+1)/2); - if(radiusIndex>=confparams_cpr->maxRangeRadius) - radiusIndex = confparams_cpr->maxRangeRadius - 1; - intervals[radiusIndex]++; - } - } - //compute the appropriate number - size_t targetCount = totalSampleSize*confparams_cpr->predThreshold; - size_t sum = 0; - for(i=0;imaxRangeRadius;i++) - { - sum += intervals[i]; - if(sum>targetCount) - break; - } - if(i>=confparams_cpr->maxRangeRadius) - i = confparams_cpr->maxRangeRadius-1; - unsigned int accIntervals = 2*(i+1); - unsigned int powerOf2 = roundUpToPowerOf2(accIntervals); - - if(powerOf2<32) - powerOf2 = 32; - - free(intervals); - //printf("accIntervals=%d, powerOf2=%d\n", accIntervals, powerOf2); - return powerOf2; -} - - -void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr(unsigned char* newByteData, double *oriData, double globalPrecision, -size_t dataLength, size_t *outSize, double min, double max) -{ - size_t pwrLength = dataLength%confparams_cpr->segment_size==0?dataLength/confparams_cpr->segment_size:dataLength/confparams_cpr->segment_size+1; - double* pwrErrBound = (double*)malloc(sizeof(double)*pwrLength); - size_t pwrErrBoundBytes_size = sizeof(unsigned char)*pwrLength*2; - unsigned char* pwrErrBoundBytes = (unsigned char*)malloc(pwrErrBoundBytes_size); - - compute_segment_precisions_double_1D(oriData, dataLength, pwrErrBound, pwrErrBoundBytes, globalPrecision); - - unsigned int quantization_intervals; - if(exe_params->optQuantMode==1) - { - quantization_intervals = optimize_intervals_double_1D_pwr(oriData, dataLength, pwrErrBound); - updateQuantizationInfo(quantization_intervals); - } - else - quantization_intervals = exe_params->intvCapacity; - size_t i = 0, j = 0; - int reqLength; - double realPrecision = pwrErrBound[j++]; - double medianValue = 0; - double radius = fabs(max)curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce); - memcpy(preDataBytes,vce->curBytes,8); - addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce); - listAdd_double(last3CmprsData, vce->data); - //printf("%.30G\n",last3CmprsData[0]); - - //add the second data - type[1] = 0; - addDBA_Data(resiBitLengthArray, (unsigned char)resiBitsLength); - compressSingleDoubleValue(vce, spaceFillingValue[1], realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength); - updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce); - memcpy(preDataBytes,vce->curBytes,8); - addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce); - listAdd_double(last3CmprsData, vce->data); - //printf("%.30G\n",last3CmprsData[0]); - - int state; - double checkRadius; - double curData; - double pred; - double predAbsErr; - checkRadius = (exe_params->intvCapacity-1)*realPrecision; - double interval = 2*realPrecision; - int updateReqLength = 0; //a marker: 1 means already updated - - for(i=2;isegment_size==0) - { - realPrecision = pwrErrBound[j++]; - checkRadius = (exe_params->intvCapacity-1)*realPrecision; - interval = 2*realPrecision; - updateReqLength = 0; - } - //pred = 2*last3CmprsData[0] - last3CmprsData[1]; - pred = last3CmprsData[0]; - predAbsErr = fabs(curData - pred); - if(predAbsErr=pred) - { - type[i] = exe_params->intvRadius+state; - pred = pred + state*interval; - } - else //curDataintvRadius-state; - pred = pred - state*interval; - } - listAdd_double(last3CmprsData, pred); - continue; - } - - //unpredictable data processing - if(updateReqLength==0) - { - computeReqLength_double(realPrecision, radExpo, &reqLength, &medianValue); - reqBytesLength = reqLength/8; - resiBitsLength = reqLength%8; - updateReqLength = 1; - } - - type[i] = 0; - addDBA_Data(resiBitLengthArray, (unsigned char)resiBitsLength); - - compressSingleDoubleValue(vce, curData, realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength); - updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce); - memcpy(preDataBytes,vce->curBytes,8); - addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce); - - listAdd_double(last3CmprsData, vce->data); - }//end of for - -// char* expSegmentsInBytes; -// int expSegmentsInBytes_size = convertESCToBytes(esc, &expSegmentsInBytes); - int exactDataNum = exactLeadNumArray->size; - - TightDataPointStorageD* tdps; - - new_TightDataPointStorageD2(&tdps, dataLength, exactDataNum, - type, exactMidByteArray->array, exactMidByteArray->size, - exactLeadNumArray->array, - resiBitArray->array, resiBitArray->size, - resiBitLengthArray->array, resiBitLengthArray->size, - realPrecision, medianValue, (char)reqLength, quantization_intervals, pwrErrBoundBytes, pwrErrBoundBytes_size, radExpo); - -//sdi:Debug -/* int sum =0; - for(i=0;itypeArray, tdps->typeArray_size, type_); -// printf("tdps->typeArray_size=%d\n", tdps->typeArray_size); - - //free memory - free_DBA(resiBitLengthArray); - free_DIA(exactLeadNumArray); - free_DIA(resiBitArray); - free(type); - - convertTDPStoFlatBytes_double(tdps, newByteData, outSize); - - int doubleSize=sizeof(double); - if(*outSize>dataLength*doubleSize) - { - size_t k = 0, i; - tdps->isLossless = 1; - size_t totalByteLength = 1 + exe_params->SZ_SIZE_TYPE + 1 + doubleSize*dataLength; - //*newByteData = (unsigned char*)malloc(totalByteLength); comment by tickduan - - unsigned char dsLengthBytes[exe_params->SZ_SIZE_TYPE]; - intToBytes_bigEndian(dsLengthBytes, dataLength);//4 - newByteData[k++] = versionNumber; - - if(exe_params->SZ_SIZE_TYPE==4) - { - newByteData[k++] = 16; //=00010000 - } - else - { - newByteData[k++] = 80; - } - for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)//4 or 8 - newByteData[k++] = dsLengthBytes[i]; - - - if(sysEndianType==BIG_ENDIAN_SYSTEM) - memcpy(newByteData+4+exe_params->SZ_SIZE_TYPE, oriData, dataLength*doubleSize); - else - { - unsigned char* p = newByteData+4+exe_params->SZ_SIZE_TYPE; - for(i=0;idataSeriesLength; - int* standGroupID = (int*)malloc(dataLength*sizeof(int)); - - size_t i; - standGroupID[0] = groupID[0]+GROUP_COUNT; //plus an offset such that it would not be a negative number. - char lastGroupIDValue = groupID[0], curGroupIDValue; - int offset = 2*(GROUP_COUNT + 2); - for(i=1; ipwrErrBoundBytes = out; //groupIDArray - tdps->pwrErrBoundBytes_size = outSize; - - free(standGroupID); -} - -TightDataPointStorageD* SZ_compress_double_1D_MDQ_pwrGroup(double* oriData, size_t dataLength, int errBoundMode, - double absErrBound, double relBoundRatio, double pwrErrRatio, double valueRangeSize, double medianValue_f) -{ - size_t i; - double *posGroups, *negGroups, *groups; - double pos_01_group = 0, neg_01_group = 0; //[0,1] and [-1,0] - int *posFlags, *negFlags, *flags; - int pos_01_flag = 0, neg_01_flag = 0; - createRangeGroups_double(&posGroups, &negGroups, &posFlags, &negFlags); - size_t nbBins = (size_t)(1/pwrErrRatio); - if(nbBins%2==1) - nbBins++; - exe_params->intvRadius = nbBins; - - int reqLength, status; - double medianValue = medianValue_f; - double realPrecision = (double)getRealPrecision_double(valueRangeSize, errBoundMode, absErrBound, relBoundRatio, &status); - if(realPrecision<0) - realPrecision = pwrErrRatio; - double realGroupPrecision; //precision (error) based on group ID - getPrecisionReqLength_double(realPrecision); - short radExpo = getExponent_double(valueRangeSize/2); - short lastGroupNum = 0, groupNum, grpNum = 0; - - double* groupErrorBounds = generateGroupErrBounds(errBoundMode, realPrecision, pwrErrRatio); - exe_params->intvRadius = generateGroupMaxIntervalCount(groupErrorBounds); - - computeReqLength_double(realPrecision, radExpo, &reqLength, &medianValue); - - int* type = (int*) malloc(dataLength*sizeof(int)); - char *groupID = (char*) malloc(dataLength*sizeof(char)); - char *gp = groupID; - - double* spaceFillingValue = oriData; - - DynamicIntArray *exactLeadNumArray; - new_DIA(&exactLeadNumArray, DynArrayInitLen); - - DynamicByteArray *exactMidByteArray; - new_DBA(&exactMidByteArray, DynArrayInitLen); - - DynamicIntArray *resiBitArray; - new_DIA(&resiBitArray, DynArrayInitLen); - - unsigned char preDataBytes[8]; - intToBytes_bigEndian(preDataBytes, 0); - - int reqBytesLength = reqLength/8; - int resiBitsLength = reqLength%8; - - DoubleValueCompressElement *vce = (DoubleValueCompressElement*)malloc(sizeof(DoubleValueCompressElement)); - LossyCompressionElement *lce = (LossyCompressionElement*)malloc(sizeof(LossyCompressionElement)); - - int state; - double curData, decValue; - double pred; - double predAbsErr; - double interval = 0; - - //add the first data - type[0] = 0; - compressSingleDoubleValue(vce, spaceFillingValue[0], realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength); - updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce); - memcpy(preDataBytes,vce->curBytes,8); - addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce); - - curData = spaceFillingValue[0]; - groupNum = computeGroupNum_double(vce->data); - - if(curData > 0 && groupNum >= 0) - { - groups = posGroups; - flags = posFlags; - grpNum = groupNum; - } - else if(curData < 0 && groupNum >= 0) - { - groups = negGroups; - flags = negFlags; - grpNum = groupNum; - } - else if(curData >= 0 && groupNum == -1) - { - groups = &pos_01_group; - flags = &pos_01_flag; - grpNum = 0; - } - else //curData < 0 && groupNum == -1 - { - groups = &neg_01_group; - flags = &neg_01_flag; - grpNum = 0; - } - - listAdd_double_group(groups, flags, groupNum, spaceFillingValue[0], vce->data, gp); - gp++; - - for(i=1;i 0 && groupNum >= 0) - { - groups = posGroups; - flags = posFlags; - grpNum = groupNum; - } - else if(curData < 0 && groupNum >= 0) - { - groups = negGroups; - flags = negFlags; - grpNum = groupNum; - } - else if(curData >= 0 && groupNum == -1) - { - groups = &pos_01_group; - flags = &pos_01_flag; - grpNum = 0; - } - else //curData < 0 && groupNum == -1 - { - groups = &neg_01_group; - flags = &neg_01_flag; - grpNum = 0; - } - - if(groupNum>=GROUP_COUNT) - { - type[i] = 0; - compressSingleDoubleValue(vce, curData, realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength); - updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce); - memcpy(preDataBytes,vce->curBytes,8); - addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce); - listAdd_double_group(groups, flags, lastGroupNum, curData, vce->data, gp); //set the group number to be last one in order to get the groupID array as smooth as possible. - } - else if(flags[grpNum]==0) //the dec value may not be in the same group - { - type[i] = 0; - compressSingleDoubleValue(vce, curData, realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength); - updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce); - memcpy(preDataBytes,vce->curBytes,8); - addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce); - //decGroupNum = computeGroupNum_double(vce->data); - - //if(decGroupNum < groupNum) - // decValue = curData>0?pow(2, groupNum):-pow(2, groupNum); - //else if(decGroupNum > groupNum) - // decValue = curData>0?pow(2, groupNum+1):-pow(2, groupNum+1); - //else - // decValue = vce->data; - - decValue = vce->data; - listAdd_double_group(groups, flags, groupNum, curData, decValue, gp); - lastGroupNum = curData>0?groupNum + 2: -(groupNum+2); - } - else //if flags[groupNum]==1, the dec value must be in the same group - { - pred = groups[grpNum]; - predAbsErr = fabs(curData - pred); - realGroupPrecision = groupErrorBounds[grpNum]; //compute real error bound - interval = realGroupPrecision*2; - state = (predAbsErr/realGroupPrecision+1)/2; - if(curData>=pred) - { - type[i] = exe_params->intvRadius+state; - decValue = pred + state*interval; - } - else //curDataintvRadius-state; - decValue = pred - state*interval; - } - //decGroupNum = computeGroupNum_double(pred); - - if((decValue>0&&curData<0)||(decValue<0&&curData>=0)) - decValue = 0; - //else - //{ - // if(decGroupNum < groupNum) - // decValue = curData>0?pow(2, groupNum):-pow(2, groupNum); - // else if(decGroupNum > groupNum) - // decValue = curData>0?pow(2, groupNum+1):-pow(2, groupNum+1); - // else - // decValue = pred; - //} - - if(fabs(curData-decValue)>realGroupPrecision) - { - type[i] = 0; - compressSingleDoubleValue(vce, curData, realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength); - updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce); - memcpy(preDataBytes,vce->curBytes,8); - addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce); - - decValue = vce->data; - } - - listAdd_double_group(groups, flags, groupNum, curData, decValue, gp); - lastGroupNum = curData>=0?groupNum + 2: -(groupNum+2); - } - gp++; - - } - - int exactDataNum = exactLeadNumArray->size; - - TightDataPointStorageD* tdps; - - //combineTypeAndGroupIDArray(nbBins, dataLength, &type, groupID); - - new_TightDataPointStorageD(&tdps, dataLength, exactDataNum, - type, exactMidByteArray->array, exactMidByteArray->size, - exactLeadNumArray->array, - resiBitArray->array, resiBitArray->size, - resiBitsLength, - realPrecision, medianValue, (char)reqLength, nbBins, NULL, 0, radExpo); - - compressGroupIDArray_double(groupID, tdps); - - free(posGroups); - free(negGroups); - free(posFlags); - free(negFlags); - free(groupID); - free(groupErrorBounds); - - free_DIA(exactLeadNumArray); - free_DIA(resiBitArray); - free(type); - free(vce); - free(lce); - free(exactMidByteArray); //exactMidByteArray->array has been released in free_TightDataPointStorageD(tdps); - - return tdps; -} - -void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwrgroup(unsigned char* newByteData, double *oriData, -size_t dataLength, double absErrBound, double relBoundRatio, double pwrErrRatio, double valueRangeSize, double medianValue_f, size_t *outSize) -{ - TightDataPointStorageD* tdps = SZ_compress_double_1D_MDQ_pwrGroup(oriData, dataLength, confparams_cpr->errorBoundMode, - absErrBound, relBoundRatio, pwrErrRatio, - valueRangeSize, medianValue_f); - - convertTDPStoFlatBytes_double(tdps, newByteData, outSize); - - if(*outSize > 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(double)*dataLength) - SZ_compress_args_double_StoreOriData(oriData, dataLength, newByteData, outSize); - - free_TightDataPointStorageD(tdps); -} - -#include - -void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log(unsigned char* newByteData, double *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, double min, double max){ - - double * log_data = (double *) malloc(dataLength * sizeof(double)); - - unsigned char * signs = (unsigned char *) malloc(dataLength); - memset(signs, 0, dataLength); - // preprocess - double max_abs_log_data; - if(min == 0) max_abs_log_data = fabs(log2(fabs(max))); - else if(max == 0) max_abs_log_data = fabs(log2(fabs(min))); - else max_abs_log_data = fabs(log2(fabs(min))) > fabs(log2(fabs(max))) ? fabs(log2(fabs(min))) : fabs(log2(fabs(max))); - double min_log_data = max_abs_log_data; - bool positive = true; - for(size_t i=0; i 0){ - log_data[i] = log2(log_data[i]); - if(log_data[i] > max_abs_log_data) max_abs_log_data = log_data[i]; - if(log_data[i] < min_log_data) min_log_data = log_data[i]; - } - } - - double valueRangeSize, medianValue_f; - computeRangeSize_double(log_data, dataLength, &valueRangeSize, &medianValue_f); - if(fabs(min_log_data) > max_abs_log_data) max_abs_log_data = fabs(min_log_data); - double realPrecision = log2(1.0 + pwrErrRatio) - max_abs_log_data * 2.23e-16; - for(size_t i=0; iminLogValue = min_log_data - 1.0001*realPrecision; - free(log_data); - if(!positive){ - unsigned char * comp_signs; - // compress signs - unsigned long signSize = sz_lossless_compress(ZSTD_COMPRESSOR, 3, signs, dataLength, &comp_signs); - tdps->pwrErrBoundBytes = comp_signs; - tdps->pwrErrBoundBytes_size = signSize; - } - else{ - tdps->pwrErrBoundBytes = NULL; - tdps->pwrErrBoundBytes_size = 0; - } - free(signs); - - convertTDPStoFlatBytes_double(tdps, newByteData, outSize); - if(*outSize>1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(double)*dataLength) - SZ_compress_args_double_StoreOriData(oriData, dataLength, newByteData, outSize); - - free_TightDataPointStorageD(tdps); -} - -void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log_MSST19(unsigned char* newByteData, double *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, double valueRangeSize, double medianValue_f, - unsigned char* signs, bool* positive, double min, double max, double nearZero){ - double multiplier = pow((1+pwrErrRatio), -3.0001); - for(int i=0; iminLogValue = nearZero / ((1+pwrErrRatio)*(1+pwrErrRatio)); - if(!(*positive)){ - unsigned char * comp_signs; - // compress signs - unsigned long signSize = sz_lossless_compress(ZSTD_COMPRESSOR, 3, signs, dataLength, &comp_signs); - tdps->pwrErrBoundBytes = comp_signs; - tdps->pwrErrBoundBytes_size = signSize; - } - else{ - tdps->pwrErrBoundBytes = NULL; - tdps->pwrErrBoundBytes_size = 0; - } - free(signs); - - convertTDPStoFlatBytes_double(tdps, newByteData, outSize); - if(*outSize > 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(double)*dataLength) - SZ_compress_args_double_StoreOriData(oriData, dataLength, newByteData, outSize); - - free_TightDataPointStorageD(tdps); -} \ No newline at end of file diff --git a/deps/SZ/sz/src/sz_double_ts.c b/deps/SZ/sz/src/sz_double_ts.c deleted file mode 100644 index 3c9b184ee6..0000000000 --- a/deps/SZ/sz/src/sz_double_ts.c +++ /dev/null @@ -1,191 +0,0 @@ -/** - * @file sz_double_ts.c - * @author Sheng Di and Dingwen Tao - * @date Aug, 2016 - * @brief - * (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - - -#include -#include -#include -#include -#include -#include "sz.h" -#include "CompressElement.h" -#include "DynamicByteArray.h" -#include "DynamicIntArray.h" -#include "TightDataPointStorageD.h" -#include "zlib.h" -#include "rw.h" -#include "sz_double_ts.h" - -unsigned int optimize_intervals_double_1D_ts(double *oriData, size_t dataLength, double* preData, double realPrecision) -{ - size_t i = 0, radiusIndex; - double pred_value = 0, pred_err; - size_t *intervals = (size_t*)malloc(confparams_cpr->maxRangeRadius*sizeof(size_t)); - memset(intervals, 0, confparams_cpr->maxRangeRadius*sizeof(size_t)); - size_t totalSampleSize = dataLength/confparams_cpr->sampleDistance; - for(i=2;isampleDistance==0) - { - pred_value = preData[i]; - pred_err = fabs(pred_value - oriData[i]); - radiusIndex = (unsigned long)((pred_err/realPrecision+1)/2); - if(radiusIndex>=confparams_cpr->maxRangeRadius) - radiusIndex = confparams_cpr->maxRangeRadius - 1; - intervals[radiusIndex]++; - } - } - //compute the appropriate number - size_t targetCount = totalSampleSize*confparams_cpr->predThreshold; - size_t sum = 0; - for(i=0;imaxRangeRadius;i++) - { - sum += intervals[i]; - if(sum>targetCount) - break; - } - if(i>=confparams_cpr->maxRangeRadius) - i = confparams_cpr->maxRangeRadius-1; - - unsigned int accIntervals = 2*(i+1); - unsigned int powerOf2 = roundUpToPowerOf2(accIntervals); - - if(powerOf2<32) - powerOf2 = 32; - - free(intervals); - return powerOf2; -} - -TightDataPointStorageD* SZ_compress_double_1D_MDQ_ts(double *oriData, size_t dataLength, sz_multisteps* multisteps, -double realPrecision, double valueRangeSize, double medianValue_d) -{ - double* preStepData = (double*)(multisteps->hist_data); - //store the decompressed data - //double* decData = (double*)malloc(sizeof(double)*dataLength); - //memset(decData, 0, sizeof(double)*dataLength); - double* decData = preStepData; - - unsigned int quantization_intervals; - if(exe_params->optQuantMode==1) - quantization_intervals = optimize_intervals_double_1D_ts(oriData, dataLength, preStepData, realPrecision); - else - quantization_intervals = exe_params->intvCapacity; - updateQuantizationInfo(quantization_intervals); - - size_t i; - int reqLength; - double medianValue = medianValue_d; - short radExpo = getExponent_double(valueRangeSize/2); - - computeReqLength_double(realPrecision, radExpo, &reqLength, &medianValue); - - int* type = (int*) malloc(dataLength*sizeof(int)); - - double* spaceFillingValue = oriData; // - - DynamicIntArray *exactLeadNumArray; - new_DIA(&exactLeadNumArray, DynArrayInitLen); - - DynamicByteArray *exactMidByteArray; - new_DBA(&exactMidByteArray, DynArrayInitLen); - - DynamicIntArray *resiBitArray; - new_DIA(&resiBitArray, DynArrayInitLen); - - unsigned char preDataBytes[8]; - longToBytes_bigEndian(preDataBytes, 0); - - int reqBytesLength = reqLength/8; - int resiBitsLength = reqLength%8; - - DoubleValueCompressElement *vce = (DoubleValueCompressElement*)malloc(sizeof(DoubleValueCompressElement)); - LossyCompressionElement *lce = (LossyCompressionElement*)malloc(sizeof(LossyCompressionElement)); - - //add the first data - type[0] = 0; - compressSingleDoubleValue(vce, spaceFillingValue[0], realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength); - updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce); - memcpy(preDataBytes,vce->curBytes,8); - addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce); - decData[0] = vce->data; - - //add the second data - type[1] = 0; - compressSingleDoubleValue(vce, spaceFillingValue[1], realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength); - updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce); - memcpy(preDataBytes,vce->curBytes,8); - addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce); - decData[1] = vce->data; - - int state = 0; - double checkRadius = 0; - double curData = 0; - double pred = 0; - double predAbsErr = 0; - checkRadius = (exe_params->intvCapacity-1)*realPrecision; - double interval = 2*realPrecision; - - for(i=2;i=pred) - { - type[i] = exe_params->intvRadius+state; - pred = pred + state*interval; - } - else //curDataintvRadius-state; - pred = pred - state*interval; - } - decData[i] = pred; - continue; - } - - //unpredictable data processing - type[i] = 0; - compressSingleDoubleValue(vce, curData, realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength); - updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce); - memcpy(preDataBytes,vce->curBytes,8); - addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce); - decData[i] = vce->data; - }//end of for - - size_t exactDataNum = exactLeadNumArray->size; - - TightDataPointStorageD* tdps; - - new_TightDataPointStorageD(&tdps, dataLength, exactDataNum, - type, exactMidByteArray->array, exactMidByteArray->size, - exactLeadNumArray->array, - resiBitArray->array, resiBitArray->size, - resiBitsLength, - realPrecision, medianValue, (char)reqLength, quantization_intervals, NULL, 0, 0); - - //free memory - free_DIA(exactLeadNumArray); - free_DIA(resiBitArray); - free(type); - free(vce); - free(lce); - free(exactMidByteArray); //exactMidByteArray->array has been released in free_TightDataPointStorageF(tdps); - - //memcpy(preStepData, decData, dataLength*sizeof(double)); //update the data - //free(decData); - - return tdps; -} - - diff --git a/deps/SZ/sz/src/sz_float.c b/deps/SZ/sz/src/sz_float.c index 083ab46c0b..ecd2ce56e8 100644 --- a/deps/SZ/sz/src/sz_float.c +++ b/deps/SZ/sz/src/sz_float.c @@ -19,9 +19,7 @@ #include "DynamicByteArray.h" #include "TightDataPointStorageF.h" #include "sz_float.h" -#include "sz_float_pwr.h" #include "szd_float.h" -#include "szd_float_pwr.h" #include "zlib.h" #include "rw.h" #include "utility.h" @@ -53,11 +51,6 @@ void computeReqLength_float(double realPrecision, short rangeExpo, int* reqLengt } } -inline short computeReqLength_float_MSST19(double realPrecision) -{ - short reqExpo = getPrecisionReqLength_float(realPrecision); - return 9-reqExpo; -} unsigned int optimize_intervals_float_1D(float *oriData, size_t dataLength, double realPrecision) { @@ -240,7 +233,7 @@ TightDataPointStorageF* SZ_compress_float_1D_MDQ(float *oriData, exactLeadNumArray->array, resiBitArray->array, resiBitArray->size, resiBitsLength, - realPrecision, medianValue, (char)reqLength, quantization_intervals, NULL, 0, 0); + realPrecision, medianValue, (char)reqLength, quantization_intervals, 0); //sdi:Debug /* int sum =0; @@ -286,176 +279,16 @@ bool SZ_compress_args_float_NoCkRngeNoGzip_1D( unsigned char* newByteData, float return true; } -/*MSST19*/ -TightDataPointStorageF* SZ_compress_float_1D_MDQ_MSST19(float *oriData, -size_t dataLength, double realPrecision, float valueRangeSize, float medianValue_f) -{ - //struct ClockPoint clockPointBuild; - //TimeDurationStart("build", &clockPointBuild); - unsigned int quantization_intervals; - if(exe_params->optQuantMode==1) - quantization_intervals = optimize_intervals_float_1D_opt_MSST19(oriData, dataLength, realPrecision); - else - quantization_intervals = exe_params->intvCapacity; - //updateQuantizationInfo(quantization_intervals); - int intvRadius = quantization_intervals/2; - - double* precisionTable = (double*)malloc(sizeof(double) * quantization_intervals); - double inv = 2.0-pow(2, -(confparams_cpr->plus_bits)); - for(int i=0; i30000 && i<40000) -// printf("%d %.30G\n", i, test); - } - //float smallest_precision = precisionTable[0], largest_precision = precisionTable[quantization_intervals-1]; - struct TopLevelTableWideInterval levelTable; - MultiLevelCacheTableWideIntervalBuild(&levelTable, precisionTable, quantization_intervals, realPrecision, confparams_cpr->plus_bits); - - size_t i; - int reqLength; - float medianValue = medianValue_f; - //float medianInverse = 1 / medianValue_f; - //short radExpo = getExponent_float(valueRangeSize/2); - - reqLength = computeReqLength_float_MSST19(realPrecision); - - int* type = (int*) malloc(dataLength*sizeof(int)); - - float* spaceFillingValue = oriData; // - - DynamicIntArray *exactLeadNumArray; - new_DIA(&exactLeadNumArray, dataLength/2/8); - - DynamicByteArray *exactMidByteArray; - new_DBA(&exactMidByteArray, dataLength/2); - - DynamicIntArray *resiBitArray; - new_DIA(&resiBitArray, DynArrayInitLen); - - unsigned char preDataBytes[4]; - intToBytes_bigEndian(preDataBytes, 0); - - int reqBytesLength = reqLength/8; - int resiBitsLength = reqLength%8; - float last3CmprsData[3] = {0}; - - //size_t miss=0, hit=0; - - FloatValueCompressElement *vce = (FloatValueCompressElement*)malloc(sizeof(FloatValueCompressElement)); - LossyCompressionElement *lce = (LossyCompressionElement*)malloc(sizeof(LossyCompressionElement)); - - //add the first data - type[0] = 0; - compressSingleFloatValue_MSST19(vce, spaceFillingValue[0], realPrecision, reqLength, reqBytesLength, resiBitsLength); - updateLossyCompElement_Float(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce); - memcpy(preDataBytes,vce->curBytes,4); - addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce); - listAdd_float(last3CmprsData, vce->data); - //miss++; - - //add the second data - type[1] = 0; - compressSingleFloatValue_MSST19(vce, spaceFillingValue[1], realPrecision, reqLength, reqBytesLength, resiBitsLength); - updateLossyCompElement_Float(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce); - memcpy(preDataBytes,vce->curBytes,4); - addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce); - listAdd_float(last3CmprsData, vce->data); - //miss++; - - int state; - //double checkRadius; - float curData; - float pred = vce->data; - - double predRelErrRatio; - - const uint64_t top = levelTable.topIndex, base = levelTable.baseIndex; - const uint64_t range = top - base; - const int bits = levelTable.bits; - uint64_t* const buffer = (uint64_t*)&predRelErrRatio; - const int shift = 52-bits; - uint64_t expoIndex, mantiIndex; - uint16_t* tables[range+1]; - for(int i=0; i<=range; i++){ - tables[i] = levelTable.subTables[i].table; - } - - for(i=2;i> 52) - base; - if(expoIndex <= range){ - mantiIndex = (*buffer & 0x000fffffffffffff) >> shift; - state = tables[expoIndex][mantiIndex]; - }else{ - state = 0; - } - - if(state) - { - type[i] = state; - pred *= precisionTable[state]; - //hit++; - continue; - } - - //unpredictable data processing - type[i] = 0; - compressSingleFloatValue_MSST19(vce, curData, realPrecision, reqLength, reqBytesLength, resiBitsLength); - updateLossyCompElement_Float(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce); - memcpy(preDataBytes,vce->curBytes,4); - addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce); - pred = vce->data; - //miss++; -#ifdef HAVE_TIMECMPR - if(confparams_cpr->szMode == SZ_TEMPORAL_COMPRESSION) - decData[i] = vce->data; -#endif - - }//end of for - -// printf("miss:%d, hit:%d\n", miss, hit); - - size_t exactDataNum = exactLeadNumArray->size; - - TightDataPointStorageF* tdps; - - new_TightDataPointStorageF(&tdps, dataLength, exactDataNum, - type, exactMidByteArray->array, exactMidByteArray->size, - exactLeadNumArray->array, - resiBitArray->array, resiBitArray->size, - resiBitsLength, - realPrecision, medianValue, (char)reqLength, quantization_intervals, NULL, 0, 0); - tdps->plus_bits = confparams_cpr->plus_bits; - - //free memory - free_DIA(exactLeadNumArray); - free_DIA(resiBitArray); - free(type); - free(vce); - free(lce); - free(exactMidByteArray); //exactMidByteArray->array has been released in free_TightDataPointStorageF(tdps); - free(precisionTable); - freeTopLevelTableWideInterval(&levelTable); - return tdps; -} - void SZ_compress_args_float_withinRange(unsigned char* newByteData, float *oriData, size_t dataLength, size_t *outSize) { TightDataPointStorageF* tdps = (TightDataPointStorageF*) malloc(sizeof(TightDataPointStorageF)); - tdps->rtypeArray = NULL; - tdps->typeArray = NULL; tdps->leadNumArray = NULL; tdps->residualMidBits = NULL; tdps->allSameData = 1; tdps->dataSeriesLength = dataLength; tdps->exactMidBytes = (unsigned char*)malloc(sizeof(unsigned char)*4); - tdps->pwrErrBoundBytes = NULL; tdps->isLossless = 0; float value = oriData[0]; floatToBytes(tdps->exactMidBytes, value); @@ -487,9 +320,6 @@ int SZ_compress_args_float(float *oriData, size_t r1, unsigned char* newByteData printf("error, input elements count=%d less than %d, so need not do compress.\n", dataLength, MIN_NUM_OF_ELEMENTS); return SZ_LITTER_ELEMENT; } - - if(params->pw_relBoundRatio < 0.000009999) - params->accelerate_pw_rel_compression = 0; float valueRangeSize = 0, medianValue = 0; unsigned char * signs = NULL; @@ -497,17 +327,7 @@ int SZ_compress_args_float(float *oriData, size_t r1, unsigned char* newByteData float nearZero = 0.0; float min = 0; - // calc min - if(params->errorBoundMode == PW_REL && params->accelerate_pw_rel_compression) - { - signs = (unsigned char *) malloc(dataLength); - memset(signs, 0, dataLength); - min = computeRangeSize_float_MSST19(oriData, dataLength, &valueRangeSize, &medianValue, signs, &positive, &nearZero); - free(signs); // tickduan move to here - signs = NULL; - } - else - min = computeRangeSize_float(oriData, dataLength, &valueRangeSize, &medianValue); + min = computeRangeSize_float(oriData, dataLength, &valueRangeSize, &medianValue); // calc max float max = min+valueRangeSize; @@ -557,23 +377,13 @@ int SZ_compress_args_float(float *oriData, size_t r1, unsigned char* newByteData tmpByteData = (unsigned char*)malloc(r1*sizeof(float) + 1024); } - if(params->errorBoundMode>=PW_REL) + // compress core algorithm + if(!SZ_compress_args_float_NoCkRngeNoGzip_1D(tmpByteData, oriData, r1, realPrecision, &tmpOutSize, valueRangeSize, medianValue)) { - if(params->accelerate_pw_rel_compression && params->maxRangeRadius <= 32768) - SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_pre_log_MSST19(tmpByteData, oriData, params->pw_relBoundRatio, r1, &tmpOutSize, valueRangeSize, medianValue, signs, &positive, min, max, nearZero); - else - SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_pre_log(tmpByteData, oriData, params->pw_relBoundRatio, r1, &tmpOutSize, min, max); - } - else - { - // compress core algorithm - if(!SZ_compress_args_float_NoCkRngeNoGzip_1D(tmpByteData, oriData, r1, realPrecision, &tmpOutSize, valueRangeSize, medianValue)) - { - *outSize = 0; - if(twoStage) - free(tmpByteData); - return SZ_ALGORITHM_ERR; - } + *outSize = 0; + if(twoStage) + free(tmpByteData); + return SZ_ALGORITHM_ERR; } //cost_end(" sz_first_compress"); @@ -597,58 +407,6 @@ int SZ_compress_args_float(float *oriData, size_t r1, unsigned char* newByteData return SZ_SUCCESS; } - -unsigned int optimize_intervals_float_1D_opt_MSST19(float *oriData, size_t dataLength, double realPrecision) -{ - size_t i = 0, radiusIndex; - float pred_value = 0; - double pred_err; - size_t *intervals = (size_t*)malloc(confparams_cpr->maxRangeRadius*sizeof(size_t)); - memset(intervals, 0, confparams_cpr->maxRangeRadius*sizeof(size_t)); - size_t totalSampleSize = 0;//dataLength/confparams_cpr->sampleDistance; - - float * data_pos = oriData + 2; - float divider = log2(1+realPrecision)*2; - int tempIndex = 0; - while(data_pos - oriData < dataLength){ - if(*data_pos == 0){ - data_pos += confparams_cpr->sampleDistance; - continue; - } - tempIndex++; - totalSampleSize++; - pred_value = data_pos[-1]; - pred_err = fabs((double)*data_pos / pred_value); - radiusIndex = (unsigned long)fabs(log2(pred_err)/divider+0.5); - if(radiusIndex>=confparams_cpr->maxRangeRadius) - radiusIndex = confparams_cpr->maxRangeRadius - 1; - intervals[radiusIndex]++; - - data_pos += confparams_cpr->sampleDistance; - } - //compute the appropriate number - size_t targetCount = totalSampleSize*confparams_cpr->predThreshold; - size_t sum = 0; - for(i=0;imaxRangeRadius;i++) - { - sum += intervals[i]; - if(sum>targetCount) - break; - } - if(i>=confparams_cpr->maxRangeRadius) - i = confparams_cpr->maxRangeRadius-1; - - unsigned int accIntervals = 2*(i+1); - unsigned int powerOf2 = roundUpToPowerOf2(accIntervals); - - if(powerOf2<32) - powerOf2 = 32; - - free(intervals); - return powerOf2; -} - - unsigned int optimize_intervals_float_1D_opt(float *oriData, size_t dataLength, double realPrecision) { size_t i = 0, radiusIndex; diff --git a/deps/SZ/sz/src/sz_float_pwr.c b/deps/SZ/sz/src/sz_float_pwr.c deleted file mode 100644 index 24cbb6c78b..0000000000 --- a/deps/SZ/sz/src/sz_float_pwr.c +++ /dev/null @@ -1,736 +0,0 @@ -/** - * @file sz_float_pwr.c - * @author Sheng Di, Dingwen Tao, Xin Liang, Xiangyu Zou, Tao Lu, Wen Xia, Xuan Wang, Weizhe Zhang - * @date Aug, 2016 - * @brief SZ_Init, Compression and Decompression functions - * This file contains the compression/decompression functions related to point-wise relative errors - * (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - - -#include -#include -#include -#include -#include -#include "sz.h" -#include "CompressElement.h" -#include "DynamicByteArray.h" -#include "DynamicIntArray.h" -#include "TightDataPointStorageF.h" -#include "sz_float.h" -#include "sz_float_pwr.h" -#include "zlib.h" -#include "rw.h" -#include "utility.h" - -void compute_segment_precisions_float_1D(float *oriData, size_t dataLength, float* pwrErrBound, unsigned char* pwrErrBoundBytes, double globalPrecision) -{ - size_t i = 0, j = 0, k = 0; - float realPrecision = oriData[0]!=0?fabs(confparams_cpr->pw_relBoundRatio*oriData[0]):confparams_cpr->pw_relBoundRatio; - float approxPrecision; - unsigned char realPrecBytes[4]; - float curPrecision; - float curValue; - float sum = 0; - for(i=0;isegment_size==0&&i>0) - { - //get two first bytes of the realPrecision - if(confparams_cpr->pwr_type==SZ_PWR_AVG_TYPE) - { - realPrecision = sum/confparams_cpr->segment_size; - sum = 0; - } - realPrecision *= confparams_cpr->pw_relBoundRatio; - - if(confparams_cpr->errorBoundMode==ABS_AND_PW_REL||confparams_cpr->errorBoundMode==REL_AND_PW_REL) - realPrecision = realPrecisionerrorBoundMode==ABS_OR_PW_REL||confparams_cpr->errorBoundMode==REL_OR_PW_REL) - realPrecision = realPrecisionpwr_type) - { - case SZ_PWR_MIN_TYPE: - if(realPrecision>curPrecision) - realPrecision = curPrecision; - break; - case SZ_PWR_AVG_TYPE: - sum += curPrecision; - break; - case SZ_PWR_MAX_TYPE: - if(realPrecisionpwr_type==SZ_PWR_AVG_TYPE) - { - int size = dataLength%confparams_cpr->segment_size==0?confparams_cpr->segment_size:dataLength%confparams_cpr->segment_size; - realPrecision = sum/size; - } - if(confparams_cpr->errorBoundMode==ABS_AND_PW_REL||confparams_cpr->errorBoundMode==REL_AND_PW_REL) - realPrecision = realPrecisionerrorBoundMode==ABS_OR_PW_REL||confparams_cpr->errorBoundMode==REL_OR_PW_REL) - realPrecision = realPrecisionmaxRangeRadius*sizeof(int)); - memset(intervals, 0, confparams_cpr->maxRangeRadius*sizeof(int)); - int totalSampleSize = dataLength/confparams_cpr->sampleDistance; - for(i=2;isegment_size==0) - realPrecision = pwrErrBound[j++]; - if(i%confparams_cpr->sampleDistance==0) - { - //pred_value = 2*oriData[i-1] - oriData[i-2]; - pred_value = oriData[i-1]; - pred_err = fabs(pred_value - oriData[i]); - radiusIndex = (unsigned long)((pred_err/realPrecision+1)/2); - if(radiusIndex>=confparams_cpr->maxRangeRadius) - radiusIndex = confparams_cpr->maxRangeRadius - 1; - intervals[radiusIndex]++; - } - } - //compute the appropriate number - size_t targetCount = totalSampleSize*confparams_cpr->predThreshold; - size_t sum = 0; - for(i=0;imaxRangeRadius;i++) - { - sum += intervals[i]; - if(sum>targetCount) - break; - } - if(i>=confparams_cpr->maxRangeRadius) - i = confparams_cpr->maxRangeRadius-1; - - unsigned int accIntervals = 2*(i+1); - unsigned int powerOf2 = roundUpToPowerOf2(accIntervals); - - if(powerOf2<32) - powerOf2 = 32; - - free(intervals); - //printf("accIntervals=%d, powerOf2=%d\n", accIntervals, powerOf2); - return powerOf2; -} - -void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr(unsigned char* newByteData, float *oriData, double globalPrecision, - size_t dataLength, size_t *outSize, float min, float max) -{ - size_t pwrLength = dataLength%confparams_cpr->segment_size==0?dataLength/confparams_cpr->segment_size:dataLength/confparams_cpr->segment_size+1; - float* pwrErrBound = (float*)malloc(sizeof(float)*pwrLength); - size_t pwrErrBoundBytes_size = sizeof(unsigned char)*pwrLength*2; - unsigned char* pwrErrBoundBytes = (unsigned char*)malloc(pwrErrBoundBytes_size); - - compute_segment_precisions_float_1D(oriData, dataLength, pwrErrBound, pwrErrBoundBytes, globalPrecision); - - unsigned int quantization_intervals; - if(exe_params->optQuantMode==1) - { - quantization_intervals = optimize_intervals_float_1D_pwr(oriData, dataLength, pwrErrBound); - updateQuantizationInfo(quantization_intervals); - } - else - quantization_intervals = exe_params->intvCapacity; - size_t i = 0, j = 0; - int reqLength; - float realPrecision = pwrErrBound[j++]; - float medianValue = 0; - float radius = fabs(max)curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce); - memcpy(preDataBytes,vce->curBytes,4); - addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce); - listAdd_float(last3CmprsData, vce->data); - //printf("%.30G\n",last3CmprsData[0]); - - //add the second data - type[1] = 0; - addDBA_Data(resiBitLengthArray, (unsigned char)resiBitsLength); - compressSingleFloatValue(vce, spaceFillingValue[1], realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength); - updateLossyCompElement_Float(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce); - memcpy(preDataBytes,vce->curBytes,4); - addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce); - listAdd_float(last3CmprsData, vce->data); - //printf("%.30G\n",last3CmprsData[0]); - - int state; - double checkRadius; - float curData; - float pred; - double predAbsErr; - checkRadius = (exe_params->intvCapacity-1)*realPrecision; - double interval = 2*realPrecision; - int updateReqLength = 0; //a marker: 1 means already updated - - for(i=2;isegment_size==0) - { - realPrecision = pwrErrBound[j++]; - checkRadius = (exe_params->intvCapacity-1)*realPrecision; - interval = 2*realPrecision; - updateReqLength = 0; - } - //pred = 2*last3CmprsData[0] - last3CmprsData[1]; - pred = last3CmprsData[0]; - predAbsErr = fabs(curData - pred); - if(predAbsErr=pred) - { - type[i] = exe_params->intvRadius+state; - pred = pred + state*interval; - } - else //curDataintvRadius-state; - pred = pred - state*interval; - } - listAdd_float(last3CmprsData, pred); - continue; - } - - //unpredictable data processing - if(updateReqLength==0) - { - computeReqLength_float(realPrecision, radExpo, &reqLength, &medianValue); - reqBytesLength = reqLength/8; - resiBitsLength = reqLength%8; - updateReqLength = 1; - } - - type[i] = 0; - addDBA_Data(resiBitLengthArray, (unsigned char)resiBitsLength); - - compressSingleFloatValue(vce, curData, realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength); - updateLossyCompElement_Float(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce); - memcpy(preDataBytes,vce->curBytes,4); - addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce); - - listAdd_float(last3CmprsData, vce->data); - }//end of for - - int exactDataNum = exactLeadNumArray->size; - - TightDataPointStorageF* tdps; - - new_TightDataPointStorageF2(&tdps, dataLength, exactDataNum, - type, exactMidByteArray->array, exactMidByteArray->size, - exactLeadNumArray->array, - resiBitArray->array, resiBitArray->size, - resiBitLengthArray->array, resiBitLengthArray->size, - realPrecision, medianValue, (char)reqLength, quantization_intervals, pwrErrBoundBytes, pwrErrBoundBytes_size, radExpo); - - - //free memory - free_DBA(resiBitLengthArray); - free_DIA(exactLeadNumArray); - free_DIA(resiBitArray); - free(type); - - convertTDPStoFlatBytes_float(tdps, newByteData, outSize); - - int floatSize=sizeof(float); - if(*outSize>dataLength*floatSize) - { - size_t k = 0, i; - tdps->isLossless = 1; - size_t totalByteLength = 1 + exe_params->SZ_SIZE_TYPE + 1 + floatSize*dataLength; - //*newByteData = (unsigned char*)malloc(totalByteLength); comment by tickduan - - unsigned char dsLengthBytes[exe_params->SZ_SIZE_TYPE]; - intToBytes_bigEndian(dsLengthBytes, dataLength);//4 - - newByteData[k++] = versionNumber; - - if(exe_params->SZ_SIZE_TYPE==4) - { - newByteData[k++] = 16; //=00010000 - } - else - { - newByteData[k++] = 80; - } - for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)//4 or 8 - newByteData[k++] = dsLengthBytes[i]; - - if(sysEndianType==BIG_ENDIAN_SYSTEM) - memcpy(newByteData+4+exe_params->SZ_SIZE_TYPE, oriData, dataLength*floatSize); - else - { - unsigned char* p = newByteData+4+exe_params->SZ_SIZE_TYPE; - for(i=0;idataSeriesLength; - int* standGroupID = (int*)malloc(dataLength*sizeof(int)); - - size_t i; - standGroupID[0] = groupID[0]+GROUP_COUNT; //plus an offset such that it would not be a negative number. - char lastGroupIDValue = groupID[0], curGroupIDValue; - int offset = 2*(GROUP_COUNT + 2); - for(i=1; ipwrErrBoundBytes = out; //groupIDArray - tdps->pwrErrBoundBytes_size = outSize; - - free(standGroupID); -} - -TightDataPointStorageF* SZ_compress_float_1D_MDQ_pwrGroup(float* oriData, size_t dataLength, int errBoundMode, -double absErrBound, double relBoundRatio, double pwrErrRatio, float valueRangeSize, float medianValue_f) -{ - size_t i; - float *posGroups, *negGroups, *groups; - float pos_01_group = 0, neg_01_group = 0; //[0,1] and [-1,0] - int *posFlags, *negFlags, *flags; - int pos_01_flag = 0, neg_01_flag = 0; - createRangeGroups_float(&posGroups, &negGroups, &posFlags, &negFlags); - size_t nbBins = (size_t)(1/pwrErrRatio); - if(nbBins%2==1) - nbBins++; - exe_params->intvRadius = nbBins; - - int reqLength, status; - float medianValue = medianValue_f; - float realPrecision = (float)getRealPrecision_float(valueRangeSize, errBoundMode, absErrBound, relBoundRatio, &status); - if(realPrecision<0) - realPrecision = pwrErrRatio; - float realGroupPrecision; //precision (error) based on group ID - getPrecisionReqLength_float(realPrecision); - short radExpo = getExponent_float(valueRangeSize/2); - short lastGroupNum = 0, groupNum, grpNum = 0; - - double* groupErrorBounds = generateGroupErrBounds(errBoundMode, realPrecision, pwrErrRatio); - exe_params->intvRadius = generateGroupMaxIntervalCount(groupErrorBounds); - - computeReqLength_float(realPrecision, radExpo, &reqLength, &medianValue); - - int* type = (int*) malloc(dataLength*sizeof(int)); - char *groupID = (char*) malloc(dataLength*sizeof(char)); - char *gp = groupID; - - float* spaceFillingValue = oriData; - - DynamicIntArray *exactLeadNumArray; - new_DIA(&exactLeadNumArray, DynArrayInitLen); - - DynamicByteArray *exactMidByteArray; - new_DBA(&exactMidByteArray, DynArrayInitLen); - - DynamicIntArray *resiBitArray; - new_DIA(&resiBitArray, DynArrayInitLen); - - unsigned char preDataBytes[4]; - intToBytes_bigEndian(preDataBytes, 0); - - int reqBytesLength = reqLength/8; - int resiBitsLength = reqLength%8; - - FloatValueCompressElement *vce = (FloatValueCompressElement*)malloc(sizeof(FloatValueCompressElement)); - LossyCompressionElement *lce = (LossyCompressionElement*)malloc(sizeof(LossyCompressionElement)); - - int state; - float curData, decValue; - float pred; - float predAbsErr; - double interval = 0; - - //add the first data - type[0] = 0; - compressSingleFloatValue(vce, spaceFillingValue[0], realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength); - updateLossyCompElement_Float(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce); - memcpy(preDataBytes,vce->curBytes,4); - addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce); - - curData = spaceFillingValue[0]; - groupNum = computeGroupNum_float(vce->data); - - if(curData > 0 && groupNum >= 0) - { - groups = posGroups; - flags = posFlags; - grpNum = groupNum; - } - else if(curData < 0 && groupNum >= 0) - { - groups = negGroups; - flags = negFlags; - grpNum = groupNum; - } - else if(curData >= 0 && groupNum == -1) - { - groups = &pos_01_group; - flags = &pos_01_flag; - grpNum = 0; - } - else //curData < 0 && groupNum == -1 - { - groups = &neg_01_group; - flags = &neg_01_flag; - grpNum = 0; - } - - listAdd_float_group(groups, flags, groupNum, spaceFillingValue[0], vce->data, gp); - gp++; - - for(i=1;i 0 && groupNum >= 0) - { - groups = posGroups; - flags = posFlags; - grpNum = groupNum; - } - else if(curData < 0 && groupNum >= 0) - { - groups = negGroups; - flags = negFlags; - grpNum = groupNum; - } - else if(curData >= 0 && groupNum == -1) - { - groups = &pos_01_group; - flags = &pos_01_flag; - grpNum = 0; - } - else //curData < 0 && groupNum == -1 - { - groups = &neg_01_group; - flags = &neg_01_flag; - grpNum = 0; - } - - if(groupNum>=GROUP_COUNT) - { - type[i] = 0; - compressSingleFloatValue(vce, curData, realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength); - updateLossyCompElement_Float(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce); - memcpy(preDataBytes,vce->curBytes,4); - addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce); - listAdd_float_group(groups, flags, lastGroupNum, curData, vce->data, gp); //set the group number to be last one in order to get the groupID array as smooth as possible. - } - else if(flags[grpNum]==0) //the dec value may not be in the same group - { - type[i] = 0; - compressSingleFloatValue(vce, curData, realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength); - updateLossyCompElement_Float(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce); - memcpy(preDataBytes,vce->curBytes,4); - addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce); - //decGroupNum = computeGroupNum_float(vce->data); - - //if(decGroupNum < groupNum) - // decValue = curData>0?pow(2, groupNum):-pow(2, groupNum); - //else if(decGroupNum > groupNum) - // decValue = curData>0?pow(2, groupNum+1):-pow(2, groupNum+1); - //else - // decValue = vce->data; - - decValue = vce->data; - listAdd_float_group(groups, flags, groupNum, curData, decValue, gp); - lastGroupNum = curData>0?groupNum + 2: -(groupNum+2); - } - else //if flags[groupNum]==1, the dec value must be in the same group - { - pred = groups[grpNum]; - predAbsErr = fabs(curData - pred); - realGroupPrecision = groupErrorBounds[grpNum]; //compute real error bound - interval = realGroupPrecision*2; - state = (predAbsErr/realGroupPrecision+1)/2; - if(curData>=pred) - { - type[i] = exe_params->intvRadius+state; - decValue = pred + state*interval; - } - else //curDataintvRadius-state; - decValue = pred - state*interval; - } - //decGroupNum = computeGroupNum_float(pred); - - if((decValue>0&&curData<0)||(decValue<0&&curData>=0)) - decValue = 0; - //else - //{ - // if(decGroupNum < groupNum) - // decValue = curData>0?pow(2, groupNum):-pow(2, groupNum); - // else if(decGroupNum > groupNum) - // decValue = curData>0?pow(2, groupNum+1):-pow(2, groupNum+1); - // else - // decValue = pred; - //} - - if(fabs(curData-decValue)>realGroupPrecision) - { - type[i] = 0; - compressSingleFloatValue(vce, curData, realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength); - updateLossyCompElement_Float(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce); - memcpy(preDataBytes,vce->curBytes,4); - addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce); - - decValue = vce->data; - } - - listAdd_float_group(groups, flags, groupNum, curData, decValue, gp); - lastGroupNum = curData>=0?groupNum + 2: -(groupNum+2); - } - gp++; - - } - - int exactDataNum = exactLeadNumArray->size; - - TightDataPointStorageF* tdps; - - //combineTypeAndGroupIDArray(nbBins, dataLength, &type, groupID); - - new_TightDataPointStorageF(&tdps, dataLength, exactDataNum, - type, exactMidByteArray->array, exactMidByteArray->size, - exactLeadNumArray->array, - resiBitArray->array, resiBitArray->size, - resiBitsLength, - realPrecision, medianValue, (char)reqLength, nbBins, NULL, 0, radExpo); - - compressGroupIDArray_float(groupID, tdps); - - free(posGroups); - free(negGroups); - free(posFlags); - free(negFlags); - free(groupID); - free(groupErrorBounds); - - free_DIA(exactLeadNumArray); - free_DIA(resiBitArray); - free(type); - free(vce); - free(lce); - free(exactMidByteArray); //exactMidByteArray->array has been released in free_TightDataPointStorageF(tdps); - - return tdps; -} - -void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwrgroup(unsigned char* newByteData, float *oriData, -size_t dataLength, double absErrBound, double relBoundRatio, double pwrErrRatio, float valueRangeSize, float medianValue_f, size_t *outSize) -{ - TightDataPointStorageF* tdps = SZ_compress_float_1D_MDQ_pwrGroup(oriData, dataLength, confparams_cpr->errorBoundMode, - absErrBound, relBoundRatio, pwrErrRatio, - valueRangeSize, medianValue_f); - - convertTDPStoFlatBytes_float(tdps, newByteData, outSize); - - free_TightDataPointStorageF(tdps); -} - -#include - -void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_pre_log(unsigned char* newByteData, float *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, float min, float max){ - - float * log_data = (float *) malloc(dataLength * sizeof(float)); - - unsigned char * signs = (unsigned char *) malloc(dataLength); - memset(signs, 0, dataLength); - // preprocess - float max_abs_log_data; - if(min == 0) max_abs_log_data = fabs(log2(fabs(max))); - else if(max == 0) max_abs_log_data = fabs(log2(fabs(min))); - else max_abs_log_data = fabs(log2(fabs(min))) > fabs(log2(fabs(max))) ? fabs(log2(fabs(min))) : fabs(log2(fabs(max))); - float min_log_data = max_abs_log_data; - bool positive = true; - for(size_t i=0; i 0){ - log_data[i] = log2(log_data[i]); - if(log_data[i] > max_abs_log_data) max_abs_log_data = log_data[i]; - if(log_data[i] < min_log_data) min_log_data = log_data[i]; - } - } - - float valueRangeSize, medianValue_f; - computeRangeSize_float(log_data, dataLength, &valueRangeSize, &medianValue_f); - if(fabs(min_log_data) > max_abs_log_data) max_abs_log_data = fabs(min_log_data); - double realPrecision = log2(1.0 + pwrErrRatio) - max_abs_log_data * 1.2e-7; - for(size_t i=0; iminLogValue = min_log_data - 1.0001*realPrecision; - free(log_data); - if(!positive){ - unsigned char * comp_signs; - // compress signs - unsigned long signSize = sz_lossless_compress(ZSTD_COMPRESSOR, 3, signs, dataLength, &comp_signs); - tdps->pwrErrBoundBytes = comp_signs; - tdps->pwrErrBoundBytes_size = signSize; - } - else{ - tdps->pwrErrBoundBytes = NULL; - tdps->pwrErrBoundBytes_size = 0; - } - free(signs); - - convertTDPStoFlatBytes_float(tdps, newByteData, outSize); - - free_TightDataPointStorageF(tdps); -} - - -void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_pre_log_MSST19(unsigned char* newByteData, float *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, float valueRangeSize, float medianValue_f, - unsigned char* signs, bool* positive, float min, float max, float nearZero){ - float multiplier = pow((1+pwrErrRatio), -3.0001); - for(int i=0; iminLogValue = nearZero / ((1+pwrErrRatio)*(1+pwrErrRatio)); - if(!(*positive)){ - unsigned char * comp_signs; - // compress signs - unsigned long signSize = sz_lossless_compress(ZSTD_COMPRESSOR, 3, signs, dataLength, &comp_signs); - tdps->pwrErrBoundBytes = comp_signs; - tdps->pwrErrBoundBytes_size = signSize; - } - else{ - tdps->pwrErrBoundBytes = NULL; - tdps->pwrErrBoundBytes_size = 0; - } - free(signs); - - convertTDPStoFlatBytes_float(tdps, newByteData, outSize); - - free_TightDataPointStorageF(tdps); -} diff --git a/deps/SZ/sz/src/szd_double.c b/deps/SZ/sz/src/szd_double.c index 9f67f754d6..91866319f6 100644 --- a/deps/SZ/sz/src/szd_double.c +++ b/deps/SZ/sz/src/szd_double.c @@ -13,10 +13,8 @@ #include "szd_double.h" #include "TightDataPointStorageD.h" #include "sz.h" -#include "Huffman.h" -#include "szd_double_pwr.h" -#include "szd_double_ts.h" #include "utility.h" +#include "Huffman.h" int SZ_decompress_args_double(double* newData, size_t r1, unsigned char* cmpBytes, size_t cmpSize, int compressionType, double* hist_data, sz_exedata* pde_exe, sz_params* pde_params) @@ -68,8 +66,12 @@ int SZ_decompress_args_double(double* newData, size_t r1, unsigned char* cmpByte // calc postion //pde_params->sol_ID = szTmpBytes[1+3-2+14-4]; //szTmpBytes: version(3bytes), samebyte(1byte), [14]:sol_ID=SZ or SZ_Transpose //TODO: convert szTmpBytes to double array. - TightDataPointStorageD* tdps; + TightDataPointStorageD* tdps = NULL; int errBoundMode = new_TightDataPointStorageD_fromFlatBytes(&tdps, szTmpBytes, tmpSize, pde_exe, pde_params); + if(tdps == NULL) + { + return SZ_FORMAT_ERR; + } int dim = r1; int doubleSize = sizeof(double); @@ -124,7 +126,6 @@ void decompressDataSeries_double_1D(double* data, size_t dataSeriesLength, doubl //*data = (double*)malloc(sizeof(double)*dataSeriesLength); comment by tickduan int* type = (int*)malloc(dataSeriesLength*sizeof(int)); - HuffmanTree* huffmanTree = createHuffmanTree(tdps->stateNum); decode_withTree(huffmanTree, tdps->typeArray, dataSeriesLength, type); SZ_ReleaseHuffman(huffmanTree); @@ -143,8 +144,10 @@ void decompressDataSeries_double_1D(double* data, size_t dataSeriesLength, doubl resiBitsLength = tdps->reqLength%8; medianValue = tdps->medianValue; + int type_; - for (i = 0; i < dataSeriesLength; i++) { + for (i = 0; i < dataSeriesLength; i++) + { type_ = type[i]; switch (type_) { case 0: @@ -203,108 +206,6 @@ void decompressDataSeries_double_1D(double* data, size_t dataSeriesLength, doubl return; } -/*MSST19*/ -void decompressDataSeries_double_1D_MSST19(double* data, size_t dataSeriesLength, TightDataPointStorageD* tdps) -{ - //updateQuantizationInfo(tdps->intervals); - int intvRadius = tdps->intervals/2; - int intvCapacity = tdps->intervals; - size_t i, j, k = 0, p = 0, l = 0; // k is to track the location of residual_bit - // in resiMidBits, p is to track the - // byte_index of resiMidBits, l is for - // leadNum - unsigned char* leadNum; - //double interval = tdps->realPrecision*2; - - convertByteArray2IntArray_fast_2b(tdps->exactDataNum, tdps->leadNumArray, tdps->leadNumArray_size, &leadNum); - //*data = (double*)malloc(sizeof(double)*dataSeriesLength); comment by tickduan - - int* type = (int*)malloc(dataSeriesLength*sizeof(int)); - - HuffmanTree* huffmanTree = createHuffmanTree(tdps->stateNum); - decode_withTree_MSST19(huffmanTree, tdps->typeArray, dataSeriesLength, type, tdps->max_bits); - //decode_withTree(huffmanTree, tdps->typeArray, dataSeriesLength, type); - SZ_ReleaseHuffman(huffmanTree); - unsigned char preBytes[8]; - unsigned char curBytes[8]; - - memset(preBytes, 0, 8); - - size_t curByteIndex = 0; - int reqBytesLength, resiBitsLength, resiBits; - unsigned char leadingNum; - double exactData, predValue = 0; - reqBytesLength = tdps->reqLength/8; - resiBitsLength = tdps->reqLength%8; - //float threshold = tdps->minLogValue; - double* precisionTable = (double*)malloc(sizeof(double) * intvCapacity); - double inv = 2.0-pow(2, -(tdps->plus_bits)); - for(int i=0; irealPrecision), inv*(i - intvRadius)); - precisionTable[i] = test; - } - - int type_; - for (i = 0; i < dataSeriesLength; i++) { - type_ = type[i]; - switch (type_) { - case 0: - // compute resiBits - resiBits = 0; - if (resiBitsLength != 0) { - int kMod8 = k % 8; - int rightMovSteps = getRightMovingSteps(kMod8, resiBitsLength); - if (rightMovSteps > 0) { - int code = getRightMovingCode(kMod8, resiBitsLength); - resiBits = (tdps->residualMidBits[p] & code) >> rightMovSteps; - } else if (rightMovSteps < 0) { - int code1 = getLeftMovingCode(kMod8); - int code2 = getRightMovingCode(kMod8, resiBitsLength); - int leftMovSteps = -rightMovSteps; - rightMovSteps = 8 - leftMovSteps; - resiBits = (tdps->residualMidBits[p] & code1) << leftMovSteps; - p++; - resiBits = resiBits - | ((tdps->residualMidBits[p] & code2) >> rightMovSteps); - } else // rightMovSteps == 0 - { - int code = getRightMovingCode(kMod8, resiBitsLength); - resiBits = (tdps->residualMidBits[p] & code); - p++; - } - k += resiBitsLength; - } - - // recover the exact data - memset(curBytes, 0, 8); - leadingNum = leadNum[l++]; - memcpy(curBytes, preBytes, leadingNum); - for (j = leadingNum; j < reqBytesLength; j++) - curBytes[j] = tdps->exactMidBytes[curByteIndex++]; - if (resiBitsLength != 0) { - unsigned char resiByte = (unsigned char) (resiBits << (8 - resiBitsLength)); - curBytes[reqBytesLength] = resiByte; - } - - exactData = bytesToDouble(curBytes); - data[i] = exactData; - memcpy(preBytes,curBytes,8); - predValue = data[i]; - break; - default: - //predValue = 2 * data[i-1] - data[i-2]; - //predValue = data[i-1]; - predValue = fabs(predValue) * precisionTable[type_]; - data[i] = predValue; - break; - } - } - - free(precisionTable); - free(leadNum); - free(type); - return; -} void getSnapshotData_double_1D(double* data, size_t dataSeriesLength, TightDataPointStorageD* tdps, int errBoundMode, int compressionType, double* hist_data, sz_params* pde_params) { @@ -315,24 +216,10 @@ void getSnapshotData_double_1D(double* data, size_t dataSeriesLength, TightDataP //*data = (double*)malloc(sizeof(double)*dataSeriesLength); comment by tickduan for (i = 0; i < dataSeriesLength; i++) data[i] = value; - } else { - if (tdps->rtypeArray == NULL) { - if(errBoundMode < PW_REL) - { - decompressDataSeries_double_1D(data, dataSeriesLength, hist_data, tdps); - } - else - { - if(confparams_dec->accelerate_pw_rel_compression) - decompressDataSeries_double_1D_pwr_pre_log_MSST19(data, dataSeriesLength, tdps); - else - decompressDataSeries_double_1D_pwr_pre_log(data, dataSeriesLength, tdps); - //decompressDataSeries_double_1D_pwrgroup(data, dataSeriesLength, tdps); - } - return; - } else { - //TODO - } + } + else + { + decompressDataSeries_double_1D(data, dataSeriesLength, hist_data, tdps); } } diff --git a/deps/SZ/sz/src/szd_double_pwr.c b/deps/SZ/sz/src/szd_double_pwr.c deleted file mode 100644 index c865129790..0000000000 --- a/deps/SZ/sz/src/szd_double_pwr.c +++ /dev/null @@ -1,371 +0,0 @@ -/** - * @file szd_double_pwr.c - * @author Sheng Di, Dingwen Tao, Xin Liang, Xiangyu Zou, Tao Lu, Wen Xia, Xuan Wang, Weizhe Zhang - * @date Feb, 2019 - * @brief - * (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -#include -#include -#include -#include -#include "TightDataPointStorageD.h" -#include "CompressElement.h" -#include "sz.h" -#include "Huffman.h" -#include "sz_double_pwr.h" -#include "utility.h" -//#include "rw.h" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wchar-subscripts" - -void decompressDataSeries_double_1D_pwr(double* data, size_t dataSeriesLength, TightDataPointStorageD* tdps) -{ - updateQuantizationInfo(tdps->intervals); - unsigned char tmpPrecBytes[8] = {0}; //used when needing to convert bytes to double values - unsigned char* bp = tdps->pwrErrBoundBytes; - size_t i, j, k = 0, p = 0, l = 0; // k is to track the location of residual_bit - // in resiMidBits, p is to track the - // byte_index of resiMidBits, l is for - // leadNum - - unsigned char* leadNum; - double interval = 0;// = (double)tdps->realPrecision*2; - - convertByteArray2IntArray_fast_2b(tdps->exactDataNum, tdps->leadNumArray, tdps->leadNumArray_size, &leadNum); - //*data = (double*)malloc(sizeof(double)*dataSeriesLength); comment by tickduan - - int* type = (int*)malloc(dataSeriesLength*sizeof(int)); - - HuffmanTree* huffmanTree = createHuffmanTree(tdps->stateNum); - decode_withTree(huffmanTree, tdps->typeArray, dataSeriesLength, type); - SZ_ReleaseHuffman(huffmanTree); - - unsigned char preBytes[8]; - unsigned char curBytes[8]; - - memset(preBytes, 0, 8); - - size_t curByteIndex = 0; - int reqLength = 0, reqBytesLength = 0, resiBitsLength = 0, resiBits = 0; - unsigned char leadingNum; - double medianValue, exactData, predValue = 0, realPrecision = 0; - - medianValue = tdps->medianValue; - - int type_, updateReqLength = 0; - for (i = 0; i < dataSeriesLength; i++) - { - if(i%tdps->segment_size==0) - { - tmpPrecBytes[0] = *(bp++); - tmpPrecBytes[1] = *(bp++); - memset(&tmpPrecBytes[2], 0, 6*sizeof(unsigned char)); - - realPrecision = bytesToDouble(tmpPrecBytes); - interval = realPrecision*2; - updateReqLength = 0; - } - type_ = type[i]; - switch (type_) { - case 0: - // compute resiBits - if(updateReqLength==0) - { - computeReqLength_double(realPrecision, tdps->radExpo, &reqLength, &medianValue); - reqBytesLength = reqLength/8; - resiBitsLength = reqLength%8; - updateReqLength = 1; - } - resiBits = 0; - if (resiBitsLength != 0) { - int kMod8 = k % 8; - int rightMovSteps = getRightMovingSteps(kMod8, resiBitsLength); - if (rightMovSteps > 0) { - int code = getRightMovingCode(kMod8, resiBitsLength); - resiBits = (tdps->residualMidBits[p] & code) >> rightMovSteps; - } else if (rightMovSteps < 0) { - int code1 = getLeftMovingCode(kMod8); - int code2 = getRightMovingCode(kMod8, resiBitsLength); - int leftMovSteps = -rightMovSteps; - rightMovSteps = 8 - leftMovSteps; - resiBits = (tdps->residualMidBits[p] & code1) << leftMovSteps; - p++; - resiBits = resiBits - | ((tdps->residualMidBits[p] & code2) >> rightMovSteps); - } else // rightMovSteps == 0 - { - int code = getRightMovingCode(kMod8, resiBitsLength); - resiBits = (tdps->residualMidBits[p] & code); - p++; - } - k += resiBitsLength; - } - - // recover the exact data - memset(curBytes, 0, 8); - leadingNum = leadNum[l++]; - memcpy(curBytes, preBytes, leadingNum); - for (j = leadingNum; j < reqBytesLength; j++) - curBytes[j] = tdps->exactMidBytes[curByteIndex++]; - if (resiBitsLength != 0) { - unsigned char resiByte = (unsigned char) (resiBits << (8 - resiBitsLength)); - curBytes[reqBytesLength] = resiByte; - } - - exactData = bytesToDouble(curBytes); - data[i] = exactData + medianValue; - memcpy(preBytes,curBytes,8); - break; - default: - //predValue = 2 * data[i-1] - data[i-2]; - predValue = data[i-1]; - data[i] = predValue + (type_-exe_params->intvRadius)*interval; - break; - } - //printf("%.30G\n",data[i]); - } - free(leadNum); - free(type); - return; -} - - - -void decompressDataSeries_double_1D_pwrgroup(double* data, size_t dataSeriesLength, TightDataPointStorageD* tdps) -{ - double *posGroups, *negGroups, *groups; - double pos_01_group, neg_01_group; - int *posFlags, *negFlags; - - updateQuantizationInfo(tdps->intervals); - - unsigned char* leadNum; - double interval;// = (double)tdps->realPrecision*2; - - convertByteArray2IntArray_fast_2b(tdps->exactDataNum, tdps->leadNumArray, tdps->leadNumArray_size, &leadNum); - - //*data = (double*)malloc(sizeof(double)*dataSeriesLength); comment by tickduan - - int* type = (int*)malloc(dataSeriesLength*sizeof(int)); - - HuffmanTree* huffmanTree = createHuffmanTree(tdps->stateNum); - decode_withTree(huffmanTree, tdps->typeArray, dataSeriesLength, type); - SZ_ReleaseHuffman(huffmanTree); - - createRangeGroups_double(&posGroups, &negGroups, &posFlags, &negFlags); - - double realGroupPrecision; - double realPrecision = tdps->realPrecision; - char* groupID = decompressGroupIDArray(tdps->pwrErrBoundBytes, tdps->dataSeriesLength); - - //note that the groupID values here are [1,2,3,....,18] or [-1,-2,...,-18] - - double* groupErrorBounds = generateGroupErrBounds(confparams_dec->errorBoundMode, realPrecision, confparams_dec->pw_relBoundRatio); - exe_params->intvRadius = generateGroupMaxIntervalCount(groupErrorBounds); - - size_t nbBins = (size_t)(1/confparams_dec->pw_relBoundRatio + 0.5); - if(nbBins%2==1) - nbBins++; - exe_params->intvRadius = nbBins; - - - - unsigned char preBytes[8]; - unsigned char curBytes[8]; - - memset(preBytes, 0, 8); - - size_t curByteIndex = 0; - int reqLength, reqBytesLength = 0, resiBitsLength = 0, resiBits; - unsigned char leadingNum; - double medianValue, exactData, curValue, predValue; - - medianValue = tdps->medianValue; - - size_t i, j, k = 0, p = 0, l = 0; // k is to track the location of residual_bit - // in resiMidBits, p is to track the - // byte_index of resiMidBits, l is for - // leadNum - - int type_, updateReqLength = 0; - char rawGrpID = 0, indexGrpID = 0; - for (i = 0; i < dataSeriesLength; i++) - { - rawGrpID = groupID[i]; - - if(rawGrpID >= 2) - { - groups = posGroups; - indexGrpID = rawGrpID - 2; - } - else if(rawGrpID <= -2) - { - groups = negGroups; - indexGrpID = -rawGrpID - 2; } - else if(rawGrpID == 1) - { - groups = &pos_01_group; - indexGrpID = 0; - } - else //rawGrpID == -1 - { - groups = &neg_01_group; - indexGrpID = 0; - } - - type_ = type[i]; - switch (type_) { - case 0: - // compute resiBits - if(updateReqLength==0) - { - computeReqLength_double(realPrecision, tdps->radExpo, &reqLength, &medianValue); - reqBytesLength = reqLength/8; - resiBitsLength = reqLength%8; - updateReqLength = 1; - } - resiBits = 0; - if (resiBitsLength != 0) { - int kMod8 = k % 8; - int rightMovSteps = getRightMovingSteps(kMod8, resiBitsLength); - if (rightMovSteps > 0) { - int code = getRightMovingCode(kMod8, resiBitsLength); - resiBits = (tdps->residualMidBits[p] & code) >> rightMovSteps; - } else if (rightMovSteps < 0) { - int code1 = getLeftMovingCode(kMod8); - int code2 = getRightMovingCode(kMod8, resiBitsLength); - int leftMovSteps = -rightMovSteps; - rightMovSteps = 8 - leftMovSteps; - resiBits = (tdps->residualMidBits[p] & code1) << leftMovSteps; - p++; - resiBits = resiBits - | ((tdps->residualMidBits[p] & code2) >> rightMovSteps); - } else // rightMovSteps == 0 - { - int code = getRightMovingCode(kMod8, resiBitsLength); - resiBits = (tdps->residualMidBits[p] & code); - p++; - } - k += resiBitsLength; - } - - // recover the exact data - memset(curBytes, 0, 8); - leadingNum = leadNum[l++]; - memcpy(curBytes, preBytes, leadingNum); - for (j = leadingNum; j < reqBytesLength; j++) - curBytes[j] = tdps->exactMidBytes[curByteIndex++]; - if (resiBitsLength != 0) { - unsigned char resiByte = (unsigned char) (resiBits << (8 - resiBitsLength)); - curBytes[reqBytesLength] = resiByte; - } - - exactData = bytesToDouble(curBytes); - exactData = exactData + medianValue; - data[i] = exactData; - memcpy(preBytes,curBytes,8); - - groups[indexGrpID] = exactData; - - break; - default: - predValue = groups[indexGrpID]; //Here, groups[indexGrpID] is the previous value. - realGroupPrecision = groupErrorBounds[indexGrpID]; - interval = realGroupPrecision*2; - - curValue = predValue + (type_-exe_params->intvRadius)*interval; - - //groupNum = computeGroupNum_double(curValue); - - if((curValue>0&&rawGrpID<0)||(curValue<0&&rawGrpID>0)) - curValue = 0; - //else - //{ - // realGrpID = fabs(rawGrpID)-2; - // if(groupNum0?pow(2,realGrpID):-pow(2,realGrpID); - // else if(groupNum>realGrpID) - // curValue = rawGrpID>0?pow(2,groupNum):-pow(2,groupNum); - //} - - data[i] = curValue; - groups[indexGrpID] = curValue; - break; - } - } - - free(leadNum); - free(type); - - free(posGroups); - free(negGroups); - free(posFlags); - free(negFlags); - free(groupErrorBounds); - free(groupID); -} - -void decompressDataSeries_double_1D_pwr_pre_log(double* data, size_t dataSeriesLength, TightDataPointStorageD* tdps) { - - decompressDataSeries_double_1D(data, dataSeriesLength, NULL, tdps); - double threshold = tdps->minLogValue; - if(tdps->pwrErrBoundBytes_size > 0){ - unsigned char * signs; - sz_lossless_decompress(ZSTD_COMPRESSOR, tdps->pwrErrBoundBytes, tdps->pwrErrBoundBytes_size, &signs, dataSeriesLength); - - for(size_t i=0; iminLogValue; - uint64_t* ptr; - - if(tdps->pwrErrBoundBytes_size > 0){ - unsigned char * signs = NULL; - if(tdps->pwrErrBoundBytes_size==0) - { - signs = (unsigned char*)malloc(dataSeriesLength); - memset(signs, 0, dataSeriesLength); - } - else - sz_lossless_decompress(ZSTD_COMPRESSOR, tdps->pwrErrBoundBytes, tdps->pwrErrBoundBytes_size, &signs, dataSeriesLength); - for(size_t i=0; i= 0){ - data[i] = 0; - continue; - } - if(signs[i]){ - ptr = (uint64_t*)data + i; - *ptr |= 0x8000000000000000; - } - } - free(signs); - } - else{ - for(size_t i=0; isol_ID = szTmpBytes[1+3-2+14-4]; //szTmpBytes: version(1bytes), samebyte(1byte), [14-4]:sol_ID=SZ or SZ_Transpose //TODO: convert szTmpBytes to data array. - TightDataPointStorageF* tdps; + TightDataPointStorageF* tdps = NULL; int errBoundMode = new_TightDataPointStorageF_fromFlatBytes(&tdps, szTmpBytes, tmpSize, pde_exe, pde_params); + if(tdps == NULL) + { + return SZ_FORMAT_ERR; + } - //writeByteData(tdps->typeArray, tdps->typeArray_size, "decompress-typebytes.tbt"); int floatSize = sizeof(float); if(tdps->isLossless) { @@ -145,14 +126,14 @@ void decompressDataSeries_float_1D(float* data, size_t dataSeriesLength, float* // leadNum unsigned char* leadNum; float interval = tdps->realPrecision*2; - + convertByteArray2IntArray_fast_2b(tdps->exactDataNum, tdps->leadNumArray, tdps->leadNumArray_size, &leadNum); //data = (float*)malloc(sizeof(float)*dataSeriesLength); // comment by tickduan - - int* type = (int*)malloc(dataSeriesLength*sizeof(int)); + // type tree + int* types = (int*)malloc(dataSeriesLength*sizeof(int)); HuffmanTree* huffmanTree = createHuffmanTree(tdps->stateNum); - decode_withTree(huffmanTree, tdps->typeArray, dataSeriesLength, type); + decode_withTree(huffmanTree, tdps->typeArray, dataSeriesLength, types); SZ_ReleaseHuffman(huffmanTree); unsigned char preBytes[4]; @@ -167,12 +148,15 @@ void decompressDataSeries_float_1D(float* data, size_t dataSeriesLength, float* reqBytesLength = tdps->reqLength/8; resiBitsLength = tdps->reqLength%8; - medianValue = tdps->medianValue; + medianValue = tdps->medianValue; - int type_; - for (i = 0; i < dataSeriesLength; i++) { - type_ = type[i]; - switch (type_) { + // decompress core + int type; + for (i = 0; i < dataSeriesLength; i++) + { + type = types[i]; + switch (type) + { case 0: // compute resiBits resiBits = 0; @@ -218,147 +202,30 @@ void decompressDataSeries_float_1D(float* data, size_t dataSeriesLength, float* default: //predValue = 2 * data[i-1] - data[i-2]; predValue = data[i-1]; - data[i] = predValue + (float)(type_-intvRadius)*interval; + data[i] = predValue + (float)(type - intvRadius) * interval; break; } //printf("%.30G\n",data[i]); } free(leadNum); - free(type); + free(types); return; } -/*MSST19*/ -void decompressDataSeries_float_1D_MSST19(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps) -{ - //updateQuantizationInfo(tdps->intervals); - int intvRadius = tdps->intervals/2; - int intvCapacity = tdps->intervals; - size_t i, j, k = 0, p = 0, l = 0; // k is to track the location of residual_bit - // in resiMidBits, p is to track the - // byte_index of resiMidBits, l is for - // leadNum - unsigned char* leadNum; - //double interval = tdps->realPrecision*2; - - convertByteArray2IntArray_fast_2b(tdps->exactDataNum, tdps->leadNumArray, tdps->leadNumArray_size, &leadNum); - // *data = (float*)malloc(sizeof(float)*dataSeriesLength); comment by tickduan - - int* type = (int*)malloc(dataSeriesLength*sizeof(int)); - - HuffmanTree* huffmanTree = createHuffmanTree(tdps->stateNum); - decode_withTree_MSST19(huffmanTree, tdps->typeArray, dataSeriesLength, type, tdps->max_bits); - SZ_ReleaseHuffman(huffmanTree); - unsigned char preBytes[4]; - unsigned char curBytes[4]; - - memset(preBytes, 0, 4); - - size_t curByteIndex = 0; - int reqBytesLength, resiBitsLength, resiBits; - unsigned char leadingNum; - float exactData, predValue = 0; - reqBytesLength = tdps->reqLength/8; - resiBitsLength = tdps->reqLength%8; - //float threshold = tdps->minLogValue; - double* precisionTable = (double*)malloc(sizeof(double) * intvCapacity); - double inv = 2.0-pow(2, -(tdps->plus_bits)); - for(int i=0; irealPrecision), inv*(i - intvRadius)); - precisionTable[i] = test; - } - - int type_; - for (i = 0; i < dataSeriesLength; i++) { - type_ = type[i]; - switch (type_) { - case 0: - // compute resiBits - resiBits = 0; - if (resiBitsLength != 0) { - int kMod8 = k % 8; - int rightMovSteps = getRightMovingSteps(kMod8, resiBitsLength); - if (rightMovSteps > 0) { - int code = getRightMovingCode(kMod8, resiBitsLength); - resiBits = (tdps->residualMidBits[p] & code) >> rightMovSteps; - } else if (rightMovSteps < 0) { - int code1 = getLeftMovingCode(kMod8); - int code2 = getRightMovingCode(kMod8, resiBitsLength); - int leftMovSteps = -rightMovSteps; - rightMovSteps = 8 - leftMovSteps; - resiBits = (tdps->residualMidBits[p] & code1) << leftMovSteps; - p++; - resiBits = resiBits - | ((tdps->residualMidBits[p] & code2) >> rightMovSteps); - } else // rightMovSteps == 0 - { - int code = getRightMovingCode(kMod8, resiBitsLength); - resiBits = (tdps->residualMidBits[p] & code); - p++; - } - k += resiBitsLength; - } - - // recover the exact data - memset(curBytes, 0, 4); - leadingNum = leadNum[l++]; - memcpy(curBytes, preBytes, leadingNum); - for (j = leadingNum; j < reqBytesLength; j++) - curBytes[j] = tdps->exactMidBytes[curByteIndex++]; - if (resiBitsLength != 0) { - unsigned char resiByte = (unsigned char) (resiBits << (8 - resiBitsLength)); - curBytes[reqBytesLength] = resiByte; - } - - exactData = bytesToFloat(curBytes); - data[i] = exactData; - memcpy(preBytes,curBytes,4); - predValue = data[i]; - break; - default: - //predValue = 2 * data[i-1] - data[i-2]; - //predValue = data[i-1]; - predValue = fabs(predValue) * precisionTable[type_]; - data[i] = predValue; - break; - } - //printf("%.30G\n",data[i]); - } - - free(precisionTable); - free(leadNum); - free(type); - return; -} - - void getSnapshotData_float_1D(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps, int errBoundMode, int compressionType, float* hist_data, sz_params* pde_params) { size_t i; - if (tdps->allSameData) { + if (tdps->allSameData) + { float value = bytesToFloat(tdps->exactMidBytes); //*data = (float*)malloc(sizeof(float)*dataSeriesLength); commnet by tickduan for (i = 0; i < dataSeriesLength; i++) data[i] = value; - } else { - if (tdps->rtypeArray == NULL) { - if(errBoundMode < PW_REL) - { - decompressDataSeries_float_1D(data, dataSeriesLength, hist_data, tdps); - } - else - { - if(pde_params->accelerate_pw_rel_compression) - decompressDataSeries_float_1D_pwr_pre_log_MSST19(data, dataSeriesLength, tdps); - else - decompressDataSeries_float_1D_pwr_pre_log(data, dataSeriesLength, tdps); - //decompressDataSeries_float_1D_pwrgroup(data, dataSeriesLength, tdps); - } - return; - } else { //the special version supporting one value to reserve - //TODO - } + } + else + { + decompressDataSeries_float_1D(data, dataSeriesLength, hist_data, tdps); } } diff --git a/deps/SZ/sz/src/szd_float_pwr.c b/deps/SZ/sz/src/szd_float_pwr.c deleted file mode 100644 index 0eb296ee45..0000000000 --- a/deps/SZ/sz/src/szd_float_pwr.c +++ /dev/null @@ -1,372 +0,0 @@ -/** - * @file szd_float_pwr.c - * @author Sheng Di, Dingwen Tao, Xin Liang, Xiangyu Zou, Tao Lu, Wen Xia, Xuan Wang, Weizhe Zhang - * @date Feb., 2019 - * @brief - * (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -#include -#include -#include -#include -#include "TightDataPointStorageF.h" -#include "CompressElement.h" -#include "sz.h" -#include "Huffman.h" -#include "sz_float_pwr.h" -#include "utility.h" -//#include "rw.h" -// -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wchar-subscripts" - - -void decompressDataSeries_float_1D_pwr(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps) -{ - updateQuantizationInfo(tdps->intervals); - unsigned char tmpPrecBytes[4] = {0}; //used when needing to convert bytes to float values - unsigned char* bp = tdps->pwrErrBoundBytes; - size_t i, j, k = 0, p = 0, l = 0; // k is to track the location of residual_bit - // in resiMidBits, p is to track the - // byte_index of resiMidBits, l is for - // leadNum - unsigned char* leadNum; - float interval = 0;// = (float)tdps->realPrecision*2; - - convertByteArray2IntArray_fast_2b(tdps->exactDataNum, tdps->leadNumArray, tdps->leadNumArray_size, &leadNum); - - //*data = (float*)malloc(sizeof(float)*dataSeriesLength); // comment by tickduan - - int* type = (int*)malloc(dataSeriesLength*sizeof(int)); - - HuffmanTree* huffmanTree = createHuffmanTree(tdps->stateNum); - decode_withTree(huffmanTree, tdps->typeArray, dataSeriesLength, type); - SZ_ReleaseHuffman(huffmanTree); - - //sdi:Debug - //writeUShortData(type, dataSeriesLength, "decompressStateBytes.sb"); - - unsigned char preBytes[4]; - unsigned char curBytes[4]; - - memset(preBytes, 0, 4); - - size_t curByteIndex = 0; - int reqLength = 0, reqBytesLength = 0, resiBitsLength = 0, resiBits = 0; - unsigned char leadingNum; - float medianValue, exactData, predValue = 0, realPrecision = 0; - - medianValue = tdps->medianValue; - - int type_, updateReqLength = 0; - for (i = 0; i < dataSeriesLength; i++) - { - if(i%tdps->segment_size==0) - { - tmpPrecBytes[0] = *(bp++); - tmpPrecBytes[1] = *(bp++); - tmpPrecBytes[2] = 0; - tmpPrecBytes[3] = 0; - realPrecision = bytesToFloat(tmpPrecBytes); - interval = realPrecision*2; - updateReqLength = 0; - } - - type_ = type[i]; - switch (type_) { - case 0: - // compute resiBits - if(updateReqLength==0) - { - computeReqLength_float(realPrecision, tdps->radExpo, &reqLength, &medianValue); - reqBytesLength = reqLength/8; - resiBitsLength = reqLength%8; - updateReqLength = 1; - } - resiBits = 0; - if (resiBitsLength != 0) { - int kMod8 = k % 8; - int rightMovSteps = getRightMovingSteps(kMod8, resiBitsLength); - if (rightMovSteps > 0) { - int code = getRightMovingCode(kMod8, resiBitsLength); - resiBits = (tdps->residualMidBits[p] & code) >> rightMovSteps; - } else if (rightMovSteps < 0) { - int code1 = getLeftMovingCode(kMod8); - int code2 = getRightMovingCode(kMod8, resiBitsLength); - int leftMovSteps = -rightMovSteps; - rightMovSteps = 8 - leftMovSteps; - resiBits = (tdps->residualMidBits[p] & code1) << leftMovSteps; - p++; - resiBits = resiBits - | ((tdps->residualMidBits[p] & code2) >> rightMovSteps); - } else // rightMovSteps == 0 - { - int code = getRightMovingCode(kMod8, resiBitsLength); - resiBits = (tdps->residualMidBits[p] & code); - p++; - } - k += resiBitsLength; - } - - // recover the exact data - memset(curBytes, 0, 4); - leadingNum = leadNum[l++]; - memcpy(curBytes, preBytes, leadingNum); - for (j = leadingNum; j < reqBytesLength; j++) - curBytes[j] = tdps->exactMidBytes[curByteIndex++]; - if (resiBitsLength != 0) { - unsigned char resiByte = (unsigned char) (resiBits << (8 - resiBitsLength)); - curBytes[reqBytesLength] = resiByte; - } - - exactData = bytesToFloat(curBytes); - data[i] = exactData + medianValue; - memcpy(preBytes,curBytes,4); - break; - default: - //predValue = 2 * data[i-1] - data[i-2]; - predValue = data[i-1]; - data[i] = predValue + (type_-exe_params->intvRadius)*interval; - break; - } - //printf("%.30G\n",data[i]); - } - free(leadNum); - free(type); - return; -} - - -void decompressDataSeries_float_1D_pwrgroup(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps) -{ - float *posGroups, *negGroups, *groups; - float pos_01_group, neg_01_group; - int *posFlags, *negFlags; - - updateQuantizationInfo(tdps->intervals); - - unsigned char* leadNum; - double interval;// = (float)tdps->realPrecision*2; - - convertByteArray2IntArray_fast_2b(tdps->exactDataNum, tdps->leadNumArray, tdps->leadNumArray_size, &leadNum); - - //*data = (float*)malloc(sizeof(float)*dataSeriesLength); // comment by tickduan - - int* type = (int*)malloc(dataSeriesLength*sizeof(int)); - - HuffmanTree* huffmanTree = createHuffmanTree(tdps->stateNum); - decode_withTree(huffmanTree, tdps->typeArray, dataSeriesLength, type); - SZ_ReleaseHuffman(huffmanTree); - - createRangeGroups_float(&posGroups, &negGroups, &posFlags, &negFlags); - - float realGroupPrecision; - float realPrecision = tdps->realPrecision; - char* groupID = decompressGroupIDArray(tdps->pwrErrBoundBytes, tdps->dataSeriesLength); - - //note that the groupID values here are [1,2,3,....,18] or [-1,-2,...,-18] - - double* groupErrorBounds = generateGroupErrBounds(confparams_dec->errorBoundMode, realPrecision, confparams_dec->pw_relBoundRatio); - exe_params->intvRadius = generateGroupMaxIntervalCount(groupErrorBounds); - - size_t nbBins = (size_t)(1/confparams_dec->pw_relBoundRatio + 0.5); - if(nbBins%2==1) - nbBins++; - exe_params->intvRadius = nbBins; - - unsigned char preBytes[4]; - unsigned char curBytes[4]; - - memset(preBytes, 0, 4); - - size_t curByteIndex = 0; - int reqLength, reqBytesLength = 0, resiBitsLength = 0, resiBits; - unsigned char leadingNum; - float medianValue, exactData, curValue, predValue; - - medianValue = tdps->medianValue; - - size_t i, j, k = 0, p = 0, l = 0; // k is to track the location of residual_bit - // in resiMidBits, p is to track the - // byte_index of resiMidBits, l is for - // leadNum - - int type_, updateReqLength = 0; - char rawGrpID = 0, indexGrpID = 0; - for (i = 0; i < dataSeriesLength; i++) - { - rawGrpID = groupID[i]; - - if(rawGrpID >= 2) - { - groups = posGroups; - indexGrpID = rawGrpID - 2; - } - else if(rawGrpID <= -2) - { - groups = negGroups; - indexGrpID = -rawGrpID - 2; } - else if(rawGrpID == 1) - { - groups = &pos_01_group; - indexGrpID = 0; - } - else //rawGrpID == -1 - { - groups = &neg_01_group; - indexGrpID = 0; - } - - type_ = type[i]; - switch (type_) { - case 0: - // compute resiBits - if(updateReqLength==0) - { - computeReqLength_float(realPrecision, tdps->radExpo, &reqLength, &medianValue); - reqBytesLength = reqLength/8; - resiBitsLength = reqLength%8; - updateReqLength = 1; - } - resiBits = 0; - if (resiBitsLength != 0) { - int kMod8 = k % 8; - int rightMovSteps = getRightMovingSteps(kMod8, resiBitsLength); - if (rightMovSteps > 0) { - int code = getRightMovingCode(kMod8, resiBitsLength); - resiBits = (tdps->residualMidBits[p] & code) >> rightMovSteps; - } else if (rightMovSteps < 0) { - int code1 = getLeftMovingCode(kMod8); - int code2 = getRightMovingCode(kMod8, resiBitsLength); - int leftMovSteps = -rightMovSteps; - rightMovSteps = 8 - leftMovSteps; - resiBits = (tdps->residualMidBits[p] & code1) << leftMovSteps; - p++; - resiBits = resiBits - | ((tdps->residualMidBits[p] & code2) >> rightMovSteps); - } else // rightMovSteps == 0 - { - int code = getRightMovingCode(kMod8, resiBitsLength); - resiBits = (tdps->residualMidBits[p] & code); - p++; - } - k += resiBitsLength; - } - - // recover the exact data - memset(curBytes, 0, 4); - leadingNum = leadNum[l++]; - memcpy(curBytes, preBytes, leadingNum); - for (j = leadingNum; j < reqBytesLength; j++) - curBytes[j] = tdps->exactMidBytes[curByteIndex++]; - if (resiBitsLength != 0) { - unsigned char resiByte = (unsigned char) (resiBits << (8 - resiBitsLength)); - curBytes[reqBytesLength] = resiByte; - } - - exactData = bytesToFloat(curBytes); - exactData = exactData + medianValue; - data[i] = exactData; - memcpy(preBytes,curBytes,4); - - groups[indexGrpID] = exactData; - - break; - default: - predValue = groups[indexGrpID]; //Here, groups[indexGrpID] is the previous value. - realGroupPrecision = groupErrorBounds[indexGrpID]; - interval = realGroupPrecision*2; - - curValue = predValue + (type_-exe_params->intvRadius)*interval; - - //groupNum = computeGroupNum_float(curValue); - - if((curValue>0&&rawGrpID<0)||(curValue<0&&rawGrpID>0)) - curValue = 0; - //else - //{ - // realGrpID = fabs(rawGrpID)-2; - // if(groupNum0?pow(2,realGrpID):-pow(2,realGrpID); - // else if(groupNum>realGrpID) - // curValue = rawGrpID>0?pow(2,groupNum):-pow(2,groupNum); - //} - - data[i] = curValue; - groups[indexGrpID] = curValue; - break; - } - } - - free(leadNum); - free(type); - - free(posGroups); - free(negGroups); - free(posFlags); - free(negFlags); - free(groupErrorBounds); - free(groupID); -} - -void decompressDataSeries_float_1D_pwr_pre_log(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps) { - - decompressDataSeries_float_1D(data, dataSeriesLength, NULL, tdps); - float threshold = tdps->minLogValue; - if(tdps->pwrErrBoundBytes_size > 0){ - unsigned char * signs; - sz_lossless_decompress(ZSTD_COMPRESSOR, tdps->pwrErrBoundBytes, tdps->pwrErrBoundBytes_size, &signs, dataSeriesLength); - for(size_t i=0; iminLogValue; - uint32_t* ptr; - - if(tdps->pwrErrBoundBytes_size > 0){ - unsigned char * signs = NULL; - if(tdps->pwrErrBoundBytes_size==0) - { - signs = (unsigned char*)malloc(dataSeriesLength); - memset(signs, 0, dataSeriesLength); - } - else - sz_lossless_decompress(ZSTD_COMPRESSOR, tdps->pwrErrBoundBytes, tdps->pwrErrBoundBytes_size, &signs, dataSeriesLength); - for(size_t i=0; i= 0){ - data[i] = 0; - continue; - } - if(signs[i]){ - ptr = (uint32_t*)data + i; - *ptr |= 0x80000000; - } - } - free(signs); - } - else{ - for(size_t i=0; i