From 9ee3360601d81a13870d3f785a090d326b2c091b Mon Sep 17 00:00:00 2001 From: tickduan <417921451@qq.com> Date: Fri, 2 Jul 2021 10:08:31 +0800 Subject: [PATCH] compare with td finished --- deps/SZ/sz/include/TightDataPointStorageD.h | 2 +- deps/SZ/sz/include/TightDataPointStorageF.h | 3 +- deps/SZ/sz/include/TightDataPointStorageI.h | 66 - deps/SZ/sz/include/callZlib.h | 2 +- deps/SZ/sz/include/defines.h | 7 +- deps/SZ/sz/include/exafelSZ.h | 57 - deps/SZ/sz/include/sz.h | 108 +- deps/SZ/sz/include/sz_double.h | 20 +- deps/SZ/sz/include/sz_double_pwr.h | 8 +- deps/SZ/sz/include/sz_double_ts.h | 27 - deps/SZ/sz/include/sz_float.h | 29 +- deps/SZ/sz/include/sz_float_pwr.h | 8 +- deps/SZ/sz/include/sz_float_ts.h | 27 - deps/SZ/sz/include/sz_int16.h | 48 - deps/SZ/sz/include/sz_int32.h | 48 - deps/SZ/sz/include/sz_int64.h | 48 - deps/SZ/sz/include/sz_int8.h | 48 - deps/SZ/sz/include/sz_omp.h | 47 - deps/SZ/sz/include/sz_opencl.h | 6 +- deps/SZ/sz/include/sz_uint16.h | 48 - deps/SZ/sz/include/sz_uint32.h | 48 - deps/SZ/sz/include/sz_uint64.h | 48 - deps/SZ/sz/include/sz_uint8.h | 48 - deps/SZ/sz/include/szd_double.h | 8 +- deps/SZ/sz/include/szd_float.h | 16 +- deps/SZ/sz/include/szd_float_pwr.h | 8 +- deps/SZ/sz/include/szd_float_ts.h | 25 - deps/SZ/sz/include/szd_int16.h | 38 - deps/SZ/sz/include/szd_int32.h | 38 - deps/SZ/sz/include/szd_int64.h | 35 - deps/SZ/sz/include/szd_int8.h | 38 - deps/SZ/sz/include/szd_uint16.h | 38 - deps/SZ/sz/include/szd_uint32.h | 38 - deps/SZ/sz/include/szd_uint64.h | 35 - deps/SZ/sz/include/szd_uint8.h | 38 - deps/SZ/sz/include/szf.h | 102 -- deps/SZ/sz/include/td_sz.h | 1 - deps/SZ/sz/include/utility.h | 20 +- deps/SZ/sz/src/TightDataPointStorageD.c | 18 +- deps/SZ/sz/src/TightDataPointStorageF.c | 129 +- deps/SZ/sz/src/VarSet.c | 8 +- deps/SZ/sz/src/callZlib.c | 10 +- deps/SZ/sz/src/conf.c | 40 +- deps/SZ/sz/src/dataCompression.c | 8 +- deps/SZ/sz/src/exafelSZ.c | 597 -------- deps/SZ/sz/src/rw.c | 132 +- deps/SZ/sz/src/sz.c | 424 +----- deps/SZ/sz/src/sz_double.c | 601 +------- deps/SZ/sz/src/sz_double_pwr.c | 23 +- deps/SZ/sz/src/sz_float.c | 1416 +------------------ deps/SZ/sz/src/sz_float_pwr.c | 33 +- deps/SZ/sz/src/sz_float_ts.c | 207 --- deps/SZ/sz/src/sz_omp.c | 986 ------------- deps/SZ/sz/src/szd_double.c | 92 +- deps/SZ/sz/src/szd_double_pwr.c | 44 +- deps/SZ/sz/src/szd_double_ts.c | 114 -- deps/SZ/sz/src/szd_float.c | 542 +------ deps/SZ/sz/src/szd_float_pwr.c | 44 +- deps/SZ/sz/src/szd_float_ts.c | 115 -- deps/SZ/sz/src/szf.c | 570 -------- deps/SZ/sz/src/td_sz.c | 49 +- deps/SZ/sz/src/utility.c | 583 +------- deps/SZ/zstd/compress/zstd_compress.c | 3 - src/common/src/ttypes.c | 11 +- src/kit/taospack/taospack.c | 77 +- src/util/inc/tscompression.h | 109 +- src/util/src/tcompression.c | 67 +- 67 files changed, 675 insertions(+), 7656 deletions(-) delete mode 100644 deps/SZ/sz/include/TightDataPointStorageI.h delete mode 100644 deps/SZ/sz/include/exafelSZ.h delete mode 100644 deps/SZ/sz/include/sz_double_ts.h delete mode 100644 deps/SZ/sz/include/sz_float_ts.h delete mode 100644 deps/SZ/sz/include/sz_int16.h delete mode 100644 deps/SZ/sz/include/sz_int32.h delete mode 100644 deps/SZ/sz/include/sz_int64.h delete mode 100644 deps/SZ/sz/include/sz_int8.h delete mode 100644 deps/SZ/sz/include/sz_omp.h delete mode 100644 deps/SZ/sz/include/sz_uint16.h delete mode 100644 deps/SZ/sz/include/sz_uint32.h delete mode 100644 deps/SZ/sz/include/sz_uint64.h delete mode 100644 deps/SZ/sz/include/sz_uint8.h delete mode 100644 deps/SZ/sz/include/szd_float_ts.h delete mode 100644 deps/SZ/sz/include/szd_int16.h delete mode 100644 deps/SZ/sz/include/szd_int32.h delete mode 100644 deps/SZ/sz/include/szd_int64.h delete mode 100644 deps/SZ/sz/include/szd_int8.h delete mode 100644 deps/SZ/sz/include/szd_uint16.h delete mode 100644 deps/SZ/sz/include/szd_uint32.h delete mode 100644 deps/SZ/sz/include/szd_uint64.h delete mode 100644 deps/SZ/sz/include/szd_uint8.h delete mode 100644 deps/SZ/sz/include/szf.h delete mode 100644 deps/SZ/sz/src/exafelSZ.c delete mode 100644 deps/SZ/sz/src/sz_float_ts.c delete mode 100644 deps/SZ/sz/src/sz_omp.c delete mode 100644 deps/SZ/sz/src/szd_double_ts.c delete mode 100644 deps/SZ/sz/src/szd_float_ts.c delete mode 100644 deps/SZ/sz/src/szf.c diff --git a/deps/SZ/sz/include/TightDataPointStorageD.h b/deps/SZ/sz/include/TightDataPointStorageD.h index 61f05d8467..11d10bd8e4 100644 --- a/deps/SZ/sz/include/TightDataPointStorageD.h +++ b/deps/SZ/sz/include/TightDataPointStorageD.h @@ -88,7 +88,7 @@ void new_TightDataPointStorageD2(TightDataPointStorageD **self, void convertTDPStoBytes_double(TightDataPointStorageD* tdps, unsigned char* bytes, unsigned char* dsLengthBytes, unsigned char sameByte); void convertTDPStoBytes_double_reserve(TightDataPointStorageD* tdps, unsigned char* bytes, unsigned char* dsLengthBytes, unsigned char sameByte); -void convertTDPStoFlatBytes_double(TightDataPointStorageD *tdps, unsigned char** bytes, size_t *size); +void convertTDPStoFlatBytes_double(TightDataPointStorageD *tdps, unsigned char* bytes, size_t *size); void convertTDPStoFlatBytes_double_args(TightDataPointStorageD *tdps, unsigned char* bytes, size_t *size); void free_TightDataPointStorageD(TightDataPointStorageD *tdps); diff --git a/deps/SZ/sz/include/TightDataPointStorageF.h b/deps/SZ/sz/include/TightDataPointStorageF.h index 6a6583e4e6..eec780cf11 100644 --- a/deps/SZ/sz/include/TightDataPointStorageF.h +++ b/deps/SZ/sz/include/TightDataPointStorageF.h @@ -92,8 +92,7 @@ void new_TightDataPointStorageF2(TightDataPointStorageF **self, unsigned char* pwrErrBoundBytes, size_t pwrErrBoundBytes_size, unsigned char radExpo); void convertTDPStoBytes_float(TightDataPointStorageF* tdps, unsigned char* bytes, unsigned char* dsLengthBytes, unsigned char sameByte); -void convertTDPStoBytes_float_reserve(TightDataPointStorageF* tdps, unsigned char* bytes, unsigned char* dsLengthBytes, unsigned char sameByte); -void convertTDPStoFlatBytes_float(TightDataPointStorageF *tdps, unsigned char** bytes, size_t *size); +void convertTDPStoFlatBytes_float(TightDataPointStorageF *tdps, unsigned char* bytes, size_t *size); void convertTDPStoFlatBytes_float_args(TightDataPointStorageF *tdps, unsigned char* bytes, size_t *size); void free_TightDataPointStorageF(TightDataPointStorageF *tdps); diff --git a/deps/SZ/sz/include/TightDataPointStorageI.h b/deps/SZ/sz/include/TightDataPointStorageI.h deleted file mode 100644 index 6a773b96e0..0000000000 --- a/deps/SZ/sz/include/TightDataPointStorageI.h +++ /dev/null @@ -1,66 +0,0 @@ -/** - * @file TightDataPointStorageI.h - * @author Sheng Di and Dingwen Tao - * @date Aug, 2017 - * @brief Header file for the tight data point storage (TDPS). - * (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -#ifndef _TightDataPointStorageI_H -#define _TightDataPointStorageI_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include "pub.h" - -typedef struct TightDataPointStorageI -{ - size_t dataSeriesLength; - int allSameData; - double realPrecision; //it's used as the pwrErrBoundRatio when errBoundMode==PW_REL - size_t exactDataNum; - long minValue; - int exactByteSize; - int dataTypeSize; //the size of data type, e.g., it's 4 when data type is int32_t - - int stateNum; - int allNodes; - - unsigned char* typeArray; //its size is dataSeriesLength/4 (or xxx/4+1) - size_t typeArray_size; - - unsigned char* exactDataBytes; - size_t exactDataBytes_size; - - unsigned int intervals; //quantization_intervals - - unsigned char isLossless; //a mark to denote whether it's lossless compression (1 is yes, 0 is no) - -} TightDataPointStorageI; - -int computeRightShiftBits(int exactByteSize, int dataType); -int convertDataTypeSizeCode(int dataTypeSizeCode); -int convertDataTypeSize(int dataTypeSize); - -void new_TightDataPointStorageI_Empty(TightDataPointStorageI **self); -int new_TightDataPointStorageI_fromFlatBytes(TightDataPointStorageI **self, unsigned char* flatBytes, size_t flatBytesLength); -void new_TightDataPointStorageI(TightDataPointStorageI **self, - size_t dataSeriesLength, size_t exactDataNum, int byteSize, - int* type, unsigned char* exactDataBytes, size_t exactDataBytes_size, - double realPrecision, long minValue, int intervals, int dataType); - -void convertTDPStoBytes_int(TightDataPointStorageI* tdps, unsigned char* bytes, unsigned char sameByte); -void convertTDPStoFlatBytes_int(TightDataPointStorageI *tdps, unsigned char** bytes, size_t *size); -void convertTDPStoFlatBytes_int_args(TightDataPointStorageI *tdps, unsigned char* bytes, size_t *size); -void free_TightDataPointStorageI(TightDataPointStorageI *tdps); -void free_TightDataPointStorageI2(TightDataPointStorageI *tdps); - -#ifdef __cplusplus -} -#endif - -#endif /* ----- #ifndef _TightDataPointStorageI_H ----- */ diff --git a/deps/SZ/sz/include/callZlib.h b/deps/SZ/sz/include/callZlib.h index 1aede548c6..7249d0d173 100644 --- a/deps/SZ/sz/include/callZlib.h +++ b/deps/SZ/sz/include/callZlib.h @@ -26,7 +26,7 @@ unsigned long zlib_compress(unsigned char* data, unsigned long dataLength, unsig unsigned long zlib_compress2(unsigned char* data, unsigned long dataLength, unsigned char** compressBytes, int level); unsigned long zlib_compress3(unsigned char* data, unsigned long dataLength, unsigned char* compressBytes, int level); unsigned long zlib_compress4(unsigned char* data, unsigned long dataLength, unsigned char** compressBytes, int level); -unsigned long zlib_compress5(unsigned char* data, unsigned long dataLength, unsigned char** compressBytes, int level); +unsigned long zlib_compress5(unsigned char* data, unsigned long dataLength, unsigned char* compressBytes, int level); unsigned long zlib_uncompress4(unsigned char* compressBytes, unsigned long cmpSize, unsigned char** oriData, unsigned long targetOriSize); unsigned long zlib_uncompress5(unsigned char* compressBytes, unsigned long cmpSize, unsigned char** oriData, unsigned long targetOriSize); diff --git a/deps/SZ/sz/include/defines.h b/deps/SZ/sz/include/defines.h index aea3171c88..b9879551a6 100644 --- a/deps/SZ/sz/include/defines.h +++ b/deps/SZ/sz/include/defines.h @@ -24,7 +24,7 @@ //prediction mode of temporal dimension based compression #define SZ_PREVIOUS_VALUE_ESTIMATE 0 -#define MIN_NUM_OF_ELEMENTS 0 //if the # elements <= 20, skip the compression +#define MIN_NUM_OF_ELEMENTS 50 //if the # elements <= 20, skip the compression #define SZ_ABS 0 #define REL 1 @@ -81,13 +81,14 @@ #define SZ_PERIO_TEMPORAL_COMPRESSION 2 //SUCCESS returning status -#define SZ_SCES 0 //successful -#define SZ_NSCS -1 //Not successful +#define SZ_SUCCESS 0 //successful +#define SZ_FAILED -1 //Not successful #define SZ_FERR -2 //Failed to open input file #define SZ_TERR -3 //wrong data type (should be only float or double) #define SZ_DERR -4 //dimension error #define SZ_MERR -5 //sz_mode error #define SZ_BERR -6 //bound-mode error (should be only SZ_ABS, REL, ABS_AND_REL, ABS_OR_REL, or PW_REL) +#define SZ_LITTER_ELEMENT -7 #define SZ_MAINTAIN_VAR_DATA 0 #define SZ_DESTROY_WHOLE_VARSET 1 diff --git a/deps/SZ/sz/include/exafelSZ.h b/deps/SZ/sz/include/exafelSZ.h deleted file mode 100644 index 06e9921a28..0000000000 --- a/deps/SZ/sz/include/exafelSZ.h +++ /dev/null @@ -1,57 +0,0 @@ -#ifndef EXAFELSZ_H -#define EXAFELSZ_H -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include - -typedef struct exafelSZ_params{ - //uint8_t *peaks; - uint16_t *peaksSegs; - uint16_t *peaksRows; - uint16_t *peaksCols; - uint64_t numPeaks; - - uint8_t *calibPanel; - - uint8_t binSize; //Binning: (pr->binSize x pr->binSize) to (1 x 1) - double tolerance; //SZ pr->tolerance - uint8_t szDim; //1D/2D/3D compression/decompression - //uint8_t szBlockSize; //Currently unused - uint8_t peakSize; //MUST BE ODD AND NOT EVEN! Each peak will have size of: (peakSize x peakSize) - - // uint64_t nEvents; - // uint64_t panels; - // uint64_t rows; - // uint64_t cols; - - //CALCULATED VARIBALES: - uint64_t binnedRows; - uint64_t binnedCols; - uint8_t peakRadius; //Will be calculated using peakSize - -} exafelSZ_params; - - -void exafelSZ_params_process(exafelSZ_params*pr, size_t panels, size_t rows, size_t cols); -void exafelSZ_params_checkDecomp(exafelSZ_params*pr, size_t panels, size_t rows, size_t cols); -void exafelSZ_params_checkComp(exafelSZ_params*pr, size_t panels, size_t rows, size_t cols); - -unsigned char * exafelSZ_Compress(void* _pr, - void* _origData, - size_t events, size_t panels, size_t rows, size_t cols, - size_t *compressedSize); - -void* exafelSZ_Decompress(void *_pr, - unsigned char*_compressedBuffer, - size_t events, size_t panels, size_t rows, size_t cols, - size_t compressedSize); - -#ifdef __cplusplus -} -#endif - -#endif /* ----- #ifndef _EXAFELSZ_H ----- */ - diff --git a/deps/SZ/sz/include/sz.h b/deps/SZ/sz/include/sz.h index 7e241d9951..d44ace61ae 100644 --- a/deps/SZ/sz/include/sz.h +++ b/deps/SZ/sz/include/sz.h @@ -22,29 +22,12 @@ #include "Huffman.h" #include "TightDataPointStorageD.h" #include "TightDataPointStorageF.h" -#include "TightDataPointStorageI.h" #include "conf.h" #include "dataCompression.h" #include "ByteToolkit.h" #include "TypeManager.h" -#include "sz_int8.h" -#include "sz_int16.h" -#include "sz_int32.h" -#include "sz_int64.h" -#include "sz_uint8.h" -#include "sz_uint16.h" -#include "sz_uint32.h" -#include "sz_uint64.h" #include "sz_float.h" #include "sz_double.h" -#include "szd_int8.h" -#include "szd_int16.h" -#include "szd_int32.h" -#include "szd_int64.h" -#include "szd_uint8.h" -#include "szd_uint16.h" -#include "szd_uint32.h" -#include "szd_uint64.h" #include "szd_float.h" #include "szd_double.h" #include "sz_float_pwr.h" @@ -53,13 +36,10 @@ #include "callZlib.h" #include "rw.h" #include "pastri.h" -#include "sz_float_ts.h" -#include "szd_float_ts.h" #include "utility.h" #include "CacheTable.h" #include "MultiLevelCacheTable.h" #include "MultiLevelCacheTableWideInterval.h" -#include "exafelSZ.h" #include "sz_stats.h" #ifdef _WIN32 @@ -208,86 +188,26 @@ extern pastri_params pastri_par; //sz.h HuffmanTree* SZ_Reset(); -int SZ_Init(const char *configFilePath); - -int SZ_Init_Params(sz_params *params); - -size_t computeDataLength(size_t r5, size_t r4, size_t r3, size_t r2, size_t r1); - -int computeDimension(size_t r5, size_t r4, size_t r3, size_t r2, size_t r1); - -int SZ_compress_args_float_subblock(unsigned char* compressedBytes, float *oriData, -size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, -size_t s5, size_t s4, size_t s3, size_t s2, size_t s1, -size_t e5, size_t e4, size_t e3, size_t e2, size_t e1, -size_t *outSize, int errBoundMode, double absErr_Bound, double relBoundRatio); - -int SZ_compress_args_double_subblock(unsigned char* compressedBytes, double *oriData, -size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, -size_t s5, size_t s4, size_t s3, size_t s2, size_t s1, -size_t e5, size_t e4, size_t e3, size_t e2, size_t e1, -size_t *outSize, int errBoundMode, double absErr_Bound, double relBoundRatio); - -unsigned char *SZ_compress(int dataType, void *data, size_t *outSize, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1); - -unsigned char* SZ_compress_args(int dataType, void *data, size_t *outSize, int errBoundMode, double absErrBound, -double relBoundRatio, double pwrBoundRatio, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1); - -int SZ_compress_args2(int dataType, void *data, unsigned char* compressed_bytes, size_t *outSize, -int errBoundMode, double absErrBound, double relBoundRatio, double pwrBoundRatio, -size_t r5, size_t r4, size_t r3, size_t r2, size_t r1); - -int SZ_compress_args3(int dataType, void *data, unsigned char* compressed_bytes, size_t *outSize, int errBoundMode, double absErrBound, double relBoundRatio, -size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, -size_t s5, size_t s4, size_t s3, size_t s2, size_t s1, -size_t e5, size_t e4, size_t e3, size_t e2, size_t e1); - -unsigned char *SZ_compress_rev_args(int dataType, void *data, void *reservedValue, size_t *outSize, int errBoundMode, double absErrBound, double relBoundRatio, -size_t r5, size_t r4, size_t r3, size_t r2, size_t r1); - -int SZ_compress_rev_args2(int dataType, void *data, void *reservedValue, unsigned char* compressed_bytes, size_t *outSize, int errBoundMode, double absErrBound, double relBoundRatio, -size_t r5, size_t r4, size_t r3, size_t r2, size_t r1); -unsigned char *SZ_compress_rev(int dataType, void *data, void *reservedValue, size_t *outSize, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1); - -void SZ_Create_ParamsExe(sz_params** conf_params, sz_exedata** exe_params); - -void *SZ_decompress(int dataType, unsigned char *bytes, size_t byteLength, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1); -size_t SZ_decompress_args(int dataType, unsigned char *bytes, size_t byteLength, void* decompressed_array, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1); - -sz_metadata* SZ_getMetadata(unsigned char* bytes, sz_exedata* pde_exe); -void SZ_printMetadata(sz_metadata* metadata); - - -void filloutDimArray(size_t* dim, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1); - -size_t compute_total_batch_size(); - -void SZ_registerVar(int var_id, char* varName, int dataType, void* data, - int errBoundMode, double absErrBound, double relBoundRatio, double pwRelBoundRatio, - size_t r5, size_t r4, size_t r3, size_t r2, size_t r1); - -int SZ_deregisterVar_ID(int var_id); -int SZ_deregisterVar(char* varName); -int SZ_deregisterAllVars(); - -int SZ_compress_ts_select_var(int cmprType, unsigned char* var_ids, unsigned char var_count, unsigned char** newByteData, size_t *outSize); -int SZ_compress_ts(int cmprType, unsigned char** newByteData, size_t *outSize); -void SZ_decompress_ts_select_var(unsigned char* var_ids, unsigned char var_count, unsigned char *bytes, size_t bytesLength); -void SZ_decompress_ts(unsigned char *bytes, size_t byteLength); void SZ_Finalize(); +int SZ_Init(const char *configFilePath); +int SZ_Init_Params(sz_params *params); + +// +// compress output data to outData and return outSize +// +size_t SZ_compress_args(int dataType, void *data, size_t r1, unsigned char* outData, sz_params* params); + + +// +// decompress output data to outData and return outSize +// +size_t SZ_decompress(int dataType, unsigned char *bytes, size_t byteLength, size_t r1, unsigned char* outData); + void convertSZParamsToBytes(sz_params* params, unsigned char* result); void convertBytesToSZParams(unsigned char* bytes, sz_params* params, sz_exedata* pde_exe); -unsigned char* SZ_compress_customize(const char* appName, void* userPara, int dataType, void* data, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize, int *status); - -unsigned char* SZ_compress_customize_threadsafe(const char* cmprName, void* userPara, int dataType, void* data, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize, int *status); - -void* SZ_decompress_customize(const char* appName, void* userPara, int dataType, unsigned char* bytes, size_t byteLength, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, int* status); - -void* SZ_decompress_customize_threadsafe(const char* cmprName, void* userPara, int dataType, unsigned char* bytes, size_t byteLength, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, int *status); - #ifdef __cplusplus } #endif diff --git a/deps/SZ/sz/include/sz_double.h b/deps/SZ/sz/include/sz_double.h index e71f5e77e2..e18e712c82 100644 --- a/deps/SZ/sz/include/sz_double.h +++ b/deps/SZ/sz/include/sz_double.h @@ -28,29 +28,17 @@ unsigned int optimize_intervals_double_1D_opt(double *oriData, size_t dataLength unsigned int optimize_intervals_double_1D_opt_MSST19(double *oriData, size_t dataLength, double realPrecision); TightDataPointStorageD* SZ_compress_double_1D_MDQ(double *oriData, size_t dataLength, double realPrecision, double valueRangeSize, double medianValue_d); -void SZ_compress_args_double_StoreOriData(double* oriData, size_t dataLength, unsigned char** newByteData, size_t *outSize); +void SZ_compress_args_double_StoreOriData(double* oriData, size_t dataLength, unsigned char* newByteData, size_t *outSize); -char SZ_compress_args_double_NoCkRngeNoGzip_1D(int cmprType, unsigned char** newByteData, double *oriData, size_t dataLength, double realPrecision, size_t *outSize, double valueRangeSize, double medianValue_d); +char SZ_compress_args_double_NoCkRngeNoGzip_1D( unsigned char* newByteData, double *oriData, size_t dataLength, double realPrecision, size_t *outSize, double valueRangeSize, double medianValue_d); TightDataPointStorageD* SZ_compress_double_1D_MDQ_MSST19(double *oriData, size_t dataLength, double realPrecision, double valueRangeSize, double medianValue_f); -void SZ_compress_args_double_withinRange(unsigned char** newByteData, double *oriData, size_t dataLength, size_t *outSize); +void SZ_compress_args_double_withinRange(unsigned char* newByteData, double *oriData, size_t dataLength, size_t *outSize); -/*int SZ_compress_args_double_wRngeNoGzip(unsigned char** newByteData, double *oriData, -size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize, -int errBoundMode, double absErr_Bound, double relBoundRatio, double pwrErrRatio);*/ -int SZ_compress_args_double(int cmprType, int withRegression, unsigned char** newByteData, double *oriData, -size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize, -int errBoundMode, double absErr_Bound, double relBoundRatio, double pwRelBoundRatio); +int SZ_compress_args_double(double *oriData, size_t r1, unsigned char* newByteData, size_t *outSize, sz_params* params); -void SZ_compress_args_double_NoCkRnge_1D_subblock(unsigned char* compressedBytes, double *oriData, double realPrecision, size_t *outSize, double valueRangeSize, double medianValue_d, -size_t r1, size_t s1, size_t e1); - -unsigned int optimize_intervals_double_1D_subblock(double *oriData, double realPrecision, size_t r1, size_t s1, size_t e1); - -TightDataPointStorageD* SZ_compress_double_1D_MDQ_subblock(double *oriData, double realPrecision, double valueRangeSize, double medianValue_d, -size_t r1, size_t s1, size_t e1); #ifdef __cplusplus diff --git a/deps/SZ/sz/include/sz_double_pwr.h b/deps/SZ/sz/include/sz_double_pwr.h index b53db479c3..93d53c7c29 100644 --- a/deps/SZ/sz/include/sz_double_pwr.h +++ b/deps/SZ/sz/include/sz_double_pwr.h @@ -19,17 +19,17 @@ extern "C" { void compute_segment_precisions_double_1D(double *oriData, size_t dataLength, double* pwrErrBound, unsigned char* pwrErrBoundBytes, double globalPrecision); unsigned int optimize_intervals_double_1D_pwr(double *oriData, size_t dataLength, double* pwrErrBound); -void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr(unsigned char** newByteData, double *oriData, double globalPrecision, size_t dataLength, size_t *outSize, double min, double max); +void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr(unsigned char* newByteData, double *oriData, double globalPrecision, size_t dataLength, size_t *outSize, double min, double max); void createRangeGroups_double(double** posGroups, double** negGroups, int** posFlags, int** negFlags); void compressGroupIDArray_double(char* groupID, TightDataPointStorageD* tdps); TightDataPointStorageD* SZ_compress_double_1D_MDQ_pwrGroup(double* oriData, size_t dataLength, int errBoundMode, double absErrBound, double relBoundRatio, double pwrErrRatio, double valueRangeSize, double medianValue_f); -void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwrgroup(unsigned char** newByteData, double *oriData, +void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwrgroup(unsigned char* newByteData, double *oriData, size_t dataLength, double absErrBound, double relBoundRatio, double pwrErrRatio, double valueRangeSize, double medianValue_f, size_t *outSize); -void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log(unsigned char** newByteData, double *oriData, double globalPrecision, size_t dataLength, size_t *outSize, double min, double max); +void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log(unsigned char* newByteData, double *oriData, double globalPrecision, size_t dataLength, size_t *outSize, double min, double max); -void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log_MSST19(unsigned char** newByteData, double *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, double valueRangeSize, double medianValue_f, +void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log_MSST19(unsigned char* newByteData, double *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, double valueRangeSize, double medianValue_f, unsigned char* signs, bool* positive, double min, double max, double nearZero); #ifdef __cplusplus diff --git a/deps/SZ/sz/include/sz_double_ts.h b/deps/SZ/sz/include/sz_double_ts.h deleted file mode 100644 index 581d20ddf5..0000000000 --- a/deps/SZ/sz/include/sz_double_ts.h +++ /dev/null @@ -1,27 +0,0 @@ -/** - * @file sz_double_ts.h - * @author Sheng Di - * @date May, 2018 - * @brief Header file for the sz_double_ts.c. - * (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ -#include "TightDataPointStorageD.h" - -#ifndef _SZ_Double_TS_H -#define _SZ_Double_TS_H - -#ifdef __cplusplus -extern "C" { -#endif -unsigned int optimize_intervals_double_1D_ts(double *oriData, size_t dataLength, double* preData, double realPrecision); - -TightDataPointStorageD* SZ_compress_double_1D_MDQ_ts(double *oriData, size_t dataLength, sz_multisteps* multisteps, -double realPrecision, double valueRangeSize, double medianValue_d); - -#ifdef __cplusplus -} -#endif - -#endif /* ----- #ifndef _SZ_Double_TS_H ----- */ - diff --git a/deps/SZ/sz/include/sz_float.h b/deps/SZ/sz/include/sz_float.h index 493f3d1b81..3aded6da33 100644 --- a/deps/SZ/sz/include/sz_float.h +++ b/deps/SZ/sz/include/sz_float.h @@ -29,9 +29,9 @@ unsigned int optimize_intervals_float_1D_opt_MSST19(float *oriData, size_t dataL TightDataPointStorageF* SZ_compress_float_1D_MDQ(float *oriData, size_t dataLength, float realPrecision, float valueRangeSize, float medianValue_f); -void SZ_compress_args_float_StoreOriData(float* oriData, size_t dataLength, unsigned char** newByteData, size_t *outSize); +void SZ_compress_args_float_StoreOriData(float* oriData, size_t dataLength, unsigned char* newByteData, size_t *outSize); -char SZ_compress_args_float_NoCkRngeNoGzip_1D(int cmprType, unsigned char** newByteData, float *oriData, +char SZ_compress_args_float_NoCkRngeNoGzip_1D( unsigned char* newByteData, float *oriData, size_t dataLength, double realPrecision, size_t *outSize, float valueRangeSize, float medianValue_f); size_t SZ_compress_float_1D_MDQ_RA_block(float * block_ori_data, float * mean, size_t dim_0, size_t block_dim_0, double realPrecision, int * type, float * unpredictable_data); @@ -40,36 +40,15 @@ size_t SZ_compress_float_1D_MDQ_RA_block_1D_pred(float * block_ori_data, float * void SZ_blocked_regression(float * block_ori_data, size_t dim_0, size_t dim_1, size_t dim_2, size_t block_dim_0, size_t block_dim_1, size_t block_dim_2, float *params); unsigned char * SZ_compress_float_1D_MDQ_RA(float *oriData, size_t r1, double realPrecision, size_t * comp_size); -unsigned char * SZ_compress_float_1D_MDQ_decompression_random_access_with_blocked_regression(float *oriData, size_t r1, double realPrecision, size_t * comp_size); TightDataPointStorageF* SZ_compress_float_1D_MDQ_MSST19(float *oriData, size_t dataLength, double realPrecision, float valueRangeSize, float medianValue_f); -void SZ_compress_args_float_withinRange(unsigned char** newByteData, float *oriData, size_t dataLength, size_t *outSize); - -/*int SZ_compress_args_float_wRngeNoGzip(unsigned char** newByteData, float *oriData, -size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize, -int errBoundMode, double absErr_Bound, double relBoundRatio, double pwrErrRatio);*/ - -int SZ_compress_args_float(int cmprType, int withRegression, unsigned char** newByteData, float *oriData, -size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize, -int errBoundMode, double absErr_Bound, double relBoundRatio, double pwRelBoundRatio); - -int SZ_compress_args_float_subblock(unsigned char* compressedBytes, float *oriData, -size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, -size_t s5, size_t s4, size_t s3, size_t s2, size_t s1, -size_t e5, size_t e4, size_t e3, size_t e2, size_t e1, -size_t *outSize, int errBoundMode, double absErr_Bound, double relBoundRatio); - -void SZ_compress_args_float_NoCkRnge_1D_subblock(unsigned char* compressedBytes, float *oriData, double realPrecision, size_t *outSize, float valueRangeSize, float medianValue_f, -size_t r1, size_t s1, size_t e1); +void SZ_compress_args_float_withinRange(unsigned char* newByteData, float *oriData, size_t dataLength, size_t *outSize); -unsigned int optimize_intervals_float_1D_subblock(float *oriData, double realPrecision, size_t r1, size_t s1, size_t e1); - -TightDataPointStorageF* SZ_compress_float_1D_MDQ_subblock(float *oriData, double realPrecision, float valueRangeSize, float medianValue_f, -size_t r1, size_t s1, size_t e1); +int SZ_compress_args_float(float *oriData, size_t r1, unsigned char* newByteData, size_t *outSize, sz_params* params); #ifdef __cplusplus } diff --git a/deps/SZ/sz/include/sz_float_pwr.h b/deps/SZ/sz/include/sz_float_pwr.h index ac7554145e..7fd75044ae 100644 --- a/deps/SZ/sz/include/sz_float_pwr.h +++ b/deps/SZ/sz/include/sz_float_pwr.h @@ -19,7 +19,7 @@ extern "C" { void compute_segment_precisions_float_1D(float *oriData, size_t dataLength, float* pwrErrBound, unsigned char* pwrErrBoundBytes, double globalPrecision); unsigned int optimize_intervals_float_1D_pwr(float *oriData, size_t dataLength, float* pwrErrBound); -void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr(unsigned char** newByteData, float *oriData, double globalPrecision, size_t dataLength, size_t *outSize, float min, float max); +void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr(unsigned char* newByteData, float *oriData, double globalPrecision, size_t dataLength, size_t *outSize, float min, float max); void createRangeGroups_float(float** posGroups, float** negGroups, int** posFlags, int** negFlags); @@ -28,11 +28,11 @@ int* generateGroupLowerBounds(); TightDataPointStorageF* SZ_compress_float_1D_MDQ_pwrGroup(float* oriData, size_t dataLength, int errBoundMode, double absErrBound, double relBoundRatio, double pwrErrRatio, float valueRangeSize, float medianValue_f); -void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwrgroup(unsigned char** newByteData, float *oriData, +void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwrgroup(unsigned char* newByteData, float *oriData, size_t dataLength, double absErrBound, double relBoundRatio, double pwrErrRatio, float valueRangeSize, float medianValue_f, size_t *outSize); -void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_pre_log(unsigned char** newByteData, float *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, float min, float max); -void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_pre_log_MSST19(unsigned char** newByteData, float *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, float valueRangeSize, float medianValue_f, +void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_pre_log(unsigned char* newByteData, float *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, float min, float max); +void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_pre_log_MSST19(unsigned char* newByteData, float *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, float valueRangeSize, float medianValue_f, unsigned char* signs, bool* positive, float min, float max, float nearZero); #ifdef __cplusplus diff --git a/deps/SZ/sz/include/sz_float_ts.h b/deps/SZ/sz/include/sz_float_ts.h deleted file mode 100644 index 9f2301da99..0000000000 --- a/deps/SZ/sz/include/sz_float_ts.h +++ /dev/null @@ -1,27 +0,0 @@ -/** - * @file sz_float_ts.h - * @author Sheng Di - * @date May, 2018 - * @brief Header file for the sz_float_ts.c. - * (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ -#include "TightDataPointStorageF.h" - -#ifndef _SZ_Float_TS_H -#define _SZ_Float_TS_H - -#ifdef __cplusplus -extern "C" { -#endif -unsigned int optimize_intervals_float_1D_ts(float *oriData, size_t dataLength, float* preData, double realPrecision); - -TightDataPointStorageF* SZ_compress_float_1D_MDQ_ts(float *oriData, size_t dataLength, sz_multisteps* multisteps, -double realPrecision, float valueRangeSize, float medianValue_f); - -#ifdef __cplusplus -} -#endif - -#endif /* ----- #ifndef _SZ_Float_TS_H ----- */ - diff --git a/deps/SZ/sz/include/sz_int16.h b/deps/SZ/sz/include/sz_int16.h deleted file mode 100644 index 0ad62c5b90..0000000000 --- a/deps/SZ/sz/include/sz_int16.h +++ /dev/null @@ -1,48 +0,0 @@ -/** - * @file sz_int16.h - * @author Sheng Di - * @date July, 2017 - * @brief Header file for the sz_int16.c. - * (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -#ifndef _SZ_Int16_H -#define _SZ_Int16_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include - -unsigned int optimize_intervals_int16_1D(int16_t *oriData, size_t dataLength, double realPrecision); -unsigned int optimize_intervals_int16_2D(int16_t *oriData, size_t r1, size_t r2, double realPrecision); -unsigned int optimize_intervals_int16_3D(int16_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision); -unsigned int optimize_intervals_int16_4D(int16_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision); -TightDataPointStorageI* SZ_compress_int16_1D_MDQ(int16_t *oriData, size_t dataLength, double realPrecision, int64_t valueRangeSize, int64_t minValue); -void SZ_compress_args_int16_StoreOriData(int16_t* oriData, size_t dataLength, TightDataPointStorageI* tdps, unsigned char** newByteData, size_t *outSize); -void SZ_compress_args_int16_NoCkRngeNoGzip_1D(unsigned char** newByteData, int16_t *oriData, -size_t dataLength, double realPrecision, size_t *outSize, int64_t valueRangeSize, int16_t minValue); -TightDataPointStorageI* SZ_compress_int16_2D_MDQ(int16_t *oriData, size_t r1, size_t r2, double realPrecision, int64_t valueRangeSize, int64_t minValue); -TightDataPointStorageI* SZ_compress_int16_3D_MDQ(int16_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, int64_t valueRangeSize, int64_t minValue); -void SZ_compress_args_int16_NoCkRngeNoGzip_3D(unsigned char** newByteData, int16_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, size_t *outSize, int64_t valueRangeSize, int64_t minValue); -TightDataPointStorageI* SZ_compress_int16_4D_MDQ(int16_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision, int64_t valueRangeSize, int64_t minValue); -void SZ_compress_args_int16_NoCkRngeNoGzip_4D(unsigned char** newByteData, int16_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision, -size_t *outSize, int64_t valueRangeSize, int64_t minValue); -void SZ_compress_args_int16_withinRange(unsigned char** newByteData, int16_t *oriData, size_t dataLength, size_t *outSize); - -int SZ_compress_args_int16_wRngeNoGzip(unsigned char** newByteData, int16_t *oriData, -size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize, -int errBoundMode, double absErr_Bound, double relBoundRatio); - -int SZ_compress_args_int16(unsigned char** newByteData, int16_t *oriData, -size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize, -int errBoundMode, double absErr_Bound, double relBoundRatio); - -#ifdef __cplusplus -} -#endif - -#endif /* ----- #ifndef _SZ_Int16_H ----- */ - diff --git a/deps/SZ/sz/include/sz_int32.h b/deps/SZ/sz/include/sz_int32.h deleted file mode 100644 index a87825d0fc..0000000000 --- a/deps/SZ/sz/include/sz_int32.h +++ /dev/null @@ -1,48 +0,0 @@ -/** - * @file sz_int32.h - * @author Sheng Di - * @date July, 2017 - * @brief Header file for the sz_int32.c. - * (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -#ifndef _SZ_Int32_H -#define _SZ_Int32_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include - -unsigned int optimize_intervals_int32_1D(int32_t *oriData, size_t dataLength, double realPrecision); -unsigned int optimize_intervals_int32_2D(int32_t *oriData, size_t r1, size_t r2, double realPrecision); -unsigned int optimize_intervals_int32_3D(int32_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision); -unsigned int optimize_intervals_int32_4D(int32_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision); -TightDataPointStorageI* SZ_compress_int32_1D_MDQ(int32_t *oriData, size_t dataLength, double realPrecision, int64_t valueRangeSize, int64_t minValue); -void SZ_compress_args_int32_StoreOriData(int32_t* oriData, size_t dataLength, TightDataPointStorageI* tdps, unsigned char** newByteData, size_t *outSize); -void SZ_compress_args_int32_NoCkRngeNoGzip_1D(unsigned char** newByteData, int32_t *oriData, -size_t dataLength, double realPrecision, size_t *outSize, int64_t valueRangeSize, int32_t minValue); -TightDataPointStorageI* SZ_compress_int32_2D_MDQ(int32_t *oriData, size_t r1, size_t r2, double realPrecision, int64_t valueRangeSize, int64_t minValue); -TightDataPointStorageI* SZ_compress_int32_3D_MDQ(int32_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, int64_t valueRangeSize, int64_t minValue); -void SZ_compress_args_int32_NoCkRngeNoGzip_3D(unsigned char** newByteData, int32_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, size_t *outSize, int64_t valueRangeSize, int64_t minValue); -TightDataPointStorageI* SZ_compress_int32_4D_MDQ(int32_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision, int64_t valueRangeSize, int64_t minValue); -void SZ_compress_args_int32_NoCkRngeNoGzip_4D(unsigned char** newByteData, int32_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision, -size_t *outSize, int64_t valueRangeSize, int64_t minValue); -void SZ_compress_args_int32_withinRange(unsigned char** newByteData, int32_t *oriData, size_t dataLength, size_t *outSize); - -int SZ_compress_args_int32_wRngeNoGzip(unsigned char** newByteData, int32_t *oriData, -size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize, -int errBoundMode, double absErr_Bound, double relBoundRatio); - -int SZ_compress_args_int32(unsigned char** newByteData, int32_t *oriData, -size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize, -int errBoundMode, double absErr_Bound, double relBoundRatio); - -#ifdef __cplusplus -} -#endif - -#endif /* ----- #ifndef _SZ_Int32_H ----- */ - diff --git a/deps/SZ/sz/include/sz_int64.h b/deps/SZ/sz/include/sz_int64.h deleted file mode 100644 index b7213b2315..0000000000 --- a/deps/SZ/sz/include/sz_int64.h +++ /dev/null @@ -1,48 +0,0 @@ -/** - * @file sz_int64.h - * @author Sheng Di - * @date July, 2017 - * @brief Header file for the sz_int64.c. - * (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -#ifndef _SZ_Int64_H -#define _SZ_Int64_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include - -unsigned int optimize_intervals_int64_1D(int64_t *oriData, size_t dataLength, double realPrecision); -unsigned int optimize_intervals_int64_2D(int64_t *oriData, size_t r1, size_t r2, double realPrecision); -unsigned int optimize_intervals_int64_3D(int64_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision); -unsigned int optimize_intervals_int64_4D(int64_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision); -TightDataPointStorageI* SZ_compress_int64_1D_MDQ(int64_t *oriData, size_t dataLength, double realPrecision, int64_t valueRangeSize, int64_t minValue); -void SZ_compress_args_int64_StoreOriData(int64_t* oriData, size_t dataLength, TightDataPointStorageI* tdps, unsigned char** newByteData, size_t *outSize); -void SZ_compress_args_int64_NoCkRngeNoGzip_1D(unsigned char** newByteData, int64_t *oriData, -size_t dataLength, double realPrecision, size_t *outSize, int64_t valueRangeSize, int64_t minValue); -TightDataPointStorageI* SZ_compress_int64_2D_MDQ(int64_t *oriData, size_t r1, size_t r2, double realPrecision, int64_t valueRangeSize, int64_t minValue); -TightDataPointStorageI* SZ_compress_int64_3D_MDQ(int64_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, int64_t valueRangeSize, int64_t minValue); -void SZ_compress_args_int64_NoCkRngeNoGzip_3D(unsigned char** newByteData, int64_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, size_t *outSize, int64_t valueRangeSize, int64_t minValue); -TightDataPointStorageI* SZ_compress_int64_4D_MDQ(int64_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision, int64_t valueRangeSize, int64_t minValue); -void SZ_compress_args_int64_NoCkRngeNoGzip_4D(unsigned char** newByteData, int64_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision, -size_t *outSize, int64_t valueRangeSize, int64_t minValue); -void SZ_compress_args_int64_withinRange(unsigned char** newByteData, int64_t *oriData, size_t dataLength, size_t *outSize); - -int SZ_compress_args_int64_wRngeNoGzip(unsigned char** newByteData, int64_t *oriData, -size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize, -int errBoundMode, double absErr_Bound, double relBoundRatio); - -int SZ_compress_args_int64(unsigned char** newByteData, int64_t *oriData, -size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize, -int errBoundMode, double absErr_Bound, double relBoundRatio); - -#ifdef __cplusplus -} -#endif - -#endif /* ----- #ifndef _SZ_Int64_H ----- */ - diff --git a/deps/SZ/sz/include/sz_int8.h b/deps/SZ/sz/include/sz_int8.h deleted file mode 100644 index c6ce758a79..0000000000 --- a/deps/SZ/sz/include/sz_int8.h +++ /dev/null @@ -1,48 +0,0 @@ -/** - * @file sz_int8.h - * @author Sheng Di - * @date July, 2017 - * @brief Header file for the sz_int8.c. - * (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -#ifndef _SZ_Int8_H -#define _SZ_Int8_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include - -unsigned int optimize_intervals_int8_1D(int8_t *oriData, size_t dataLength, double realPrecision); -unsigned int optimize_intervals_int8_2D(int8_t *oriData, size_t r1, size_t r2, double realPrecision); -unsigned int optimize_intervals_int8_3D(int8_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision); -unsigned int optimize_intervals_int8_4D(int8_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision); -TightDataPointStorageI* SZ_compress_int8_1D_MDQ(int8_t *oriData, size_t dataLength, double realPrecision, int64_t valueRangeSize, int64_t minValue); -void SZ_compress_args_int8_StoreOriData(int8_t* oriData, size_t dataLength, TightDataPointStorageI* tdps, unsigned char** newByteData, size_t *outSize); -void SZ_compress_args_int8_NoCkRngeNoGzip_1D(unsigned char** newByteData, int8_t *oriData, -size_t dataLength, double realPrecision, size_t *outSize, int64_t valueRangeSize, int8_t minValue); -TightDataPointStorageI* SZ_compress_int8_2D_MDQ(int8_t *oriData, size_t r1, size_t r2, double realPrecision, int64_t valueRangeSize, int64_t minValue); -TightDataPointStorageI* SZ_compress_int8_3D_MDQ(int8_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, int64_t valueRangeSize, int64_t minValue); -void SZ_compress_args_int8_NoCkRngeNoGzip_3D(unsigned char** newByteData, int8_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, size_t *outSize, int64_t valueRangeSize, int64_t minValue); -TightDataPointStorageI* SZ_compress_int8_4D_MDQ(int8_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision, int64_t valueRangeSize, int64_t minValue); -void SZ_compress_args_int8_NoCkRngeNoGzip_4D(unsigned char** newByteData, int8_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision, -size_t *outSize, int64_t valueRangeSize, int64_t minValue); -void SZ_compress_args_int8_withinRange(unsigned char** newByteData, int8_t *oriData, size_t dataLength, size_t *outSize); - -int SZ_compress_args_int8_wRngeNoGzip(unsigned char** newByteData, int8_t *oriData, -size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize, -int errBoundMode, double absErr_Bound, double relBoundRatio); - -int SZ_compress_args_int8(unsigned char** newByteData, int8_t *oriData, -size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize, -int errBoundMode, double absErr_Bound, double relBoundRatio); - -#ifdef __cplusplus -} -#endif - -#endif /* ----- #ifndef _SZ_Int8_H ----- */ - diff --git a/deps/SZ/sz/include/sz_omp.h b/deps/SZ/sz/include/sz_omp.h deleted file mode 100644 index cb83acbe9a..0000000000 --- a/deps/SZ/sz/include/sz_omp.h +++ /dev/null @@ -1,47 +0,0 @@ -/** - * @file sz_omp.h - * @author Xin Liang - * @date July, 2017 - * @brief Header file for the sz_omp.c. - * (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -#include -#include -#ifdef _OPENMP -#include "omp.h" -#endif -#include "sz.h" - -#ifndef _SZ_OMP_H -#define _SZ_OMP_H - -#ifdef __cplusplus -extern "C" { -#endif - -unsigned char * SZ_compress_float_1D_MDQ_openmp(float *oriData, size_t r1, double realPrecision, size_t * comp_size); -unsigned char * SZ_compress_float_2D_MDQ_openmp(float *oriData, size_t r1, size_t r2, double realPrecision, size_t * comp_size); -unsigned char * SZ_compress_float_3D_MDQ_openmp(float *oriData, size_t r1, size_t r2, size_t r3, float realPrecision, size_t * comp_size); - -void decompressDataSeries_float_1D_openmp(float** data, size_t r1, unsigned char* comp_data); -void decompressDataSeries_float_3D_openmp(float** data, size_t r1, size_t r2, size_t r3, unsigned char* comp_data); -void decompressDataSeries_float_2D_openmp(float** data, size_t r1, size_t r2, unsigned char* comp_data); - -unsigned char * SZ_compress_double_1D_MDQ_openmp(double *oriData, size_t r1, double realPrecision, size_t * comp_size); -unsigned char * SZ_compress_double_2D_MDQ_openmp(double *oriData, size_t r1, size_t r2, double realPrecision, size_t * comp_size); -unsigned char * SZ_compress_double_3D_MDQ_openmp(double *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, size_t * comp_size); - -void decompressDataSeries_double_1D_openmp(double** data, size_t r1, unsigned char* comp_data); -void decompressDataSeries_double_2D_openmp(double** data, size_t r1, size_t r2, unsigned char* comp_data); -void decompressDataSeries_double_3D_openmp(double** data, size_t r1, size_t r2, size_t r3, unsigned char* comp_data); - -//void Huffman_init_openmp(HuffmanTree* huffmanTree, int *s, size_t length, int thread_num); -void Huffman_init_openmp(HuffmanTree* huffmanTree, int *s, size_t length, int thread_num, size_t * freq); - -#ifdef __cplusplus -} -#endif - -#endif /* ----- #ifndef _SZ_OMP_H ----- */ diff --git a/deps/SZ/sz/include/sz_opencl.h b/deps/SZ/sz/include/sz_opencl.h index 693256161c..b4ee35bb82 100644 --- a/deps/SZ/sz/include/sz_opencl.h +++ b/deps/SZ/sz/include/sz_opencl.h @@ -20,7 +20,7 @@ extern "C" { * sz_opencl_* function. * * \param[out] state the sz opencl state - * \return SZ_SCES for success or SZ_NCES on error + * \return SZ_SUCCESS for success or SZ_NCES on error */ int sz_opencl_init(struct sz_opencl_state** state); @@ -28,7 +28,7 @@ extern "C" { * deinitializes an opencl state * * \param[in] state the sz opencl state - * \return SZ_SCES + * \return SZ_SUCCESS */ int sz_opencl_release(struct sz_opencl_state** state); @@ -53,7 +53,7 @@ extern "C" { * confirms that the sz opencl state is ready to use by performing a vector addition * * \param[in] state the sz opencl state - * \return SZ_SCES if the opencl implementation is functioning + * \return SZ_SUCCESS if the opencl implementation is functioning */ int sz_opencl_check(struct sz_opencl_state*); diff --git a/deps/SZ/sz/include/sz_uint16.h b/deps/SZ/sz/include/sz_uint16.h deleted file mode 100644 index eb2319772f..0000000000 --- a/deps/SZ/sz/include/sz_uint16.h +++ /dev/null @@ -1,48 +0,0 @@ -/** - * @file sz_uint16.h - * @author Sheng Di - * @date Nov, 2017 - * @brief Header file for the sz_uint16.c. - * (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -#ifndef _SZ_UInt16_H -#define _SZ_UInt16_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include - -unsigned int optimize_intervals_uint16_1D(uint16_t *oriData, size_t dataLength, double realPrecision); -unsigned int optimize_intervals_uint16_2D(uint16_t *oriData, size_t r1, size_t r2, double realPrecision); -unsigned int optimize_intervals_uint16_3D(uint16_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision); -unsigned int optimize_intervals_uint16_4D(uint16_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision); -TightDataPointStorageI* SZ_compress_uint16_1D_MDQ(uint16_t *oriData, size_t dataLength, double realPrecision, int64_t valueRangeSize, int64_t minValue); -void SZ_compress_args_uint16_StoreOriData(uint16_t* oriData, size_t dataLength, TightDataPointStorageI* tdps, unsigned char** newByteData, size_t *outSize); -void SZ_compress_args_uint16_NoCkRngeNoGzip_1D(unsigned char** newByteData, uint16_t *oriData, -size_t dataLength, double realPrecision, size_t *outSize, int64_t valueRangeSize, uint16_t minValue); -TightDataPointStorageI* SZ_compress_uint16_2D_MDQ(uint16_t *oriData, size_t r1, size_t r2, double realPrecision, int64_t valueRangeSize, int64_t minValue); -TightDataPointStorageI* SZ_compress_uint16_3D_MDQ(uint16_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, int64_t valueRangeSize, int64_t minValue); -void SZ_compress_args_uint16_NoCkRngeNoGzip_3D(unsigned char** newByteData, uint16_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, size_t *outSize, int64_t valueRangeSize, int64_t minValue); -TightDataPointStorageI* SZ_compress_uint16_4D_MDQ(uint16_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision, int64_t valueRangeSize, int64_t minValue); -void SZ_compress_args_uint16_NoCkRngeNoGzip_4D(unsigned char** newByteData, uint16_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision, -size_t *outSize, int64_t valueRangeSize, int64_t minValue); -void SZ_compress_args_uint16_withinRange(unsigned char** newByteData, uint16_t *oriData, size_t dataLength, size_t *outSize); - -int SZ_compress_args_uint16_wRngeNoGzip(unsigned char** newByteData, uint16_t *oriData, -size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize, -int errBoundMode, double absErr_Bound, double relBoundRatio); - -int SZ_compress_args_uint16(unsigned char** newByteData, uint16_t *oriData, -size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize, -int errBoundMode, double absErr_Bound, double relBoundRatio); - -#ifdef __cplusplus -} -#endif - -#endif /* ----- #ifndef _SZ_UInt16_H ----- */ - diff --git a/deps/SZ/sz/include/sz_uint32.h b/deps/SZ/sz/include/sz_uint32.h deleted file mode 100644 index 8adb31d3fc..0000000000 --- a/deps/SZ/sz/include/sz_uint32.h +++ /dev/null @@ -1,48 +0,0 @@ -/** - * @file sz_uint32.h - * @author Sheng Di - * @date July, 2017 - * @brief Header file for the sz_uint32.c. - * (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -#ifndef _SZ_UInt32_H -#define _SZ_UInt32_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include - -unsigned int optimize_intervals_uint32_1D(uint32_t *oriData, size_t dataLength, double realPrecision); -unsigned int optimize_intervals_uint32_2D(uint32_t *oriData, size_t r1, size_t r2, double realPrecision); -unsigned int optimize_intervals_uint32_3D(uint32_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision); -unsigned int optimize_intervals_uint32_4D(uint32_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision); -TightDataPointStorageI* SZ_compress_uint32_1D_MDQ(uint32_t *oriData, size_t dataLength, double realPrecision, int64_t valueRangeSize, int64_t minValue); -void SZ_compress_args_uint32_StoreOriData(uint32_t* oriData, size_t dataLength, TightDataPointStorageI* tdps, unsigned char** newByteData, size_t *outSize); -void SZ_compress_args_uint32_NoCkRngeNoGzip_1D(unsigned char** newByteData, uint32_t *oriData, -size_t dataLength, double realPrecision, size_t *outSize, int64_t valueRangeSize, uint32_t minValue); -TightDataPointStorageI* SZ_compress_uint32_2D_MDQ(uint32_t *oriData, size_t r1, size_t r2, double realPrecision, int64_t valueRangeSize, int64_t minValue); -TightDataPointStorageI* SZ_compress_uint32_3D_MDQ(uint32_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, int64_t valueRangeSize, int64_t minValue); -void SZ_compress_args_uint32_NoCkRngeNoGzip_3D(unsigned char** newByteData, uint32_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, size_t *outSize, int64_t valueRangeSize, int64_t minValue); -TightDataPointStorageI* SZ_compress_uint32_4D_MDQ(uint32_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision, int64_t valueRangeSize, int64_t minValue); -void SZ_compress_args_uint32_NoCkRngeNoGzip_4D(unsigned char** newByteData, uint32_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision, -size_t *outSize, int64_t valueRangeSize, int64_t minValue); -void SZ_compress_args_uint32_withinRange(unsigned char** newByteData, uint32_t *oriData, size_t dataLength, size_t *outSize); - -int SZ_compress_args_uint32_wRngeNoGzip(unsigned char** newByteData, uint32_t *oriData, -size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize, -int errBoundMode, double absErr_Bound, double relBoundRatio); - -int SZ_compress_args_uint32(unsigned char** newByteData, uint32_t *oriData, -size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize, -int errBoundMode, double absErr_Bound, double relBoundRatio); - -#ifdef __cplusplus -} -#endif - -#endif /* ----- #ifndef _SZ_UInt32_H ----- */ - diff --git a/deps/SZ/sz/include/sz_uint64.h b/deps/SZ/sz/include/sz_uint64.h deleted file mode 100644 index 7717aa2d5f..0000000000 --- a/deps/SZ/sz/include/sz_uint64.h +++ /dev/null @@ -1,48 +0,0 @@ -/** - * @file sz_uint64.h - * @author Sheng Di - * @date July, 2017 - * @brief Header file for the sz_uint64.c. - * (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -#ifndef _SZ_UInt64_H -#define _SZ_UInt64_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include - -unsigned int optimize_intervals_uint64_1D(uint64_t *oriData, size_t dataLength, double realPrecision); -unsigned int optimize_intervals_uint64_2D(uint64_t *oriData, size_t r1, size_t r2, double realPrecision); -unsigned int optimize_intervals_uint64_3D(uint64_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision); -unsigned int optimize_intervals_uint64_4D(uint64_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision); -TightDataPointStorageI* SZ_compress_uint64_1D_MDQ(uint64_t *oriData, size_t dataLength, double realPrecision, uint64_t valueRangeSize, uint64_t minValue); -void SZ_compress_args_uint64_StoreOriData(uint64_t* oriData, size_t dataLength, TightDataPointStorageI* tdps, unsigned char** newByteData, size_t *outSize); -void SZ_compress_args_uint64_NoCkRngeNoGzip_1D(unsigned char** newByteData, uint64_t *oriData, -size_t dataLength, double realPrecision, size_t *outSize, uint64_t valueRangeSize, uint64_t minValue); -TightDataPointStorageI* SZ_compress_uint64_2D_MDQ(uint64_t *oriData, size_t r1, size_t r2, double realPrecision, uint64_t valueRangeSize, uint64_t minValue); -TightDataPointStorageI* SZ_compress_uint64_3D_MDQ(uint64_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, uint64_t valueRangeSize, uint64_t minValue); -void SZ_compress_args_uint64_NoCkRngeNoGzip_3D(unsigned char** newByteData, uint64_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, size_t *outSize, uint64_t valueRangeSize, uint64_t minValue); -TightDataPointStorageI* SZ_compress_uint64_4D_MDQ(uint64_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision, uint64_t valueRangeSize, uint64_t minValue); -void SZ_compress_args_uint64_NoCkRngeNoGzip_4D(unsigned char** newByteData, uint64_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision, -size_t *outSize, uint64_t valueRangeSize, uint64_t minValue); -void SZ_compress_args_uint64_withinRange(unsigned char** newByteData, uint64_t *oriData, size_t dataLength, size_t *outSize); - -int SZ_compress_args_uint64_wRngeNoGzip(unsigned char** newByteData, uint64_t *oriData, -size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize, -int errBoundMode, double absErr_Bound, double relBoundRatio); - -int SZ_compress_args_uint64(unsigned char** newByteData, uint64_t *oriData, -size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize, -int errBoundMode, double absErr_Bound, double relBoundRatio); - -#ifdef __cplusplus -} -#endif - -#endif /* ----- #ifndef _SZ_UInt64_H ----- */ - diff --git a/deps/SZ/sz/include/sz_uint8.h b/deps/SZ/sz/include/sz_uint8.h deleted file mode 100644 index 9de3a117b5..0000000000 --- a/deps/SZ/sz/include/sz_uint8.h +++ /dev/null @@ -1,48 +0,0 @@ -/** - * @file sz_uint8.h - * @author Sheng Di - * @date July, 2017 - * @brief Header file for the sz_uint8.c. - * (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -#ifndef _SZ_UInt8_H -#define _SZ_UInt8_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include - -unsigned int optimize_intervals_uint8_1D(uint8_t *oriData, size_t dataLength, double realPrecision); -unsigned int optimize_intervals_uint8_2D(uint8_t *oriData, size_t r1, size_t r2, double realPrecision); -unsigned int optimize_intervals_uint8_3D(uint8_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision); -unsigned int optimize_intervals_uint8_4D(uint8_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision); -TightDataPointStorageI* SZ_compress_uint8_1D_MDQ(uint8_t *oriData, size_t dataLength, double realPrecision, int64_t valueRangeSize, int64_t minValue); -void SZ_compress_args_uint8_StoreOriData(uint8_t* oriData, size_t dataLength, TightDataPointStorageI* tdps, unsigned char** newByteData, size_t *outSize); -void SZ_compress_args_uint8_NoCkRngeNoGzip_1D(unsigned char** newByteData, uint8_t *oriData, -size_t dataLength, double realPrecision, size_t *outSize, int64_t valueRangeSize, uint8_t minValue); -TightDataPointStorageI* SZ_compress_uint8_2D_MDQ(uint8_t *oriData, size_t r1, size_t r2, double realPrecision, int64_t valueRangeSize, int64_t minValue); -TightDataPointStorageI* SZ_compress_uint8_3D_MDQ(uint8_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, int64_t valueRangeSize, int64_t minValue); -void SZ_compress_args_uint8_NoCkRngeNoGzip_3D(unsigned char** newByteData, uint8_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, size_t *outSize, int64_t valueRangeSize, int64_t minValue); -TightDataPointStorageI* SZ_compress_uint8_4D_MDQ(uint8_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision, int64_t valueRangeSize, int64_t minValue); -void SZ_compress_args_uint8_NoCkRngeNoGzip_4D(unsigned char** newByteData, uint8_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision, -size_t *outSize, int64_t valueRangeSize, int64_t minValue); -void SZ_compress_args_uint8_withinRange(unsigned char** newByteData, uint8_t *oriData, size_t dataLength, size_t *outSize); - -int SZ_compress_args_uint8_wRngeNoGzip(unsigned char** newByteData, uint8_t *oriData, -size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize, -int errBoundMode, double absErr_Bound, double relBoundRatio); - -int SZ_compress_args_uint8(unsigned char** newByteData, uint8_t *oriData, -size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize, -int errBoundMode, double absErr_Bound, double relBoundRatio); - -#ifdef __cplusplus -} -#endif - -#endif /* ----- #ifndef _SZ_UInt8_H ----- */ - diff --git a/deps/SZ/sz/include/szd_double.h b/deps/SZ/sz/include/szd_double.h index 3219110f93..a314e0b3fa 100644 --- a/deps/SZ/sz/include/szd_double.h +++ b/deps/SZ/sz/include/szd_double.h @@ -16,13 +16,13 @@ extern "C" { #include "TightDataPointStorageD.h" -void decompressDataSeries_double_1D(double** data, size_t dataSeriesLength, double* hist_data, TightDataPointStorageD* tdps); +void decompressDataSeries_double_1D(double* data, size_t dataSeriesLength, double* hist_data, TightDataPointStorageD* tdps); -void decompressDataSeries_double_1D_MSST19(double** data, size_t dataSeriesLength, TightDataPointStorageD* tdps); +void decompressDataSeries_double_1D_MSST19(double* data, size_t dataSeriesLength, TightDataPointStorageD* tdps); -void getSnapshotData_double_1D(double** data, size_t dataSeriesLength, TightDataPointStorageD* tdps, int errBoundMode, int compressionType, double* hist_data, sz_params* pde_params); +void getSnapshotData_double_1D(double* data, size_t dataSeriesLength, TightDataPointStorageD* tdps, int errBoundMode, int compressionType, double* hist_data, sz_params* pde_params); -int SZ_decompress_args_double(double** newData, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, unsigned char* cmpBytes, size_t cmpSize, int compressionType, double* hist_data, sz_exedata* pde_exe, sz_params* pde_params); +int SZ_decompress_args_double(double* newData, size_t r1, unsigned char* cmpBytes, size_t cmpSize, int compressionType, double* hist_data, sz_exedata* pde_exe, sz_params* pde_params); #ifdef __cplusplus } diff --git a/deps/SZ/sz/include/szd_float.h b/deps/SZ/sz/include/szd_float.h index 279588cd6a..224853d081 100644 --- a/deps/SZ/sz/include/szd_float.h +++ b/deps/SZ/sz/include/szd_float.h @@ -16,24 +16,16 @@ extern "C" { #include "TightDataPointStorageF.h" -void decompressDataSeries_float_1D(float** data, size_t dataSeriesLength, float* hist_data, TightDataPointStorageF* tdps); +void decompressDataSeries_float_1D(float* data, size_t dataSeriesLength, float* hist_data, TightDataPointStorageF* tdps); -void decompressDataSeries_float_1D_MSST19(float** data, size_t dataSeriesLength, TightDataPointStorageF* tdps); +void decompressDataSeries_float_1D_MSST19(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps); -void getSnapshotData_float_1D(float** data, size_t dataSeriesLength, TightDataPointStorageF* tdps, int errBoundMode, int compressionType, float* hist_data, sz_params* pde_params); +void getSnapshotData_float_1D(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps, int errBoundMode, int compressionType, float* hist_data, sz_params* pde_params); -size_t decompressDataSeries_float_1D_RA_block(float * data, float mean, size_t dim_0, size_t block_dim_0, double realPrecision, int * type, float * unpredictable_data); - -int SZ_decompress_args_float(float** newData, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, unsigned char* cmpBytes, size_t cmpSize, int compressionType, float* hist_data, sz_exedata* pde_exe, sz_params* pde_params); +int SZ_decompress_args_float(float* newData, size_t r1, unsigned char* cmpBytes, size_t cmpSize, int compressionType, float* hist_data, sz_exedata* pde_exe, sz_params* pde_params); -void decompressDataSeries_float_1D_decompression_given_areas_with_blocked_regression(float** data, size_t r1, size_t s1, size_t e1, unsigned char* comp_data); -int SZ_decompress_args_randomaccess_float(float** newData, -size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, -size_t s5, size_t s4, size_t s3, size_t s2, size_t s1, // start point -size_t e5, size_t e4, size_t e3, size_t e2, size_t e1, // end point -unsigned char* cmpBytes, size_t cmpSize); #ifdef __cplusplus } diff --git a/deps/SZ/sz/include/szd_float_pwr.h b/deps/SZ/sz/include/szd_float_pwr.h index 5d5848605f..769686dc17 100644 --- a/deps/SZ/sz/include/szd_float_pwr.h +++ b/deps/SZ/sz/include/szd_float_pwr.h @@ -14,13 +14,13 @@ extern "C" { #endif -void decompressDataSeries_float_1D_pwr(float** data, size_t dataSeriesLength, TightDataPointStorageF* tdps); +void decompressDataSeries_float_1D_pwr(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps); char* decompressGroupIDArray(unsigned char* bytes, size_t dataLength); -void decompressDataSeries_float_1D_pwrgroup(float** data, size_t dataSeriesLength, TightDataPointStorageF* tdps); -void decompressDataSeries_float_1D_pwr_pre_log(float** data, size_t dataSeriesLength, TightDataPointStorageF* tdps); +void decompressDataSeries_float_1D_pwrgroup(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps); +void decompressDataSeries_float_1D_pwr_pre_log(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps); -void decompressDataSeries_float_1D_pwr_pre_log_MSST19(float** data, size_t dataSeriesLength, TightDataPointStorageF* tdps); +void decompressDataSeries_float_1D_pwr_pre_log_MSST19(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps); #ifdef __cplusplus } diff --git a/deps/SZ/sz/include/szd_float_ts.h b/deps/SZ/sz/include/szd_float_ts.h deleted file mode 100644 index 88ea07f4a8..0000000000 --- a/deps/SZ/sz/include/szd_float_ts.h +++ /dev/null @@ -1,25 +0,0 @@ -/** - * @file szd_float_ts.h - * @author Sheng Di - * @date July, 2017 - * @brief Header file for the szd_float_ts.c. - * (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -#ifndef _SZD_Float_TS_H -#define _SZD_Float_TS_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "TightDataPointStorageF.h" - -void decompressDataSeries_float_1D_ts(float** data, size_t dataSeriesLength, float* hist_data, TightDataPointStorageF* tdps); - -#ifdef __cplusplus -} -#endif - -#endif /* ----- #ifndef _SZD_Float_TS_H ----- */ diff --git a/deps/SZ/sz/include/szd_int16.h b/deps/SZ/sz/include/szd_int16.h deleted file mode 100644 index a55a3d0487..0000000000 --- a/deps/SZ/sz/include/szd_int16.h +++ /dev/null @@ -1,38 +0,0 @@ -/** - * @file szd_int16.h - * @author Sheng Di - * @date July, 2017 - * @brief Header file for the szd_int16.c. - * (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -#ifndef _SZD_Int16_H -#define _SZD_Int16_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "TightDataPointStorageI.h" - -#define SZ_INT16_MIN -32768 -#define SZ_INT16_MAX 32767 - -void decompressDataSeries_int16_1D(int16_t** data, size_t dataSeriesLength, TightDataPointStorageI* tdps); -void decompressDataSeries_int16_2D(int16_t** data, size_t r1, size_t r2, TightDataPointStorageI* tdps); -void decompressDataSeries_int16_3D(int16_t** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageI* tdps); -void decompressDataSeries_int16_4D(int16_t** data, size_t r1, size_t r2, size_t r3, size_t r4, TightDataPointStorageI* tdps); - -void getSnapshotData_int16_1D(int16_t** data, size_t dataSeriesLength, TightDataPointStorageI* tdps, int errBoundMode); -void getSnapshotData_int16_2D(int16_t** data, size_t r1, size_t r2, TightDataPointStorageI* tdps, int errBoundMode); -void getSnapshotData_int16_3D(int16_t** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageI* tdps, int errBoundMode); -void getSnapshotData_int16_4D(int16_t** data, size_t r1, size_t r2, size_t r3, size_t r4, TightDataPointStorageI* tdps, int errBoundMode); - -int SZ_decompress_args_int16(int16_t** newData, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, unsigned char* cmpBytes, size_t cmpSize); - -#ifdef __cplusplus -} -#endif - -#endif /* ----- #ifndef _SZD_Int16_H ----- */ diff --git a/deps/SZ/sz/include/szd_int32.h b/deps/SZ/sz/include/szd_int32.h deleted file mode 100644 index 233901f54e..0000000000 --- a/deps/SZ/sz/include/szd_int32.h +++ /dev/null @@ -1,38 +0,0 @@ -/** - * @file szd_int32.h - * @author Sheng Di - * @date July, 2017 - * @brief Header file for the szd_int32.c. - * (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -#ifndef _SZD_Int32_H -#define _SZD_Int32_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "TightDataPointStorageI.h" - -#define SZ_INT32_MIN -2147483648 -#define SZ_INT32_MAX 2147483647 - -void decompressDataSeries_int32_1D(int32_t** data, size_t dataSeriesLength, TightDataPointStorageI* tdps); -void decompressDataSeries_int32_2D(int32_t** data, size_t r1, size_t r2, TightDataPointStorageI* tdps); -void decompressDataSeries_int32_3D(int32_t** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageI* tdps); -void decompressDataSeries_int32_4D(int32_t** data, size_t r1, size_t r2, size_t r3, size_t r4, TightDataPointStorageI* tdps); - -void getSnapshotData_int32_1D(int32_t** data, size_t dataSeriesLength, TightDataPointStorageI* tdps, int errBoundMode); -void getSnapshotData_int32_2D(int32_t** data, size_t r1, size_t r2, TightDataPointStorageI* tdps, int errBoundMode); -void getSnapshotData_int32_3D(int32_t** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageI* tdps, int errBoundMode); -void getSnapshotData_int32_4D(int32_t** data, size_t r1, size_t r2, size_t r3, size_t r4, TightDataPointStorageI* tdps, int errBoundMode); - -int SZ_decompress_args_int32(int32_t** newData, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, unsigned char* cmpBytes, size_t cmpSize); - -#ifdef __cplusplus -} -#endif - -#endif /* ----- #ifndef _SZD_Int32_H ----- */ diff --git a/deps/SZ/sz/include/szd_int64.h b/deps/SZ/sz/include/szd_int64.h deleted file mode 100644 index 5dcb97ac9b..0000000000 --- a/deps/SZ/sz/include/szd_int64.h +++ /dev/null @@ -1,35 +0,0 @@ -/** - * @file szd_int64.h - * @author Sheng Di - * @date July, 2017 - * @brief Header file for the szd_int64.c. - * (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -#ifndef _SZD_Int64_H -#define _SZD_Int64_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "TightDataPointStorageI.h" - -void decompressDataSeries_int64_1D(int64_t** data, size_t dataSeriesLength, TightDataPointStorageI* tdps); -void decompressDataSeries_int64_2D(int64_t** data, size_t r1, size_t r2, TightDataPointStorageI* tdps); -void decompressDataSeries_int64_3D(int64_t** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageI* tdps); -void decompressDataSeries_int64_4D(int64_t** data, size_t r1, size_t r2, size_t r3, size_t r4, TightDataPointStorageI* tdps); - -void getSnapshotData_int64_1D(int64_t** data, size_t dataSeriesLength, TightDataPointStorageI* tdps, int errBoundMode); -void getSnapshotData_int64_2D(int64_t** data, size_t r1, size_t r2, TightDataPointStorageI* tdps, int errBoundMode); -void getSnapshotData_int64_3D(int64_t** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageI* tdps, int errBoundMode); -void getSnapshotData_int64_4D(int64_t** data, size_t r1, size_t r2, size_t r3, size_t r4, TightDataPointStorageI* tdps, int errBoundMode); - -int SZ_decompress_args_int64(int64_t** newData, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, unsigned char* cmpBytes, size_t cmpSize); - -#ifdef __cplusplus -} -#endif - -#endif /* ----- #ifndef _SZD_Int64_H ----- */ diff --git a/deps/SZ/sz/include/szd_int8.h b/deps/SZ/sz/include/szd_int8.h deleted file mode 100644 index c6186f866d..0000000000 --- a/deps/SZ/sz/include/szd_int8.h +++ /dev/null @@ -1,38 +0,0 @@ -/** - * @file szd_int8.h - * @author Sheng Di - * @date July, 2017 - * @brief Header file for the szd_int8.c. - * (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -#ifndef _SZD_Int8_H -#define _SZD_Int8_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "TightDataPointStorageI.h" - -#define SZ_INT8_MIN -128 -#define SZ_INT8_MAX 127 - -void decompressDataSeries_int8_1D(int8_t** data, size_t dataSeriesLength, TightDataPointStorageI* tdps); -void decompressDataSeries_int8_2D(int8_t** data, size_t r1, size_t r2, TightDataPointStorageI* tdps); -void decompressDataSeries_int8_3D(int8_t** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageI* tdps); -void decompressDataSeries_int8_4D(int8_t** data, size_t r1, size_t r2, size_t r3, size_t r4, TightDataPointStorageI* tdps); - -void getSnapshotData_int8_1D(int8_t** data, size_t dataSeriesLength, TightDataPointStorageI* tdps, int errBoundMode); -void getSnapshotData_int8_2D(int8_t** data, size_t r1, size_t r2, TightDataPointStorageI* tdps, int errBoundMode); -void getSnapshotData_int8_3D(int8_t** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageI* tdps, int errBoundMode); -void getSnapshotData_int8_4D(int8_t** data, size_t r1, size_t r2, size_t r3, size_t r4, TightDataPointStorageI* tdps, int errBoundMode); - -int SZ_decompress_args_int8(int8_t** newData, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, unsigned char* cmpBytes, size_t cmpSize); - -#ifdef __cplusplus -} -#endif - -#endif /* ----- #ifndef _SZD_Int8_H ----- */ diff --git a/deps/SZ/sz/include/szd_uint16.h b/deps/SZ/sz/include/szd_uint16.h deleted file mode 100644 index dcd3ed8307..0000000000 --- a/deps/SZ/sz/include/szd_uint16.h +++ /dev/null @@ -1,38 +0,0 @@ -/** - * @file szd_uint16.h - * @author Sheng Di - * @date July, 2017 - * @brief Header file for the szd_uint16.c. - * (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -#ifndef _SZD_UInt16_H -#define _SZD_UInt16_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "TightDataPointStorageI.h" - -#define SZ_UINT16_MIN 0 -#define SZ_UINT16_MAX 65535 - -void decompressDataSeries_uint16_1D(uint16_t** data, size_t dataSeriesLength, TightDataPointStorageI* tdps); -void decompressDataSeries_uint16_2D(uint16_t** data, size_t r1, size_t r2, TightDataPointStorageI* tdps); -void decompressDataSeries_uint16_3D(uint16_t** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageI* tdps); -void decompressDataSeries_uint16_4D(uint16_t** data, size_t r1, size_t r2, size_t r3, size_t r4, TightDataPointStorageI* tdps); - -void getSnapshotData_uint16_1D(uint16_t** data, size_t dataSeriesLength, TightDataPointStorageI* tdps, int errBoundMode); -void getSnapshotData_uint16_2D(uint16_t** data, size_t r1, size_t r2, TightDataPointStorageI* tdps, int errBoundMode); -void getSnapshotData_uint16_3D(uint16_t** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageI* tdps, int errBoundMode); -void getSnapshotData_uint16_4D(uint16_t** data, size_t r1, size_t r2, size_t r3, size_t r4, TightDataPointStorageI* tdps, int errBoundMode); - -int SZ_decompress_args_uint16(uint16_t** newData, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, unsigned char* cmpBytes, size_t cmpSize); - -#ifdef __cplusplus -} -#endif - -#endif /* ----- #ifndef _SZD_Int16_H ----- */ diff --git a/deps/SZ/sz/include/szd_uint32.h b/deps/SZ/sz/include/szd_uint32.h deleted file mode 100644 index 88ff5708a8..0000000000 --- a/deps/SZ/sz/include/szd_uint32.h +++ /dev/null @@ -1,38 +0,0 @@ -/** - * @file szd_uint32.h - * @author Sheng Di - * @date July, 2017 - * @brief Header file for the szd_uint32.c. - * (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -#ifndef _SZD_UInt32_H -#define _SZD_UInt32_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "TightDataPointStorageI.h" - -#define SZ_UINT32_MIN 0 -#define SZ_UINT32_MAX 4294967295 - -void decompressDataSeries_uint32_1D(uint32_t** data, size_t dataSeriesLength, TightDataPointStorageI* tdps); -void decompressDataSeries_uint32_2D(uint32_t** data, size_t r1, size_t r2, TightDataPointStorageI* tdps); -void decompressDataSeries_uint32_3D(uint32_t** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageI* tdps); -void decompressDataSeries_uint32_4D(uint32_t** data, size_t r1, size_t r2, size_t r3, size_t r4, TightDataPointStorageI* tdps); - -void getSnapshotData_uint32_1D(uint32_t** data, size_t dataSeriesLength, TightDataPointStorageI* tdps, int errBoundMode); -void getSnapshotData_uint32_2D(uint32_t** data, size_t r1, size_t r2, TightDataPointStorageI* tdps, int errBoundMode); -void getSnapshotData_uint32_3D(uint32_t** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageI* tdps, int errBoundMode); -void getSnapshotData_uint32_4D(uint32_t** data, size_t r1, size_t r2, size_t r3, size_t r4, TightDataPointStorageI* tdps, int errBoundMode); - -int SZ_decompress_args_uint32(uint32_t** newData, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, unsigned char* cmpBytes, size_t cmpSize); - -#ifdef __cplusplus -} -#endif - -#endif /* ----- #ifndef _SZD_UInt32_H ----- */ diff --git a/deps/SZ/sz/include/szd_uint64.h b/deps/SZ/sz/include/szd_uint64.h deleted file mode 100644 index 6992c68ebc..0000000000 --- a/deps/SZ/sz/include/szd_uint64.h +++ /dev/null @@ -1,35 +0,0 @@ -/** - * @file szd_uint64.h - * @author Sheng Di - * @date July, 2017 - * @brief Header file for the szd_uint64.c. - * (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -#ifndef _SZD_UInt64_H -#define _SZD_UInt64_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "TightDataPointStorageI.h" - -void decompressDataSeries_uint64_1D(uint64_t** data, size_t dataSeriesLength, TightDataPointStorageI* tdps); -void decompressDataSeries_uint64_2D(uint64_t** data, size_t r1, size_t r2, TightDataPointStorageI* tdps); -void decompressDataSeries_uint64_3D(uint64_t** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageI* tdps); -void decompressDataSeries_uint64_4D(uint64_t** data, size_t r1, size_t r2, size_t r3, size_t r4, TightDataPointStorageI* tdps); - -void getSnapshotData_uint64_1D(uint64_t** data, size_t dataSeriesLength, TightDataPointStorageI* tdps, int errBoundMode); -void getSnapshotData_uint64_2D(uint64_t** data, size_t r1, size_t r2, TightDataPointStorageI* tdps, int errBoundMode); -void getSnapshotData_uint64_3D(uint64_t** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageI* tdps, int errBoundMode); -void getSnapshotData_uint64_4D(uint64_t** data, size_t r1, size_t r2, size_t r3, size_t r4, TightDataPointStorageI* tdps, int errBoundMode); - -int SZ_decompress_args_uint64(uint64_t** newData, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, unsigned char* cmpBytes, size_t cmpSize); - -#ifdef __cplusplus -} -#endif - -#endif /* ----- #ifndef _SZD_UInt64_H ----- */ diff --git a/deps/SZ/sz/include/szd_uint8.h b/deps/SZ/sz/include/szd_uint8.h deleted file mode 100644 index 2366c7e08e..0000000000 --- a/deps/SZ/sz/include/szd_uint8.h +++ /dev/null @@ -1,38 +0,0 @@ -/** - * @file szd_uint8.h - * @author Sheng Di - * @date July, 2017 - * @brief Header file for the szd_uint8.c. - * (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -#ifndef _SZD_UInt8_H -#define _SZD_UInt8_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "TightDataPointStorageI.h" - -#define SZ_UINT8_MIN 0 -#define SZ_UINT8_MAX 255 - -void decompressDataSeries_uint8_1D(uint8_t** data, size_t dataSeriesLength, TightDataPointStorageI* tdps); -void decompressDataSeries_uint8_2D(uint8_t** data, size_t r1, size_t r2, TightDataPointStorageI* tdps); -void decompressDataSeries_uint8_3D(uint8_t** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageI* tdps); -void decompressDataSeries_uint8_4D(uint8_t** data, size_t r1, size_t r2, size_t r3, size_t r4, TightDataPointStorageI* tdps); - -void getSnapshotData_uint8_1D(uint8_t** data, size_t dataSeriesLength, TightDataPointStorageI* tdps, int errBoundMode); -void getSnapshotData_uint8_2D(uint8_t** data, size_t r1, size_t r2, TightDataPointStorageI* tdps, int errBoundMode); -void getSnapshotData_uint8_3D(uint8_t** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageI* tdps, int errBoundMode); -void getSnapshotData_uint8_4D(uint8_t** data, size_t r1, size_t r2, size_t r3, size_t r4, TightDataPointStorageI* tdps, int errBoundMode); - -int SZ_decompress_args_uint8(uint8_t** newData, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, unsigned char* cmpBytes, size_t cmpSize); - -#ifdef __cplusplus -} -#endif - -#endif /* ----- #ifndef _SZD_UInt8_H ----- */ diff --git a/deps/SZ/sz/include/szf.h b/deps/SZ/sz/include/szf.h deleted file mode 100644 index 7cf2e838da..0000000000 --- a/deps/SZ/sz/include/szf.h +++ /dev/null @@ -1,102 +0,0 @@ -/** - * @file szf.h - * @author Sheng Di - * @date July, 2017 - * @brief Header file for the szf.c. - * (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -#ifndef _SZF_H -#define _SZF_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include - -//szf.c -void sz_init_c_(char *configFile,int *len,int *ierr); -void sz_finalize_c_(); -void SZ_writeData_inBinary_d1_Float_(float* data, char *fileName, int *len); -void sz_compress_d1_float_(float* data, unsigned char *bytes, size_t *outSize, size_t *r1); -void sz_compress_d1_float_rev_(float* data, float *reservedValue, unsigned char *bytes, size_t *outSize, size_t *r1); -void sz_compress_d2_float_(float* data, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2); -void sz_compress_d2_float_rev_(float* data, float *reservedValue, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2); -void sz_compress_d3_float_(float* data, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3); -void sz_compress_d3_float_rev_(float* data, float *reservedValue, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3); -void sz_compress_d4_float_(float* data, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3, size_t *r4); -void sz_compress_d4_float_rev_(float* data, float *reservedValue, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3, size_t *r4); -void sz_compress_d5_float_(float* data, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5); -void sz_compress_d5_float_rev_(float* data, float *reservedValue, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5); - -void sz_compress_d1_double_(double* data, unsigned char *bytes, size_t *outSize, size_t *r1); -void sz_compress_d1_double_rev_(double* data, double *reservedValue, unsigned char *bytes, size_t *outSize, size_t *r1); -void sz_compress_d2_double_(double* data, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2); -void sz_compress_d2_double_rev_(double* data, double *reservedValue, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2); -void sz_compress_d3_double_(double* data, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3); -void sz_compress_d3_double_rev_(double* data, double *reservedValue, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3); -void sz_compress_d4_double_(double* data, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3, size_t *r4); -void sz_compress_d4_double_rev_(double* data, double *reservedValue, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3, size_t *r4); -void sz_compress_d5_double_(double* data, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5); -void sz_compress_d5_double_rev_(double* data, double *reservedValue, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5); - -void sz_compress_d1_float_args_(float* data, unsigned char *bytes, size_t *outSize, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1); -void sz_compress_d2_float_args_(float* data, unsigned char *bytes, size_t *outSize, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2); -void sz_compress_d3_float_args_(float* data, unsigned char *bytes, size_t *outSize, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2, size_t *r3); -void sz_compress_d4_float_args_(float* data, unsigned char *bytes, size_t *outSize, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4); -void sz_compress_d5_float_args_(float* data, unsigned char *bytes, size_t *outSize, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5); -void sz_compress_d1_double_args_(double* data, unsigned char *bytes, size_t *outSize, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1); -void sz_compress_d2_double_args_(double* data, unsigned char *bytes, size_t *outSize, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2); -void sz_compress_d3_double_args_(double* data, unsigned char *bytes, size_t *outSize, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2, size_t *r3); -void sz_compress_d4_double_args_(double* data, unsigned char *bytes, size_t *outSize, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4); -void sz_compress_d5_double_args_(double* data, unsigned char *bytes, size_t *outSize, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5); - -void sz_compress_d1_float_rev_args_(float* data, float *reservedValue, unsigned char *bytes, size_t *outSize, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1); -void sz_compress_d2_float_rev_args_(float* data, float *reservedValue, unsigned char *bytes, size_t *outSize, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2); -void sz_compress_d3_float_rev_args_(float* data, float *reservedValue, unsigned char *bytes, size_t *outSize, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2, size_t *r3); -void sz_compress_d4_float_rev_args_(float* data, float *reservedValue, unsigned char *bytes, size_t *outSize, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4); -void sz_compress_d5_float_rev_args_(float* data, float *reservedValue, unsigned char *bytes, size_t *outSize, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5); -void sz_compress_d1_double_rev_args_(double* data, float *reservedValue, unsigned char *bytes, size_t *outSize, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1); -void sz_compress_d2_double_rev_args_(double* data, float *reservedValue, unsigned char *bytes, size_t *outSize, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2); -void sz_compress_d3_double_rev_args_(double* data, float *reservedValue, unsigned char *bytes, size_t *outSize, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2, size_t *r3); -void sz_compress_d4_double_rev_args_(double* data, double *reservedValue, unsigned char *bytes, size_t *outSize, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4); -void sz_compress_d5_double_rev_args_(double* data, double *reservedValue, unsigned char *bytes, size_t *outSize, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5); - -void sz_decompress_d1_float_(unsigned char *bytes, size_t *byteLength, float *data, size_t *r1); -void sz_decompress_d2_float_(unsigned char *bytes, size_t *byteLength, float *data, size_t *r1, size_t *r2); -void sz_decompress_d3_float_(unsigned char *bytes, size_t *byteLength, float *data, size_t *r1, size_t *r2, size_t *r3); -void sz_decompress_d4_float_(unsigned char *bytes, size_t *byteLength, float *data, size_t *r1, size_t *r2, size_t *r3, size_t *r4); -void sz_decompress_d5_float_(unsigned char *bytes, size_t *byteLength, float *data, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5); -void sz_decompress_d1_double_(unsigned char *bytes, size_t *byteLength, double *data, size_t *r1); -void sz_decompress_d2_double_(unsigned char *bytes, size_t *byteLength, double *data, size_t *r1, size_t *r2); -void sz_decompress_d3_double_(unsigned char *bytes, size_t *byteLength, double *data, size_t *r1, size_t *r2, size_t *r3); -void sz_decompress_d4_double_(unsigned char *bytes, size_t *byteLength, double *data, size_t *r1, size_t *r2, size_t *r3, size_t *r4); -void sz_decompress_d5_double_(unsigned char *bytes, size_t *byteLength, double *data, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5); - -void sz_batchaddVar_d1_float_(int var_id, char* varName, int *len, float* data, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1); -void sz_batchaddvar_d2_float_(int var_id, char* varName, int *len, float* data, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2); -void sz_batchaddvar_d3_float_(int var_id, char* varName, int *len, float* data, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2, size_t *r3); -void sz_batchaddvar_d4_float_(int var_id, char* varName, int *len, float* data, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4); -void sz_batchaddvar_d5_float_(int var_id, char* varName, int *len, float* data, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5); -void sz_batchaddvar_d1_double_(int var_id, char* varName, int *len, double* data, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1); -void sz_batchaddvar_d2_double_(int var_id, char* varName, int *len, double* data, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2); -void sz_batchaddvar_d3_double_(int var_id, char* varName, int *len, double* data, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2, size_t *r3); -void sz_batchaddvar_d4_double_(int var_id, char* varName, int *len, double* data, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4); -void sz_batchaddvar_d5_double_(int var_id, char* varName, int *len, double* data, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5); -void sz_batchdelvar_c_(char* varName, int *len, int *errState); -void sz_batch_compress_c_(unsigned char* bytes, size_t *outSize); -void sz_batch_decompress_c_(unsigned char* bytes, size_t *byteLength, int *ierr); -void sz_getvardim_c_(char* varName, int *len, int *dim, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5); -void compute_total_batch_size_c_(size_t *totalSize); -void sz_getvardata_float_(char* varName, int *len, float* data); -void sz_getvardata_double_(char* varName, int *len, double* data); -void sz_freevarset_c_(int *mode); - -#ifdef __cplusplus -} -#endif - -#endif /* ----- #ifndef _SZF_H ----- */ - diff --git a/deps/SZ/sz/include/td_sz.h b/deps/SZ/sz/include/td_sz.h index ea110ead53..db2b47faea 100644 --- a/deps/SZ/sz/include/td_sz.h +++ b/deps/SZ/sz/include/td_sz.h @@ -3,7 +3,6 @@ #define _TD_SZ_H #include "defines.h" - #ifdef __cplusplus extern "C" { #endif diff --git a/deps/SZ/sz/include/utility.h b/deps/SZ/sz/include/utility.h index f4ae641589..48192c12ed 100644 --- a/deps/SZ/sz/include/utility.h +++ b/deps/SZ/sz/include/utility.h @@ -16,27 +16,11 @@ extern "C" { #endif -//sihuan added: use a assistant struct to do sorting and swap that are easy to implement: should -//consider optimizing the performance later. -typedef struct sort_ast_particle{ - int64_t id; - float var[6]; -} sort_ast_particle; - -int compare_struct(const void* obj1, const void* obj2);//sihuan added: the compare function in the qsort parameter for 2 structures -void reorder_vars(SZ_VarSet* vset);//sihuan added: reorder the variables increasingly by their index -size_t intersectAndsort(int64_t* preIndex, size_t preLen, SZ_VarSet* curVar, size_t dataLen, unsigned char* bitarray); -//sihuan added: find intersection and keep new var sorted by id -void write_reordered_tofile(SZ_VarSet* curVar, size_t dataLen); -//sihuan added: write the reordered input to files for further decompression validation -float calculate_delta_t(size_t size);//sihuan added int is_lossless_compressed_data(unsigned char* compressedBytes, size_t cmpSize); -unsigned long sz_lossless_compress(int losslessCompressor, int level, unsigned char* data, unsigned long dataLength, unsigned char** compressBytes); +unsigned long sz_lossless_compress(int losslessCompressor, int level, unsigned char* data, unsigned long dataLength, unsigned char* compressBytes); unsigned long sz_lossless_decompress(int losslessCompressor, unsigned char* compressBytes, unsigned long cmpSize, unsigned char** oriData, unsigned long targetOriSize); -unsigned long sz_lossless_decompress65536bytes(int losslessCompressor, unsigned char* compressBytes, unsigned long cmpSize, unsigned char** oriData); -void* detransposeData(void* data, int dataType, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1); -void* transposeData(void* data, int dataType, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1); + #ifdef __cplusplus } diff --git a/deps/SZ/sz/src/TightDataPointStorageD.c b/deps/SZ/sz/src/TightDataPointStorageD.c index bad891cdfc..b8303446a0 100644 --- a/deps/SZ/sz/src/TightDataPointStorageD.c +++ b/deps/SZ/sz/src/TightDataPointStorageD.c @@ -572,7 +572,7 @@ void convertTDPStoBytes_double_reserve(TightDataPointStorageD* tdps, unsigned ch } //Convert TightDataPointStorageD to bytes... -void convertTDPStoFlatBytes_double(TightDataPointStorageD *tdps, unsigned char** bytes, size_t *size) +void convertTDPStoFlatBytes_double(TightDataPointStorageD *tdps, unsigned char* bytes, size_t *size) { size_t i, k = 0; unsigned char dsLengthBytes[8]; @@ -596,20 +596,20 @@ void convertTDPStoFlatBytes_double(TightDataPointStorageD *tdps, unsigned char** if(tdps->allSameData==1) { size_t totalByteLength = 3 + 1 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + tdps->exactMidBytes_size; - *bytes = (unsigned char *)malloc(sizeof(unsigned char)*totalByteLength); + //bytes = (unsigned char *)malloc(sizeof(unsigned char)*totalByteLength); // comment by tickduan for (i = 0; i < 3; i++)//3 - (*bytes)[k++] = versionNumber[i]; - (*bytes)[k++] = sameByte; + bytes[k++] = versionNumber[i]; + bytes[k++] = sameByte; - convertSZParamsToBytes(confparams_cpr, &((*bytes)[k])); + convertSZParamsToBytes(confparams_cpr, &(bytes[k])); k = k + MetaDataByteLength_double; for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++) - (*bytes)[k++] = dsLengthBytes[i]; + bytes[k++] = dsLengthBytes[i]; for (i = 0; i < tdps->exactMidBytes_size; i++) - (*bytes)[k++] = tdps->exactMidBytes[i]; + bytes[k++] = tdps->exactMidBytes[i]; *size = totalByteLength; } @@ -635,9 +635,9 @@ void convertTDPStoFlatBytes_double(TightDataPointStorageD *tdps, unsigned char** if(confparams_cpr->errorBoundMode == PW_REL && confparams_cpr->accelerate_pw_rel_compression) totalByteLength += (1+1); // for MSST19 - *bytes = (unsigned char *)malloc(sizeof(unsigned char)*totalByteLength); + //*bytes = (unsigned char *)malloc(sizeof(unsigned char)*totalByteLength); comment by tickduan - convertTDPStoBytes_double(tdps, *bytes, dsLengthBytes, sameByte); + convertTDPStoBytes_double(tdps, bytes, dsLengthBytes, sameByte); *size = totalByteLength; } diff --git a/deps/SZ/sz/src/TightDataPointStorageF.c b/deps/SZ/sz/src/TightDataPointStorageF.c index 2a236ab36b..da019f08b5 100644 --- a/deps/SZ/sz/src/TightDataPointStorageF.c +++ b/deps/SZ/sz/src/TightDataPointStorageF.c @@ -470,116 +470,8 @@ void convertTDPStoBytes_float(TightDataPointStorageF* tdps, unsigned char* bytes } } -/*deprecated*/ -void convertTDPStoBytes_float_reserve(TightDataPointStorageF* tdps, unsigned char* bytes, unsigned char* dsLengthBytes, unsigned char sameByte) -{ - size_t i, k = 0; - unsigned char intervalsBytes[4]; - unsigned char typeArrayLengthBytes[8]; - unsigned char rTypeLengthBytes[8]; - unsigned char exactLengthBytes[8]; - unsigned char exactMidBytesLength[8]; - unsigned char realPrecisionBytes[8]; - unsigned char reservedValueBytes[4]; - - unsigned char medianValueBytes[4]; - - unsigned char segment_sizeBytes[8]; - unsigned char pwrErrBoundBytes_sizeBytes[4]; - unsigned char max_quant_intervals_Bytes[4]; - - for(i = 0;i<3;i++)//3 - bytes[k++] = versionNumber[i]; - bytes[k++] = sameByte; //1 - - convertSZParamsToBytes(confparams_cpr, &(bytes[k])); - k = k + MetaDataByteLength; - - for(i = 0;iSZ_SIZE_TYPE;i++)//ST - bytes[k++] = dsLengthBytes[i]; - - - intToBytes_bigEndian(max_quant_intervals_Bytes, confparams_cpr->max_quant_intervals); - for(i = 0;i<4;i++)//4 - bytes[k++] = max_quant_intervals_Bytes[i]; - - if(confparams_cpr->errorBoundMode>=PW_REL) - { - bytes[k++] = tdps->radExpo; //1 byte - - sizeToBytes(segment_sizeBytes, confparams_cpr->segment_size); - for(i = 0;iSZ_SIZE_TYPE;i++)//ST - bytes[k++] = segment_sizeBytes[i]; - - intToBytes_bigEndian(pwrErrBoundBytes_sizeBytes, tdps->pwrErrBoundBytes_size); - for(i = 0;i<4;i++)//4 - bytes[k++] = pwrErrBoundBytes_sizeBytes[i]; - } - - intToBytes_bigEndian(intervalsBytes, tdps->intervals); - for(i = 0;i<4;i++)//4 - bytes[k++] = intervalsBytes[i]; - - floatToBytes(medianValueBytes, tdps->medianValue); - for (i = 0; i < 4; i++)// 4 - bytes[k++] = medianValueBytes[i]; - - bytes[k++] = tdps->reqLength; //1 byte - - floatToBytes(realPrecisionBytes, tdps->realPrecision); - for (i = 0; i < 8; i++)// 8 - bytes[k++] = realPrecisionBytes[i]; - - sizeToBytes(typeArrayLengthBytes, tdps->typeArray_size); - for(i = 0;iSZ_SIZE_TYPE;i++)//ST - bytes[k++] = typeArrayLengthBytes[i]; - - sizeToBytes(rTypeLengthBytes, tdps->rtypeArray_size); - for(i = 0;iSZ_SIZE_TYPE;i++)//ST - bytes[k++] = rTypeLengthBytes[i]; - - sizeToBytes(exactLengthBytes, tdps->exactDataNum); - for(i = 0;iSZ_SIZE_TYPE;i++)//ST - bytes[k++] = exactLengthBytes[i]; - - sizeToBytes(exactMidBytesLength, tdps->exactMidBytes_size); - for(i = 0;iSZ_SIZE_TYPE;i++)//ST - bytes[k++] = exactMidBytesLength[i]; - - floatToBytes(reservedValueBytes, tdps->reservedValue); - for (i = 0; i < 4; i++)// 4 - bytes[k++] = reservedValueBytes[i]; - - memcpy(&(bytes[k]), tdps->rtypeArray, tdps->rtypeArray_size); - k += tdps->rtypeArray_size; - - if(confparams_cpr->errorBoundMode>=PW_REL) - { - floatToBytes(exactMidBytesLength, tdps->minLogValue); - for(i=0;i<4;i++) - bytes[k++] = exactMidBytesLength[i]; - } - - memcpy(&(bytes[k]), tdps->typeArray, tdps->typeArray_size); - k += tdps->typeArray_size; - if(confparams_cpr->errorBoundMode>=PW_REL) - { - memcpy(&(bytes[k]), tdps->pwrErrBoundBytes, tdps->pwrErrBoundBytes_size); - k += tdps->pwrErrBoundBytes_size; - } - memcpy(&(bytes[k]), tdps->leadNumArray, tdps->leadNumArray_size); - k += tdps->leadNumArray_size; - memcpy(&(bytes[k]), tdps->exactMidBytes, tdps->exactMidBytes_size); - k += tdps->exactMidBytes_size; - if(tdps->residualMidBits!=NULL) - { - memcpy(&(bytes[k]), tdps->residualMidBits, tdps->residualMidBits_size); - k += tdps->residualMidBits_size; - } -} - //convert TightDataPointStorageD to bytes... -void convertTDPStoFlatBytes_float(TightDataPointStorageF *tdps, unsigned char** bytes, size_t *size) +void convertTDPStoFlatBytes_float(TightDataPointStorageF *tdps, unsigned char* bytes, size_t *size) { size_t i, k = 0; unsigned char dsLengthBytes[8]; @@ -605,20 +497,23 @@ void convertTDPStoFlatBytes_float(TightDataPointStorageF *tdps, unsigned char** if(tdps->allSameData==1) { size_t totalByteLength = 3 + 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + tdps->exactMidBytes_size; - *bytes = (unsigned char *)malloc(sizeof(unsigned char)*totalByteLength); + //*bytes = (unsigned char *)malloc(sizeof(unsigned char)*totalByteLength); // not need malloc comment by tickduan + + // check output buffer enough + for (i = 0; i < 3; i++)//3 - (*bytes)[k++] = versionNumber[i]; - (*bytes)[k++] = sameByte; + bytes[k++] = versionNumber[i]; + bytes[k++] = sameByte; - convertSZParamsToBytes(confparams_cpr, &((*bytes)[k])); + convertSZParamsToBytes(confparams_cpr, &(bytes[k])); k = k + MetaDataByteLength; for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++) - (*bytes)[k++] = dsLengthBytes[i]; + bytes[k++] = dsLengthBytes[i]; for (i = 0; i < tdps->exactMidBytes_size; i++) - (*bytes)[k++] = tdps->exactMidBytes[i]; + bytes[k++] = tdps->exactMidBytes[i]; *size = totalByteLength; } @@ -642,9 +537,9 @@ void convertTDPStoFlatBytes_float(TightDataPointStorageF *tdps, unsigned char** if(confparams_cpr->errorBoundMode == PW_REL && confparams_cpr->accelerate_pw_rel_compression) totalByteLength += (1+1); // for MSST19 - *bytes = (unsigned char *)malloc(sizeof(unsigned char)*totalByteLength); + //*bytes = (unsigned char *)malloc(sizeof(unsigned char)*totalByteLength); // comment by tickduan - convertTDPStoBytes_float(tdps, *bytes, dsLengthBytes, sameByte); + convertTDPStoBytes_float(tdps, bytes, dsLengthBytes, sameByte); *size = totalByteLength; } diff --git a/deps/SZ/sz/src/VarSet.c b/deps/SZ/sz/src/VarSet.c index 719c6774fd..d97828be14 100644 --- a/deps/SZ/sz/src/VarSet.c +++ b/deps/SZ/sz/src/VarSet.c @@ -119,7 +119,7 @@ int SZ_batchDelVar(char* varName) int SZ_batchDelVar_ID_vset(SZ_VarSet* vset, int var_id) { - int delSuccess = SZ_NSCS; + int delSuccess = SZ_FAILED; SZ_Variable* p = vset->header; SZ_Variable* q = p->next; while(q != NULL) @@ -130,7 +130,7 @@ int SZ_batchDelVar_ID_vset(SZ_VarSet* vset, int var_id) //free_Variable_all(q); free_Variable_keepOriginalData(q); vset->count --; - delSuccess = SZ_SCES; + delSuccess = SZ_SUCCESS; if(q->next==NULL) //means that q is the last variable vset->lastVar = p; break; @@ -145,7 +145,7 @@ int SZ_batchDelVar_ID_vset(SZ_VarSet* vset, int var_id) int SZ_batchDelVar_vset(SZ_VarSet* vset, char* varName) { - int delSuccess = SZ_NSCS; + int delSuccess = SZ_FAILED; SZ_Variable* p = vset->header; SZ_Variable* q = p->next; while(q != NULL) @@ -157,7 +157,7 @@ int SZ_batchDelVar_vset(SZ_VarSet* vset, char* varName) //free_Variable_all(q); free_Variable_keepOriginalData(q); vset->count --; - delSuccess = SZ_SCES; + delSuccess = SZ_SUCCESS; break; } p = p->next; diff --git a/deps/SZ/sz/src/callZlib.c b/deps/SZ/sz/src/callZlib.c index 4e4bb6f272..e2c6cae4f3 100644 --- a/deps/SZ/sz/src/callZlib.c +++ b/deps/SZ/sz/src/callZlib.c @@ -23,7 +23,7 @@ #define CHECK_ERR(err, msg) { \ if (err != Z_OK && err != Z_STREAM_END) { \ fprintf(stderr, "%s error: %d\n", msg, err); \ - return SZ_NSCS; \ + return SZ_FAILED; \ } \ } @@ -202,7 +202,7 @@ unsigned long zlib_compress4(unsigned char* data, unsigned long dataLength, unsi return c_stream.total_out; } -unsigned long zlib_compress5(unsigned char* data, unsigned long dataLength, unsigned char** compressBytes, int level) +unsigned long zlib_compress5(unsigned char* data, unsigned long dataLength, unsigned char* compressBytes, int level) { int ret, flush; unsigned have; @@ -222,7 +222,7 @@ unsigned long zlib_compress5(unsigned char* data, unsigned long dataLength, unsi size_t p_size = 0, av_in = 0; uLong estCmpLen = deflateBound(&strm, dataLength); - *compressBytes = (unsigned char*)malloc(sizeof(unsigned char)*estCmpLen); + //*compressBytes = (unsigned char*)malloc(sizeof(unsigned char)*estCmpLen); // comment by tickduan no need malloc unsigned char* out = *compressBytes; /* compress until end of file */ @@ -296,7 +296,7 @@ unsigned long zlib_uncompress2 (unsigned char* compressBytes, unsigned long cmpS { printf("Error: zlib_uncompress2: stream.avail_in != cmpSize"); //exit(1); - return SZ_NSCS; //-1 + return SZ_FAILED; //-1 } stream.next_out = *oriData; @@ -309,7 +309,7 @@ unsigned long zlib_uncompress2 (unsigned char* compressBytes, unsigned long cmpS if (err != Z_OK) { printf("Error: zlib_uncompress2: err != Z_OK\n"); - return SZ_NSCS; + return SZ_FAILED; } err = inflate(&stream, Z_FINISH); diff --git a/deps/SZ/sz/src/conf.c b/deps/SZ/sz/src/conf.c index cfd918bf76..f9549c8ad7 100644 --- a/deps/SZ/sz/src/conf.c +++ b/deps/SZ/sz/src/conf.c @@ -135,13 +135,13 @@ int SZ_ReadConf(const char* sz_cfgFile) { confparams_cpr->protectValueRange = 0; - return SZ_SCES; + return SZ_SUCCESS; } if (access(sz_cfgFile, F_OK) != 0) { printf("[SZ] Configuration file NOT accessible.\n"); - return SZ_NSCS; + return SZ_FAILED; } //printf("[SZ] Reading SZ configuration file (%s) ...\n", sz_cfgFile); @@ -149,7 +149,7 @@ int SZ_ReadConf(const char* sz_cfgFile) { if (ini == NULL) { printf("[SZ] Iniparser failed to parse the conf. file.\n"); - return SZ_NSCS; + return SZ_FAILED; } endianTypeString = iniparser_getstring(ini, "ENV:dataEndianType", "LITTLE_ENDIAN_DATA"); @@ -161,7 +161,7 @@ int SZ_ReadConf(const char* sz_cfgFile) { { printf("Error: Wrong dataEndianType: please set it correctly in sz.config.\n"); iniparser_freedict(ini); - return SZ_NSCS; + return SZ_FAILED; } // Reading/setting detection parameters @@ -178,7 +178,7 @@ int SZ_ReadConf(const char* sz_cfgFile) { else{ printf("[SZ] Error: wrong solution name (please check sz.config file), sol=%s\n", sol_name); iniparser_freedict(ini); - return SZ_NSCS; + return SZ_FAILED; } if(confparams_cpr->sol_ID==SZ || confparams_cpr->sol_ID==SZ_Transpose) @@ -208,7 +208,7 @@ int SZ_ReadConf(const char* sz_cfgFile) { { printf("Error: quantization_intervals must be an even number!\n"); iniparser_freedict(ini); - return SZ_NSCS; + return SZ_FAILED; } confparams_cpr->predThreshold = (float)iniparser_getdouble(ini, "PARAMETER:predThreshold", 0); @@ -219,7 +219,7 @@ int SZ_ReadConf(const char* sz_cfgFile) { { printf("[SZ] Error: Null szMode setting (please check sz.config file)\n"); iniparser_freedict(ini); - return SZ_NSCS; + return SZ_FAILED; } else if(strcmp(modeBuf, "SZ_BEST_SPEED")==0) confparams_cpr->szMode = SZ_BEST_SPEED; @@ -231,7 +231,7 @@ int SZ_ReadConf(const char* sz_cfgFile) { { printf("[SZ] Error: Wrong szMode setting (please check sz.config file)\n"); iniparser_freedict(ini); - return SZ_NSCS; + return SZ_FAILED; } modeBuf = iniparser_getstring(ini, "PARAMETER:losslessCompressor", "ZSTD_COMPRESSOR"); @@ -244,7 +244,7 @@ int SZ_ReadConf(const char* sz_cfgFile) { printf("[SZ] Error: Wrong losslessCompressor setting (please check sz.config file)\n");\ printf("No Such a lossless compressor: %s\n", modeBuf); iniparser_freedict(ini); - return SZ_NSCS; + return SZ_FAILED; } modeBuf = iniparser_getstring(ini, "PARAMETER:withLinearRegression", "YES"); @@ -258,7 +258,7 @@ int SZ_ReadConf(const char* sz_cfgFile) { { printf("[SZ] Error: Null Gzip mode setting (please check sz.config file)\n"); iniparser_freedict(ini); - return SZ_NSCS; + return SZ_FAILED; } else if(strcmp(modeBuf, "Gzip_NO_COMPRESSION")==0) confparams_cpr->gzipMode = 0; @@ -271,7 +271,7 @@ int SZ_ReadConf(const char* sz_cfgFile) { else { printf("[SZ] Error: Wrong gzip Mode (please check sz.config file)\n"); - return SZ_NSCS; + return SZ_FAILED; } modeBuf = iniparser_getstring(ini, "PARAMETER:zstdMode", "Zstd_HIGH_SPEED"); @@ -279,7 +279,7 @@ int SZ_ReadConf(const char* sz_cfgFile) { { printf("[SZ] Error: Null Zstd mode setting (please check sz.config file)\n"); iniparser_freedict(ini); - return SZ_NSCS; + return SZ_FAILED; } else if(strcmp(modeBuf, "Zstd_BEST_SPEED")==0) confparams_cpr->gzipMode = 1; @@ -294,7 +294,7 @@ int SZ_ReadConf(const char* sz_cfgFile) { else { printf("[SZ] Error: Wrong zstd Mode (please check sz.config file)\n"); - return SZ_NSCS; + return SZ_FAILED; } modeBuf = iniparser_getstring(ini, "PARAMETER:protectValueRange", "YES"); @@ -313,7 +313,7 @@ int SZ_ReadConf(const char* sz_cfgFile) { { printf("[SZ] Error: Null error bound setting (please check sz.config file)\n"); iniparser_freedict(ini); - return SZ_NSCS; + return SZ_FAILED; } else if(strcmp(errBoundMode,"ABS")==0||strcmp(errBoundMode,"abs")==0) confparams_cpr->errorBoundMode=SZ_ABS; @@ -343,7 +343,7 @@ int SZ_ReadConf(const char* sz_cfgFile) { { printf("[SZ] Error: Wrong error bound mode (please check sz.config file)\n"); iniparser_freedict(ini); - return SZ_NSCS; + return SZ_FAILED; } confparams_cpr->absErrBound = (double)iniparser_getdouble(ini, "PARAMETER:absErrBound", 0); @@ -366,7 +366,7 @@ int SZ_ReadConf(const char* sz_cfgFile) { { printf("[SZ] Error: Wrong pwr_type setting (please check sz.config file).\n"); iniparser_freedict(ini); - return SZ_NSCS; + return SZ_FAILED; } else //by default confparams_cpr->pwr_type = SZ_PWR_AVG_TYPE; @@ -385,7 +385,7 @@ int SZ_ReadConf(const char* sz_cfgFile) { } iniparser_freedict(ini); - return SZ_SCES; + return SZ_SUCCESS; } /*-------------------------------------------------------------------------*/ @@ -400,12 +400,12 @@ int SZ_ReadConf(const char* sz_cfgFile) { /*-------------------------------------------------------------------------*/ int SZ_LoadConf(const char* sz_cfgFile) { int res = SZ_ReadConf(sz_cfgFile); - if (res != SZ_SCES) + if (res != SZ_SUCCESS) { printf("[SZ] ERROR: Impossible to read configuration.\n"); - return SZ_NSCS; + return SZ_FAILED; } - return SZ_SCES; + return SZ_SUCCESS; } int checkVersion(char* version) diff --git a/deps/SZ/sz/src/dataCompression.c b/deps/SZ/sz/src/dataCompression.c index c630659d38..15a5825fff 100644 --- a/deps/SZ/sz/src/dataCompression.c +++ b/deps/SZ/sz/src/dataCompression.c @@ -284,7 +284,7 @@ float max_f(float a, float b) double getRealPrecision_double(double valueRangeSize, int errBoundMode, double absErrBound, double relBoundRatio, int *status) { - int state = SZ_SCES; + int state = SZ_SUCCESS; double precision = 0; if(errBoundMode==SZ_ABS||errBoundMode==ABS_OR_PW_REL||errBoundMode==ABS_AND_PW_REL) precision = absErrBound; @@ -307,7 +307,7 @@ double getRealPrecision_double(double valueRangeSize, int errBoundMode, double a double getRealPrecision_float(float valueRangeSize, int errBoundMode, double absErrBound, double relBoundRatio, int *status) { - int state = SZ_SCES; + int state = SZ_SUCCESS; double precision = 0; if(errBoundMode==SZ_ABS||errBoundMode==ABS_OR_PW_REL||errBoundMode==ABS_AND_PW_REL) precision = absErrBound; @@ -330,7 +330,7 @@ double getRealPrecision_float(float valueRangeSize, int errBoundMode, double abs double getRealPrecision_int(long valueRangeSize, int errBoundMode, double absErrBound, double relBoundRatio, int *status) { - int state = SZ_SCES; + int state = SZ_SUCCESS; double precision = 0; if(errBoundMode==SZ_ABS||errBoundMode==ABS_OR_PW_REL||errBoundMode==ABS_AND_PW_REL) precision = absErrBound; @@ -651,7 +651,7 @@ int getPredictionCoefficients(int layers, int dimension, int **coeff_array, int printf("Error: dimension must be no greater than 3 in the current version.\n"); *status = SZ_DERR; } - *status = SZ_SCES; + *status = SZ_SUCCESS; return size; } diff --git a/deps/SZ/sz/src/exafelSZ.c b/deps/SZ/sz/src/exafelSZ.c deleted file mode 100644 index f90d8ce74a..0000000000 --- a/deps/SZ/sz/src/exafelSZ.c +++ /dev/null @@ -1,597 +0,0 @@ -#ifdef __cplusplus -extern "C" { -#endif - -#include "sz.h" - -void exafelSZ_params_process(exafelSZ_params*pr, size_t panels, size_t rows, size_t cols){ - pr->binnedRows=(rows+pr->binSize-1)/pr->binSize; - pr->binnedCols=(cols+pr->binSize-1)/pr->binSize; - - pr->peakRadius=(pr->peakSize-1)/2; -} - -void exafelSZ_params_checkDecomp(exafelSZ_params*pr, size_t panels, size_t rows, size_t cols){ - if(pr->calibPanel==NULL){ - printf("ERROR: calibPanel is NULL : calibPanel=%ld\n",(long)pr->calibPanel); - assert(0); - } - if(pr->binSize<1 || pr->tolerance<0 || pr->szDim<1 || pr->szDim>3){ - printf("ERROR: Something wrong with the following:\n"); - printf("binSize=%d\n",(int)pr->binSize); - printf("tolerance=%d\n",(int)pr->tolerance); - printf("szDim=%d\n",(int)pr->szDim); - assert(0); - } - if(!(pr->peakSize%2)){ - printf("ERROR: peakSize = %d cannot be even. It must be odd!\n",(int)pr->peakSize); - assert(0); - } - //if(nEvents<1 || panels<1 || rows<1 || cols<1){ - if(panels<1 || rows<1 || cols<1){ - printf("ERROR: Something wrong with the following:\n"); - printf("panels=%d\n",(int)panels); - printf("rows=%d\n",(int)rows); - printf("cols=%d\n",(int)cols); - assert(0); - } -} - -void exafelSZ_params_checkComp(exafelSZ_params*pr, size_t panels, size_t rows, size_t cols){ - if(pr->peaksSegs==NULL || pr->peaksRows==NULL || pr->peaksCols==NULL){ - printf("ERROR: One or more of the following are NULL : peaksSegs , peaksRows , peaksCols\n"); - assert(0); - } - exafelSZ_params_checkDecomp(pr, panels, rows, cols); -} - -void exafelSZ_params_print(exafelSZ_params*pr){ - printf("Configuration (exafelSZ_params) :\n"); - printf("binSize: %d\n",pr->binSize); - printf("tolerance:%e\n",pr->tolerance); - printf("szDim:%d\n",pr->szDim); - printf("peakSize:%d\n",pr->peakSize); - //printf("nEvents:%d\n",pr->nEvents); - //printf("panels:%d\n",pr->panels); - //printf("rows:%d\n",pr->rows); - //printf("cols:%d\n",pr->cols); - printf("\n"); - printf("CALCULATED VARIABLES\n"); - printf("binnedRows:%ld\n",pr->binnedRows); - printf("binnedCols:%ld\n",pr->binnedCols); - printf("peakRadius:%d\n",pr->peakRadius); - printf("\n"); - // outs<<"Configuration (exafelSZ_params) : "<binnedRows*pr->binnedCols*sizeof(float)) ; - //float *binnedData=(float*)malloc(nEvents*panels*rows*cols*sizeof(float)) ; - - size_t e,p,r,c,pk,ri,ci,br,bc,roii,bi; - /* - printf("AMG : exafelSZ_Compress : pr->numPeaks = %d\n",pr->numPeaks); - printf("S:\n"); - for(e=0;enumPeaks;e++) - printf("%d ",pr->peaksSegs[e]); - printf("\nR:\n"); - for(e=0;enumPeaks;e++) - printf("%d ",pr->peaksRows[e]); - printf("\nC:\n"); - for(e=0;enumPeaks;e++) - printf("%d ",pr->peaksCols[e]); - printf("\n"); - */ - - //Generate the ROI mask: NOTE: 0 means affirmative in ROI mask! This comes from the python scripts! - //First, initialize with calibration panel: - for(e=0;ecalibPanel[calcIdx_2D(r,c,cols)]; //calibPanel is a single segment copied over all the event(image) - roiM[calcIdx_4D(e,p,r,c,panels,rows,cols)]=pr->calibPanel[calcIdx_3D(p,r,c,rows,cols)]; //calibPanel is as big as the event(image) itself - } - } - } - } - //uint64_t peaksBytePos=0; //Position in the peaks buffer - //Now process the peaks and generate the mask: - uint64_t nPeaksTotal=0; //Total number of peaks - for(e=0;epeaks[peaksBytePos]); - //peaksBytePos+=8; - - //peaksBytePos+=8;//Skip the second one! This is due to the problem in Python. - - nPeaksTotal+=pr->numPeaks; - for(pk=0;pknumPeaks;pk++){ - //uint16_t p_=*(uint16_t*)(&pr->peaks[peaksBytePos]); //Panel for the current peak - //peaksBytePos+=2; - //uint16_t r_=*(uint16_t*)(&pr->peaks[peaksBytePos]); //Row for the current peak - //peaksBytePos+=2; - //uint16_t c_=*(uint16_t*)(&pr->peaks[peaksBytePos]); //Col for the current peak - //peaksBytePos+=2; - - uint16_t p_=pr->peaksSegs[pk]; - uint16_t r_=pr->peaksRows[pk]; - uint16_t c_=pr->peaksCols[pk]; - - if(p_>=panels){ - printf("ERROR: Peak coordinate out of bounds: Panel=%d, Valid range: 0,%d\n",(int)p_,(int)panels-1); - assert(0); - printf("Skipping this peak...\n"); - continue; - } - if(r_>=rows){ - printf("ERROR: Peak coordinate out of bounds: Row=%d, Valid range: 0,%d\n",(int)r_,(int)rows-1); - assert(0); - printf("Skipping this peak...\n"); - continue; - } - if(c_>=cols){ - printf("ERROR: Peak coordinate out of bounds: Col=%d, Valid range: 0,%d\n",(int)c_,(int)cols-1); - assert(0); - printf("Skipping this peak...\n"); - continue; - } - - for(ri=r_-pr->peakRadius;ri<=r_+pr->peakRadius;ri++){ //ri: row index. Just a temporary variable. - for(ci=c_-pr->peakRadius;ci<=c_+pr->peakRadius;ci++){ //ci: column index. Just a temporary variable. - if(ribinnedRows;r++){ //Row of the binnedData - for(c=0;cbinnedCols;c++){ //Column of the binnedData - float sum=0; - int nPts=0; - for(br=0;brbinSize;br++) //Bin Row (from origData) - for(bc=0;bcbinSize;bc++) //Bin Column (from origData) - if(r*pr->binSize+brbinSize+bcbinSize+br<<" "<binSize+bc<binSize+br,c*pr->binSize+bc,panels,rows,cols)]; - nPts++; - } - // cout<<"p:"<binnedRows,pr->binnedCols)]=sum/nPts; - } - } - } - } - - //Additional compression using SZ: - size_t szCompressedSize=0; - unsigned char* szComp; - - switch(pr->szDim){ - case 1: - // szComp=sz_compress_3D(binnedData, 0, 0, nEvents * panels * pr->binnedRows * pr->binnedCols, pr->tolerance, szCompressedSize); //1D - szComp=SZ_compress_args(SZ_FLOAT, binnedData, &szCompressedSize, SZ_ABS, pr->tolerance, 0, 0, 0, 0,0,0, nEvents * panels * pr->binnedRows * pr->binnedCols); - break; - case 2: - // szComp=sz_compress_3D(binnedData, 0, nEvents * panels * pr->binnedRows, pr->binnedCols, pr->tolerance, szCompressedSize); //2D - szComp=SZ_compress_args(SZ_FLOAT, binnedData, &szCompressedSize, SZ_ABS, pr->tolerance, 0, 0, 0, 0,0, nEvents * panels * pr->binnedRows, pr->binnedCols); - break; - case 3: - // szComp=sz_compress_3D(binnedData, nEvents * panels, pr->binnedRows, pr->binnedCols, pr->tolerance, szCompressedSize); //3D - szComp=SZ_compress_args(SZ_FLOAT, binnedData, &szCompressedSize, SZ_ABS, pr->tolerance, 0, 0, 0, 0, nEvents * panels, pr->binnedRows, pr->binnedCols); - break; - default: - printf("ERROR: Wrong szDim : %d It must be 1,2 or 3.\n",(int)pr->szDim); - assert(0); - } - - /* - Compressed buffer format: (Types are indicated in parenthesis) - WRITE: nPeaksTotal(uint64_t) (Total number of peaks in this batch) - for(e=0;epeaks[peaksBytePos]); - //peaksBytePos+=8; - ////peaksBytePos+=8;//Skip the second one. This is due to the error in Python! - - //*(uint64_t*)(&compressedBuffer[bytePos])=nPeaks; - *(uint64_t*)(&compressedBuffer[bytePos])=pr->numPeaks; - bytePos+=8; - //for(pk=0;pknumPeaks;pk++){ - //*(uint16_t*)(&compressedBuffer[bytePos])=*(uint16_t*)(&pr->peaks[peaksBytePos]); //Panel for the current peak - //bytePos+=2; - //peaksBytePos+=2; - //*(uint16_t*)(&compressedBuffer[bytePos])=*(uint16_t*)(&pr->peaks[peaksBytePos]); //Row for the current peak - //bytePos+=2; - //peaksBytePos+=2; - //*(uint16_t*)(&compressedBuffer[bytePos])=*(uint16_t*)(&pr->peaks[peaksBytePos]); //Column for the current peak - //bytePos+=2; - //peaksBytePos+=2; - - *(uint16_t*)(&compressedBuffer[bytePos])=pr->peaksSegs[pk]; //Panel for the current peak - bytePos+=2; - *(uint16_t*)(&compressedBuffer[bytePos])=pr->peaksRows[pk]; //Row for the current peak - bytePos+=2; - *(uint16_t*)(&compressedBuffer[bytePos])=pr->peaksCols[pk]; //Column for the current peak - bytePos+=2; - } - } - // cout<<"peaks"<=rows*cols){ - printf("ERROR: calcIdx_2D(r,c,cols) = calcIdx_2D(%d,%d,%d) = %d",(int)r,(int)c,(int)cols,(int)calcIdx_2D(r,c,cols)); - printf(" is NOT in the correct range: [0,%ld]",(int)rows*cols-1); - assert(0); - } - if(calcIdx_4D(e,p,r,c,panels,rows,cols)<0 ||calcIdx_4D(e,p,r,c,panels,rows,cols)>=nEvents*panels*rows*cols){ - printf("ERROR: calcIdx_4D(e,p,r,c,panels,rows,cols) = calcIdx_4D(%d,%d,%d,%d,%d,%d,%d) = %d",(int)e,(int)p,(int)r,(int)c,(int)panels,(int)rows,(int)cols,(int)calcIdx_4D(e,p,r,c,panels,rows,cols)); - assert(0); - } - //roiM[calcIdx_4D(e,p,r,c,panels,rows,cols)]=pr->calibPanel[calcIdx_2D(r,c,cols)]; //calibPanel is a single segment copied over all the event(image) - roiM[calcIdx_4D(e,p,r,c,panels,rows,cols)]=pr->calibPanel[calcIdx_3D(p,r,c,rows,cols)]; //calibPanel is as big as the event(image) itself - } - } - } - } - uint64_t peaksBytePos=0; //Position in the peaks buffer - //Now process the peaks and generate the mask: - for(e=0;e=panels){ - printf("ERROR: Peak coordinate out of bounds: Panel=%d, Valid range: 0,%d\n",(int)p_,(int)panels-1); - assert(0); - printf("Skipping this peak...\n"); - continue; - } - if(r_>=rows){ - printf("ERROR: Peak coordinate out of bounds: Row=%d, Valid range: 0,%d\n",(int)r_,(int)rows-1); - assert(0); - printf("Skipping this peak...\n"); - continue; - } - if(c_>=cols){ - printf("ERROR: Peak coordinate out of bounds: Col=%d, Valid range: 0,%d\n",(int)c_,(int)cols-1); - assert(0); - printf("Skipping this peak...\n"); - continue; - } - - for(ri=r_-pr->peakRadius;ri<=r_+pr->peakRadius;ri++){ //ri: row index. Just a temporary variable. - for(ci=c_-pr->peakRadius;ci<=c_+pr->peakRadius;ci++){ //ci: column index. Just a temporary variable. - if(ri>=0 && ri=0 && ciszDim){ - case 1: - szDecomp=SZ_decompress(SZ_FLOAT,szComp,_szCompressedSize,0,0,0,0, nEvents * panels * pr->binnedRows * pr->binnedCols); - break; - case 2: - szDecomp=SZ_decompress(SZ_FLOAT,szComp,_szCompressedSize,0,0,0, nEvents * panels * pr->binnedRows, pr->binnedCols); - break; - case 3: - szDecomp=SZ_decompress(SZ_FLOAT,szComp,_szCompressedSize,0,0,nEvents * panels, pr->binnedRows, pr->binnedCols); - break; - default: - printf("ERROR: Wrong szDim : %d It must be 1,2 or 3.\n",(int)pr->szDim); - assert(0); - } - //szDecomp=(void*)malloc(nEvents*panels*rows*cols*sizeof(float)); - - // double max_err = 0; - // for(int i=0; ibinnedRows * pr->binnedCols; i++){ - // double err = fabs(szDecomp[i]-binnedData[i]); - // if(err > max_err) max_err = err; - // } - // cout << "Max err = \t\t\t" << max_err << endl; - - - //De-binning: - for(e=0;ebinnedRows;r++) //Row of the binnedData - for(c=0;cbinnedCols;c++) //Column of the binnedData - for(br=0;brbinSize;br++) //Bin Row (from origData) - for(bc=0;bcbinSize;bc++) //Bin Column (from origData) - if(r*pr->binSize+brbinSize+bcbinSize+br,c*pr->binSize+bc,panels,rows,cols)] = szDecomp[calcIdx_4D(e,p,r,c,panels,pr->binnedRows,pr->binnedCols)]; - } - //Restore ROI: - uint64_t current=0; - for(e=0;eSZ_SIZE_TYPE = sizeof(size_t); @@ -66,7 +73,7 @@ int SZ_Init(const char *configFilePath) { initSZ_TSC(); } - return SZ_SCES; + return SZ_SUCCESS; } int SZ_Init_Params(sz_params *params) @@ -84,71 +91,12 @@ int SZ_Init_Params(sz_params *params) if(params->quantization_intervals%2!=0) { printf("Error: quantization_intervals must be an even number!\n"); - return SZ_NSCS; + return SZ_FAILED; } - return SZ_SCES; + return SZ_SUCCESS; } -int computeDimension(size_t r5, size_t r4, size_t r3, size_t r2, size_t r1) -{ - int dimension; - if(r1==0) - { - dimension = 0; - } - else if(r2==0) - { - dimension = 1; - } - else if(r3==0) - { - dimension = 2; - } - else if(r4==0) - { - dimension = 3; - } - else if(r5==0) - { - dimension = 4; - } - else - { - dimension = 5; - } - return dimension; -} - -size_t computeDataLength(size_t r5, size_t r4, size_t r3, size_t r2, size_t r1) -{ - size_t dataLength; - if(r1==0) - { - dataLength = 0; - } - else if(r2==0) - { - dataLength = r1; - } - else if(r3==0) - { - dataLength = r1*r2; - } - else if(r4==0) - { - dataLength = r1*r2*r3; - } - else if(r5==0) - { - dataLength = r1*r2*r3*r4; - } - else - { - dataLength = r1*r2*r3*r4*r5; - } - return dataLength; -} /*-------------------------------------------------------------------------*/ /** @@ -160,72 +108,34 @@ size_t computeDataLength(size_t r5, size_t r4, size_t r3, size_t r2, size_t r1) **/ /*-------------------------------------------------------------------------*/ -unsigned char* SZ_compress_args(int dataType, void *data, size_t *outSize, int errBoundMode, double absErrBound, -double relBoundRatio, double pwrBoundRatio, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1) + +// +// compress output data to outData and return outSize +// +size_t SZ_compress_args(int dataType, void *data, size_t r1, unsigned char* outData, sz_params* params) { - if(confparams_cpr == NULL) - SZ_Init(NULL); - else if(exe_params == NULL) - { - exe_params = (sz_exedata*)malloc(sizeof(sz_exedata)); - memset(exe_params, 0, sizeof(sz_exedata)); - } - if(exe_params->intvCapacity == 0) - { - exe_params->intvCapacity = confparams_cpr->maxRangeRadius*2; - exe_params->intvRadius = confparams_cpr->maxRangeRadius; - exe_params->optQuantMode = 1; - } - - confparams_cpr->dataType = dataType; + size_t outSize = 0; if(dataType==SZ_FLOAT) { - unsigned char *newByteData = NULL; - - SZ_compress_args_float(-1, confparams_cpr->withRegression, &newByteData, (float *)data, r5, r4, r3, r2, r1, - outSize, errBoundMode, absErrBound, relBoundRatio, pwrBoundRatio); - - return newByteData; + SZ_compress_args_float((float *)data, r1, outData, &outSize, params); } else if(dataType==SZ_DOUBLE) { - unsigned char *newByteData; - SZ_compress_args_double(-1, confparams_cpr->withRegression, &newByteData, (double *)data, r5, r4, r3, r2, r1, - outSize, errBoundMode, absErrBound, relBoundRatio, pwrBoundRatio); - - return newByteData; + SZ_compress_args_double((double *)data, r1, outData, &outSize, params); } else { - printf("Error: dataType can only be SZ_FLOAT, SZ_DOUBLE, SZ_INT8/16/32/64 or SZ_UINT8/16/32/64.\n"); - return NULL; + printf("Error: dataType can only be SZ_FLOAT, SZ_DOUBLE .\n"); + return 0; } + + return outSize; } - - -unsigned char *SZ_compress(int dataType, void *data, size_t *outSize, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1) -{ - unsigned char *newByteData = SZ_compress_args(dataType, data, outSize, confparams_cpr->errorBoundMode, confparams_cpr->absErrBound, confparams_cpr->relBoundRatio, - confparams_cpr->pw_relBoundRatio, r5, r4, r3, r2, r1); - return newByteData; -} - -////////////////// -/*-------------------------------------------------------------------------*/ -/** - @brief Perform Compression - @param data data to be compressed - @param reservedValue the reserved value - @param outSize the size (in bytes) after compression - @param r5,r4,r3,r2,r1 the sizes of each dimension (supporting only 5 dimensions at most in this version. - @return compressed data (in binary stream) - - **/ -/*-------------------------------------------------------------------------*/ - - -void *SZ_decompress(int dataType, unsigned char *bytes, size_t byteLength, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1) +// +// decompress output data to outData and return outSize +// +size_t SZ_decompress(int dataType, unsigned char *bytes, size_t byteLength, size_t r1, unsigned char* outData) { sz_exedata de_exe; memset(&de_exe, 0, sizeof(sz_exedata)); @@ -234,80 +144,37 @@ void *SZ_decompress(int dataType, unsigned char *bytes, size_t byteLength, size_ sz_params de_params; memset(&de_params, 0, sizeof(sz_params)); - - int x = 1; - char *y = (char*)&x; - if(*y==1) - sysEndianType = LITTLE_ENDIAN_SYSTEM; - else //=0 - sysEndianType = BIG_ENDIAN_SYSTEM; - + size_t outSize = 0; + if(dataType == SZ_FLOAT) { - float *newFloatData; - SZ_decompress_args_float(&newFloatData, r5, r4, r3, r2, r1, bytes, byteLength, 0, NULL, &de_exe, &de_params); - return newFloatData; + int status = SZ_decompress_args_float((float*)outData, r1, bytes, byteLength, 0, NULL, &de_exe, &de_params); + if(status == SZ_SUCCESS) + { + return r1*sizeof(float); + } + return 0; } else if(dataType == SZ_DOUBLE) - { - double *newDoubleData; - SZ_decompress_args_double(&newDoubleData, r5, r4, r3, r2, r1, bytes, byteLength, 0, NULL, &de_exe, &de_params); - return newDoubleData; + { + int status = SZ_decompress_args_double((double*)outData, r1, bytes, byteLength, 0, NULL, &de_exe, &de_params); + if(status == SZ_SUCCESS) + { + return r1*sizeof(double); + } + return 0; } else { - printf("Error: data type cannot be the types other than SZ_FLOAT or SZ_DOUBLE\n"); - return NULL; + printf("Error: data type cannot be the types other than SZ_FLOAT or SZ_DOUBLE\n"); } + + return outSize; } -/** - * - * - * return number of elements or -1 if any errors - * */ -size_t SZ_decompress_args(int dataType, unsigned char *bytes, size_t byteLength, void* decompressed_array, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1) -{ - //size_t i; - size_t nbEle = computeDataLength(r5,r4,r3,r2,r1); - - if(dataType == SZ_FLOAT) - { - float* data = (float *)SZ_decompress(dataType, bytes, byteLength, r5, r4, r3, r2, r1); - float* data_array = (float *)decompressed_array; - memcpy(data_array, data, nbEle*sizeof(float)); - //for(i=0;imetadata_file!=NULL) -// fclose(sz_tsc->metadata_file); -//#endif } - -/** - * - * Inits the compressor for SZ_compress_customize - * - * with SZ_Init(NULL) if not previously initialized and no params passed - * with SZ_InitParam(userPara) otherwise if params are passed - * and doesn't not initialize otherwise - * - * @param sz_params* userPara : the user configuration or null - * @param sz_params* confparams : the current configuration - */ -static void sz_maybe_init_with_user_params(struct sz_params* userPara, struct sz_params* current_params) { - if(userPara==NULL && current_params == NULL) - SZ_Init(NULL); - else if(userPara != NULL) - SZ_Init_Params((sz_params*)userPara); -} - - -/** - * - * The interface for the user-customized compression method - * - * @param char* comprName : the name of the specific compression approach - * @param void* userPara : the pointer of the user-customized data stracture containing the cusotmized compressors' requried input parameters - * @param int dataType : data type (SZ_FLOAT, SZ_DOUBLE, SZ_INT8, SZ_UINT8, SZ_INT16, SZ_UINT16, ....) - * @param void* data : input dataset - * @param size_t r5 : the size of dimension 5 - * @param size_t r4 : the size of dimension 4 - * @param size_t r3 : the size of dimension 3 - * @param size_t r2 : the size of dimension 2 - * @param size_t r1 : the size of dimension 1 - * @param size_t outSize : the number of bytes after compression - * @param int *status : the execution status of the compression operation (success: SZ_SCES or fail: SZ_NSCS) - * - * */ -unsigned char* SZ_compress_customize(const char* cmprName, void* userPara, int dataType, void* data, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize, int *status) -{ - unsigned char* result = NULL; - if(strcmp(cmprName, "SZ2.0")==0 || strcmp(cmprName, "SZ2.1")==0 || strcmp(cmprName, "SZ")==0) - { - sz_maybe_init_with_user_params(userPara, confparams_cpr); - result = SZ_compress(dataType, data, outSize, r5, r4, r3, r2, r1); - *status = SZ_SCES; - } - else if(strcmp(cmprName, "SZ1.4")==0) - { - sz_maybe_init_with_user_params(userPara, confparams_cpr); - confparams_cpr->withRegression = SZ_NO_REGRESSION; - - result = SZ_compress(dataType, data, outSize, r5, r4, r3, r2, r1); - *status = SZ_SCES; - } - else if(strcmp(cmprName, "SZ_Transpose")==0) - { - void* transData = transposeData(data, dataType, r5, r4, r3, r2, r1); - sz_maybe_init_with_user_params(userPara, confparams_cpr); - size_t n = computeDataLength(r5, r4, r3, r2, r1); - result = SZ_compress(dataType, transData, outSize, 0, 0, 0, 0, n); - } - else if(strcmp(cmprName, "ExaFEL")==0){ - assert(dataType==SZ_FLOAT); - assert(r5==0); - result = exafelSZ_Compress(userPara,data, r4, r3, r2, r1,outSize); - *status = SZ_SCES; - } - else - { - *status = SZ_NSCS; - } - return result; -} - -unsigned char* SZ_compress_customize_threadsafe(const char* cmprName, void* userPara, int dataType, void* data, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize, int *status) -{ - unsigned char* result = NULL; - if(strcmp(cmprName, "SZ2.0")==0 || strcmp(cmprName, "SZ2.1")==0 || strcmp(cmprName, "SZ")==0) - { - SZ_Init(NULL); - struct sz_params* para = (struct sz_params*)userPara; - - if(dataType==SZ_FLOAT) - { - SZ_compress_args_float(-1, SZ_WITH_LINEAR_REGRESSION, &result, (float *)data, r5, r4, r3, r2, r1, - outSize, para->errorBoundMode, para->absErrBound, para->relBoundRatio, para->pw_relBoundRatio); - } - else if(dataType==SZ_DOUBLE) - { - SZ_compress_args_double(-1, SZ_WITH_LINEAR_REGRESSION, &result, (double *)data, r5, r4, r3, r2, r1, - outSize, para->errorBoundMode, para->absErrBound, para->relBoundRatio, para->pw_relBoundRatio); - } - - *status = SZ_SCES; - return result; - } - else if(strcmp(cmprName, "SZ1.4")==0) - { - SZ_Init(NULL); - struct sz_params* para = (struct sz_params*)userPara; - - if(dataType==SZ_FLOAT) - { - SZ_compress_args_float(-1, SZ_NO_REGRESSION, &result, (float *)data, r5, r4, r3, r2, r1, - outSize, para->errorBoundMode, para->absErrBound, para->relBoundRatio, para->pw_relBoundRatio); - } - else if(dataType==SZ_DOUBLE) - { - SZ_compress_args_double(-1, SZ_NO_REGRESSION, &result, (double *)data, r5, r4, r3, r2, r1, - outSize, para->errorBoundMode, para->absErrBound, para->relBoundRatio, para->pw_relBoundRatio); - } - - *status = SZ_SCES; - return result; - } - else if(strcmp(cmprName, "SZ_Transpose")==0) - { - void* transData = transposeData(data, dataType, r5, r4, r3, r2, r1); - struct sz_params* para = (struct sz_params*)userPara; - - size_t n = computeDataLength(r5, r4, r3, r2, r1); - - result = SZ_compress_args(dataType, transData, outSize, para->errorBoundMode, para->absErrBound, para->relBoundRatio, para->pw_relBoundRatio, 0, 0, 0, 0, n); - - *status = SZ_SCES; - } - else if(strcmp(cmprName, "ExaFEL")==0){ //not sure if this part is thread safe! - assert(dataType==SZ_FLOAT); - assert(r5==0); - result = exafelSZ_Compress(userPara,data, r4, r3, r2, r1,outSize); - *status = SZ_SCES; - } - else - { - *status = SZ_NSCS; - } - return result; -} - - -/** - * - * The interface for the user-customized decompression method - * - * @param char* comprName : the name of the specific compression approach - * @param void* userPara : the pointer of the user-customized data stracture containing the cusotmized compressors' requried input parameters - * @param int dataType : data type (SZ_FLOAT, SZ_DOUBLE, SZ_INT8, SZ_UINT8, SZ_INT16, SZ_UINT16, ....) - * @param unsigned char* bytes : input bytes (the compressed data) - * @param size_t r5 : the size of dimension 5 - * @param size_t r4 : the size of dimension 4 - * @param size_t r3 : the size of dimension 3 - * @param size_t r2 : the size of dimension 2 - * @param size_t r1 : the size of dimension 1 - * @param int *status : the execution status of the compression operation (success: SZ_SCES or fail: SZ_NSCS) - * - * */ -void* SZ_decompress_customize(const char* cmprName, void* userPara, int dataType, unsigned char* bytes, size_t byteLength, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, int *status) -{ - void* result = NULL; - if(strcmp(cmprName, "SZ2.0")==0 || strcmp(cmprName, "SZ")==0 || strcmp(cmprName, "SZ1.4")==0) - { - result = SZ_decompress(dataType, bytes, byteLength, r5, r4, r3, r2, r1); - * status = SZ_SCES; - } - else if(strcmp(cmprName, "SZ_Transpose")==0) - { - size_t n = computeDataLength(r5, r4, r3, r2, r1); - void* tmpData = SZ_decompress(dataType, bytes, byteLength, 0, 0, 0, 0, n); - result = detransposeData(tmpData, dataType, r5, r4, r3, r2, r1); - } - else if(strcmp(cmprName, "ExaFEL")==0){ - assert(dataType==SZ_FLOAT); - assert(r5==0); - result = exafelSZ_Decompress(userPara,bytes, r4, r3, r2, r1,byteLength); - *status = SZ_SCES; - } - else - { - *status = SZ_NSCS; - } - return result; -} - - -void* SZ_decompress_customize_threadsafe(const char* cmprName, void* userPara, int dataType, unsigned char* bytes, size_t byteLength, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, int *status) -{ - return SZ_decompress_customize(cmprName, userPara, dataType, bytes, byteLength, r5, r4, r3, r2, r1, status); -} diff --git a/deps/SZ/sz/src/sz_double.c b/deps/SZ/sz/src/sz_double.c index c1bc32d70d..088568fdfa 100644 --- a/deps/SZ/sz/src/sz_double.c +++ b/deps/SZ/sz/src/sz_double.c @@ -100,169 +100,9 @@ unsigned int optimize_intervals_double_1D(double *oriData, size_t dataLength, do return powerOf2; } -unsigned int optimize_intervals_double_2D(double *oriData, size_t r1, size_t r2, double realPrecision) -{ - size_t i,j, index; - size_t radiusIndex; - double pred_value = 0, pred_err; - size_t *intervals = (size_t*)malloc(confparams_cpr->maxRangeRadius*sizeof(size_t)); - memset(intervals, 0, confparams_cpr->maxRangeRadius*sizeof(size_t)); - size_t totalSampleSize = (r1-1)*(r2-1)/confparams_cpr->sampleDistance; - for(i=1;isampleDistance==0) - { - index = i*r2+j; - pred_value = oriData[index-1] + oriData[index-r2] - oriData[index-r2-1]; - pred_err = fabs(pred_value - oriData[index]); - radiusIndex = (unsigned long)((pred_err/realPrecision+1)/2); - if(radiusIndex>=confparams_cpr->maxRangeRadius) - radiusIndex = confparams_cpr->maxRangeRadius - 1; - intervals[radiusIndex]++; - } - } - } - //compute the appropriate number - size_t targetCount = totalSampleSize*confparams_cpr->predThreshold; - size_t sum = 0; - for(i=0;imaxRangeRadius;i++) - { - sum += intervals[i]; - if(sum>targetCount) - break; - } - if(i>=confparams_cpr->maxRangeRadius) - i = confparams_cpr->maxRangeRadius-1; - unsigned int accIntervals = 2*(i+1); - unsigned int powerOf2 = roundUpToPowerOf2(accIntervals); - //printf("confparams_cpr->maxRangeRadius = %d, accIntervals=%d, powerOf2=%d\n", confparams_cpr->maxRangeRadius, accIntervals, powerOf2); - - if(powerOf2<32) - powerOf2 = 32; - - free(intervals); - return powerOf2; -} - -unsigned int optimize_intervals_double_3D(double *oriData, size_t r1, size_t r2, size_t r3, double realPrecision) -{ - size_t i,j,k, index; - size_t radiusIndex; - size_t r23=r2*r3; - double pred_value = 0, pred_err; - size_t *intervals = (size_t*)malloc(confparams_cpr->maxRangeRadius*sizeof(size_t)); - memset(intervals, 0, confparams_cpr->maxRangeRadius*sizeof(size_t)); - size_t totalSampleSize = (r1-1)*(r2-1)*(r3-1)/confparams_cpr->sampleDistance; - for(i=1;isampleDistance==0) - { - index = i*r23+j*r3+k; - pred_value = oriData[index-1] + oriData[index-r3] + oriData[index-r23] - - oriData[index-1-r23] - oriData[index-r3-1] - oriData[index-r3-r23] + oriData[index-r3-r23-1]; - pred_err = fabs(pred_value - oriData[index]); - radiusIndex = (pred_err/realPrecision+1)/2; - if(radiusIndex>=confparams_cpr->maxRangeRadius) - radiusIndex = confparams_cpr->maxRangeRadius - 1; - intervals[radiusIndex]++; - } - } - - } - } - //compute the appropriate number - size_t targetCount = totalSampleSize*confparams_cpr->predThreshold; - size_t sum = 0; - for(i=0;imaxRangeRadius;i++) - { - sum += intervals[i]; - if(sum>targetCount) - break; - } - if(i>=confparams_cpr->maxRangeRadius) - i = confparams_cpr->maxRangeRadius-1; - - unsigned int accIntervals = 2*(i+1); - unsigned int powerOf2 = roundUpToPowerOf2(accIntervals); - - if(powerOf2<32) - powerOf2 = 32; - - free(intervals); - //printf("confparams_cpr->maxRangeRadius = %d, accIntervals=%d, powerOf2=%d\n", confparams_cpr->maxRangeRadius, accIntervals, powerOf2); - return powerOf2; -} - -unsigned int optimize_intervals_double_4D(double *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision) -{ - size_t i,j,k,l, index; - size_t radiusIndex; - size_t r234=r2*r3*r4; - size_t r34=r3*r4; - double pred_value = 0, pred_err; - size_t *intervals = (size_t*)malloc(confparams_cpr->maxRangeRadius*sizeof(size_t)); - memset(intervals, 0, confparams_cpr->maxRangeRadius*sizeof(size_t)); - size_t totalSampleSize = (r1-1)*(r2-1)*(r3-1)*(r4-1)/confparams_cpr->sampleDistance; - for(i=1;isampleDistance==0) - { - index = i*r234+j*r34+k*r4+l; - pred_value = oriData[index-1] + oriData[index-r3] + oriData[index-r34] - - oriData[index-1-r34] - oriData[index-r4-1] - oriData[index-r4-r34] + oriData[index-r4-r34-1]; - pred_err = fabs(pred_value - oriData[index]); - radiusIndex = (unsigned long)((pred_err/realPrecision+1)/2); - if(radiusIndex>=confparams_cpr->maxRangeRadius) - radiusIndex = confparams_cpr->maxRangeRadius - 1; - intervals[radiusIndex]++; - } - } - } - } - } - //compute the appropriate number - size_t targetCount = totalSampleSize*confparams_cpr->predThreshold; - size_t sum = 0; - for(i=0;imaxRangeRadius;i++) - { - sum += intervals[i]; - if(sum>targetCount) - break; - } - if(i>=confparams_cpr->maxRangeRadius) - i = confparams_cpr->maxRangeRadius-1; - - unsigned int accIntervals = 2*(i+1); - unsigned int powerOf2 = roundUpToPowerOf2(accIntervals); - - if(powerOf2<32) - powerOf2 = 32; - - free(intervals); - return powerOf2; -} - TightDataPointStorageD* SZ_compress_double_1D_MDQ(double *oriData, size_t dataLength, double realPrecision, double valueRangeSize, double medianValue_d) { -#ifdef HAVE_TIMECMPR - double* decData = NULL; - if(confparams_cpr->szMode == SZ_TEMPORAL_COMPRESSION) - decData = (double*)(multisteps->hist_data); -#endif - unsigned int quantization_intervals; if(exe_params->optQuantMode==1) quantization_intervals = optimize_intervals_double_1D_opt(oriData, dataLength, realPrecision); @@ -307,11 +147,7 @@ size_t dataLength, double realPrecision, double valueRangeSize, double medianVal updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce); memcpy(preDataBytes,vce->curBytes,8); addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce); - listAdd_double(last3CmprsData, vce->data); -#ifdef HAVE_TIMECMPR - if(confparams_cpr->szMode == SZ_TEMPORAL_COMPRESSION) - decData[0] = vce->data; -#endif + listAdd_double(last3CmprsData, vce->data); //add the second data type[1] = 0; @@ -320,10 +156,7 @@ size_t dataLength, double realPrecision, double valueRangeSize, double medianVal memcpy(preDataBytes,vce->curBytes,8); addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce); listAdd_double(last3CmprsData, vce->data); -#ifdef HAVE_TIMECMPR - if(confparams_cpr->szMode == SZ_TEMPORAL_COMPRESSION) - decData[1] = vce->data; -#endif + int state; double checkRadius; double curData; @@ -354,10 +187,6 @@ size_t dataLength, double realPrecision, double valueRangeSize, double medianVal pred = pred - state*interval; } //listAdd_double(last3CmprsData, pred); -#ifdef HAVE_TIMECMPR - if(confparams_cpr->szMode == SZ_TEMPORAL_COMPRESSION) - decData[i] = pred; -#endif continue; } @@ -370,11 +199,7 @@ size_t dataLength, double realPrecision, double valueRangeSize, double medianVal //listAdd_double(last3CmprsData, vce->data); pred = vce->data; - -#ifdef HAVE_TIMECMPR - if(confparams_cpr->szMode == SZ_TEMPORAL_COMPRESSION) - decData[i] = vce->data; -#endif + }//end of for @@ -403,7 +228,7 @@ size_t dataLength, double realPrecision, double valueRangeSize, double medianVal return tdps; } -void SZ_compress_args_double_StoreOriData(double* oriData, size_t dataLength, unsigned char** newByteData, size_t *outSize) +void SZ_compress_args_double_StoreOriData(double* oriData, size_t dataLength, unsigned char* newByteData, size_t *outSize) { int doubleSize = sizeof(double); size_t k = 0, i; @@ -413,25 +238,25 @@ void SZ_compress_args_double_StoreOriData(double* oriData, size_t dataLength, un unsigned char dsLengthBytes[8]; for (i = 0; i < 3; i++)//3 - (*newByteData)[k++] = versionNumber[i]; + newByteData[k++] = versionNumber[i]; if(exe_params->SZ_SIZE_TYPE==4)//1 - (*newByteData)[k++] = 16; //00010000 + newByteData[k++] = 16; //00010000 else - (*newByteData)[k++] = 80; //01010000: 01000000 indicates the SZ_SIZE_TYPE=8 + newByteData[k++] = 80; //01010000: 01000000 indicates the SZ_SIZE_TYPE=8 - convertSZParamsToBytes(confparams_cpr, &((*newByteData)[k])); + convertSZParamsToBytes(confparams_cpr, &(newByteData[k])); k = k + MetaDataByteLength_double; sizeToBytes(dsLengthBytes,dataLength); for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)//ST: 4 or 8 - (*newByteData)[k++] = dsLengthBytes[i]; + newByteData[k++] = dsLengthBytes[i]; if(sysEndianType==BIG_ENDIAN_SYSTEM) - memcpy((*newByteData)+4+MetaDataByteLength_double+exe_params->SZ_SIZE_TYPE, oriData, dataLength*doubleSize); + memcpy(newByteData+4+MetaDataByteLength_double+exe_params->SZ_SIZE_TYPE, oriData, dataLength*doubleSize); else { - unsigned char* p = (*newByteData)+4+MetaDataByteLength_double+exe_params->SZ_SIZE_TYPE; + unsigned char* p = newByteData+4+MetaDataByteLength_double+exe_params->SZ_SIZE_TYPE; for(i=0;iszMode == SZ_TEMPORAL_COMPRESSION) - { - int timestep = sz_tsc->currentStep; - if(cmprType == SZ_PERIO_TEMPORAL_COMPRESSION) - { - if(timestep % confparams_cpr->snapshotCmprStep != 0) - { - tdps = SZ_compress_double_1D_MDQ_ts(oriData, dataLength, multisteps, realPrecision, valueRangeSize, medianValue_d); - compressionType = 1; //time-series based compression - } - else - { - tdps = SZ_compress_double_1D_MDQ(oriData, dataLength, realPrecision, valueRangeSize, medianValue_d); - compressionType = 0; //snapshot-based compression - multisteps->lastSnapshotStep = timestep; - } - } - else if(cmprType == SZ_FORCE_SNAPSHOT_COMPRESSION) - { - tdps = SZ_compress_double_1D_MDQ(oriData, dataLength, realPrecision, valueRangeSize, medianValue_d); - compressionType = 0; //snapshot-based compression - multisteps->lastSnapshotStep = timestep; - } - else if(cmprType == SZ_FORCE_TEMPORAL_COMPRESSION) - { - tdps = SZ_compress_double_1D_MDQ_ts(oriData, dataLength, multisteps, realPrecision, valueRangeSize, medianValue_d); - compressionType = 1; //time-series based compression - } - - } - else -#endif - tdps = SZ_compress_double_1D_MDQ(oriData, dataLength, realPrecision, valueRangeSize, medianValue_d); + tdps = SZ_compress_double_1D_MDQ(oriData, dataLength, realPrecision, valueRangeSize, medianValue_d); convertTDPStoFlatBytes_double(tdps, newByteData, outSize); @@ -492,11 +284,6 @@ size_t dataLength, double realPrecision, size_t *outSize, double valueRangeSize, TightDataPointStorageD* SZ_compress_double_1D_MDQ_MSST19(double *oriData, size_t dataLength, double realPrecision, double valueRangeSize, double medianValue_f) { -#ifdef HAVE_TIMECMPR - double* decData = NULL; - if(confparams_cpr->szMode == SZ_TEMPORAL_COMPRESSION) - decData = (double*)(multisteps->hist_data); -#endif //struct ClockPoint clockPointBuild; //TimeDurationStart("build", &clockPointBuild); @@ -558,11 +345,7 @@ size_t dataLength, double realPrecision, double valueRangeSize, double medianVal memcpy(preDataBytes,vce->curBytes,8); addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce); listAdd_double(last3CmprsData, vce->data); - //miss++; -#ifdef HAVE_TIMECMPR - if(confparams_cpr->szMode == SZ_TEMPORAL_COMPRESSION) - decData[0] = vce->data; -#endif + //miss++; //add the second data type[1] = 0; @@ -572,10 +355,7 @@ size_t dataLength, double realPrecision, double valueRangeSize, double medianVal addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce); listAdd_double(last3CmprsData, vce->data); //miss++; -#ifdef HAVE_TIMECMPR - if(confparams_cpr->szMode == SZ_TEMPORAL_COMPRESSION) - decData[1] = vce->data; -#endif + int state; //double checkRadius; double curData; @@ -623,10 +403,6 @@ size_t dataLength, double realPrecision, double valueRangeSize, double medianVal addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce); pred = vce->data; //miss++; -#ifdef HAVE_TIMECMPR - if(confparams_cpr->szMode == SZ_TEMPORAL_COMPRESSION) - decData[i] = vce->data; -#endif }//end of for @@ -657,7 +433,7 @@ size_t dataLength, double realPrecision, double valueRangeSize, double medianVal } -void SZ_compress_args_double_withinRange(unsigned char** newByteData, double *oriData, size_t dataLength, size_t *outSize) +void SZ_compress_args_double_withinRange(unsigned char* newByteData, double *oriData, size_t dataLength, size_t *outSize) { TightDataPointStorageD* tdps = (TightDataPointStorageD*) malloc(sizeof(TightDataPointStorageD)); tdps->rtypeArray = NULL; @@ -685,35 +461,30 @@ void SZ_compress_args_double_withinRange(unsigned char** newByteData, double *or free_TightDataPointStorageD(tdps); } -int SZ_compress_args_double(int cmprType, int withRegression, unsigned char** newByteData, double *oriData, -size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize, -int errBoundMode, double absErr_Bound, double relBoundRatio, double pwRelBoundRatio) -{ - confparams_cpr->errorBoundMode = errBoundMode; - if(errBoundMode==PW_REL) - { - confparams_cpr->pw_relBoundRatio = pwRelBoundRatio; - } - - int status = SZ_SCES; - size_t dataLength = computeDataLength(r5,r4,r3,r2,r1); - - if(dataLength <= MIN_NUM_OF_ELEMENTS) - { - *newByteData = SZ_skip_compress_double(oriData, dataLength, outSize); - return status; - } +int SZ_compress_args_double(double *oriData, size_t r1, unsigned char* newByteData, size_t *outSize, sz_params* params) +{ + int status = SZ_SUCCESS; + size_t dataLength = r1; + double valueRangeSize = 0, medianValue = 0; unsigned char * signs = NULL; bool positive = true; double nearZero = 0.0; double min = 0; - if(pwRelBoundRatio < 0.000009999) - confparams_cpr->accelerate_pw_rel_compression = 0; + if(params->pw_relBoundRatio < 0.000009999) + params->accelerate_pw_rel_compression = 0; + + // check at least elements count + if(dataLength <= MIN_NUM_OF_ELEMENTS) + { + printf("error, double input elements count=%d less than %d, so need not do compress.\n", dataLength, MIN_NUM_OF_ELEMENTS); + return SZ_LITTER_ELEMENT; + } + - if(confparams_cpr->errorBoundMode == PW_REL && confparams_cpr->accelerate_pw_rel_compression == 1) + if(params->errorBoundMode == PW_REL && params->accelerate_pw_rel_compression == 1) { signs = (unsigned char *) malloc(dataLength); memset(signs, 0, dataLength); @@ -722,81 +493,72 @@ int errBoundMode, double absErr_Bound, double relBoundRatio, double pwRelBoundRa else min = computeRangeSize_double(oriData, dataLength, &valueRangeSize, &medianValue); double max = min+valueRangeSize; - confparams_cpr->dmin = min; - confparams_cpr->dmax = max; + params->dmin = min; + params->dmax = max; double realPrecision = 0; - if(confparams_cpr->errorBoundMode==PSNR) + if(params->errorBoundMode==PSNR) { - confparams_cpr->errorBoundMode = SZ_ABS; - realPrecision = confparams_cpr->absErrBound = computeABSErrBoundFromPSNR(confparams_cpr->psnr, (double)confparams_cpr->predThreshold, valueRangeSize); + params->errorBoundMode = SZ_ABS; + realPrecision = params->absErrBound = computeABSErrBoundFromPSNR(params->psnr, (double)params->predThreshold, valueRangeSize); } - else if(confparams_cpr->errorBoundMode==NORM) //norm error = sqrt(sum((xi-xi_)^2)) + else if(params->errorBoundMode==NORM) //norm error = sqrt(sum((xi-xi_)^2)) { - confparams_cpr->errorBoundMode = SZ_ABS; - realPrecision = confparams_cpr->absErrBound = computeABSErrBoundFromNORM_ERR(confparams_cpr->normErr, dataLength); + params->errorBoundMode = SZ_ABS; + realPrecision = params->absErrBound = computeABSErrBoundFromNORM_ERR(params->normErr, dataLength); //printf("realPrecision=%lf\n", realPrecision); } else { - realPrecision = getRealPrecision_double(valueRangeSize, errBoundMode, absErr_Bound, relBoundRatio, &status); - confparams_cpr->absErrBound = realPrecision; + realPrecision = getRealPrecision_double(valueRangeSize, params->errorBoundMode, params->absErrBound, params->relBoundRatio, &status); + params->absErrBound = realPrecision; } if(valueRangeSize <= realPrecision) { - if(confparams_cpr->errorBoundMode>=PW_REL && confparams_cpr->accelerate_pw_rel_compression == 1) + if(params->errorBoundMode>=PW_REL && params->accelerate_pw_rel_compression == 1) free(signs); SZ_compress_args_double_withinRange(newByteData, oriData, dataLength, outSize); } else { size_t tmpOutSize = 0; - unsigned char* tmpByteData; - if (r2==0) + unsigned char* tmpByteData = newByteData; + if(params->szMode != SZ_BEST_SPEED) { - if(confparams_cpr->errorBoundMode>=PW_REL) - { - if(confparams_cpr->accelerate_pw_rel_compression && confparams_cpr->maxRangeRadius <= 32768) - SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log_MSST19(&tmpByteData, oriData, pwRelBoundRatio, r1, &tmpOutSize, valueRangeSize, medianValue, signs, &positive, min, max, nearZero); - else - SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log(&tmpByteData, oriData, pwRelBoundRatio, r1, &tmpOutSize, min, max); - //SZ_compress_args_double_NoCkRngeNoGzip_1D_pwrgroup(&tmpByteData, oriData, r1, absErr_Bound, relBoundRatio, pwRelBoundRatio, valueRangeSize, medianValue, &tmpOutSize); - } + tmpByteData = (unsigned char*)malloc(r1*sizeof(double)*1.2); + } + + if(params->errorBoundMode>=PW_REL) + { + if(params->accelerate_pw_rel_compression && params->maxRangeRadius <= 32768) + SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log_MSST19(tmpByteData, oriData, params->pw_relBoundRatio, r1, &tmpOutSize, valueRangeSize, medianValue, signs, &positive, min, max, nearZero); else -#ifdef HAVE_TIMECMPR - if(confparams_cpr->szMode == SZ_TEMPORAL_COMPRESSION) - multisteps->compressionType = SZ_compress_args_double_NoCkRngeNoGzip_1D(cmprType, &tmpByteData, oriData, r1, realPrecision, &tmpOutSize, valueRangeSize, medianValue); - else -#endif - { - SZ_compress_args_double_NoCkRngeNoGzip_1D(cmprType, &tmpByteData, oriData, r1, realPrecision, &tmpOutSize, valueRangeSize, medianValue); - if(tmpOutSize>=dataLength*sizeof(double) + 3 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + 1) - SZ_compress_args_double_StoreOriData(oriData, dataLength, &tmpByteData, &tmpOutSize); - } + SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log(tmpByteData, oriData, params->pw_relBoundRatio, r1, &tmpOutSize, min, max); } else { - printf("Error: doesn't support 5 dimensions for now.\n"); - status = SZ_DERR; + SZ_compress_args_double_NoCkRngeNoGzip_1D(tmpByteData, oriData, r1, realPrecision, &tmpOutSize, valueRangeSize, medianValue); + if(tmpOutSize>=dataLength*sizeof(double) + 3 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + 1) + SZ_compress_args_double_StoreOriData(oriData, dataLength, tmpByteData, &tmpOutSize); } - + // //Call Gzip to do the further compression. // - if(confparams_cpr->szMode==SZ_BEST_SPEED) + if(params->szMode==SZ_BEST_SPEED) { *outSize = tmpOutSize; - *newByteData = tmpByteData; + //*newByteData = tmpByteData; } - else if(confparams_cpr->szMode==SZ_BEST_COMPRESSION || confparams_cpr->szMode==SZ_DEFAULT_COMPRESSION || confparams_cpr->szMode==SZ_TEMPORAL_COMPRESSION) + else if(params->szMode==SZ_BEST_COMPRESSION || params->szMode==SZ_DEFAULT_COMPRESSION || params->szMode==SZ_TEMPORAL_COMPRESSION) { - *outSize = sz_lossless_compress(confparams_cpr->losslessCompressor, confparams_cpr->gzipMode, tmpByteData, tmpOutSize, newByteData); + *outSize = sz_lossless_compress(params->losslessCompressor, params->gzipMode, tmpByteData, tmpOutSize, newByteData); free(tmpByteData); } else { - printf("Error: Wrong setting of confparams_cpr->szMode in the double compression.\n"); + printf("Error: Wrong setting of params->szMode in the double compression.\n"); status = SZ_MERR; } } @@ -804,245 +566,6 @@ int errBoundMode, double absErr_Bound, double relBoundRatio, double pwRelBoundRa return status; } -//TODO -int SZ_compress_args_double_subblock(unsigned char* compressedBytes, double *oriData, -size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, -size_t s5, size_t s4, size_t s3, size_t s2, size_t s1, -size_t e5, size_t e4, size_t e3, size_t e2, size_t e1, -size_t *outSize, int errBoundMode, double absErr_Bound, double relBoundRatio) -{ - int status = SZ_SCES; - double valueRangeSize = 0, medianValue = 0; - computeRangeSize_double_subblock(oriData, &valueRangeSize, &medianValue, r5, r4, r3, r2, r1, s5, s4, s3, s2, s1, e5, e4, e3, e2, e1); - - double realPrecision = getRealPrecision_double(valueRangeSize, errBoundMode, absErr_Bound, relBoundRatio, &status); - - if(valueRangeSize <= realPrecision) - { - //TODO - //SZ_compress_args_double_withinRange_subblock(); - } - else - { - if (r2==0) - { - //TODO - if(errBoundMode==PW_REL) - { - //TODO - //SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_subblock(); - printf ("Current subblock version does not support point-wise relative error bound.\n"); - } - else - SZ_compress_args_double_NoCkRnge_1D_subblock(compressedBytes, oriData, realPrecision, outSize, valueRangeSize, medianValue, r1, s1, e1); - } - else - { - printf("Error: doesn't support 5 dimensions for now.\n"); - status = SZ_DERR; //dimension error - } - } - return status; -} - -void SZ_compress_args_double_NoCkRnge_1D_subblock(unsigned char* compressedBytes, double *oriData, double realPrecision, size_t *outSize, double valueRangeSize, double medianValue_d, -size_t r1, size_t s1, size_t e1) -{ - TightDataPointStorageD* tdps = SZ_compress_double_1D_MDQ_subblock(oriData, realPrecision, valueRangeSize, medianValue_d, r1, s1, e1); - - if (confparams_cpr->szMode==SZ_BEST_SPEED) - convertTDPStoFlatBytes_double_args(tdps, compressedBytes, outSize); - else if(confparams_cpr->szMode==SZ_BEST_COMPRESSION || confparams_cpr->szMode==SZ_DEFAULT_COMPRESSION) - { - unsigned char *tmpCompBytes; - size_t tmpOutSize; - convertTDPStoFlatBytes_double(tdps, &tmpCompBytes, &tmpOutSize); - *outSize = zlib_compress3(tmpCompBytes, tmpOutSize, compressedBytes, confparams_cpr->gzipMode); - free(tmpCompBytes); - } - else - { - printf ("Error: Wrong setting of confparams_cpr->szMode in the double compression.\n"); - } - - //TODO -// if(*outSize>dataLength*sizeof(double)) -// SZ_compress_args_double_StoreOriData(oriData, dataLength, newByteData, outSize); - - free_TightDataPointStorageD(tdps); -} - -unsigned int optimize_intervals_double_1D_subblock(double *oriData, double realPrecision, size_t r1, size_t s1, size_t e1) -{ - size_t dataLength = e1 - s1 + 1; - oriData = oriData + s1; - - size_t i = 0; - unsigned long radiusIndex; - double pred_value = 0, pred_err; - int *intervals = (int*)malloc(confparams_cpr->maxRangeRadius*sizeof(int)); - memset(intervals, 0, confparams_cpr->maxRangeRadius*sizeof(int)); - size_t totalSampleSize = dataLength/confparams_cpr->sampleDistance; - for(i=2;isampleDistance==0) - { - pred_value = 2*oriData[i-1] - oriData[i-2]; - //pred_value = oriData[i-1]; - pred_err = fabs(pred_value - oriData[i]); - radiusIndex = (unsigned long)((pred_err/realPrecision+1)/2); - if(radiusIndex>=confparams_cpr->maxRangeRadius) - radiusIndex = confparams_cpr->maxRangeRadius - 1; - intervals[radiusIndex]++; - } - } - //compute the appropriate number - size_t targetCount = totalSampleSize*confparams_cpr->predThreshold; - size_t sum = 0; - for(i=0;imaxRangeRadius;i++) - { - sum += intervals[i]; - if(sum>targetCount) - break; - } - - if(i>=confparams_cpr->maxRangeRadius) - i = confparams_cpr->maxRangeRadius-1; - unsigned int accIntervals = 2*(i+1); - unsigned int powerOf2 = roundUpToPowerOf2(accIntervals); - - if(powerOf2<32) - powerOf2 = 32; - - free(intervals); - return powerOf2; -} - - -TightDataPointStorageD* SZ_compress_double_1D_MDQ_subblock(double *oriData, double realPrecision, double valueRangeSize, double medianValue_d, -size_t r1, size_t s1, size_t e1) -{ - size_t dataLength = e1 - s1 + 1; - - unsigned int quantization_intervals; - if(exe_params->optQuantMode==1) - quantization_intervals = optimize_intervals_double_1D_subblock(oriData, realPrecision, r1, s1, e1); - else - quantization_intervals = exe_params->intvCapacity; - //updateQuantizationInfo(quantization_intervals); - int intvRadius = quantization_intervals/2; - - size_t i; - int reqLength; - double medianValue = medianValue_d; - short radExpo = getExponent_double(valueRangeSize/2); - - computeReqLength_double(realPrecision, radExpo, &reqLength, &medianValue); - - int* type = (int*) malloc(dataLength*sizeof(int)); - - double* spaceFillingValue = oriData + s1; // - - DynamicIntArray *exactLeadNumArray; - new_DIA(&exactLeadNumArray, DynArrayInitLen); - - DynamicByteArray *exactMidByteArray; - new_DBA(&exactMidByteArray, DynArrayInitLen); - - DynamicIntArray *resiBitArray; - new_DIA(&resiBitArray, DynArrayInitLen); - - type[0] = 0; - - unsigned char preDataBytes[8]; - longToBytes_bigEndian(preDataBytes, 0); - - int reqBytesLength = reqLength/8; - int resiBitsLength = reqLength%8; - double last3CmprsData[3] = {0}; - - DoubleValueCompressElement *vce = (DoubleValueCompressElement*)malloc(sizeof(DoubleValueCompressElement)); - LossyCompressionElement *lce = (LossyCompressionElement*)malloc(sizeof(LossyCompressionElement)); - - //add the first data - compressSingleDoubleValue(vce, spaceFillingValue[0], realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength); - updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce); - memcpy(preDataBytes,vce->curBytes,8); - addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce); - listAdd_double(last3CmprsData, vce->data); - - //add the second data - type[1] = 0; - compressSingleDoubleValue(vce, spaceFillingValue[1], realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength); - updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce); - memcpy(preDataBytes,vce->curBytes,8); - addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce); - listAdd_double(last3CmprsData, vce->data); - - int state; - double checkRadius; - double curData; - double pred; - double predAbsErr; - checkRadius = (quantization_intervals-1)*realPrecision; - double interval = 2*realPrecision; - - for(i=2;i=pred) - { - type[i] = intvRadius+state; - pred = pred + state*interval; - } - else //curDatacurBytes, preDataBytes, reqBytesLength, resiBitsLength, lce); - memcpy(preDataBytes,vce->curBytes,8); - addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce); - - listAdd_double(last3CmprsData, vce->data); - }//end of for - - size_t exactDataNum = exactLeadNumArray->size; - - TightDataPointStorageD* tdps; - - new_TightDataPointStorageD(&tdps, dataLength, exactDataNum, - type, exactMidByteArray->array, exactMidByteArray->size, - exactLeadNumArray->array, - resiBitArray->array, resiBitArray->size, - resiBitsLength, - realPrecision, medianValue, (char)reqLength, quantization_intervals, NULL, 0, 0); - - //free memory - free_DIA(exactLeadNumArray); - free_DIA(resiBitArray); - free(type); - free(vce); - free(lce); - free(exactMidByteArray); //exactMidByteArray->array has been released in free_TightDataPointStorageF(tdps); - - return tdps; -} - unsigned int optimize_intervals_double_1D_opt_MSST19(double *oriData, size_t dataLength, double realPrecision) { diff --git a/deps/SZ/sz/src/sz_double_pwr.c b/deps/SZ/sz/src/sz_double_pwr.c index 78ff9a3da2..6d9b2f837d 100644 --- a/deps/SZ/sz/src/sz_double_pwr.c +++ b/deps/SZ/sz/src/sz_double_pwr.c @@ -17,7 +17,6 @@ #include "sz.h" #include "CompressElement.h" #include "DynamicByteArray.h" -#include "DynamicIntArray.h" #include "TightDataPointStorageD.h" #include "sz_double.h" #include "sz_double_pwr.h" @@ -149,7 +148,7 @@ unsigned int optimize_intervals_double_1D_pwr(double *oriData, size_t dataLength } -void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr(unsigned char** newByteData, double *oriData, double globalPrecision, +void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr(unsigned char* newByteData, double *oriData, double globalPrecision, size_t dataLength, size_t *outSize, double min, double max) { size_t pwrLength = dataLength%confparams_cpr->segment_size==0?dataLength/confparams_cpr->segment_size:dataLength/confparams_cpr->segment_size+1; @@ -327,25 +326,25 @@ size_t dataLength, size_t *outSize, double min, double max) unsigned char dsLengthBytes[exe_params->SZ_SIZE_TYPE]; intToBytes_bigEndian(dsLengthBytes, dataLength);//4 for (i = 0; i < 3; i++)//3 - (*newByteData)[k++] = versionNumber[i]; + newByteData[k++] = versionNumber[i]; if(exe_params->SZ_SIZE_TYPE==4) { - (*newByteData)[k++] = 16; //=00010000 + newByteData[k++] = 16; //=00010000 } else { - (*newByteData)[k++] = 80; + newByteData[k++] = 80; } for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)//4 or 8 - (*newByteData)[k++] = dsLengthBytes[i]; + newByteData[k++] = dsLengthBytes[i]; if(sysEndianType==BIG_ENDIAN_SYSTEM) - memcpy((*newByteData)+4+exe_params->SZ_SIZE_TYPE, oriData, dataLength*doubleSize); + memcpy(newByteData+4+exe_params->SZ_SIZE_TYPE, oriData, dataLength*doubleSize); else { - unsigned char* p = (*newByteData)+4+exe_params->SZ_SIZE_TYPE; + unsigned char* p = newByteData+4+exe_params->SZ_SIZE_TYPE; for(i=0;ierrorBoundMode, @@ -659,7 +658,7 @@ size_t dataLength, double absErrBound, double relBoundRatio, double pwrErrRatio, #include -void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log(unsigned char** newByteData, double *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, double min, double max){ +void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log(unsigned char* newByteData, double *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, double min, double max){ double * log_data = (double *) malloc(dataLength * sizeof(double)); @@ -719,7 +718,7 @@ void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log(unsigned char** newBy free_TightDataPointStorageD(tdps); } -void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log_MSST19(unsigned char** newByteData, double *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, double valueRangeSize, double medianValue_f, +void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log_MSST19(unsigned char* newByteData, double *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, double valueRangeSize, double medianValue_f, unsigned char* signs, bool* positive, double min, double max, double nearZero){ double multiplier = pow((1+pwrErrRatio), -3.0001); for(int i=0; imaxRangeRadius*sizeof(size_t)); - memset(intervals, 0, confparams_cpr->maxRangeRadius*sizeof(size_t)); - size_t totalSampleSize = (r1-1)*(r2-1)/confparams_cpr->sampleDistance; - - //float max = oriData[0]; - //float min = oriData[0]; - - for(i=1;isampleDistance==0) - { - index = i*r2+j; - pred_value = oriData[index-1] + oriData[index-r2] - oriData[index-r2-1]; - pred_err = fabs(pred_value - oriData[index]); - radiusIndex = (unsigned long)((pred_err/realPrecision+1)/2); - if(radiusIndex>=confparams_cpr->maxRangeRadius) - radiusIndex = confparams_cpr->maxRangeRadius - 1; - intervals[radiusIndex]++; - - // if (max < oriData[index]) max = oriData[index]; - // if (min > oriData[index]) min = oriData[index]; - } - } - } - //compute the appropriate number - size_t targetCount = totalSampleSize*confparams_cpr->predThreshold; - size_t sum = 0; - for(i=0;imaxRangeRadius;i++) - { - sum += intervals[i]; - if(sum>targetCount) - break; - } - if(i>=confparams_cpr->maxRangeRadius) - i = confparams_cpr->maxRangeRadius-1; - unsigned int accIntervals = 2*(i+1); - unsigned int powerOf2 = roundUpToPowerOf2(accIntervals); - - if(powerOf2<32) - powerOf2 = 32; - - // struct timeval costStart, costEnd; - // double cost_est = 0; - // - // gettimeofday(&costStart, NULL); - // - // //compute estimate of bit-rate and distortion - // double est_br = 0; - // double est_psnr = 0; - // double c1 = log2(targetCount)+1; - // double c2 = -20.0*log10(realPrecision) + 20.0*log10(max-min) + 10.0*log10(3); - // - // for (i = 0; i < powerOf2/2; i++) - // { - // int count = intervals[i]; - // if (count != 0) - // est_br += count*log2(count); - // est_psnr += count; - // } - // - // //compute estimate of bit-rate - // est_br -= c1*est_psnr; - // est_br /= totalSampleSize; - // est_br = -est_br; - // - // //compute estimate of psnr - // est_psnr /= totalSampleSize; - // printf ("sum of P(i) = %lf\n", est_psnr); - // est_psnr = -10.0*log10(est_psnr); - // est_psnr += c2; - // - // printf ("estimate bitrate = %.2f\n", est_br); - // printf ("estimate psnr = %.2f\n",est_psnr); - // - // gettimeofday(&costEnd, NULL); - // cost_est = ((costEnd.tv_sec*1000000+costEnd.tv_usec)-(costStart.tv_sec*1000000+costStart.tv_usec))/1000000.0; - // - // printf ("analysis time = %f\n", cost_est); - - free(intervals); - //printf("confparams_cpr->maxRangeRadius = %d, accIntervals=%d, powerOf2=%d\n", confparams_cpr->maxRangeRadius, accIntervals, powerOf2); - return powerOf2; -} - -unsigned int optimize_intervals_float_3D(float *oriData, size_t r1, size_t r2, size_t r3, double realPrecision) -{ - size_t i,j,k, index; - size_t radiusIndex; - size_t r23=r2*r3; - float pred_value = 0, pred_err; - size_t *intervals = (size_t*)malloc(confparams_cpr->maxRangeRadius*sizeof(size_t)); - memset(intervals, 0, confparams_cpr->maxRangeRadius*sizeof(size_t)); - size_t totalSampleSize = (r1-1)*(r2-1)*(r3-1)/confparams_cpr->sampleDistance; - - //float max = oriData[0]; - //float min = oriData[0]; - - for(i=1;isampleDistance==0) - { - index = i*r23+j*r3+k; - pred_value = oriData[index-1] + oriData[index-r3] + oriData[index-r23] - - oriData[index-1-r23] - oriData[index-r3-1] - oriData[index-r3-r23] + oriData[index-r3-r23-1]; - pred_err = fabs(pred_value - oriData[index]); - radiusIndex = (pred_err/realPrecision+1)/2; - if(radiusIndex>=confparams_cpr->maxRangeRadius) - { - radiusIndex = confparams_cpr->maxRangeRadius - 1; - //printf("radiusIndex=%d\n", radiusIndex); - } - intervals[radiusIndex]++; - - // if (max < oriData[index]) max = oriData[index]; - // if (min > oriData[index]) min = oriData[index]; - } - } - } - } - //compute the appropriate number - size_t targetCount = totalSampleSize*confparams_cpr->predThreshold; - size_t sum = 0; - for(i=0;imaxRangeRadius;i++) - { - sum += intervals[i]; - if(sum>targetCount) - break; - } - if(i>=confparams_cpr->maxRangeRadius) - i = confparams_cpr->maxRangeRadius-1; - unsigned int accIntervals = 2*(i+1); - unsigned int powerOf2 = roundUpToPowerOf2(accIntervals); - - if(powerOf2<32) - powerOf2 = 32; - - // struct timeval costStart, costEnd; - // double cost_est = 0; - // - // gettimeofday(&costStart, NULL); - // - // //compute estimate of bit-rate and distortion - // double est_br = 0; - // double est_psnr = 0; - // double c1 = log2(targetCount)+1; - // double c2 = -20.0*log10(realPrecision) + 20.0*log10(max-min) + 10.0*log10(3); - // - // for (i = 0; i < powerOf2/2; i++) - // { - // int count = intervals[i]; - // if (count != 0) - // est_br += count*log2(count); - // est_psnr += count; - // } - // - // //compute estimate of bit-rate - // est_br -= c1*est_psnr; - // est_br /= totalSampleSize; - // est_br = -est_br; - // - // //compute estimate of psnr - // est_psnr /= totalSampleSize; - // printf ("sum of P(i) = %lf\n", est_psnr); - // est_psnr = -10.0*log10(est_psnr); - // est_psnr += c2; - // - // printf ("estimate bitrate = %.2f\n", est_br); - // printf ("estimate psnr = %.2f\n",est_psnr); - // - // gettimeofday(&costEnd, NULL); - // cost_est = ((costEnd.tv_sec*1000000+costEnd.tv_usec)-(costStart.tv_sec*1000000+costStart.tv_usec))/1000000.0; - // - // printf ("analysis time = %f\n", cost_est); - - free(intervals); - //printf("targetCount=%d, sum=%d, totalSampleSize=%d, ratio=%f, accIntervals=%d, powerOf2=%d\n", targetCount, sum, totalSampleSize, (double)sum/(double)totalSampleSize, accIntervals, powerOf2); - return powerOf2; -} - - -unsigned int optimize_intervals_float_4D(float *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision) -{ - size_t i,j,k,l, index; - size_t radiusIndex; - size_t r234=r2*r3*r4; - size_t r34=r3*r4; - float pred_value = 0, pred_err; - size_t *intervals = (size_t*)malloc(confparams_cpr->maxRangeRadius*sizeof(size_t)); - memset(intervals, 0, confparams_cpr->maxRangeRadius*sizeof(size_t)); - size_t totalSampleSize = (r1-1)*(r2-1)*(r3-1)*(r4-1)/confparams_cpr->sampleDistance; - for(i=1;isampleDistance==0) - { - index = i*r234+j*r34+k*r4+l; - pred_value = oriData[index-1] + oriData[index-r3] + oriData[index-r34] - - oriData[index-1-r34] - oriData[index-r4-1] - oriData[index-r4-r34] + oriData[index-r4-r34-1]; - pred_err = fabs(pred_value - oriData[index]); - radiusIndex = (unsigned long)((pred_err/realPrecision+1)/2); - if(radiusIndex>=confparams_cpr->maxRangeRadius) - radiusIndex = confparams_cpr->maxRangeRadius - 1; - intervals[radiusIndex]++; - } - } - } - } - } - //compute the appropriate number - size_t targetCount = totalSampleSize*confparams_cpr->predThreshold; - size_t sum = 0; - for(i=0;imaxRangeRadius;i++) - { - sum += intervals[i]; - if(sum>targetCount) - break; - } - if(i>=confparams_cpr->maxRangeRadius) - i = confparams_cpr->maxRangeRadius-1; - - unsigned int accIntervals = 2*(i+1); - unsigned int powerOf2 = roundUpToPowerOf2(accIntervals); - - if(powerOf2<32) - powerOf2 = 32; - - free(intervals); - return powerOf2; -} - TightDataPointStorageF* SZ_compress_float_1D_MDQ(float *oriData, size_t dataLength, float realPrecision, float valueRangeSize, float medianValue_f) { -#ifdef HAVE_TIMECMPR - float* decData = NULL; - if(confparams_cpr->szMode == SZ_TEMPORAL_COMPRESSION) - decData = (float*)(multisteps->hist_data); -#endif - unsigned int quantization_intervals; if(exe_params->optQuantMode==1) quantization_intervals = optimize_intervals_float_1D_opt(oriData, dataLength, realPrecision); @@ -404,10 +150,6 @@ size_t dataLength, float realPrecision, float valueRangeSize, float medianValue_ memcpy(preDataBytes,vce->curBytes,4); addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce); listAdd_float(last3CmprsData, vce->data); -#ifdef HAVE_TIMECMPR - if(confparams_cpr->szMode == SZ_TEMPORAL_COMPRESSION) - decData[0] = vce->data; -#endif //add the second data type[1] = 0; @@ -416,10 +158,7 @@ size_t dataLength, float realPrecision, float valueRangeSize, float medianValue_ memcpy(preDataBytes,vce->curBytes,4); addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce); listAdd_float(last3CmprsData, vce->data); -#ifdef HAVE_TIMECMPR - if(confparams_cpr->szMode == SZ_TEMPORAL_COMPRESSION) - decData[1] = vce->data; -#endif + int state; float checkRadius; float curData; @@ -460,19 +199,11 @@ size_t dataLength, float realPrecision, float valueRangeSize, float medianValue_ addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce); //listAdd_float(last3CmprsData, vce->data); - pred = vce->data; -#ifdef HAVE_TIMECMPR - if(confparams_cpr->szMode == SZ_TEMPORAL_COMPRESSION) - decData[i] = vce->data; -#endif + pred = vce->data; } else { - //listAdd_float(last3CmprsData, pred); -#ifdef HAVE_TIMECMPR - if(confparams_cpr->szMode == SZ_TEMPORAL_COMPRESSION) - decData[i] = pred; -#endif + //listAdd_float(last3CmprsData, pred); } continue; } @@ -485,11 +216,7 @@ size_t dataLength, float realPrecision, float valueRangeSize, float medianValue_ addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce); //listAdd_float(last3CmprsData, vce->data); - pred = vce->data; -#ifdef HAVE_TIMECMPR - if(confparams_cpr->szMode == SZ_TEMPORAL_COMPRESSION) - decData[i] = vce->data; -#endif + pred = vce->data; }//end of for @@ -523,7 +250,7 @@ size_t dataLength, float realPrecision, float valueRangeSize, float medianValue_ return tdps; } -void SZ_compress_args_float_StoreOriData(float* oriData, size_t dataLength, unsigned char** newByteData, size_t *outSize) +void SZ_compress_args_float_StoreOriData(float* oriData, size_t dataLength, unsigned char* newByteData, size_t *outSize) { int floatSize=sizeof(float); size_t k = 0, i; @@ -533,74 +260,41 @@ void SZ_compress_args_float_StoreOriData(float* oriData, size_t dataLength, unsi unsigned char dsLengthBytes[8]; for (i = 0; i < 3; i++)//3 - (*newByteData)[k++] = versionNumber[i]; + newByteData[k++] = versionNumber[i]; if(exe_params->SZ_SIZE_TYPE==4)//1 - (*newByteData)[k++] = 16; //00010000 + newByteData[k++] = 16; //00010000 else - (*newByteData)[k++] = 80; //01010000: 01000000 indicates the SZ_SIZE_TYPE=8 + newByteData[k++] = 80; //01010000: 01000000 indicates the SZ_SIZE_TYPE=8 - convertSZParamsToBytes(confparams_cpr, &((*newByteData)[k])); + convertSZParamsToBytes(confparams_cpr, &(newByteData[k])); k = k + MetaDataByteLength; sizeToBytes(dsLengthBytes,dataLength); //SZ_SIZE_TYPE: 4 or 8 for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++) - (*newByteData)[k++] = dsLengthBytes[i]; + newByteData[k++] = dsLengthBytes[i]; if(sysEndianType==BIG_ENDIAN_SYSTEM) - memcpy((*newByteData)+4+MetaDataByteLength+exe_params->SZ_SIZE_TYPE, oriData, dataLength*floatSize); + memcpy(newByteData+4+MetaDataByteLength+exe_params->SZ_SIZE_TYPE, oriData, dataLength*floatSize); else { - unsigned char* p = (*newByteData)+4+MetaDataByteLength+exe_params->SZ_SIZE_TYPE; + unsigned char* p = newByteData+4+MetaDataByteLength+exe_params->SZ_SIZE_TYPE; for(i=0;iszMode == SZ_TEMPORAL_COMPRESSION) - { - int timestep = sz_tsc->currentStep; - if(cmprType == SZ_PERIO_TEMPORAL_COMPRESSION) - { - if(timestep % confparams_cpr->snapshotCmprStep != 0) - { - tdps = SZ_compress_float_1D_MDQ_ts(oriData, dataLength, multisteps, realPrecision, valueRangeSize, medianValue_f); - compressionType = 1; //time-series based compression - } - else - { - tdps = SZ_compress_float_1D_MDQ(oriData, dataLength, realPrecision, valueRangeSize, medianValue_f); - compressionType = 0; //snapshot-based compression - multisteps->lastSnapshotStep = timestep; - } - } - else if(cmprType == SZ_FORCE_SNAPSHOT_COMPRESSION) - { - tdps = SZ_compress_float_1D_MDQ(oriData, dataLength, realPrecision, valueRangeSize, medianValue_f); - compressionType = 0; //snapshot-based compression - multisteps->lastSnapshotStep = timestep; - } - else if(cmprType == SZ_FORCE_TEMPORAL_COMPRESSION) - { - tdps = SZ_compress_float_1D_MDQ_ts(oriData, dataLength, multisteps, realPrecision, valueRangeSize, medianValue_f); - compressionType = 1; //time-series based compression - } - } - else -#endif - tdps = SZ_compress_float_1D_MDQ(oriData, dataLength, realPrecision, valueRangeSize, medianValue_f); - + tdps = SZ_compress_float_1D_MDQ(oriData, dataLength, realPrecision, valueRangeSize, medianValue_f); convertTDPStoFlatBytes_float(tdps, newByteData, outSize); - if(*outSize>3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(float)*dataLength) + if(*outSize > 3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(float)*dataLength) SZ_compress_args_float_StoreOriData(oriData, dataLength, newByteData, outSize); free_TightDataPointStorageF(tdps); @@ -611,12 +305,6 @@ size_t dataLength, double realPrecision, size_t *outSize, float valueRangeSize, TightDataPointStorageF* SZ_compress_float_1D_MDQ_MSST19(float *oriData, size_t dataLength, double realPrecision, float valueRangeSize, float medianValue_f) { -#ifdef HAVE_TIMECMPR - float* decData = NULL; - if(confparams_cpr->szMode == SZ_TEMPORAL_COMPRESSION) - decData = (float*)(multisteps->hist_data); -#endif - //struct ClockPoint clockPointBuild; //TimeDurationStart("build", &clockPointBuild); unsigned int quantization_intervals; @@ -679,11 +367,7 @@ size_t dataLength, double realPrecision, float valueRangeSize, float medianValue memcpy(preDataBytes,vce->curBytes,4); addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce); listAdd_float(last3CmprsData, vce->data); - //miss++; -#ifdef HAVE_TIMECMPR - if(confparams_cpr->szMode == SZ_TEMPORAL_COMPRESSION) - decData[0] = vce->data; -#endif + //miss++; //add the second data type[1] = 0; @@ -693,10 +377,7 @@ size_t dataLength, double realPrecision, float valueRangeSize, float medianValue addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce); listAdd_float(last3CmprsData, vce->data); //miss++; -#ifdef HAVE_TIMECMPR - if(confparams_cpr->szMode == SZ_TEMPORAL_COMPRESSION) - decData[1] = vce->data; -#endif + int state; //double checkRadius; float curData; @@ -778,7 +459,7 @@ size_t dataLength, double realPrecision, float valueRangeSize, float medianValue } -void SZ_compress_args_float_withinRange(unsigned char** newByteData, float *oriData, size_t dataLength, size_t *outSize) +void SZ_compress_args_float_withinRange(unsigned char* newByteData, float *oriData, size_t dataLength, size_t *outSize) { TightDataPointStorageF* tdps = (TightDataPointStorageF*) malloc(sizeof(TightDataPointStorageF)); tdps->rtypeArray = NULL; @@ -805,23 +486,20 @@ void SZ_compress_args_float_withinRange(unsigned char** newByteData, float *oriD free_TightDataPointStorageF(tdps); } -int SZ_compress_args_float(int cmprType, int withRegression, unsigned char** newByteData, float *oriData, -size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize, -int errBoundMode, double absErr_Bound, double relBoundRatio, double pwRelBoundRatio) +int SZ_compress_args_float(float *oriData, size_t r1, unsigned char* newByteData, size_t *outSize, sz_params* params) { - confparams_cpr->errorBoundMode = errBoundMode; //this is used to print the metadata if needed... - if(errBoundMode==PW_REL) - { - confparams_cpr->pw_relBoundRatio = pwRelBoundRatio; - } - int status = SZ_SCES; - size_t dataLength = computeDataLength(r5,r4,r3,r2,r1); + int status = SZ_SUCCESS; + size_t dataLength = r1; + // check at least elements count if(dataLength <= MIN_NUM_OF_ELEMENTS) { - *newByteData = SZ_skip_compress_float(oriData, dataLength, outSize); - return status; + printf("error, input elements count=%d less than %d, so need not do compress.\n", dataLength, MIN_NUM_OF_ELEMENTS); + return SZ_LITTER_ELEMENT; } + + if(params->pw_relBoundRatio < 0.000009999) + params->accelerate_pw_rel_compression = 0; float valueRangeSize = 0, medianValue = 0; @@ -829,347 +507,91 @@ int errBoundMode, double absErr_Bound, double relBoundRatio, double pwRelBoundRa bool positive = true; float nearZero = 0.0; float min = 0; - if(pwRelBoundRatio < 0.000009999) - confparams_cpr->accelerate_pw_rel_compression = 0; - if(confparams_cpr->errorBoundMode == PW_REL && confparams_cpr->accelerate_pw_rel_compression) + + // calc min + if(params->errorBoundMode == PW_REL && params->accelerate_pw_rel_compression) { signs = (unsigned char *) malloc(dataLength); memset(signs, 0, dataLength); min = computeRangeSize_float_MSST19(oriData, dataLength, &valueRangeSize, &medianValue, signs, &positive, &nearZero); + free(signs); // tickduan move to here + signs = NULL; } else min = computeRangeSize_float(oriData, dataLength, &valueRangeSize, &medianValue); + + // calc max float max = min+valueRangeSize; - confparams_cpr->fmin = min; - confparams_cpr->fmax = max; + params->fmin = min; + params->fmax = max; double realPrecision = 0; - if(confparams_cpr->errorBoundMode==PSNR) + if(params->errorBoundMode==PSNR) { - confparams_cpr->errorBoundMode = SZ_ABS; - realPrecision = confparams_cpr->absErrBound = computeABSErrBoundFromPSNR(confparams_cpr->psnr, (double)confparams_cpr->predThreshold, (double)valueRangeSize); + params->errorBoundMode = SZ_ABS; + realPrecision = params->absErrBound = computeABSErrBoundFromPSNR(params->psnr, (double)params->predThreshold, (double)valueRangeSize); //printf("realPrecision=%lf\n", realPrecision); } - else if(confparams_cpr->errorBoundMode==NORM) //norm error = sqrt(sum((xi-xi_)^2)) + else if(params->errorBoundMode==NORM) //norm error = sqrt(sum((xi-xi_)^2)) { - confparams_cpr->errorBoundMode = SZ_ABS; - realPrecision = confparams_cpr->absErrBound = computeABSErrBoundFromNORM_ERR(confparams_cpr->normErr, dataLength); + params->errorBoundMode = SZ_ABS; + realPrecision = params->absErrBound = computeABSErrBoundFromNORM_ERR(params->normErr, dataLength); //printf("realPrecision=%lf\n", realPrecision); } else { - realPrecision = getRealPrecision_float(valueRangeSize, errBoundMode, absErr_Bound, relBoundRatio, &status); - confparams_cpr->absErrBound = realPrecision; + realPrecision = getRealPrecision_float(valueRangeSize, params->errorBoundMode, params->absErrBound, params->relBoundRatio, &status); + params->absErrBound = realPrecision; } + + // range if(valueRangeSize <= realPrecision) - { - if(confparams_cpr->errorBoundMode>=PW_REL && confparams_cpr->accelerate_pw_rel_compression == 1) - free(signs); + { SZ_compress_args_float_withinRange(newByteData, oriData, dataLength, outSize); } else { size_t tmpOutSize = 0; - unsigned char* tmpByteData; - - if (r2==0) + unsigned char* tmpByteData = newByteData; + bool twoStage = params->szMode != SZ_BEST_SPEED; + if(twoStage) { - if(confparams_cpr->errorBoundMode>=PW_REL) - { - if(confparams_cpr->accelerate_pw_rel_compression && confparams_cpr->maxRangeRadius <= 32768) - SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_pre_log_MSST19(&tmpByteData, oriData, pwRelBoundRatio, r1, &tmpOutSize, valueRangeSize, medianValue, signs, &positive, min, max, nearZero); - else - SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_pre_log(&tmpByteData, oriData, pwRelBoundRatio, r1, &tmpOutSize, min, max); - //SZ_compress_args_float_NoCkRngeNoGzip_1D_pwrgroup(&tmpByteData, oriData, r1, absErr_Bound, relBoundRatio, pwRelBoundRatio, valueRangeSize, medianValue, &tmpOutSize); - } - else -#ifdef HAVE_TIMECMPR - if(confparams_cpr->szMode == SZ_TEMPORAL_COMPRESSION) - multisteps->compressionType = SZ_compress_args_float_NoCkRngeNoGzip_1D(cmprType, &tmpByteData, oriData, r1, realPrecision, &tmpOutSize, valueRangeSize, medianValue); - else -#endif - { -#ifdef HAVE_RANDOMACCESS - if(confparams_cpr->randomAccess == 0) - { -#endif - SZ_compress_args_float_NoCkRngeNoGzip_1D(cmprType, &tmpByteData, oriData, r1, realPrecision, &tmpOutSize, valueRangeSize, medianValue); - if(tmpOutSize>=dataLength*sizeof(float) + 3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1) - SZ_compress_args_float_StoreOriData(oriData, dataLength, &tmpByteData, &tmpOutSize); -#ifdef HAVE_RANDOMACCESS - } - else - tmpByteData = SZ_compress_float_1D_MDQ_decompression_random_access_with_blocked_regression(oriData, r1, realPrecision, &tmpOutSize); -#endif - } + tmpByteData = (unsigned char*)malloc(r1*sizeof(float) + 1024); } - else + + if(params->errorBoundMode>=PW_REL) { - printf("Error: doesn't support 5 dimensions for now.\n"); - status = SZ_DERR; //dimension error + if(params->accelerate_pw_rel_compression && params->maxRangeRadius <= 32768) + SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_pre_log_MSST19(tmpByteData, oriData, params->pw_relBoundRatio, r1, &tmpOutSize, valueRangeSize, medianValue, signs, &positive, min, max, nearZero); + else + SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_pre_log(tmpByteData, oriData, params->pw_relBoundRatio, r1, &tmpOutSize, min, max); + } + else + { + SZ_compress_args_float_NoCkRngeNoGzip_1D( tmpByteData, oriData, r1, realPrecision, &tmpOutSize, valueRangeSize, medianValue); + if(tmpOutSize >= dataLength*sizeof(float) + 3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1) + SZ_compress_args_float_StoreOriData(oriData, dataLength, tmpByteData, &tmpOutSize); } // //Call Zstd or Gzip to do the further compression. // - if(confparams_cpr->szMode==SZ_BEST_SPEED) + if(!twoStage) { *outSize = tmpOutSize; - *newByteData = tmpByteData; - } - else if(confparams_cpr->szMode==SZ_BEST_COMPRESSION || confparams_cpr->szMode==SZ_DEFAULT_COMPRESSION || confparams_cpr->szMode==SZ_TEMPORAL_COMPRESSION) - { - *outSize = sz_lossless_compress(confparams_cpr->losslessCompressor, confparams_cpr->gzipMode, tmpByteData, tmpOutSize, newByteData); - free(tmpByteData); + //*newByteData = tmpByteData; } else { - printf("Error: Wrong setting of confparams_cpr->szMode in the float compression.\n"); - status = SZ_MERR; //mode error + *outSize = sz_lossless_compress(params->losslessCompressor, params->gzipMode, tmpByteData, tmpOutSize, newByteData); + free(tmpByteData); } } return status; } -//TODO -int SZ_compress_args_float_subblock(unsigned char* compressedBytes, float *oriData, -size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, -size_t s5, size_t s4, size_t s3, size_t s2, size_t s1, -size_t e5, size_t e4, size_t e3, size_t e2, size_t e1, -size_t *outSize, int errBoundMode, double absErr_Bound, double relBoundRatio) -{ - int status = SZ_SCES; - float valueRangeSize = 0, medianValue = 0; - computeRangeSize_float_subblock(oriData, &valueRangeSize, &medianValue, r5, r4, r3, r2, r1, s5, s4, s3, s2, s1, e5, e4, e3, e2, e1); - - double realPrecision = getRealPrecision_float(valueRangeSize, errBoundMode, absErr_Bound, relBoundRatio, &status); - - if(valueRangeSize <= realPrecision) - { - //TODO - //SZ_compress_args_float_withinRange_subblock(); - } - else - { - if (r2==0) - { - if(errBoundMode>=PW_REL) - { - //TODO - //SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_subblock(); - printf ("Current subblock version does not support point-wise relative error bound.\n"); - } - else - SZ_compress_args_float_NoCkRnge_1D_subblock(compressedBytes, oriData, realPrecision, outSize, valueRangeSize, medianValue, r1, s1, e1); - } - else - { - printf("Error: doesn't support 5 dimensions for now.\n"); - status = SZ_DERR; //dimension error - } - } - return status; -} - -void SZ_compress_args_float_NoCkRnge_1D_subblock(unsigned char* compressedBytes, float *oriData, double realPrecision, size_t *outSize, float valueRangeSize, float medianValue_f, -size_t r1, size_t s1, size_t e1) -{ - TightDataPointStorageF* tdps = SZ_compress_float_1D_MDQ_subblock(oriData, realPrecision, valueRangeSize, medianValue_f, r1, s1, e1); - - if (confparams_cpr->szMode==SZ_BEST_SPEED) - convertTDPStoFlatBytes_float_args(tdps, compressedBytes, outSize); - else if(confparams_cpr->szMode==SZ_BEST_COMPRESSION || confparams_cpr->szMode==SZ_DEFAULT_COMPRESSION) - { - unsigned char *tmpCompBytes; - size_t tmpOutSize; - convertTDPStoFlatBytes_float(tdps, &tmpCompBytes, &tmpOutSize); - *outSize = zlib_compress3(tmpCompBytes, tmpOutSize, compressedBytes, confparams_cpr->gzipMode); - free(tmpCompBytes); - } - else - { - printf ("Error: Wrong setting of confparams_cpr->szMode in the double compression.\n"); - } - - //TODO -// if(*outSize>dataLength*sizeof(float)) -// SZ_compress_args_float_StoreOriData(oriData, dataLength, newByteData, outSize); - - free_TightDataPointStorageF(tdps); -} - - -unsigned int optimize_intervals_float_1D_subblock(float *oriData, double realPrecision, size_t r1, size_t s1, size_t e1) -{ - size_t dataLength = e1 - s1 + 1; - oriData = oriData + s1; - - size_t i = 0; - unsigned long radiusIndex; - float pred_value = 0, pred_err; - int *intervals = (int*)malloc(confparams_cpr->maxRangeRadius*sizeof(int)); - memset(intervals, 0, confparams_cpr->maxRangeRadius*sizeof(int)); - size_t totalSampleSize = dataLength/confparams_cpr->sampleDistance; - for(i=2;isampleDistance==0) - { - pred_value = 2*oriData[i-1] - oriData[i-2]; - //pred_value = oriData[i-1]; - pred_err = fabs(pred_value - oriData[i]); - radiusIndex = (unsigned long)((pred_err/realPrecision+1)/2); - if(radiusIndex>=confparams_cpr->maxRangeRadius) - radiusIndex = confparams_cpr->maxRangeRadius - 1; - intervals[radiusIndex]++; - } - } - //compute the appropriate number - size_t targetCount = totalSampleSize*confparams_cpr->predThreshold; - size_t sum = 0; - for(i=0;imaxRangeRadius;i++) - { - sum += intervals[i]; - if(sum>targetCount) - break; - } - if(i>=confparams_cpr->maxRangeRadius) - i = confparams_cpr->maxRangeRadius-1; - - unsigned int accIntervals = 2*(i+1); - unsigned int powerOf2 = roundUpToPowerOf2(accIntervals); - - if(powerOf2<32) - powerOf2 = 32; - - free(intervals); - //printf("accIntervals=%d, powerOf2=%d\n", accIntervals, powerOf2); - return powerOf2; -} - - -TightDataPointStorageF* SZ_compress_float_1D_MDQ_subblock(float *oriData, double realPrecision, float valueRangeSize, float medianValue_f, -size_t r1, size_t s1, size_t e1) -{ - size_t dataLength = e1 - s1 + 1; - unsigned int quantization_intervals; - if(exe_params->optQuantMode==1) - quantization_intervals = optimize_intervals_float_1D_subblock(oriData, realPrecision, r1, s1, e1); - else - quantization_intervals = exe_params->intvCapacity; - //updateQuantizationInfo(quantization_intervals); - int intvRadius = quantization_intervals/2; - - size_t i; - int reqLength; - float medianValue = medianValue_f; - short radExpo = getExponent_float(valueRangeSize/2); - - computeReqLength_float(realPrecision, radExpo, &reqLength, &medianValue); - - int* type = (int*) malloc(dataLength*sizeof(int)); - - float* spaceFillingValue = oriData + s1; - - DynamicIntArray *exactLeadNumArray; - new_DIA(&exactLeadNumArray, DynArrayInitLen); - - DynamicByteArray *exactMidByteArray; - new_DBA(&exactMidByteArray, DynArrayInitLen); - - DynamicIntArray *resiBitArray; - new_DIA(&resiBitArray, DynArrayInitLen); - - type[0] = 0; - - unsigned char preDataBytes[4]; - intToBytes_bigEndian(preDataBytes, 0); - - int reqBytesLength = reqLength/8; - int resiBitsLength = reqLength%8; - float last3CmprsData[3] = {0}; - - FloatValueCompressElement *vce = (FloatValueCompressElement*)malloc(sizeof(FloatValueCompressElement)); - LossyCompressionElement *lce = (LossyCompressionElement*)malloc(sizeof(LossyCompressionElement)); - - //add the first data - compressSingleFloatValue(vce, spaceFillingValue[0], realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength); - updateLossyCompElement_Float(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce); - memcpy(preDataBytes,vce->curBytes,4); - addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce); - listAdd_float(last3CmprsData, vce->data); - - //add the second data - type[1] = 0; - compressSingleFloatValue(vce, spaceFillingValue[1], realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength); - updateLossyCompElement_Float(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce); - memcpy(preDataBytes,vce->curBytes,4); - addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce); - listAdd_float(last3CmprsData, vce->data); - - int state; - double checkRadius; - float curData; - float pred; - float predAbsErr; - checkRadius = (quantization_intervals-1)*realPrecision; - double interval = 2*realPrecision; - - for(i=2;i=pred) - { - type[i] = intvRadius+state; - pred = pred + state*interval; - } - else - { - type[i] = intvRadius-state; - pred = pred - state*interval; - } - - listAdd_float(last3CmprsData, pred); - continue; - } - - //unpredictable data processing - type[i] = 0; - compressSingleFloatValue(vce, curData, realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength); - updateLossyCompElement_Float(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce); - memcpy(preDataBytes,vce->curBytes,4); - addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce); - - listAdd_float(last3CmprsData, vce->data); - } - - size_t exactDataNum = exactLeadNumArray->size; - - TightDataPointStorageF* tdps; - - new_TightDataPointStorageF(&tdps, dataLength, exactDataNum, - type, exactMidByteArray->array, exactMidByteArray->size, - exactLeadNumArray->array, - resiBitArray->array, resiBitArray->size, - resiBitsLength, - realPrecision, medianValue, (char)reqLength, quantization_intervals, NULL, 0, 0); - - //free memory - free_DIA(exactLeadNumArray); - free_DIA(resiBitArray); - free(type); - free(vce); - free(lce); - free(exactMidByteArray); //exactMidByteArray->array has been released in free_TightDataPointStorageF(tdps); - - return tdps; -} unsigned int optimize_intervals_float_1D_opt_MSST19(float *oriData, size_t dataLength, double realPrecision) { @@ -1263,707 +685,3 @@ unsigned int optimize_intervals_float_1D_opt(float *oriData, size_t dataLength, free(intervals); return powerOf2; } - - - -size_t SZ_compress_float_1D_MDQ_RA_block(float * block_ori_data, float * mean, size_t dim_0, size_t block_dim_0, double realPrecision, int * type, float * unpredictable_data){ - - mean[0] = block_ori_data[0]; - unsigned short unpredictable_count = 0; - - float curData; - double itvNum; - double diff; - float last_over_thres = mean[0]; - float pred1D; - size_t type_index = 0; - float * data_pos = block_ori_data; - for(size_t i=0; iintvCapacity){ - if (diff < 0) itvNum = -itvNum; - type[type_index] = (int) (itvNum/2) + exe_params->intvRadius; - last_over_thres = pred1D + 2 * (type[type_index] - exe_params->intvRadius) * realPrecision; - if(fabs(curData-last_over_thres)>realPrecision){ - type[type_index] = 0; - last_over_thres = curData; - unpredictable_data[unpredictable_count ++] = curData; - } - - } - else{ - type[type_index] = 0; - unpredictable_data[unpredictable_count ++] = curData; - last_over_thres = curData; - } - type_index ++; - data_pos ++; - } - return unpredictable_count; - -} - -/*The above code is for sz 1.4.13; the following code is for sz 2.0*/ -static unsigned int optimize_intervals_float_1D_with_freq_and_dense_pos(float *oriData, size_t r1, double realPrecision, float * dense_pos, float * max_freq, float * mean_freq) -{ - float mean = 0.0; - size_t len = r1; - size_t mean_distance = (int) (sqrt(len)); - - float * data_pos = oriData; - size_t mean_count = 0; - while(data_pos - oriData < len){ - mean += *data_pos; - mean_count ++; - data_pos += mean_distance; - } - if(mean_count > 0) mean /= mean_count; - size_t range = 8192; - size_t radius = 4096; - size_t * freq_intervals = (size_t *) malloc(range*sizeof(size_t)); - memset(freq_intervals, 0, range*sizeof(size_t)); - - unsigned int maxRangeRadius = confparams_cpr->maxRangeRadius; - int sampleDistance = confparams_cpr->sampleDistance; - float predThreshold = confparams_cpr->predThreshold; - - size_t i; - size_t radiusIndex; - float pred_value = 0, pred_err; - size_t *intervals = (size_t*)malloc(maxRangeRadius*sizeof(size_t)); - memset(intervals, 0, maxRangeRadius*sizeof(size_t)); - - float mean_diff; - ptrdiff_t freq_index; - size_t freq_count = 0; - size_t sample_count = 0; - data_pos = oriData + 1; - while(data_pos - oriData < len){ - pred_value = data_pos[-1]; - pred_err = fabs(pred_value - *data_pos); - if(pred_err < realPrecision) freq_count ++; - radiusIndex = (unsigned long)((pred_err/realPrecision+1)/2); - if(radiusIndex>=maxRangeRadius) - radiusIndex = maxRangeRadius - 1; - intervals[radiusIndex]++; - - mean_diff = *data_pos - mean; - if(mean_diff > 0) freq_index = (ptrdiff_t)(mean_diff/realPrecision) + radius; - else freq_index = (ptrdiff_t)(mean_diff/realPrecision) - 1 + radius; - if(freq_index <= 0){ - freq_intervals[0] ++; - } - else if(freq_index >= range){ - freq_intervals[range - 1] ++; - } - else{ - freq_intervals[freq_index] ++; - } - data_pos += sampleDistance; - sample_count ++; - } - *max_freq = freq_count * 1.0/ sample_count; - - //compute the appropriate number - size_t targetCount = sample_count*predThreshold; - size_t sum = 0; - for(i=0;itargetCount) - break; - } - if(i>=maxRangeRadius) - i = maxRangeRadius-1; - unsigned int accIntervals = 2*(i+1); - unsigned int powerOf2 = roundUpToPowerOf2(accIntervals); - - if(powerOf2<32) - powerOf2 = 32; - - // collect frequency - size_t max_sum = 0; - size_t max_index = 0; - size_t tmp_sum; - size_t * freq_pos = freq_intervals + 1; - for(size_t i=1; i max_sum){ - max_sum = tmp_sum; - max_index = i; - } - freq_pos ++; - } - *dense_pos = mean + realPrecision * (ptrdiff_t)(max_index + 1 - radius); - *mean_freq = max_sum * 1.0 / sample_count; - - free(freq_intervals); - free(intervals); - return powerOf2; -} - - -// random access -unsigned char * SZ_compress_float_1D_MDQ_decompression_random_access_with_blocked_regression(float *oriData, size_t r1, double realPrecision, size_t * comp_size){ - - unsigned int quantization_intervals; - float sz_sample_correct_freq = -1;//0.5; //-1 - float dense_pos; - float mean_flush_freq; - unsigned char use_mean = 0; - - // calculate block dims - size_t num_x; - size_t block_size = 256; - num_x = (r1 - 1) / block_size + 1; - - size_t max_num_block_elements = block_size; - size_t num_blocks = num_x; - size_t num_elements = r1; - - int * result_type = (int *) malloc(num_blocks*max_num_block_elements * sizeof(int)); - size_t unpred_data_max_size = max_num_block_elements; - float * result_unpredictable_data = (float *) malloc(unpred_data_max_size * sizeof(float) * num_blocks); - size_t total_unpred = 0; - size_t unpredictable_count; - float * data_pos = oriData; - int * type = result_type; - float * reg_params = (float *) malloc(num_blocks * 2 * sizeof(float)); - float * reg_params_pos = reg_params; - // move regression part out - size_t params_offset_b = num_blocks; - float * pred_buffer = (float *) malloc((block_size+1)*sizeof(float)); - float * pred_buffer_pos = NULL; - float * block_data_pos_x = NULL; - for(size_t i=0; ioptQuantMode==1) - { - quantization_intervals = optimize_intervals_float_1D_with_freq_and_dense_pos(oriData, r1, realPrecision, &dense_pos, &sz_sample_correct_freq, &mean_flush_freq); - if(mean_flush_freq > 0.5 || mean_flush_freq > sz_sample_correct_freq) use_mean = 1; - updateQuantizationInfo(quantization_intervals); - } - else{ - quantization_intervals = exe_params->intvCapacity; - } - - float mean = 0; - if(use_mean){ - // compute mean - double sum = 0.0; - size_t mean_count = 0; - for(size_t i=0; i 0) mean = sum / mean_count; - } - - double tmp_realPrecision = realPrecision; - - // use two prediction buffers for higher performance - float * unpredictable_data = result_unpredictable_data; - unsigned char * indicator = (unsigned char *) malloc(num_blocks * sizeof(unsigned char)); - memset(indicator, 0, num_blocks * sizeof(unsigned char)); - unsigned char * indicator_pos = indicator; - - int intvCapacity = quantization_intervals; //exe_params->intvCapacity; - int intvRadius = intvCapacity/2; //exe_params->intvRadius; - float noise = realPrecision * 0.5; - reg_params_pos = reg_params; - - memset(pred_buffer, 0, (block_size+1)*sizeof(float)); - // select - int sample_distance = sqrt(block_size) + 1; - if(use_mean){ - for(size_t i=0; iprecision[e]){ - coeff_type[e][coeff_index] = 0; - last_coeffcients[e] = cur_coeff; - coeff_unpred_data[e][coeff_unpredictable_count[e] ++] = cur_coeff; - } - } - else{ - coeff_type[e][coeff_index] = 0; - last_coeffcients[e] = cur_coeff; - coeff_unpred_data[e][coeff_unpredictable_count[e] ++] = cur_coeff; - } - reg_params_separte[e][i] = last_coeffcients[e]; - } - coeff_index ++; - } - // pred & quantization - int * blockwise_unpred_count = (int *) malloc(num_blocks * sizeof(int)); - int * blockwise_unpred_count_pos = blockwise_unpred_count; - reg_params_pos = reg_params; - indicator_pos = indicator; - if(use_mean){ - int intvCapacity_sz = intvCapacity - 2; - type = result_type; - for(size_t i=0; itmp_realPrecision){ - type[index] = 0; - pred = curData; - unpredictable_data[block_unpredictable_count ++] = curData; - } - } - else{ - type[index] = 0; - pred = curData; - unpredictable_data[block_unpredictable_count ++] = curData; - } - index ++; - cur_data_pos ++; - } - reg_params_pos ++; - total_unpred += block_unpredictable_count; - unpredictable_data += block_unpredictable_count; - *blockwise_unpred_count_pos = block_unpredictable_count; - } - else{ - // use SZ - // SZ predication - unpredictable_count = 0; - float * cur_data_pos = pred_buffer + 1; - float curData; - float pred3D; - double itvNum, diff; - size_t index = 0; - for(size_t ii=0; iitmp_realPrecision){ - type[index] = 0; - *cur_data_pos = curData; - unpredictable_data[unpredictable_count ++] = curData; - } - } - else{ - type[index] = 0; - *cur_data_pos = curData; - unpredictable_data[unpredictable_count ++] = curData; - } - } - index ++; - cur_data_pos ++; - } - total_unpred += unpredictable_count; - unpredictable_data += unpredictable_count; - *blockwise_unpred_count_pos = unpredictable_count; - }// end SZ - blockwise_unpred_count_pos ++; - type += block_size; - indicator_pos ++; - }// end i - } - else{ - int intvCapacity_sz = intvCapacity; - type = result_type; - for(size_t i=0; itmp_realPrecision){ - type[index] = 0; - pred = curData; - unpredictable_data[block_unpredictable_count ++] = curData; - } - } - else{ - type[index] = 0; - pred = curData; - unpredictable_data[block_unpredictable_count ++] = curData; - } - index ++; - cur_data_pos ++; - } - reg_params_pos ++; - total_unpred += block_unpredictable_count; - unpredictable_data += block_unpredictable_count; - *blockwise_unpred_count_pos = block_unpredictable_count; - } - else{ - // use SZ - // SZ predication - unpredictable_count = 0; - float * cur_data_pos = pred_buffer + 1; - float curData; - float pred3D; - double itvNum, diff; - size_t index = 0; - for(size_t ii=0; iitmp_realPrecision){ - type[index] = 0; - *cur_data_pos = curData; - unpredictable_data[unpredictable_count ++] = curData; - } - } - else{ - type[index] = 0; - *cur_data_pos = curData; - unpredictable_data[unpredictable_count ++] = curData; - } - index ++; - cur_data_pos ++; - } - total_unpred += unpredictable_count; - unpredictable_data += unpredictable_count; - *blockwise_unpred_count_pos = unpredictable_count; - }// end SZ - blockwise_unpred_count_pos ++; - type += block_size; - indicator_pos ++; - }// end i - } - free(pred_buffer); - int stateNum = 2*quantization_intervals; - HuffmanTree* huffmanTree = createHuffmanTree(stateNum); - - size_t nodeCount = 0; - init(huffmanTree, result_type, num_blocks*max_num_block_elements); - size_t i = 0; - for (i = 0; i < huffmanTree->stateNum; i++) - if (huffmanTree->code[i]) nodeCount++; - nodeCount = nodeCount*2-1; - - unsigned char *treeBytes; - unsigned int treeByteSize = convert_HuffTree_to_bytes_anyStates(huffmanTree, nodeCount, &treeBytes); - - unsigned int meta_data_offset = 3 + 1 + MetaDataByteLength; - // total size metadata # elements real precision intervals nodeCount huffman block index unpredicatable count mean unpred size elements - unsigned char * result = (unsigned char *) calloc(meta_data_offset + exe_params->SZ_SIZE_TYPE + sizeof(double) + sizeof(int) + sizeof(int) + 5*treeByteSize +4*num_blocks*sizeof(int) + num_blocks * sizeof(unsigned short) + num_blocks * sizeof(unsigned short) + num_blocks * sizeof(float) + total_unpred * sizeof(float) + num_elements * sizeof(int), 1); - unsigned char * result_pos = result; - initRandomAccessBytes(result_pos); - - result_pos += meta_data_offset; - - sizeToBytes(result_pos,num_elements); //SZ_SIZE_TYPE: 4 or 8 - result_pos += exe_params->SZ_SIZE_TYPE; - - intToBytes_bigEndian(result_pos, block_size); - result_pos += sizeof(int); - doubleToBytes(result_pos, realPrecision); - result_pos += sizeof(double); - intToBytes_bigEndian(result_pos, quantization_intervals); - result_pos += sizeof(int); - intToBytes_bigEndian(result_pos, treeByteSize); - result_pos += sizeof(int); - intToBytes_bigEndian(result_pos, nodeCount); - result_pos += sizeof(int); - memcpy(result_pos, treeBytes, treeByteSize); - result_pos += treeByteSize; - free(treeBytes); - - memcpy(result_pos, &use_mean, sizeof(unsigned char)); - result_pos += sizeof(unsigned char); - memcpy(result_pos, &mean, sizeof(float)); - result_pos += sizeof(float); - size_t indicator_size = convertIntArray2ByteArray_fast_1b_to_result(indicator, num_blocks, result_pos); - result_pos += indicator_size; - - //convert the lead/mid/resi to byte stream - if(reg_count > 0){ - for(int e=0; e<2; e++){ - int stateNum = 2*coeff_intvCapacity_sz; - HuffmanTree* huffmanTree = createHuffmanTree(stateNum); - size_t nodeCount = 0; - init(huffmanTree, coeff_type[e], reg_count); - size_t i = 0; - for (i = 0; i < huffmanTree->stateNum; i++) - if (huffmanTree->code[i]) nodeCount++; - nodeCount = nodeCount*2-1; - unsigned char *treeBytes; - unsigned int treeByteSize = convert_HuffTree_to_bytes_anyStates(huffmanTree, nodeCount, &treeBytes); - doubleToBytes(result_pos, precision[e]); - result_pos += sizeof(double); - intToBytes_bigEndian(result_pos, coeff_intvRadius); - result_pos += sizeof(int); - intToBytes_bigEndian(result_pos, treeByteSize); - result_pos += sizeof(int); - intToBytes_bigEndian(result_pos, nodeCount); - result_pos += sizeof(int); - memcpy(result_pos, treeBytes, treeByteSize); - result_pos += treeByteSize; - free(treeBytes); - size_t typeArray_size = 0; - encode(huffmanTree, coeff_type[e], reg_count, result_pos + sizeof(size_t), &typeArray_size); - sizeToBytes(result_pos, typeArray_size); - result_pos += sizeof(size_t) + typeArray_size; - intToBytes_bigEndian(result_pos, coeff_unpredictable_count[e]); - result_pos += sizeof(int); - memcpy(result_pos, coeff_unpred_data[e], coeff_unpredictable_count[e]*sizeof(float)); - result_pos += coeff_unpredictable_count[e]*sizeof(float); - SZ_ReleaseHuffman(huffmanTree); - } - } - free(coeff_result_type); - free(coeff_unpredictable_data); - - //record the number of unpredictable data and also store them - memcpy(result_pos, &total_unpred, sizeof(size_t)); - result_pos += sizeof(size_t); - // record blockwise unpred data - size_t compressed_blockwise_unpred_count_size; - unsigned char * compressed_bw_unpred_count = SZ_compress_args(SZ_INT32, blockwise_unpred_count, &compressed_blockwise_unpred_count_size, SZ_ABS, 0.5, 0, 0, 0, 0, 0, 0, num_blocks); - memcpy(result_pos, &compressed_blockwise_unpred_count_size, sizeof(size_t)); - result_pos += sizeof(size_t); - memcpy(result_pos, compressed_bw_unpred_count, compressed_blockwise_unpred_count_size); - result_pos += compressed_blockwise_unpred_count_size; - free(blockwise_unpred_count); - free(compressed_bw_unpred_count); - memcpy(result_pos, result_unpredictable_data, total_unpred * sizeof(float)); - result_pos += total_unpred * sizeof(float); - - free(reg_params); - free(indicator); - free(result_unpredictable_data); - // encode type array by block - type = result_type; - size_t total_type_array_size = 0; - unsigned char * type_array_buffer = (unsigned char *) malloc(num_blocks*max_num_block_elements*sizeof(int)); - unsigned short * type_array_block_size = (unsigned short *) malloc(num_blocks*sizeof(unsigned short)); - unsigned char * type_array_buffer_pos = type_array_buffer; - unsigned short * type_array_block_size_pos = type_array_block_size; - - for(size_t i=0; isegment_size==0?dataLength/confparams_cpr->segment_size:dataLength/confparams_cpr->segment_size+1; @@ -284,8 +284,6 @@ size_t dataLength, size_t *outSize, float min, float max) listAdd_float(last3CmprsData, vce->data); }//end of for -// char* expSegmentsInBytes; -// int expSegmentsInBytes_size = convertESCToBytes(esc, &expSegmentsInBytes); int exactDataNum = exactLeadNumArray->size; TightDataPointStorageF* tdps; @@ -297,17 +295,6 @@ size_t dataLength, size_t *outSize, float min, float max) resiBitLengthArray->array, resiBitLengthArray->size, realPrecision, medianValue, (char)reqLength, quantization_intervals, pwrErrBoundBytes, pwrErrBoundBytes_size, radExpo); -//sdi:Debug -/* int sum =0; - for(i=0;itypeArray, tdps->typeArray_size, type_); -// printf("tdps->typeArray_size=%d\n", tdps->typeArray_size); //free memory free_DBA(resiBitLengthArray); @@ -328,25 +315,25 @@ size_t dataLength, size_t *outSize, float min, float max) unsigned char dsLengthBytes[exe_params->SZ_SIZE_TYPE]; intToBytes_bigEndian(dsLengthBytes, dataLength);//4 for (i = 0; i < 3; i++)//3 - (*newByteData)[k++] = versionNumber[i]; + newByteData[k++] = versionNumber[i]; if(exe_params->SZ_SIZE_TYPE==4) { - (*newByteData)[k++] = 16; //=00010000 + newByteData[k++] = 16; //=00010000 } else { - (*newByteData)[k++] = 80; + newByteData[k++] = 80; } for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)//4 or 8 - (*newByteData)[k++] = dsLengthBytes[i]; + newByteData[k++] = dsLengthBytes[i]; if(sysEndianType==BIG_ENDIAN_SYSTEM) - memcpy((*newByteData)+4+exe_params->SZ_SIZE_TYPE, oriData, dataLength*floatSize); + memcpy(newByteData+4+exe_params->SZ_SIZE_TYPE, oriData, dataLength*floatSize); else { - unsigned char* p = (*newByteData)+4+exe_params->SZ_SIZE_TYPE; + unsigned char* p = newByteData+4+exe_params->SZ_SIZE_TYPE; for(i=0;ierrorBoundMode, @@ -660,7 +647,7 @@ size_t dataLength, double absErrBound, double relBoundRatio, double pwrErrRatio, #include -void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_pre_log(unsigned char** newByteData, float *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, float min, float max){ +void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_pre_log(unsigned char* newByteData, float *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, float min, float max){ float * log_data = (float *) malloc(dataLength * sizeof(float)); @@ -722,7 +709,7 @@ void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_pre_log(unsigned char** newByt } -void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_pre_log_MSST19(unsigned char** newByteData, float *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, float valueRangeSize, float medianValue_f, +void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_pre_log_MSST19(unsigned char* newByteData, float *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, float valueRangeSize, float medianValue_f, unsigned char* signs, bool* positive, float min, float max, float nearZero){ float multiplier = pow((1+pwrErrRatio), -3.0001); for(int i=0; i -#include -#include -#include -#include -#include "sz.h" -#include "CompressElement.h" -#include "DynamicByteArray.h" -#include "DynamicIntArray.h" -#include "TightDataPointStorageF.h" -#include "zlib.h" -#include "rw.h" -#include "sz_float_ts.h" - -unsigned int optimize_intervals_float_1D_ts(float *oriData, size_t dataLength, float* preData, double realPrecision) -{ - size_t i = 0, radiusIndex; - float pred_value = 0, pred_err; - size_t *intervals = (size_t*)malloc(confparams_cpr->maxRangeRadius*sizeof(size_t)); - memset(intervals, 0, confparams_cpr->maxRangeRadius*sizeof(size_t)); - size_t totalSampleSize = dataLength/confparams_cpr->sampleDistance; - for(i=2;isampleDistance==0) - { - pred_value = preData[i]; - pred_err = fabs(pred_value - oriData[i]); - radiusIndex = (unsigned long)((pred_err/realPrecision+1)/2); - if(radiusIndex>=confparams_cpr->maxRangeRadius) - radiusIndex = confparams_cpr->maxRangeRadius - 1; - intervals[radiusIndex]++; - } - } - //compute the appropriate number - size_t targetCount = totalSampleSize*confparams_cpr->predThreshold; - size_t sum = 0; - for(i=0;imaxRangeRadius;i++) - { - sum += intervals[i]; - if(sum>targetCount) - break; - } - if(i>=confparams_cpr->maxRangeRadius) - i = confparams_cpr->maxRangeRadius-1; - - unsigned int accIntervals = 2*(i+1); - unsigned int powerOf2 = roundUpToPowerOf2(accIntervals); - - if(powerOf2<32) - powerOf2 = 32; - - free(intervals); - return powerOf2; -} - -TightDataPointStorageF* SZ_compress_float_1D_MDQ_ts(float *oriData, size_t dataLength, sz_multisteps* multisteps, -double realPrecision, float valueRangeSize, float medianValue_f) -{ - float* preStepData = (float*)(multisteps->hist_data); - - //store the decompressed data - //float* decData = (float*)malloc(sizeof(float)*dataLength); - //memset(decData, 0, sizeof(float)*dataLength); - float* decData = preStepData; - - unsigned int quantization_intervals; - if(exe_params->optQuantMode==1) - quantization_intervals = optimize_intervals_float_1D_ts(oriData, dataLength, preStepData, realPrecision); - else - quantization_intervals = exe_params->intvCapacity; - updateQuantizationInfo(quantization_intervals); - - size_t i; - int reqLength; - float medianValue = medianValue_f; - short radExpo = getExponent_float(valueRangeSize/2); - - computeReqLength_float(realPrecision, radExpo, &reqLength, &medianValue); - - int* type = (int*) malloc(dataLength*sizeof(int)); - - float* spaceFillingValue = oriData; // - - DynamicIntArray *exactLeadNumArray; - new_DIA(&exactLeadNumArray, DynArrayInitLen); - - DynamicByteArray *exactMidByteArray; - new_DBA(&exactMidByteArray, DynArrayInitLen); - - DynamicIntArray *resiBitArray; - new_DIA(&resiBitArray, DynArrayInitLen); - - unsigned char preDataBytes[4]; - intToBytes_bigEndian(preDataBytes, 0); - - int reqBytesLength = reqLength/8; - int resiBitsLength = reqLength%8; - - FloatValueCompressElement *vce = (FloatValueCompressElement*)malloc(sizeof(FloatValueCompressElement)); - LossyCompressionElement *lce = (LossyCompressionElement*)malloc(sizeof(LossyCompressionElement)); - - //add the first data - type[0] = 0; - compressSingleFloatValue(vce, spaceFillingValue[0], realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength); - updateLossyCompElement_Float(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce); - memcpy(preDataBytes,vce->curBytes,4); - addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce); - decData[0] = vce->data; - - //add the second data - type[1] = 0; - compressSingleFloatValue(vce, spaceFillingValue[1], realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength); - updateLossyCompElement_Float(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce); - memcpy(preDataBytes,vce->curBytes,4); - addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce); - decData[1] = vce->data; - - int state = 0; - double checkRadius = 0; - float curData = 0; - float pred = 0; - float predAbsErr = 0; - checkRadius = (exe_params->intvCapacity-1)*realPrecision; - double interval = 2*realPrecision; - - for(i=2;i=pred) - { - type[i] = exe_params->intvRadius+state; - pred = pred + state*interval; - } - else //curDataintvRadius-state; - pred = pred - state*interval; - } - - //double-check the prediction error in case of machine-epsilon impact - if(fabs(curData-pred)>realPrecision) - { - type[i] = 0; - compressSingleFloatValue(vce, curData, realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength); - updateLossyCompElement_Float(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce); - memcpy(preDataBytes,vce->curBytes,4); - addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce); - decData[i] = vce->data; - } - else - { - decData[i] = pred; - } - - continue; - } - - //unpredictable data processing - type[i] = 0; - compressSingleFloatValue(vce, curData, realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength); - updateLossyCompElement_Float(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce); - memcpy(preDataBytes,vce->curBytes,4); - addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce); - decData[i] = vce->data; - }//end of for - - size_t exactDataNum = exactLeadNumArray->size; - - TightDataPointStorageF* tdps; - - new_TightDataPointStorageF(&tdps, dataLength, exactDataNum, - type, exactMidByteArray->array, exactMidByteArray->size, - exactLeadNumArray->array, - resiBitArray->array, resiBitArray->size, - resiBitsLength, - realPrecision, medianValue, (char)reqLength, quantization_intervals, NULL, 0, 0); - - //free memory - free_DIA(exactLeadNumArray); - free_DIA(resiBitArray); - free(type); - free(vce); - free(lce); - free(exactMidByteArray); //exactMidByteArray->array has been released in free_TightDataPointStorageF(tdps); - - //memcpy(preStepData, decData, dataLength*sizeof(float)); //update the data - //free(decData); - - return tdps; -} - - diff --git a/deps/SZ/sz/src/sz_omp.c b/deps/SZ/sz/src/sz_omp.c deleted file mode 100644 index 362b6bf695..0000000000 --- a/deps/SZ/sz/src/sz_omp.c +++ /dev/null @@ -1,986 +0,0 @@ -/** - * @file sz_omp.c - * @author Xin Liang - * @date July, 2017 - * @brief the implementation of openMP version - * (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -#include "sz_omp.h" -#include -#include - -double sz_wtime(){ -#ifdef _OPENMP - return omp_get_wtime(); -#else - struct timespec ts; - clock_gettime(CLOCK_MONOTONIC, &ts); - - return (double)ts.tv_sec + (double)ts.tv_nsec / 1000000000.0; -#endif -} - -int sz_get_max_threads(){ -#ifdef _OPENMP - return omp_get_max_threads(); -#else - return 1; -#endif -} - -int sz_get_thread_num(){ -#ifdef _OPENMP - return omp_get_thread_num(); -#else - return 0; -#endif -} - -void sz_set_num_threads(int nthreads){ -#ifdef _OPENMP - omp_set_num_threads(nthreads); -#endif -} - -unsigned char * SZ_compress_float_1D_MDQ_openmp(float *oriData, size_t r1, double realPrecision, size_t * comp_size){ - return NULL; -} -unsigned char * SZ_compress_float_2D_MDQ_openmp(float *oriData, size_t r1, size_t r2, double realPrecision, size_t * comp_size){ - return NULL; -} - -unsigned char * SZ_compress_float_3D_MDQ_openmp(float *oriData, size_t r1, size_t r2, size_t r3, float realPrecision, size_t * comp_size){ - - float elapsed_time = 0.0; - - elapsed_time = -sz_wtime(); - unsigned int quantization_intervals; - if(exe_params->optQuantMode==1) - { - // quantization_intervals = optimize_intervals_float_3D(oriData, r1, realPrecision); - quantization_intervals = optimize_intervals_float_3D_opt(oriData, r1, r2, r3, realPrecision); - //quantization_intervals = 32768; - printf("3D number of bins: %d\nerror bound %.20f\n", quantization_intervals, realPrecision); - // exit(0); - updateQuantizationInfo(quantization_intervals); - } - else{ - quantization_intervals = exe_params->intvCapacity; - } - elapsed_time += sz_wtime(); - printf("opt interval time: %.4f\n", elapsed_time); - - elapsed_time = -sz_wtime(); - int thread_num = sz_get_max_threads(); - int thread_order = (int)log2(thread_num); - size_t num_x = 0, num_y = 0, num_z = 0; - { - int block_thread_order = thread_order / 3; - switch(thread_order % 3){ - case 0:{ - num_x = 1 << block_thread_order; - num_y = 1 << block_thread_order; - num_z = 1 << block_thread_order; - break; - } - case 1:{ - num_x = 1 << (block_thread_order + 1); - num_y = 1 << block_thread_order; - num_z = 1 << block_thread_order; - break; - } - case 2:{ - num_x = 1 << (block_thread_order + 1); - num_y = 1 << (block_thread_order + 1); - num_z = 1 << block_thread_order; - break; - } - } - thread_num = num_x * num_y * num_z; - } - sz_set_num_threads(thread_num); - // calculate block dims - printf("number of blocks: %zu %zu %zu\n", num_x, num_y, num_z); - - size_t split_index_x, split_index_y, split_index_z; - size_t early_blockcount_x, early_blockcount_y, early_blockcount_z; - size_t late_blockcount_x, late_blockcount_y, late_blockcount_z; - SZ_COMPUTE_BLOCKCOUNT(r1, num_x, split_index_x, early_blockcount_x, late_blockcount_x); - SZ_COMPUTE_BLOCKCOUNT(r2, num_y, split_index_y, early_blockcount_y, late_blockcount_y); - SZ_COMPUTE_BLOCKCOUNT(r3, num_z, split_index_z, early_blockcount_z, late_blockcount_z); - - size_t max_num_block_elements = early_blockcount_x * early_blockcount_y * early_blockcount_z; - size_t num_blocks = num_x * num_y * num_z; - size_t num_elements = r1 * r2 * r3; - // printf("max_num_block_elements %d num_blocks %d\n", max_num_block_elements, num_blocks); - - size_t dim0_offset = r2 * r3; - size_t dim1_offset = r3; - - // printf("malloc blockinfo array start\n"); - // fflush(stdout); - - size_t buffer_size = early_blockcount_y * early_blockcount_z * sizeof(float); - int * result_type = (int *) malloc(num_elements * sizeof(int)); - size_t unpred_data_max_size = max_num_block_elements; - float * result_unpredictable_data = (float *) malloc(unpred_data_max_size * sizeof(float) * num_blocks); - unsigned int * unpredictable_count = (unsigned int *) malloc(num_blocks * sizeof(unsigned int)); - float * mean = malloc(num_blocks * sizeof(float)); - float * buffer0, * buffer1; - buffer0 = (float *) malloc(buffer_size * thread_num); - buffer1 = (float *) malloc(buffer_size * thread_num); - unsigned char * result = (unsigned char *) malloc(num_elements * (sizeof(int) + sizeof(float))); - size_t * unpred_offset = (size_t *) malloc(num_blocks * sizeof(size_t)); - unsigned char * encoding_buffer = (unsigned char *) malloc(max_num_block_elements * sizeof(int) * num_blocks); - size_t * block_offset = (size_t *) malloc(num_blocks * sizeof(size_t)); - size_t *freq = (size_t *)malloc(thread_num*quantization_intervals*4*sizeof(size_t)); - memset(freq, 0, thread_num*quantization_intervals*4*sizeof(size_t)); - - size_t stateNum = quantization_intervals*2; - HuffmanTree* huffmanTree = createHuffmanTree(stateNum); - - int num_yz = num_y * num_z; - #pragma omp parallel for - for(int t=0; tcode[i]) nodeCount++; - nodeCount = nodeCount*2-1; - unsigned char *treeBytes; - unsigned int treeByteSize = convert_HuffTree_to_bytes_anyStates(huffmanTree, nodeCount, &treeBytes); - - unsigned int meta_data_offset = 3 + 1 + MetaDataByteLength; - size_t total_unpred = 0; - for(int i=0; iintvRadius = (int)((tdps->intervals - 1)/ 2); - - unsigned int tree_size = bytesToInt_bigEndian(comp_data_pos); - comp_data_pos += sizeof(unsigned int); - size_t huffman_nodes = bytesToInt_bigEndian(comp_data_pos); - huffmanTree->allNodes = huffman_nodes; - // printf("Reconstruct huffman tree with node count %ld\n", nodeCount); - // fflush(stdout); - node root = reconstruct_HuffTree_from_bytes_anyStates(huffmanTree, comp_data_pos+4, huffmanTree->allNodes); - - comp_data_pos += 4 + tree_size; - unsigned int * unpred_count = (unsigned int *) comp_data_pos; - comp_data_pos += num_blocks * sizeof(unsigned int); - float * mean_pos = (float *) comp_data_pos; - comp_data_pos += num_blocks * sizeof(float); - float * result_unpredictable_data = (float *) comp_data_pos; - size_t total_unpred = 0; - for(int i=0; ioptQuantMode==1) - { - // quantization_intervals = optimize_intervals_float_3D(oriData, r1, realPrecision); - quantization_intervals = optimize_intervals_double_3D_opt(oriData, r1, r2, r3, realPrecision); - //quantization_intervals = 32768; - printf("3D number of bins: %d\nerror bound %.20f\n", quantization_intervals, realPrecision); - // exit(0); - updateQuantizationInfo(quantization_intervals); - } - else{ - quantization_intervals = exe_params->intvCapacity; - } - elapsed_time += sz_wtime(); - printf("opt interval time: %.4f\n", elapsed_time); - - elapsed_time = -sz_wtime(); - int thread_num = sz_get_max_threads(); - int thread_order = (int)log2(thread_num); - size_t num_x = 0, num_y = 0, num_z = 0; - { - int block_thread_order = thread_order / 3; - switch(thread_order % 3){ - case 0:{ - num_x = 1 << block_thread_order; - num_y = 1 << block_thread_order; - num_z = 1 << block_thread_order; - break; - } - case 1:{ - num_x = 1 << (block_thread_order + 1); - num_y = 1 << block_thread_order; - num_z = 1 << block_thread_order; - break; - } - case 2:{ - num_x = 1 << (block_thread_order + 1); - num_y = 1 << (block_thread_order + 1); - num_z = 1 << block_thread_order; - break; - } - } - thread_num = num_x * num_y * num_z; - } - sz_set_num_threads(thread_num); - // calculate block dims - printf("number of blocks: %zu %zu %zu\n", num_x, num_y, num_z); - - size_t split_index_x, split_index_y, split_index_z; - size_t early_blockcount_x, early_blockcount_y, early_blockcount_z; - size_t late_blockcount_x, late_blockcount_y, late_blockcount_z; - SZ_COMPUTE_BLOCKCOUNT(r1, num_x, split_index_x, early_blockcount_x, late_blockcount_x); - SZ_COMPUTE_BLOCKCOUNT(r2, num_y, split_index_y, early_blockcount_y, late_blockcount_y); - SZ_COMPUTE_BLOCKCOUNT(r3, num_z, split_index_z, early_blockcount_z, late_blockcount_z); - - size_t max_num_block_elements = early_blockcount_x * early_blockcount_y * early_blockcount_z; - size_t num_blocks = num_x * num_y * num_z; - size_t num_elements = r1 * r2 * r3; - // printf("max_num_block_elements %d num_blocks %d\n", max_num_block_elements, num_blocks); - - size_t dim0_offset = r2 * r3; - size_t dim1_offset = r3; - - // printf("malloc blockinfo array start\n"); - // fflush(stdout); - - size_t buffer_size = early_blockcount_y * early_blockcount_z * sizeof(double); - int * result_type = (int *) malloc(num_elements * sizeof(int)); - size_t unpred_data_max_size = max_num_block_elements; - double * result_unpredictable_data = (double *) malloc(unpred_data_max_size * sizeof(double) * num_blocks); - unsigned int * unpredictable_count = (unsigned int *) malloc(num_blocks * sizeof(unsigned int)); - double * mean = malloc(num_blocks * sizeof(double)); - double * buffer0, * buffer1; - buffer0 = (double *) malloc(buffer_size * thread_num); - buffer1 = (double *) malloc(buffer_size * thread_num); - unsigned char * result = (unsigned char *) malloc(num_elements * (sizeof(int) + sizeof(double))); - size_t * unpred_offset = (size_t *) malloc(num_blocks * sizeof(size_t)); - unsigned char * encoding_buffer = (unsigned char *) malloc(max_num_block_elements * sizeof(int) * num_blocks); - size_t * block_offset = (size_t *) malloc(num_blocks * sizeof(size_t)); - size_t *freq = (size_t *)malloc(thread_num*quantization_intervals*4*sizeof(size_t)); - memset(freq, 0, thread_num*quantization_intervals*4*sizeof(size_t)); - - size_t stateNum = quantization_intervals*2; - HuffmanTree* huffmanTree = createHuffmanTree(stateNum); - - int num_yz = num_y * num_z; - #pragma omp parallel for - for(int t=0; tcode[i]) nodeCount++; - nodeCount = nodeCount*2-1; - unsigned char *treeBytes; - unsigned int treeByteSize = convert_HuffTree_to_bytes_anyStates(huffmanTree, nodeCount, &treeBytes); - - unsigned int meta_data_offset = 3 + 1 + MetaDataByteLength; - size_t total_unpred = 0; - for(int i=0; iintvRadius = (int)((tdps->intervals - 1)/ 2); - - unsigned int tree_size = bytesToInt_bigEndian(comp_data_pos); - comp_data_pos += sizeof(unsigned int); - size_t huffman_nodes = bytesToInt_bigEndian(comp_data_pos); - huffmanTree->allNodes = huffman_nodes; - // printf("Reconstruct huffman tree with node count %ld\n", nodeCount); - // fflush(stdout); - node root = reconstruct_HuffTree_from_bytes_anyStates(huffmanTree, comp_data_pos+4, huffmanTree->allNodes); - - comp_data_pos += 4 + tree_size; - unsigned int * unpred_count = (unsigned int *) comp_data_pos; - comp_data_pos += num_blocks * sizeof(unsigned int); - double * mean_pos = (double *) comp_data_pos; - comp_data_pos += num_blocks * sizeof(double); - double * result_unpredictable_data = (double *) comp_data_pos; - size_t total_unpred = 0; - for(int i=0; iallNodes*sizeof(size_t)); - // memset(freq, 0, thread_num*huffmanTree->allNodes*sizeof(size_t)); - size_t block_size = (length - 1)/ thread_num + 1; - size_t block_residue = length - (thread_num - 1) * block_size; - #pragma omp parallel for - for(int t=0; tallNodes; - if(id < thread_num - 1){ - for(size_t i=0; iallNodes; - for(int t=1; tallNodes; i++){ - freq[i] += freq_pos[i]; - } - freq_pos += huffmanTree->allNodes; - } - - for (i = 0; i < huffmanTree->allNodes; i++) - if (freq[i]) - qinsert(huffmanTree, new_node(huffmanTree, freq[i], i, 0, 0)); - - while (huffmanTree->qend > 2) - qinsert(huffmanTree, new_node(huffmanTree, 0, 0, qremove(huffmanTree), qremove(huffmanTree))); - - build_code(huffmanTree, huffmanTree->qq[1], 0, 0, 0); - // free(freq); -} - - - diff --git a/deps/SZ/sz/src/szd_double.c b/deps/SZ/sz/src/szd_double.c index 0bbaaff0f9..14b2b9958a 100644 --- a/deps/SZ/sz/src/szd_double.c +++ b/deps/SZ/sz/src/szd_double.c @@ -18,11 +18,11 @@ #include "szd_double_ts.h" #include "utility.h" -int SZ_decompress_args_double(double** newData, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, unsigned char* cmpBytes, -size_t cmpSize, int compressionType, double* hist_data, sz_exedata* pde_exe, sz_params* pde_params) +int SZ_decompress_args_double(double* newData, size_t r1, unsigned char* cmpBytes, + size_t cmpSize, int compressionType, double* hist_data, sz_exedata* pde_exe, sz_params* pde_params) { - int status = SZ_SCES; - size_t dataLength = computeDataLength(r5,r4,r3,r2,r1); + int status = SZ_SUCCESS; + size_t dataLength = r1; //unsigned char* tmpBytes; size_t targetUncompressSize = dataLength <<3; //i.e., *8 @@ -39,6 +39,8 @@ size_t cmpSize, int compressionType, double* hist_data, sz_exedata* pde_exe, sz_ else pde_params->szMode = SZ_BEST_SPEED; } + + if(pde_params->szMode==SZ_BEST_SPEED) { tmpSize = cmpSize; @@ -68,20 +70,20 @@ size_t cmpSize, int compressionType, double* hist_data, sz_exedata* pde_exe, sz_ TightDataPointStorageD* tdps; int errBoundMode = new_TightDataPointStorageD_fromFlatBytes(&tdps, szTmpBytes, tmpSize, pde_exe, pde_params); - int dim = computeDimension(r5,r4,r3,r2,r1); + int dim = r1; int doubleSize = sizeof(double); if(tdps->isLossless) { - *newData = (double*)malloc(doubleSize*dataLength); + // *newData = (double*)malloc(doubleSize*dataLength); comment by tickduan if(sysEndianType==BIG_ENDIAN_SYSTEM) { - memcpy(*newData, szTmpBytes+4+MetaDataByteLength_double+exe_params->SZ_SIZE_TYPE, dataLength*doubleSize); + memcpy(newData, szTmpBytes+4+MetaDataByteLength_double+exe_params->SZ_SIZE_TYPE, dataLength*doubleSize); } else { unsigned char* p = szTmpBytes+4+MetaDataByteLength_double+exe_params->SZ_SIZE_TYPE; for(i=0;isol_ID==SZ_Transpose) @@ -92,29 +94,17 @@ size_t cmpSize, int compressionType, double* hist_data, sz_exedata* pde_exe, sz_ { if(tdps->raBytes_size > 0) //v2.0 { - if (dim == 1) - getSnapshotData_double_1D(newData,r1,tdps, errBoundMode, 0, hist_data, pde_params); - else - { - printf("Error: currently support only at most 4 dimensions!\n"); - status = SZ_DERR; - } + getSnapshotData_double_1D(newData,r1,tdps, errBoundMode, 0, hist_data, pde_params); } else //1.4.13 or time-based compression { - if (dim == 1) - getSnapshotData_double_1D(newData,r1,tdps, errBoundMode, compressionType, hist_data, pde_params); - else - { - printf("Error: currently support only at most 4 dimensions!\n"); - status = SZ_DERR; - } + getSnapshotData_double_1D(newData,r1,tdps, errBoundMode, compressionType, hist_data, pde_params); } } if(pde_params->protectValueRange) { - double* nd = *newData; + double* nd = newData; double min = pde_params->dmin; double max = pde_params->dmax; for(i=0;iintervals); int intvRadius = tdps->intervals/2; @@ -147,7 +137,7 @@ void decompressDataSeries_double_1D(double** data, size_t dataSeriesLength, doub double interval = tdps->realPrecision*2; convertByteArray2IntArray_fast_2b(tdps->exactDataNum, tdps->leadNumArray, tdps->leadNumArray_size, &leadNum); - *data = (double*)malloc(sizeof(double)*dataSeriesLength); + //*data = (double*)malloc(sizeof(double)*dataSeriesLength); comment by tickduan int* type = (int*)malloc(dataSeriesLength*sizeof(int)); @@ -212,30 +202,25 @@ void decompressDataSeries_double_1D(double** data, size_t dataSeriesLength, doub } exactData = bytesToDouble(curBytes); - (*data)[i] = exactData + medianValue; + data[i] = exactData + medianValue; memcpy(preBytes,curBytes,8); break; default: - //predValue = 2 * (*data)[i-1] - (*data)[i-2]; - predValue = (*data)[i-1]; - (*data)[i] = predValue + (type_-intvRadius)*interval; + //predValue = 2 * data[i-1] - data[i-2]; + predValue = data[i-1]; + data[i] = predValue + (type_-intvRadius)*interval; break; } - //printf("%.30G\n",(*data)[i]); + //printf("%.30G\n",data[i]); } -#ifdef HAVE_TIMECMPR - if(confparams_dec->szMode == SZ_TEMPORAL_COMPRESSION) - memcpy(hist_data, (*data), dataSeriesLength*sizeof(double)); -#endif - free(leadNum); free(type); return; } /*MSST19*/ -void decompressDataSeries_double_1D_MSST19(double** data, size_t dataSeriesLength, TightDataPointStorageD* tdps) +void decompressDataSeries_double_1D_MSST19(double* data, size_t dataSeriesLength, TightDataPointStorageD* tdps) { //updateQuantizationInfo(tdps->intervals); int intvRadius = tdps->intervals/2; @@ -248,7 +233,7 @@ void decompressDataSeries_double_1D_MSST19(double** data, size_t dataSeriesLengt //double interval = tdps->realPrecision*2; convertByteArray2IntArray_fast_2b(tdps->exactDataNum, tdps->leadNumArray, tdps->leadNumArray_size, &leadNum); - *data = (double*)malloc(sizeof(double)*dataSeriesLength); + //*data = (double*)malloc(sizeof(double)*dataSeriesLength); comment by tickduan int* type = (int*)malloc(dataSeriesLength*sizeof(int)); @@ -318,52 +303,39 @@ void decompressDataSeries_double_1D_MSST19(double** data, size_t dataSeriesLengt } exactData = bytesToDouble(curBytes); - (*data)[i] = exactData; + data[i] = exactData; memcpy(preBytes,curBytes,8); - predValue = (*data)[i]; + predValue = data[i]; break; default: - //predValue = 2 * (*data)[i-1] - (*data)[i-2]; - //predValue = (*data)[i-1]; + //predValue = 2 * data[i-1] - data[i-2]; + //predValue = data[i-1]; predValue = fabs(predValue) * precisionTable[type_]; - (*data)[i] = predValue; + data[i] = predValue; break; } } -#ifdef HAVE_TIMECMPR - if(confparams_dec->szMode == SZ_TEMPORAL_COMPRESSION) - memcpy(multisteps->hist_data, (*data), dataSeriesLength*sizeof(double)); -#endif free(precisionTable); free(leadNum); free(type); return; } -void getSnapshotData_double_1D(double** data, size_t dataSeriesLength, TightDataPointStorageD* tdps, int errBoundMode, int compressionType, double* hist_data, sz_params* pde_params) +void getSnapshotData_double_1D(double* data, size_t dataSeriesLength, TightDataPointStorageD* tdps, int errBoundMode, int compressionType, double* hist_data, sz_params* pde_params) { size_t i; if (tdps->allSameData) { double value = bytesToDouble(tdps->exactMidBytes); - *data = (double*)malloc(sizeof(double)*dataSeriesLength); + + //*data = (double*)malloc(sizeof(double)*dataSeriesLength); comment by tickduan for (i = 0; i < dataSeriesLength; i++) - (*data)[i] = value; + data[i] = value; } else { if (tdps->rtypeArray == NULL) { if(errBoundMode < PW_REL) { -#ifdef HAVE_TIMECMPR - if(confparams_dec->szMode == SZ_TEMPORAL_COMPRESSION) - { - if(multisteps->compressionType == 0) //snapshot - decompressDataSeries_double_1D(data, dataSeriesLength, hist_data, tdps); - else - decompressDataSeries_double_1D_ts(data, dataSeriesLength, hist_data, tdps); - } - else -#endif - decompressDataSeries_double_1D(data, dataSeriesLength, hist_data, tdps); + decompressDataSeries_double_1D(data, dataSeriesLength, hist_data, tdps); } else { diff --git a/deps/SZ/sz/src/szd_double_pwr.c b/deps/SZ/sz/src/szd_double_pwr.c index 6b034bb8f1..c865129790 100644 --- a/deps/SZ/sz/src/szd_double_pwr.c +++ b/deps/SZ/sz/src/szd_double_pwr.c @@ -22,7 +22,7 @@ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wchar-subscripts" -void decompressDataSeries_double_1D_pwr(double** data, size_t dataSeriesLength, TightDataPointStorageD* tdps) +void decompressDataSeries_double_1D_pwr(double* data, size_t dataSeriesLength, TightDataPointStorageD* tdps) { updateQuantizationInfo(tdps->intervals); unsigned char tmpPrecBytes[8] = {0}; //used when needing to convert bytes to double values @@ -36,7 +36,7 @@ void decompressDataSeries_double_1D_pwr(double** data, size_t dataSeriesLength, double interval = 0;// = (double)tdps->realPrecision*2; convertByteArray2IntArray_fast_2b(tdps->exactDataNum, tdps->leadNumArray, tdps->leadNumArray_size, &leadNum); - *data = (double*)malloc(sizeof(double)*dataSeriesLength); + //*data = (double*)malloc(sizeof(double)*dataSeriesLength); comment by tickduan int* type = (int*)malloc(dataSeriesLength*sizeof(int)); @@ -117,16 +117,16 @@ void decompressDataSeries_double_1D_pwr(double** data, size_t dataSeriesLength, } exactData = bytesToDouble(curBytes); - (*data)[i] = exactData + medianValue; + data[i] = exactData + medianValue; memcpy(preBytes,curBytes,8); break; default: - //predValue = 2 * (*data)[i-1] - (*data)[i-2]; - predValue = (*data)[i-1]; - (*data)[i] = predValue + (type_-exe_params->intvRadius)*interval; + //predValue = 2 * data[i-1] - data[i-2]; + predValue = data[i-1]; + data[i] = predValue + (type_-exe_params->intvRadius)*interval; break; } - //printf("%.30G\n",(*data)[i]); + //printf("%.30G\n",data[i]); } free(leadNum); free(type); @@ -135,7 +135,7 @@ void decompressDataSeries_double_1D_pwr(double** data, size_t dataSeriesLength, -void decompressDataSeries_double_1D_pwrgroup(double** data, size_t dataSeriesLength, TightDataPointStorageD* tdps) +void decompressDataSeries_double_1D_pwrgroup(double* data, size_t dataSeriesLength, TightDataPointStorageD* tdps) { double *posGroups, *negGroups, *groups; double pos_01_group, neg_01_group; @@ -148,7 +148,7 @@ void decompressDataSeries_double_1D_pwrgroup(double** data, size_t dataSeriesLen convertByteArray2IntArray_fast_2b(tdps->exactDataNum, tdps->leadNumArray, tdps->leadNumArray_size, &leadNum); - *data = (double*)malloc(sizeof(double)*dataSeriesLength); + //*data = (double*)malloc(sizeof(double)*dataSeriesLength); comment by tickduan int* type = (int*)malloc(dataSeriesLength*sizeof(int)); @@ -266,7 +266,7 @@ void decompressDataSeries_double_1D_pwrgroup(double** data, size_t dataSeriesLen exactData = bytesToDouble(curBytes); exactData = exactData + medianValue; - (*data)[i] = exactData; + data[i] = exactData; memcpy(preBytes,curBytes,8); groups[indexGrpID] = exactData; @@ -292,7 +292,7 @@ void decompressDataSeries_double_1D_pwrgroup(double** data, size_t dataSeriesLen // curValue = rawGrpID>0?pow(2,groupNum):-pow(2,groupNum); //} - (*data)[i] = curValue; + data[i] = curValue; groups[indexGrpID] = curValue; break; } @@ -309,7 +309,7 @@ void decompressDataSeries_double_1D_pwrgroup(double** data, size_t dataSeriesLen free(groupID); } -void decompressDataSeries_double_1D_pwr_pre_log(double** data, size_t dataSeriesLength, TightDataPointStorageD* tdps) { +void decompressDataSeries_double_1D_pwr_pre_log(double* data, size_t dataSeriesLength, TightDataPointStorageD* tdps) { decompressDataSeries_double_1D(data, dataSeriesLength, NULL, tdps); double threshold = tdps->minLogValue; @@ -318,22 +318,22 @@ void decompressDataSeries_double_1D_pwr_pre_log(double** data, size_t dataSeries sz_lossless_decompress(ZSTD_COMPRESSOR, tdps->pwrErrBoundBytes, tdps->pwrErrBoundBytes_size, &signs, dataSeriesLength); for(size_t i=0; iminLogValue; @@ -349,12 +349,12 @@ void decompressDataSeries_double_1D_pwr_pre_log_MSST19(double** data, size_t dat else sz_lossless_decompress(ZSTD_COMPRESSOR, tdps->pwrErrBoundBytes, tdps->pwrErrBoundBytes_size, &signs, dataSeriesLength); for(size_t i=0; i= 0){ - (*data)[i] = 0; + if(data[i] < threshold && data[i] >= 0){ + data[i] = 0; continue; } if(signs[i]){ - ptr = (uint64_t*)(*data) + i; + ptr = (uint64_t*)data + i; *ptr |= 0x8000000000000000; } } @@ -362,7 +362,7 @@ void decompressDataSeries_double_1D_pwr_pre_log_MSST19(double** data, size_t dat } else{ for(size_t i=0; i -#include -#include -#include "szd_double.h" -#include "TightDataPointStorageD.h" -#include "sz.h" -#include "Huffman.h" -#include "szd_double_ts.h" - -void decompressDataSeries_double_1D_ts(double** data, size_t dataSeriesLength, double* hist_data, TightDataPointStorageD* tdps) -{ - double* lastSnapshotData = hist_data; - updateQuantizationInfo(tdps->intervals); - size_t i, j, k = 0, p = 0, l = 0; // k is to track the location of residual_bit - // in resiMidBits, p is to track the - // byte_index of resiMidBits, l is for - // leadNum - unsigned char* leadNum; - double interval = tdps->realPrecision*2; - - convertByteArray2IntArray_fast_2b(tdps->exactDataNum, tdps->leadNumArray, tdps->leadNumArray_size, &leadNum); - *data = (double*)malloc(sizeof(double)*dataSeriesLength); - - int* type = (int*)malloc(dataSeriesLength*sizeof(int)); - - HuffmanTree* huffmanTree = createHuffmanTree(tdps->stateNum); - decode_withTree(huffmanTree, tdps->typeArray, dataSeriesLength, type); - SZ_ReleaseHuffman(huffmanTree); - - unsigned char preBytes[8]; - unsigned char curBytes[8]; - - memset(preBytes, 0, 8); - - size_t curByteIndex = 0; - int reqBytesLength, resiBitsLength, resiBits; - unsigned char leadingNum; - double medianValue, exactData, predValue = 0; - - reqBytesLength = tdps->reqLength/8; - resiBitsLength = tdps->reqLength%8; - medianValue = tdps->medianValue; - - int type_; - for (i = 0; i < dataSeriesLength; i++) { - type_ = type[i]; - switch (type_) { - case 0: - // compute resiBits - resiBits = 0; - if (resiBitsLength != 0) { - int kMod8 = k % 8; - int rightMovSteps = getRightMovingSteps(kMod8, resiBitsLength); - if (rightMovSteps > 0) { - int code = getRightMovingCode(kMod8, resiBitsLength); - resiBits = (tdps->residualMidBits[p] & code) >> rightMovSteps; - } else if (rightMovSteps < 0) { - int code1 = getLeftMovingCode(kMod8); - int code2 = getRightMovingCode(kMod8, resiBitsLength); - int leftMovSteps = -rightMovSteps; - rightMovSteps = 8 - leftMovSteps; - resiBits = (tdps->residualMidBits[p] & code1) << leftMovSteps; - p++; - resiBits = resiBits - | ((tdps->residualMidBits[p] & code2) >> rightMovSteps); - } else // rightMovSteps == 0 - { - int code = getRightMovingCode(kMod8, resiBitsLength); - resiBits = (tdps->residualMidBits[p] & code); - p++; - } - k += resiBitsLength; - } - - // recover the exact data - memset(curBytes, 0, 8); - leadingNum = leadNum[l++]; - memcpy(curBytes, preBytes, leadingNum); - for (j = leadingNum; j < reqBytesLength; j++) - curBytes[j] = tdps->exactMidBytes[curByteIndex++]; - if (resiBitsLength != 0) { - unsigned char resiByte = (unsigned char) (resiBits << (8 - resiBitsLength)); - curBytes[reqBytesLength] = resiByte; - } - - exactData = bytesToDouble(curBytes); - (*data)[i] = exactData + medianValue; - memcpy(preBytes,curBytes,8); - break; - default: - //predValue = (*data)[i-1]; - if(confparams_dec->szMode == SZ_TEMPORAL_COMPRESSION) - predValue = lastSnapshotData[i]; - (*data)[i] = predValue + (type_-exe_params->intvRadius)*interval; - break; - } - //printf("%.30G\n",(*data)[i]); - } - - memcpy(hist_data, (*data), dataSeriesLength*sizeof(double)); - - free(leadNum); - free(type); - return; -} diff --git a/deps/SZ/sz/src/szd_float.c b/deps/SZ/sz/src/szd_float.c index 4e244bf374..45e5d07e1a 100644 --- a/deps/SZ/sz/src/szd_float.c +++ b/deps/SZ/sz/src/szd_float.c @@ -15,7 +15,6 @@ #include "sz.h" #include "Huffman.h" #include "szd_float_pwr.h" -#include "szd_float_ts.h" #include "utility.h" @@ -45,13 +44,13 @@ void cost_end_() * int compressionType: 1 (time-based compression) ; 0 (space-based compression) * hist_data: only valid when compressionType==1, hist_data is the historical dataset such as the data in previous time step * - * @return status SUCCESSFUL (SZ_SCES) or not (other error codes) f + * @return status SUCCESSFUL (SZ_SUCCESS) or not (other error codes) f * */ -int SZ_decompress_args_float(float** newData, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, unsigned char* cmpBytes, -size_t cmpSize, int compressionType, float* hist_data, sz_exedata* pde_exe, sz_params* pde_params) +int SZ_decompress_args_float(float* newData, size_t r1, unsigned char* cmpBytes, + size_t cmpSize, int compressionType, float* hist_data, sz_exedata* pde_exe, sz_params* pde_params) { - int status = SZ_SCES; - size_t dataLength = computeDataLength(r5,r4,r3,r2,r1); + int status = SZ_SUCCESS; + size_t dataLength = r1; //unsigned char* tmpBytes; size_t targetUncompressSize = dataLength <<2; //i.e., *4 @@ -80,9 +79,7 @@ size_t cmpSize, int compressionType, float* hist_data, sz_exedata* pde_exe, sz_p if(targetUncompressSizelosslessCompressor, cmpBytes, (unsigned long)cmpSize, &szTmpBytes, (unsigned long)targetUncompressSize+4+MetaDataByteLength+exe_params->SZ_SIZE_TYPE);// (unsigned long)targetUncompressSize+8: consider the total length under lossless compression mode is actually 3+4+1+targetUncompressSize - //szTmpBytes = (unsigned char*)malloc(sizeof(unsigned char)*tmpSize); - //memcpy(szTmpBytes, tmpBytes, tmpSize); - //free(tmpBytes); //release useless memory + } else { @@ -101,20 +98,19 @@ size_t cmpSize, int compressionType, float* hist_data, sz_exedata* pde_exe, sz_p int errBoundMode = new_TightDataPointStorageF_fromFlatBytes(&tdps, szTmpBytes, tmpSize, pde_exe, pde_params); //writeByteData(tdps->typeArray, tdps->typeArray_size, "decompress-typebytes.tbt"); - int dim = computeDimension(r5,r4,r3,r2,r1); int floatSize = sizeof(float); if(tdps->isLossless) { - *newData = (float*)malloc(floatSize*dataLength); + //*newData = (float*)malloc(floatSize*dataLength); comment by tickduan if(sysEndianType==BIG_ENDIAN_SYSTEM) { - memcpy(*newData, szTmpBytes+4+MetaDataByteLength+exe_params->SZ_SIZE_TYPE, dataLength*floatSize); + memcpy(newData, szTmpBytes+4+MetaDataByteLength+exe_params->SZ_SIZE_TYPE, dataLength*floatSize); } else { unsigned char* p = szTmpBytes+4+MetaDataByteLength+exe_params->SZ_SIZE_TYPE; for(i=0;isol_ID==SZ_Transpose) @@ -125,30 +121,18 @@ size_t cmpSize, int compressionType, float* hist_data, sz_exedata* pde_exe, sz_p { if(tdps->raBytes_size > 0) //v2.0 { - if (dim == 1) - getSnapshotData_float_1D(newData,r1,tdps, errBoundMode, 0, hist_data, pde_params); - else - { - printf("Error: currently support only at most 4 dimensions!\n"); - status = SZ_DERR; - } + getSnapshotData_float_1D(newData,r1,tdps, errBoundMode, 0, hist_data, pde_params); } else //1.4.13 or time-based compression { - if (dim == 1) - getSnapshotData_float_1D(newData,r1,tdps, errBoundMode, compressionType, hist_data, pde_params); - else - { - printf("Error: currently support only at most 4 dimensions!\n"); - status = SZ_DERR; - } + getSnapshotData_float_1D(newData,r1,tdps, errBoundMode, compressionType, hist_data, pde_params); } } //cost_start_(); if(pde_params->protectValueRange) { - float* nd = *newData; + float* nd = newData; float min = pde_params->fmin; float max = pde_params->fmax; for(i=0;iintervals); int intvRadius = tdps->intervals/2; @@ -182,7 +166,7 @@ void decompressDataSeries_float_1D(float** data, size_t dataSeriesLength, float* float interval = tdps->realPrecision*2; convertByteArray2IntArray_fast_2b(tdps->exactDataNum, tdps->leadNumArray, tdps->leadNumArray_size, &leadNum); - *data = (float*)malloc(sizeof(float)*dataSeriesLength); + //data = (float*)malloc(sizeof(float)*dataSeriesLength); // comment by tickduan int* type = (int*)malloc(dataSeriesLength*sizeof(int)); @@ -247,30 +231,25 @@ void decompressDataSeries_float_1D(float** data, size_t dataSeriesLength, float* } exactData = bytesToFloat(curBytes); - (*data)[i] = exactData + medianValue; + data[i] = exactData + medianValue; memcpy(preBytes,curBytes,4); break; default: - //predValue = 2 * (*data)[i-1] - (*data)[i-2]; - predValue = (*data)[i-1]; - (*data)[i] = predValue + (float)(type_-intvRadius)*interval; + //predValue = 2 * data[i-1] - data[i-2]; + predValue = data[i-1]; + data[i] = predValue + (float)(type_-intvRadius)*interval; break; } - //printf("%.30G\n",(*data)[i]); + //printf("%.30G\n",data[i]); } -#ifdef HAVE_TIMECMPR - if(pde_params->szMode == SZ_TEMPORAL_COMPRESSION) - memcpy(hist_data, (*data), dataSeriesLength*sizeof(float)); -#endif - free(leadNum); free(type); return; } /*MSST19*/ -void decompressDataSeries_float_1D_MSST19(float** data, size_t dataSeriesLength, TightDataPointStorageF* tdps) +void decompressDataSeries_float_1D_MSST19(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps) { //updateQuantizationInfo(tdps->intervals); int intvRadius = tdps->intervals/2; @@ -283,7 +262,7 @@ void decompressDataSeries_float_1D_MSST19(float** data, size_t dataSeriesLength, //double interval = tdps->realPrecision*2; convertByteArray2IntArray_fast_2b(tdps->exactDataNum, tdps->leadNumArray, tdps->leadNumArray_size, &leadNum); - *data = (float*)malloc(sizeof(float)*dataSeriesLength); + // *data = (float*)malloc(sizeof(float)*dataSeriesLength); comment by tickduan int* type = (int*)malloc(dataSeriesLength*sizeof(int)); @@ -352,24 +331,20 @@ void decompressDataSeries_float_1D_MSST19(float** data, size_t dataSeriesLength, } exactData = bytesToFloat(curBytes); - (*data)[i] = exactData; + data[i] = exactData; memcpy(preBytes,curBytes,4); - predValue = (*data)[i]; + predValue = data[i]; break; default: - //predValue = 2 * (*data)[i-1] - (*data)[i-2]; - //predValue = (*data)[i-1]; + //predValue = 2 * data[i-1] - data[i-2]; + //predValue = data[i-1]; predValue = fabs(predValue) * precisionTable[type_]; - (*data)[i] = predValue; + data[i] = predValue; break; } - //printf("%.30G\n",(*data)[i]); + //printf("%.30G\n",data[i]); } -#ifdef HAVE_TIMECMPR - if(confparams_dec->szMode == SZ_TEMPORAL_COMPRESSION) - memcpy(multisteps->hist_data, (*data), dataSeriesLength*sizeof(float)); -#endif free(precisionTable); free(leadNum); free(type); @@ -377,30 +352,20 @@ void decompressDataSeries_float_1D_MSST19(float** data, size_t dataSeriesLength, } -void getSnapshotData_float_1D(float** data, size_t dataSeriesLength, TightDataPointStorageF* tdps, int errBoundMode, int compressionType, float* hist_data, sz_params* pde_params) +void getSnapshotData_float_1D(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps, int errBoundMode, int compressionType, float* hist_data, sz_params* pde_params) { size_t i; if (tdps->allSameData) { float value = bytesToFloat(tdps->exactMidBytes); - *data = (float*)malloc(sizeof(float)*dataSeriesLength); + //*data = (float*)malloc(sizeof(float)*dataSeriesLength); commnet by tickduan for (i = 0; i < dataSeriesLength; i++) - (*data)[i] = value; + data[i] = value; } else { if (tdps->rtypeArray == NULL) { if(errBoundMode < PW_REL) - { -#ifdef HAVE_TIMECMPR - if(pde_params->szMode == SZ_TEMPORAL_COMPRESSION) - { - if(compressionType == 0) //snapshot - decompressDataSeries_float_1D(data, dataSeriesLength, hist_data, tdps); - else - decompressDataSeries_float_1D_ts(data, dataSeriesLength, hist_data, tdps); - } - else -#endif - decompressDataSeries_float_1D(data, dataSeriesLength, hist_data, tdps); + { + decompressDataSeries_float_1D(data, dataSeriesLength, hist_data, tdps); } else { @@ -416,446 +381,3 @@ void getSnapshotData_float_1D(float** data, size_t dataSeriesLength, TightDataPo } } } - - -size_t decompressDataSeries_float_1D_RA_block(float * data, float mean, size_t dim_0, size_t block_dim_0, double realPrecision, int * type, float * unpredictable_data){ - int intvRadius = exe_params->intvRadius; - size_t unpredictable_count = 0; - - float * cur_data_pos = data; - size_t type_index = 0; - int type_; - float last_over_thres = mean; - for(size_t i=0; i 0){ - for(int i=0; i<2; i++){ - precision[i] = bytesToDouble(comp_data_pos); - comp_data_pos += sizeof(double); - coeff_intvRadius[i] = bytesToInt_bigEndian(comp_data_pos); - comp_data_pos += sizeof(int); - unsigned int tree_size = bytesToInt_bigEndian(comp_data_pos); - comp_data_pos += sizeof(int); - int stateNum = 2*coeff_intvRadius[i]*2; - HuffmanTree* huffmanTree = createHuffmanTree(stateNum); - int nodeCount = bytesToInt_bigEndian(comp_data_pos); - node root = reconstruct_HuffTree_from_bytes_anyStates(huffmanTree, comp_data_pos+sizeof(int), nodeCount); - comp_data_pos += sizeof(int) + tree_size; - - coeff_type[i] = coeff_result_type + i * num_blocks; - size_t typeArray_size = bytesToSize(comp_data_pos); - decode(comp_data_pos + sizeof(size_t), reg_count, root, coeff_type[i]); - comp_data_pos += sizeof(size_t) + typeArray_size; - int coeff_unpred_count = bytesToInt_bigEndian(comp_data_pos); - comp_data_pos += sizeof(int); - coeff_unpred_data[i] = (float *) comp_data_pos; - comp_data_pos += coeff_unpred_count * sizeof(float); - SZ_ReleaseHuffman(huffmanTree); - } - } - float last_coefficients[2] = {0.0}; - int coeff_unpred_data_count[2] = {0}; - // decompress coeffcients - float * reg_params = (float *) malloc(2*num_blocks*sizeof(float)); - memset(reg_params, 0, 2*num_blocks*sizeof(float)); - float * reg_params_pos = reg_params; - size_t coeff_index = 0; - for(size_t i=0; i= r1) break; - *block_data_pos_x = data_pos[ii]; - block_data_pos_x ++; - } - } - - } - else{ - for(size_t i=sx; i= r1) break; - *block_data_pos_x = data_pos[ii]; - block_data_pos_x ++; - } - } - } - free(unpred_offset); - free(reg_params); - free(blockwise_unpred_count); - free(dec_buffer); - free(coeff_result_type); - - free(indicator); - free(result_type); - - // extract data - int resi_x = s1 % block_size; - *data = (float*) malloc(sizeof(float)*(e1 - s1)); - float * final_data_pos = *data; - float * block_data_pos = dec_block_data + resi_x; - for(int i=0; i<(e1 - s1); i++){ - *(final_data_pos++) = *(block_data_pos++); - } - free(dec_block_data); -} - -int SZ_decompress_args_randomaccess_float(float** newData, -size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, -size_t s5, size_t s4, size_t s3, size_t s2, size_t s1, // start point -size_t e5, size_t e4, size_t e3, size_t e2, size_t e1, // end point -unsigned char* cmpBytes, size_t cmpSize) -{ - if(confparams_dec==NULL) - confparams_dec = (sz_params*)malloc(sizeof(sz_params)); - memset(confparams_dec, 0, sizeof(sz_params)); - if(exe_params==NULL) - exe_params = (sz_exedata*)malloc(sizeof(sz_exedata)); - memset(exe_params, 0, sizeof(sz_exedata)); - - int x = 1; - char *y = (char*)&x; - if(*y==1) - sysEndianType = LITTLE_ENDIAN_SYSTEM; - else //=0 - sysEndianType = BIG_ENDIAN_SYSTEM; - - confparams_dec->randomAccess = 1; - - int status = SZ_SCES; - size_t dataLength = computeDataLength(r5,r4,r3,r2,r1); - - //unsigned char* tmpBytes; - size_t targetUncompressSize = dataLength <<2; //i.e., *4 - //tmpSize must be "much" smaller than dataLength - size_t i, tmpSize = 8+MetaDataByteLength+exe_params->SZ_SIZE_TYPE; - unsigned char* szTmpBytes; - - if(cmpSize!=8+4+MetaDataByteLength && cmpSize!=8+8+MetaDataByteLength) //4,8 means two posibilities of SZ_SIZE_TYPE - { - confparams_dec->losslessCompressor = is_lossless_compressed_data(cmpBytes, cmpSize); - if(confparams_dec->szMode!=SZ_TEMPORAL_COMPRESSION) - { - if(confparams_dec->losslessCompressor!=-1) - confparams_dec->szMode = SZ_BEST_COMPRESSION; - else - confparams_dec->szMode = SZ_BEST_SPEED; - } - - if(confparams_dec->szMode==SZ_BEST_SPEED) - { - tmpSize = cmpSize; - szTmpBytes = cmpBytes; - } - else if(confparams_dec->szMode==SZ_BEST_COMPRESSION || confparams_dec->szMode==SZ_DEFAULT_COMPRESSION || confparams_dec->szMode==SZ_TEMPORAL_COMPRESSION) - { - if(targetUncompressSizelosslessCompressor, cmpBytes, (unsigned long)cmpSize, &szTmpBytes, (unsigned long)targetUncompressSize+4+MetaDataByteLength+exe_params->SZ_SIZE_TYPE);// (unsigned long)targetUncompressSize+8: consider the total length under lossless compression mode is actually 3+4+1+targetUncompressSize - } - else - { - printf("Wrong value of confparams_dec->szMode in the double compressed bytes.\n"); - status = SZ_MERR; - return status; - } - } - else - szTmpBytes = cmpBytes; - - TightDataPointStorageF* tdps; - new_TightDataPointStorageF_fromFlatBytes(&tdps, szTmpBytes, tmpSize); - - int dim = computeDimension(r5,r4,r3,r2,r1); - int floatSize = sizeof(float); - if(tdps->isLossless) - { - *newData = (float*)malloc(floatSize*dataLength); - if(sysEndianType==BIG_ENDIAN_SYSTEM) - { - memcpy(*newData, szTmpBytes+4+MetaDataByteLength+exe_params->SZ_SIZE_TYPE, dataLength*floatSize); - } - else - { - unsigned char* p = szTmpBytes+4+MetaDataByteLength+exe_params->SZ_SIZE_TYPE; - for(i=0;irandomAccess == 0 && (s1+s2+s3+s4+s5>0 || (r5-e5+r4-e4+r3-e3+r2-e2+r1-e1 > 0))) - { - printf("Error: you specified the random access mode for decompression, but the compressed data were generate in the non-random-access way.!\n"); - status = SZ_DERR; - } - else if (dim == 1) - { - //printf("Error: random access mode doesn't support 1D yet, but only 3D.\n"); - decompressDataSeries_float_1D_decompression_given_areas_with_blocked_regression(newData, r1, s1, e1, tdps->raBytes); - //status = SZ_DERR; - } - else - { - printf("Error: currently support only at most 4 dimensions!\n"); - status = SZ_DERR; - } - } - - free_TightDataPointStorageF2(tdps); - if(confparams_dec->szMode!=SZ_BEST_SPEED && cmpSize!=8+MetaDataByteLength+exe_params->SZ_SIZE_TYPE) - free(szTmpBytes); - return status; -} -#endif diff --git a/deps/SZ/sz/src/szd_float_pwr.c b/deps/SZ/sz/src/szd_float_pwr.c index 1e9d1ee14e..0eb296ee45 100644 --- a/deps/SZ/sz/src/szd_float_pwr.c +++ b/deps/SZ/sz/src/szd_float_pwr.c @@ -23,7 +23,7 @@ #pragma GCC diagnostic ignored "-Wchar-subscripts" -void decompressDataSeries_float_1D_pwr(float** data, size_t dataSeriesLength, TightDataPointStorageF* tdps) +void decompressDataSeries_float_1D_pwr(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps) { updateQuantizationInfo(tdps->intervals); unsigned char tmpPrecBytes[4] = {0}; //used when needing to convert bytes to float values @@ -37,7 +37,7 @@ void decompressDataSeries_float_1D_pwr(float** data, size_t dataSeriesLength, Ti convertByteArray2IntArray_fast_2b(tdps->exactDataNum, tdps->leadNumArray, tdps->leadNumArray_size, &leadNum); - *data = (float*)malloc(sizeof(float)*dataSeriesLength); + //*data = (float*)malloc(sizeof(float)*dataSeriesLength); // comment by tickduan int* type = (int*)malloc(dataSeriesLength*sizeof(int)); @@ -122,16 +122,16 @@ void decompressDataSeries_float_1D_pwr(float** data, size_t dataSeriesLength, Ti } exactData = bytesToFloat(curBytes); - (*data)[i] = exactData + medianValue; + data[i] = exactData + medianValue; memcpy(preBytes,curBytes,4); break; default: - //predValue = 2 * (*data)[i-1] - (*data)[i-2]; - predValue = (*data)[i-1]; - (*data)[i] = predValue + (type_-exe_params->intvRadius)*interval; + //predValue = 2 * data[i-1] - data[i-2]; + predValue = data[i-1]; + data[i] = predValue + (type_-exe_params->intvRadius)*interval; break; } - //printf("%.30G\n",(*data)[i]); + //printf("%.30G\n",data[i]); } free(leadNum); free(type); @@ -139,7 +139,7 @@ void decompressDataSeries_float_1D_pwr(float** data, size_t dataSeriesLength, Ti } -void decompressDataSeries_float_1D_pwrgroup(float** data, size_t dataSeriesLength, TightDataPointStorageF* tdps) +void decompressDataSeries_float_1D_pwrgroup(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps) { float *posGroups, *negGroups, *groups; float pos_01_group, neg_01_group; @@ -152,7 +152,7 @@ void decompressDataSeries_float_1D_pwrgroup(float** data, size_t dataSeriesLengt convertByteArray2IntArray_fast_2b(tdps->exactDataNum, tdps->leadNumArray, tdps->leadNumArray_size, &leadNum); - *data = (float*)malloc(sizeof(float)*dataSeriesLength); + //*data = (float*)malloc(sizeof(float)*dataSeriesLength); // comment by tickduan int* type = (int*)malloc(dataSeriesLength*sizeof(int)); @@ -268,7 +268,7 @@ void decompressDataSeries_float_1D_pwrgroup(float** data, size_t dataSeriesLengt exactData = bytesToFloat(curBytes); exactData = exactData + medianValue; - (*data)[i] = exactData; + data[i] = exactData; memcpy(preBytes,curBytes,4); groups[indexGrpID] = exactData; @@ -294,7 +294,7 @@ void decompressDataSeries_float_1D_pwrgroup(float** data, size_t dataSeriesLengt // curValue = rawGrpID>0?pow(2,groupNum):-pow(2,groupNum); //} - (*data)[i] = curValue; + data[i] = curValue; groups[indexGrpID] = curValue; break; } @@ -311,7 +311,7 @@ void decompressDataSeries_float_1D_pwrgroup(float** data, size_t dataSeriesLengt free(groupID); } -void decompressDataSeries_float_1D_pwr_pre_log(float** data, size_t dataSeriesLength, TightDataPointStorageF* tdps) { +void decompressDataSeries_float_1D_pwr_pre_log(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps) { decompressDataSeries_float_1D(data, dataSeriesLength, NULL, tdps); float threshold = tdps->minLogValue; @@ -319,22 +319,22 @@ void decompressDataSeries_float_1D_pwr_pre_log(float** data, size_t dataSeriesLe unsigned char * signs; sz_lossless_decompress(ZSTD_COMPRESSOR, tdps->pwrErrBoundBytes, tdps->pwrErrBoundBytes_size, &signs, dataSeriesLength); for(size_t i=0; iminLogValue; @@ -350,12 +350,12 @@ void decompressDataSeries_float_1D_pwr_pre_log_MSST19(float** data, size_t dataS else sz_lossless_decompress(ZSTD_COMPRESSOR, tdps->pwrErrBoundBytes, tdps->pwrErrBoundBytes_size, &signs, dataSeriesLength); for(size_t i=0; i= 0){ - (*data)[i] = 0; + if(data[i] < threshold && data[i] >= 0){ + data[i] = 0; continue; } if(signs[i]){ - ptr = (uint32_t*)(*data) + i; + ptr = (uint32_t*)data + i; *ptr |= 0x80000000; } } @@ -363,7 +363,7 @@ void decompressDataSeries_float_1D_pwr_pre_log_MSST19(float** data, size_t dataS } else{ for(size_t i=0; i -#include -#include -#include "szd_float.h" -#include "TightDataPointStorageF.h" -#include "sz.h" -#include "Huffman.h" -#include "szd_float_ts.h" - -void decompressDataSeries_float_1D_ts(float** data, size_t dataSeriesLength, float* hist_data, TightDataPointStorageF* tdps) -{ - float* lastSnapshotData = hist_data; - updateQuantizationInfo(tdps->intervals); - size_t i, j, k = 0, p = 0, l = 0; // k is to track the location of residual_bit - // in resiMidBits, p is to track the - // byte_index of resiMidBits, l is for - // leadNum - unsigned char* leadNum; - double interval = tdps->realPrecision*2; - - convertByteArray2IntArray_fast_2b(tdps->exactDataNum, tdps->leadNumArray, tdps->leadNumArray_size, &leadNum); - - *data = (float*)malloc(sizeof(float)*dataSeriesLength); - - int* type = (int*)malloc(dataSeriesLength*sizeof(int)); - - HuffmanTree* huffmanTree = createHuffmanTree(tdps->stateNum); - decode_withTree(huffmanTree, tdps->typeArray, dataSeriesLength, type); - SZ_ReleaseHuffman(huffmanTree); - - unsigned char preBytes[4]; - unsigned char curBytes[4]; - - memset(preBytes, 0, 4); - - size_t curByteIndex = 0; - int reqBytesLength, resiBitsLength, resiBits; - unsigned char leadingNum; - float medianValue, exactData, predValue = 0; - - reqBytesLength = tdps->reqLength/8; - resiBitsLength = tdps->reqLength%8; - medianValue = tdps->medianValue; - - int type_; - for (i = 0; i < dataSeriesLength; i++) { - type_ = type[i]; - switch (type_) { - case 0: - // compute resiBits - resiBits = 0; - if (resiBitsLength != 0) { - int kMod8 = k % 8; - int rightMovSteps = getRightMovingSteps(kMod8, resiBitsLength); - if (rightMovSteps > 0) { - int code = getRightMovingCode(kMod8, resiBitsLength); - resiBits = (tdps->residualMidBits[p] & code) >> rightMovSteps; - } else if (rightMovSteps < 0) { - int code1 = getLeftMovingCode(kMod8); - int code2 = getRightMovingCode(kMod8, resiBitsLength); - int leftMovSteps = -rightMovSteps; - rightMovSteps = 8 - leftMovSteps; - resiBits = (tdps->residualMidBits[p] & code1) << leftMovSteps; - p++; - resiBits = resiBits - | ((tdps->residualMidBits[p] & code2) >> rightMovSteps); - } else // rightMovSteps == 0 - { - int code = getRightMovingCode(kMod8, resiBitsLength); - resiBits = (tdps->residualMidBits[p] & code); - p++; - } - k += resiBitsLength; - } - - // recover the exact data - memset(curBytes, 0, 4); - leadingNum = leadNum[l++]; - memcpy(curBytes, preBytes, leadingNum); - for (j = leadingNum; j < reqBytesLength; j++) - curBytes[j] = tdps->exactMidBytes[curByteIndex++]; - if (resiBitsLength != 0) { - unsigned char resiByte = (unsigned char) (resiBits << (8 - resiBitsLength)); - curBytes[reqBytesLength] = resiByte; - } - - exactData = bytesToFloat(curBytes); - (*data)[i] = exactData + medianValue; - memcpy(preBytes,curBytes,4); - break; - default: - //predValue = (*data)[i-1]; - if(confparams_dec->szMode == SZ_TEMPORAL_COMPRESSION) - predValue = lastSnapshotData[i]; - (*data)[i] = predValue + (type_-exe_params->intvRadius)*interval; - break; - } - //printf("%.30G\n",(*data)[i]); - } - - memcpy(hist_data, (*data), dataSeriesLength*sizeof(float)); - - free(leadNum); - free(type); - return; -} diff --git a/deps/SZ/sz/src/szf.c b/deps/SZ/sz/src/szf.c deleted file mode 100644 index a40dc38f8f..0000000000 --- a/deps/SZ/sz/src/szf.c +++ /dev/null @@ -1,570 +0,0 @@ -/** - * @file szf.c - * @author Sheng Di - * @date April, 2015 - * @brief the key C binding file to connect Fortran and C - * (C) 2015 by Mathematics and Computer Science (MCS), Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - - -#include -#include -#include -#include "sz.h" -#include "szf.h" - -//special notice: all the function names in this file must be lower-cases!! -void sz_init_c_(char *configFile,int *len,int *ierr) -{ - int i; - char s2[*len+1]; - for(i=0;i<*len;i++) - s2[i]=configFile[i]; - s2[*len]='\0'; - // printf("sconfigFile=%s\n",configFile); - *ierr = SZ_Init(s2); -} - -void sz_finalize_c_() -{ - SZ_Finalize(); -} - -//compress with config (without args in function) -void sz_compress_d1_float_(float* data, unsigned char *bytes, size_t *outSize, size_t *r1) -{ - unsigned char *tmp_bytes = SZ_compress(SZ_FLOAT, data, outSize, 0, 0, 0, 0, *r1); - memcpy(bytes, tmp_bytes, *outSize); - free(tmp_bytes); -} - -void sz_compress_d1_float_rev_(float* data, float *reservedValue, unsigned char *bytes, size_t *outSize, size_t *r1) -{ - unsigned char *tmp_bytes = SZ_compress_rev(SZ_FLOAT, data, reservedValue, outSize, 0, 0, 0, 0, *r1); - memcpy(bytes, tmp_bytes, *outSize); - free(tmp_bytes); -} - -void sz_compress_d2_float_(float* data, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2) -{ - unsigned char *tmp_bytes = SZ_compress(SZ_FLOAT, data, outSize, 0, 0, 0, *r2, *r1); - memcpy(bytes, tmp_bytes, *outSize); - free(tmp_bytes); -} - -void sz_compress_d2_float_rev_(float* data, float *reservedValue, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2) -{ - unsigned char *tmp_bytes = SZ_compress_rev(SZ_FLOAT, data, reservedValue, outSize, 0, 0, 0, *r2, *r1); - memcpy(bytes, tmp_bytes, *outSize); - free(tmp_bytes); -} - -void sz_compress_d3_float_(float* data, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3) -{ - unsigned char *tmp_bytes = SZ_compress(SZ_FLOAT, data, outSize, 0, 0, *r3, *r2, *r1); - memcpy(bytes, tmp_bytes, *outSize); - free(tmp_bytes); -} - -void sz_compress_d3_float_rev_(float* data, float *reservedValue, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3) -{ - unsigned char *tmp_bytes = SZ_compress_rev(SZ_FLOAT, data, reservedValue, outSize, 0, 0, *r3, *r2, *r1); - memcpy(bytes, tmp_bytes, *outSize); - free(tmp_bytes); -} - -void sz_compress_d4_float_(float* data, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3, size_t *r4) -{ - unsigned char *tmp_bytes = SZ_compress(SZ_FLOAT, data, outSize, 0, *r4, *r3, *r2, *r1); - memcpy(bytes, tmp_bytes, *outSize); - free(tmp_bytes); -} - -void sz_compress_d4_float_rev_(float* data, float *reservedValue, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3, size_t *r4) -{ - unsigned char *tmp_bytes = SZ_compress_rev(SZ_FLOAT, data, reservedValue, outSize, 0, *r4, *r3, *r2, *r1); - memcpy(bytes, tmp_bytes, *outSize); - free(tmp_bytes); -} - -void sz_compress_d5_float_(float* data, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5) -{ - unsigned char *tmp_bytes = SZ_compress(SZ_FLOAT, data, outSize, *r5, *r4, *r3, *r2, *r1); - memcpy(bytes, tmp_bytes, *outSize); - free(tmp_bytes); -} - -void sz_compress_d5_float_rev_(float* data, float *reservedValue, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5) -{ - unsigned char *tmp_bytes = SZ_compress_rev(SZ_FLOAT, data, reservedValue, outSize, *r5, *r4, *r3, *r2, *r1); - memcpy(bytes, tmp_bytes, *outSize); - free(tmp_bytes); -} - -void sz_compress_d1_double_(double* data, unsigned char *bytes, size_t *outSize, size_t *r1) -{ - unsigned char *tmp_bytes = SZ_compress(SZ_DOUBLE, data, outSize, 0, 0, 0, 0, *r1); - memcpy(bytes, tmp_bytes, *outSize); - free(tmp_bytes); -} - -void sz_compress_d1_double_rev_(double* data, double *reservedValue, unsigned char *bytes, size_t *outSize, size_t *r1) -{ - unsigned char *tmp_bytes = SZ_compress_rev(SZ_DOUBLE, data, reservedValue, outSize, 0, 0, 0, 0, *r1); - memcpy(bytes, tmp_bytes, *outSize); - free(tmp_bytes); -} - -void sz_compress_d2_double_(double* data, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2) -{ - unsigned char *tmp_bytes = SZ_compress(SZ_DOUBLE, data, outSize, 0, 0, 0, *r2, *r1); - memcpy(bytes, tmp_bytes, *outSize); - free(tmp_bytes); -} - -void sz_compress_d2_double_rev_(double* data, double *reservedValue, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2) -{ - unsigned char *tmp_bytes = SZ_compress_rev(SZ_DOUBLE, data, reservedValue, outSize, 0, 0, 0, *r2, *r1); - memcpy(bytes, tmp_bytes, *outSize); - free(tmp_bytes); -} - -void sz_compress_d3_double_(double* data, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3) -{ - unsigned char *tmp_bytes = SZ_compress(SZ_DOUBLE, data, outSize, 0, 0, *r3, *r2, *r1); - memcpy(bytes, tmp_bytes, *outSize); - free(tmp_bytes); -} - -void sz_compress_d3_double_rev_(double* data, double *reservedValue, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3) -{ - unsigned char *tmp_bytes = SZ_compress_rev(SZ_DOUBLE, data, reservedValue, outSize, 0, 0, *r3, *r2, *r1); - memcpy(bytes, tmp_bytes, *outSize); - free(tmp_bytes); -} - -void sz_compress_d4_double_(double* data, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3, size_t *r4) -{ - unsigned char *tmp_bytes = SZ_compress(SZ_DOUBLE, data, outSize, 0, *r4, *r3, *r2, *r1); - memcpy(bytes, tmp_bytes, *outSize); - free(tmp_bytes); -} - -void sz_compress_d4_double_rev_(double* data, double *reservedValue, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3, size_t *r4) -{ - unsigned char *tmp_bytes = SZ_compress_rev(SZ_DOUBLE, data, reservedValue, outSize, 0, *r4, *r3, *r2, *r1); - memcpy(bytes, tmp_bytes, *outSize); - free(tmp_bytes); -} - -void sz_compress_d5_double_(double* data, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5) -{ - unsigned char *tmp_bytes = SZ_compress(SZ_DOUBLE, data, outSize, *r5, *r4, *r3, *r2, *r1); - memcpy(bytes, tmp_bytes, *outSize); - free(tmp_bytes); -} - -void sz_compress_d5_double_rev_(double* data, double *reservedValue, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5) -{ - unsigned char *tmp_bytes = SZ_compress_rev(SZ_DOUBLE, data, reservedValue, outSize, *r5, *r4, *r3, *r2, *r1); - memcpy(bytes, tmp_bytes, *outSize); - free(tmp_bytes); -} - -//compress with args - -void sz_compress_d1_float_args_(float* data, unsigned char *bytes, size_t *outSize, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1) -{ - unsigned char *tmp_bytes = SZ_compress_args(SZ_FLOAT, data, outSize, *errBoundMode, *absErrBound, *relBoundRatio, 0.1, 0, 0, 0, 0, *r1); - memcpy(bytes, tmp_bytes, *outSize); - free(tmp_bytes); -} - -void sz_compress_d2_float_args_(float* data, unsigned char *bytes, size_t *outSize, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2) -{ - unsigned char *tmp_bytes = SZ_compress_args(SZ_FLOAT, data, outSize, *errBoundMode, *absErrBound, *relBoundRatio, 0.1, 0, 0, 0, *r2, *r1); - memcpy(bytes, tmp_bytes, *outSize); - free(tmp_bytes); -} - -void sz_compress_d3_float_args_(float* data, unsigned char *bytes, size_t *outSize, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2, size_t *r3) -{ - unsigned char *tmp_bytes = SZ_compress_args(SZ_FLOAT, data, outSize, *errBoundMode, *absErrBound, *relBoundRatio, 0.1, 0, 0, *r3, *r2, *r1); - memcpy(bytes, tmp_bytes, *outSize); - free(tmp_bytes); -} - -void sz_compress_d4_float_args_(float* data, unsigned char *bytes, size_t *outSize, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4) -{ - unsigned char *tmp_bytes = SZ_compress_args(SZ_FLOAT, data, outSize, *errBoundMode, *absErrBound, *relBoundRatio, 0.1, 0, *r4, *r3, *r2, *r1); - memcpy(bytes, tmp_bytes, *outSize); - free(tmp_bytes); -} - -void sz_compress_d5_float_args_(float* data, unsigned char *bytes, size_t *outSize, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5) -{ - unsigned char *tmp_bytes = SZ_compress_args(SZ_FLOAT, data, outSize, *errBoundMode, *absErrBound, *relBoundRatio, 0.1, *r5, *r4, *r3, *r2, *r1); - memcpy(bytes, tmp_bytes, *outSize); - free(tmp_bytes); -} - -void sz_compress_d1_double_args_(double* data, unsigned char *bytes, size_t *outSize, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1) -{ - unsigned char *tmp_bytes = SZ_compress_args(SZ_DOUBLE, data, outSize, *errBoundMode, *absErrBound, *relBoundRatio, 0.1, 0, 0, 0, 0, *r1); - memcpy(bytes, tmp_bytes, *outSize); - free(tmp_bytes); -} - -void sz_compress_d2_double_args_(double* data, unsigned char *bytes, size_t *outSize, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2) -{ - unsigned char *tmp_bytes = SZ_compress_args(SZ_DOUBLE, data, outSize, *errBoundMode, *absErrBound, *relBoundRatio, 0.1, 0, 0, 0, *r2, *r1); - memcpy(bytes, tmp_bytes, *outSize); - free(tmp_bytes); -} - -void sz_compress_d3_double_args_(double* data, unsigned char *bytes, size_t *outSize, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2, size_t *r3) -{ - unsigned char *tmp_bytes = SZ_compress_args(SZ_DOUBLE, data, outSize, *errBoundMode, *absErrBound, *relBoundRatio, 0.1, 0, 0, *r3, *r2, *r1); - memcpy(bytes, tmp_bytes, *outSize); - free(tmp_bytes); -} - -void sz_compress_d4_double_args_(double* data, unsigned char *bytes, size_t *outSize, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4) -{ - unsigned char *tmp_bytes = SZ_compress_args(SZ_DOUBLE, data, outSize, *errBoundMode, *absErrBound, *relBoundRatio, 0.1, 0, *r4, *r3, *r2, *r1); - memcpy(bytes, tmp_bytes, *outSize); - free(tmp_bytes); -} - -void sz_compress_d5_double_args_(double* data, unsigned char *bytes, size_t *outSize, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5) -{ - unsigned char *tmp_bytes = SZ_compress_args(SZ_DOUBLE, data, outSize, *errBoundMode, *absErrBound, *relBoundRatio, 0.1, *r5, *r4, *r3, *r2, *r1); - memcpy(bytes, tmp_bytes, *outSize); - free(tmp_bytes); -} - -//-------------- - -void sz_compress_d1_float_rev_args_(float* data, float *reservedValue, unsigned char *bytes, size_t *outSize, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1) -{ - unsigned char *tmp_bytes = SZ_compress_rev_args(SZ_FLOAT, data, reservedValue, outSize, *errBoundMode, *absErrBound, *relBoundRatio, 0, 0, 0, 0, *r1); - memcpy(bytes, tmp_bytes, *outSize); - free(tmp_bytes); -} - -void sz_compress_d2_float_rev_args_(float* data, float *reservedValue, unsigned char *bytes, size_t *outSize, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2) -{ - unsigned char *tmp_bytes = SZ_compress_rev_args(SZ_FLOAT, data, reservedValue, outSize, *errBoundMode, *absErrBound, *relBoundRatio, 0, 0, 0, *r2, *r1); - memcpy(bytes, tmp_bytes, *outSize); - free(tmp_bytes); -} - -void sz_compress_d3_float_rev_args_(float* data, float *reservedValue, unsigned char *bytes, size_t *outSize, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2, size_t *r3) -{ - unsigned char *tmp_bytes = SZ_compress_rev_args(SZ_FLOAT, data, reservedValue, outSize, *errBoundMode, *absErrBound, *relBoundRatio, 0, 0, *r3, *r2, *r1); - memcpy(bytes, tmp_bytes, *outSize); - free(tmp_bytes); -} - -void sz_compress_d4_float_rev_args_(float* data, float *reservedValue, unsigned char *bytes, size_t *outSize, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4) -{ - unsigned char *tmp_bytes = SZ_compress_rev_args(SZ_FLOAT, data, reservedValue, outSize, *errBoundMode, *absErrBound, *relBoundRatio, 0, *r4, *r3, *r2, *r1); - memcpy(bytes, tmp_bytes, *outSize); - free(tmp_bytes); -} - -void sz_compress_d5_float_rev_args_(float* data, float *reservedValue, unsigned char *bytes, size_t *outSize, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5) -{ - unsigned char *tmp_bytes = SZ_compress_rev_args(SZ_FLOAT, data, reservedValue, outSize, *errBoundMode, *absErrBound, *relBoundRatio, *r5, *r4, *r3, *r2, *r1); - memcpy(bytes, tmp_bytes, *outSize); - free(tmp_bytes); -} - -void sz_compress_d1_double_rev_args_(double* data, float *reservedValue, unsigned char *bytes, size_t *outSize, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1) -{ - unsigned char *tmp_bytes = SZ_compress_rev_args(SZ_DOUBLE, data, reservedValue, outSize, *errBoundMode, *absErrBound, *relBoundRatio, 0, 0, 0, 0, *r1); - memcpy(bytes, tmp_bytes, *outSize); - free(tmp_bytes); -} - -void sz_compress_d2_double_rev_args_(double* data, float *reservedValue, unsigned char *bytes, size_t *outSize, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2) -{ - unsigned char *tmp_bytes = SZ_compress_rev_args(SZ_DOUBLE, data, reservedValue, outSize, *errBoundMode, *absErrBound, *relBoundRatio, 0, 0, 0, *r2, *r1); - memcpy(bytes, tmp_bytes, *outSize); - free(tmp_bytes); -} - -void sz_compress_d3_double_rev_args_(double* data, float *reservedValue, unsigned char *bytes, size_t *outSize, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2, size_t *r3) -{ - unsigned char *tmp_bytes = SZ_compress_rev_args(SZ_DOUBLE, data, reservedValue, outSize, *errBoundMode, *absErrBound, *relBoundRatio, 0, 0, *r3, *r2, *r1); - memcpy(bytes, tmp_bytes, *outSize); -} - -void sz_compress_d4_double_rev_args_(double* data, double *reservedValue, unsigned char *bytes, size_t *outSize, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4) -{ - unsigned char *tmp_bytes = SZ_compress_rev_args(SZ_DOUBLE, data, reservedValue, outSize, *errBoundMode, *absErrBound, *relBoundRatio, 0, *r4, *r3, *r2, *r1); - memcpy(bytes, tmp_bytes, *outSize); - free(tmp_bytes); -} - -void sz_compress_d5_double_rev_args_(double* data, double *reservedValue, unsigned char *bytes, size_t *outSize, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5) -{ - unsigned char *tmp_bytes = SZ_compress_rev_args(SZ_DOUBLE, data, reservedValue, outSize, *errBoundMode, *absErrBound, *relBoundRatio, *r5, *r4, *r3, *r2, *r1); - memcpy(bytes, tmp_bytes, *outSize); - free(tmp_bytes); -} - -//decompress - -void sz_decompress_d1_float_(unsigned char *bytes, size_t *byteLength, float *data, size_t *r1) -{ - float *tmp_data = SZ_decompress(SZ_FLOAT, bytes, *byteLength, 0, 0, 0, 0, *r1); - memcpy(data, tmp_data, (*r1)*sizeof(float)); - free(tmp_data); -} - -void sz_decompress_d2_float_(unsigned char *bytes, size_t *byteLength, float *data, size_t *r1, size_t *r2) -{ - size_t r; - float *tmp_data = SZ_decompress(SZ_FLOAT, bytes, *byteLength, 0, 0, 0, *r2, *r1); - r=(*r1)*(*r2); - memcpy(data, tmp_data, r*sizeof(float)); - free(tmp_data); -} - -void sz_decompress_d3_float_(unsigned char *bytes, size_t *byteLength, float *data, size_t *r1, size_t *r2, size_t *r3) -{ - size_t r; - float *tmp_data = SZ_decompress(SZ_FLOAT, bytes, *byteLength, 0, 0, *r3, *r2, *r1); - r=(*r1)*(*r2)*(*r3); - memcpy(data, tmp_data, r*sizeof(float)); - free(tmp_data); -} - -void sz_decompress_d4_float_(unsigned char *bytes, size_t *byteLength, float *data, size_t *r1, size_t *r2, size_t *r3, size_t *r4) -{ - size_t r; - float *tmp_data = SZ_decompress(SZ_FLOAT, bytes, *byteLength, 0, *r4, *r3, *r2, *r1); - r=(*r1)*(*r2)*(*r3)*(*r4); - memcpy(data, tmp_data, r*sizeof(float)); - free(tmp_data); -} - -void sz_decompress_d5_float_(unsigned char *bytes, size_t *byteLength, float *data, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5) -{ - size_t r; - float *tmp_data = SZ_decompress(SZ_FLOAT, bytes, *byteLength, *r5, *r4, *r3, *r2, *r1); - r=(*r1)*(*r2)*(*r3)*(*r4)*(*r5); - memcpy(data, tmp_data, r*sizeof(float)); - free(tmp_data); -} - -void sz_decompress_d1_double_(unsigned char *bytes, size_t *byteLength, double *data, size_t *r1) -{ - double *tmp_data = SZ_decompress(SZ_DOUBLE, bytes, *byteLength, 0, 0, 0, 0, *r1); - memcpy(data, tmp_data, (*r1)*sizeof(double)); - free(tmp_data); -} - -void sz_decompress_d2_double_(unsigned char *bytes, size_t *byteLength, double *data, size_t *r1, size_t *r2) -{ - size_t r; - double *tmp_data = SZ_decompress(SZ_DOUBLE, bytes, *byteLength, 0, 0, 0, *r2, *r1); - r=(*r1)*(*r2); - memcpy(data, tmp_data, r*sizeof(double)); - free(tmp_data); -} - -void sz_decompress_d3_double_(unsigned char *bytes, size_t *byteLength, double *data, size_t *r1, size_t *r2, size_t *r3) -{ - size_t r; - double *tmp_data = SZ_decompress(SZ_DOUBLE, bytes, *byteLength, 0, 0, *r3, *r2, *r1); - r=(*r1)*(*r2)*(*r3); - memcpy(data, tmp_data, r*sizeof(double)); - free(tmp_data); -} - -void sz_decompress_d4_double_(unsigned char *bytes, size_t *byteLength, double *data, size_t *r1, size_t *r2, size_t *r3, size_t *r4) -{ - size_t r; - double *tmp_data = SZ_decompress(SZ_DOUBLE, bytes, *byteLength, 0, *r4, *r3, *r2, *r1); - r=(*r1)*(*r2)*(*r3)*(*r4); - memcpy(data, tmp_data, r*sizeof(double)); - free(tmp_data); -} - -void sz_decompress_d5_double_(unsigned char *bytes, size_t *byteLength, double *data, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5) -{ - size_t r; - double *tmp_data = SZ_decompress(SZ_DOUBLE, bytes, *byteLength, *r5, *r4, *r3, *r2, *r1); - r=(*r1)*(*r2)*(*r3)*(*r4)*(*r5); - memcpy(data, tmp_data, r*sizeof(double)); - free(tmp_data); -} - -//-----------------TODO: batch mode----------- -void sz_batchaddvar_d1_float_(int var_id, char* varName, int *len, float* data, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1) -{ - int i; - char s2[*len+1]; - for(i=0;i<*len;i++) - s2[i]=varName[i]; - s2[*len]='\0'; - SZ_batchAddVar(var_id, s2, SZ_FLOAT, data, *errBoundMode, *absErrBound, *relBoundRatio, 0.1, 0, 0, 0, 0, *r1); -} -void sz_batchaddvar_d2_float_(int var_id, char* varName, int *len, float* data, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2) -{ - int i; - char s2[*len+1]; - for(i=0;i<*len;i++) - s2[i]=varName[i]; - s2[*len]='\0'; - SZ_batchAddVar(var_id, s2, SZ_FLOAT, data, *errBoundMode, *absErrBound, *relBoundRatio, 0.1, 0, 0, 0, *r2, *r1); -} -void sz_batchaddvar_d3_float_(int var_id, char* varName, int *len, float* data, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2, size_t *r3) -{ - int i; - char s2[*len+1]; - for(i=0;i<*len;i++) - s2[i]=varName[i]; - s2[*len]='\0'; - SZ_batchAddVar(var_id, s2, SZ_FLOAT, data, *errBoundMode, *absErrBound, *relBoundRatio, 0.1, 0, 0, *r3, *r2, *r1); -} -void sz_batchaddvar_d4_float_(int var_id, char* varName, int *len, float* data, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4) -{ - int i; - char s2[*len+1]; - for(i=0;i<*len;i++) - s2[i]=varName[i]; - s2[*len]='\0'; - SZ_batchAddVar(var_id, s2, SZ_FLOAT, data, *errBoundMode, *absErrBound, *relBoundRatio, 0.1, 0, *r4, *r3, *r2, *r1); -} -void sz_batchaddvar_d5_float_(int var_id, char* varName, int *len, float* data, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5) -{ - int i; - char s2[*len+1]; - for(i=0;i<*len;i++) - s2[i]=varName[i]; - s2[*len]='\0'; - SZ_batchAddVar(var_id, s2, SZ_FLOAT, data, *errBoundMode, *absErrBound, *relBoundRatio, 0.1, *r5, *r4, *r3, *r2, *r1); -} -void sz_batchaddvar_d1_double_(int var_id, char* varName, int *len, double* data, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1) -{ - int i; - char s2[*len+1]; - for(i=0;i<*len;i++) - s2[i]=varName[i]; - s2[*len]='\0'; - SZ_batchAddVar(var_id, s2, SZ_DOUBLE, data, *errBoundMode, *absErrBound, *relBoundRatio, 0.1, 0, 0, 0, 0, *r1); -} -void sz_batchaddvar_d2_double_(int var_id, char* varName, int *len, double* data, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2) -{ - int i; - char s2[*len+1]; - for(i=0;i<*len;i++) - s2[i]=varName[i]; - s2[*len]='\0'; - SZ_batchAddVar(var_id, s2, SZ_DOUBLE, data, *errBoundMode, *absErrBound, *relBoundRatio, 0.1, 0, 0, 0, *r2, *r1); -} -void sz_batchaddvar_d3_double_(int var_id, char* varName, int *len, double* data, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2, size_t *r3) -{ - int i; - char s2[*len+1]; - for(i=0;i<*len;i++) - s2[i]=varName[i]; - s2[*len]='\0'; - SZ_batchAddVar(var_id, s2, SZ_DOUBLE, data, *errBoundMode, *absErrBound, *relBoundRatio, 0.1, 0, 0, *r3, *r2, *r1); -} -void sz_batchaddvar_d4_double_(int var_id, char* varName, int *len, double* data, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4) -{ - int i; - char s2[*len+1]; - for(i=0;i<*len;i++) - s2[i]=varName[i]; - s2[*len]='\0'; - SZ_batchAddVar(var_id, s2, SZ_DOUBLE, data, *errBoundMode, *absErrBound, *relBoundRatio, 0.1, 0, *r4, *r3, *r2, *r1); -} -void sz_batchaddvar_d5_double_(int var_id, char* varName, int *len, double* data, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5) -{ - int i; - char s2[*len+1]; - for(i=0;i<*len;i++) - s2[i]=varName[i]; - s2[*len]='\0'; - SZ_batchAddVar(var_id, s2, SZ_DOUBLE, data, *errBoundMode, *absErrBound, *relBoundRatio, 0.1, *r5, *r4, *r3, *r2, *r1); -} -void sz_batchdelvar_c_(char* varName, int *len, int *errState) -{ - int i; - char s2[*len+1]; - for(i=0;i<*len;i++) - s2[i]=varName[i]; - s2[*len]='\0'; - *errState = SZ_batchDelVar(s2); -} - -/*@deprecated*/ -void sz_batch_compress_c_(unsigned char* bytes, size_t *outSize) -{ - //unsigned char* tmp_bytes = SZ_batch_compress(outSize); - //memcpy(bytes, tmp_bytes, *outSize); - //free(tmp_bytes); -} -/*@deprecated*/ -void sz_batch_decompress_c_(unsigned char* bytes, size_t *byteLength, int *ierr) -{ - //SZ_batch_decompress(bytes, *byteLength, ierr); -} - -void sz_getvardim_c_(char* varName, int *len, int *dim, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5) -{ - int i; - char s2[*len+1]; - for(i=0;i<*len;i++) - s2[i]=varName[i]; - s2[*len]='\0'; - - SZ_getVarData(s2, r5, r4, r3, r2, r1); - *dim = computeDimension(*r5, *r4, *r3, *r2, *r1); -} - -void compute_total_batch_size_c_(size_t *totalSize) -{ - *totalSize = compute_total_batch_size(); -} - -void sz_getvardata_float_(char* varName, int *len, float* data) -{ - int i; - size_t r1, r2, r3, r4, r5; - char s2[*len+1]; - for(i=0;i<*len;i++) - s2[i]=varName[i]; - s2[*len]='\0'; - - float* tmp_data = (float*)SZ_getVarData(s2, &r5, &r4, &r3, &r2, &r1); - int size = computeDataLength(r5, r4, r3, r2, r1); - memcpy(data, tmp_data, size*sizeof(float)); - free(tmp_data); -} -void sz_getvardata_double_(char* varName, int *len, double* data) -{ - int i; - size_t r1, r2, r3, r4, r5; - char s2[*len+1]; - for(i=0;i<*len;i++) - s2[i]=varName[i]; - s2[*len]='\0'; - - double* tmp_data = (double*)SZ_getVarData(s2, &r5, &r4, &r3, &r2, &r1); - int size = computeDataLength(r5, r4, r3, r2, r1); - memcpy(data, tmp_data, size*sizeof(double)); - //free(tmp_data); -} - -void sz_freevarset_c_(int *mode) -{ - SZ_freeVarSet(*mode); -} - diff --git a/deps/SZ/sz/src/td_sz.c b/deps/SZ/sz/src/td_sz.c index 4621688930..fa6e83486c 100644 --- a/deps/SZ/sz/src/td_sz.c +++ b/deps/SZ/sz/src/td_sz.c @@ -2,21 +2,10 @@ #include #include #include -#include + #include "td_sz.h" -#include "CompressElement.h" -#include "DynamicByteArray.h" -#include "DynamicIntArray.h" -#include "TightDataPointStorageD.h" -#include "TightDataPointStorageF.h" -#include "zlib.h" -#include "rw.h" -#include "Huffman.h" -#include "conf.h" -#include "utility.h" -#include "exafelSZ.h" #include "sz.h" -#include "defines.h" + // @@ -24,15 +13,11 @@ // int tdszCompress(int type, const char * input, const int nelements, const char * output) { - size_t outSize = 0; - void* pOut = SZ_compress(type, (void*)input, &outSize, 0, 0, 0, 0, (size_t)nelements); - if(pOut == NULL) - return 0; + // check valid + sz_params comp_params = *confparams_cpr; - // copy to dest - memcpy(output, pOut, outSize); - free(pOut); - return outSize; + size_t outSize = SZ_compress_args(type, input, (size_t)nelements, (unsigned char*)output, &comp_params); + return (int)outSize; } // @@ -40,24 +25,6 @@ int tdszCompress(int type, const char * input, const int nelements, const char * // int tdszDecompress(int type, const char * input, int compressedSize, const int nelements, const char * output) { - int width = 0; - if(type == SZ_FLOAT) - width = sizeof(float); - else if(type == SZ_DOUBLE) - width = sizeof(double); - else - return 0; - - void* pOut = SZ_decompress(type, (void*)input, compressedSize, 0, 0, 0, 0, (size_t)nelements); - if(pOut == NULL) - return 0; - - - size_t outSize = nelements * width; - - // copy to dest - memcpy(output, pOut, outSize); - free(pOut); - - return outSize; + size_t outSize = SZ_decompress(type, (void*)input, compressedSize, (size_t)nelements, (unsigned char*)output); + return (int)outSize; } diff --git a/deps/SZ/sz/src/utility.c b/deps/SZ/sz/src/utility.c index 6478852206..fee015cf54 100644 --- a/deps/SZ/sz/src/utility.c +++ b/deps/SZ/sz/src/utility.c @@ -16,142 +16,6 @@ #include "callZlib.h" #include "zstd.h" -int compare_struct(const void* obj1, const void* obj2){ - struct sort_ast_particle * srt1 = (struct sort_ast_particle*)obj1; - struct sort_ast_particle * srt2 = (struct sort_ast_particle*)obj2; - return srt1->id - srt2->id; -} - -void reorder_vars(SZ_VarSet* vset){ - SZ_Variable* v[7]; - SZ_Variable* v_tmp; - int i, j; - //v[0] - for (v_tmp = vset->header->next, i = 0; i < 7; i++){ - v[i] = v_tmp; - v_tmp = v_tmp->next; - } - //printf("here"); - size_t dataLen = computeDataLength(v[0]->r5, v[0]->r4, v[0]->r3, v[0]->r2, v[0]->r1); - //sihuan debug - //printf("the data length is (in sorting): %u", dataLen); - struct sort_ast_particle* particle = (struct sort_ast_particle*) malloc(sizeof(struct sort_ast_particle)*dataLen); - - for (i = 0; i < dataLen; i++){ - particle[i].id = ((int64_t*)v[6]->data)[i]; - // printf("%llu ", particle[i].id); - for (j = 0; j < 6; j++) - particle[i].var[j] = ((float*)v[j]->data)[i]; - } - - //sihuan debug - #if 0 - printf("index before sorting: \n"); - for (i = 0; i < 5; i++){ - printf("%llu ", particle[i].id); - printf("%.5f ", ((float*)v[0]->data)[i]); - } - #endif - //printf("\n"); - //sihuan debug - //for (i = 0; i < 5; i++)//{ - //for (j = 0; j < 6; j++) - // printf("%.5f ", particle[i].var[j]); - // printf("%llu ", particle[i].id ); - ///} - //printf("\n\n"); - - - qsort(particle, dataLen, sizeof(struct sort_ast_particle), compare_struct); - for (i = 0; i < dataLen; i++){ - ((int64_t*)v[6]->data)[i] = particle[i].id; - for (j = 0; j < 6; j++) - ((float*)v[j]->data)[i] = particle[i].var[j]; - } - free(particle); - - //sihuan debug - #if 0 - for (i = 0; i < 5; i++){ - printf("%llu ", particle[i].id); - printf("%.5f ", ((float*)v[0]->data)[i]); - } - printf("\n"); - #endif -} - -size_t intersectAndsort(int64_t* preIndex, size_t preLen, SZ_VarSet* curVar, size_t dataLen, unsigned char* bitarray){ - size_t i, j, k, m, cnt; - i = j = k = m = cnt = 0; - SZ_Variable* v[7]; - SZ_Variable* v_tmp; - //v[0] - for (v_tmp = curVar->header->next, i = 0; i < 7; i++){ - v[i] = v_tmp; - v_tmp = v_tmp->next; - } - for (i = 0; i < preLen; i++) - bitarray[i] = '0'; - i = 0; - while(i < preLen && j < dataLen){ - if (preIndex[i] == ((int64_t*)v[6]->data)[j]){ - cnt++; - int64_t tmp; - tmp = ((int64_t*)v[6]->data)[k]; - ((int64_t*)v[6]->data)[k] = ((int64_t*)v[6]->data)[j]; - ((int64_t*)v[6]->data)[j] = tmp; - float data_tmp; - for (m = 0; m < 6; m++){ - data_tmp = ((float*)v[m]->data)[k]; - ((float*)v[m]->data)[k] = ((float*)v[m]->data)[j]; - ((float*)v[m]->data)[j] = data_tmp; - } - k++; i++; j++; - } - else if (preIndex[i] < ((int64_t*)v[6]->data)[j]){ - bitarray[i] = '1'; - i++; - } - else j++; - } - printf("intersect count is: %zu, i j k pre curlen is: %zu, %zu, %zu, %zu, %zu\n\n", cnt, i, j, k, preLen, dataLen); - return cnt; -} - -void write_reordered_tofile(SZ_VarSet* curVar, size_t dataLen){ - int var_index; //0 for x, 1 for y...,3 for vx...5 for vz - int i; - char outputfile_name[256]; - SZ_Variable* v[7]; SZ_Variable* v_tmp; - for (v_tmp = curVar->header->next, i = 0; i < 6; i++){ - v[i] = v_tmp; - v_tmp = v_tmp->next; - } - for (var_index = 0; var_index < 6; var_index++){ - sprintf(outputfile_name, "reordered_input_%d_%d.in", sz_tsc->currentStep, var_index); - int status_tmp; - writeFloatData_inBytes((float*)v[var_index]->data, dataLen, outputfile_name, &status_tmp); - } -} - -float calculate_delta_t(size_t size){ - SZ_Variable* v_tmp = sz_varset->header->next; - while(strcmp(v_tmp->varName, "x")) v_tmp = v_tmp->next; - float* x1 = (float*) v_tmp->data; - float* x0 = (float*) v_tmp->multisteps->hist_data; - while(strcmp(v_tmp->varName, "vx")) v_tmp = v_tmp->next; - float* vx0 = (float*) v_tmp->multisteps->hist_data; - int i, j; - double denom = 0.0; - double div = 0.0; - for (i = 0, j = 0; i < size; i++, j++){ - while(sz_tsc->bit_array[j] == '1') j++; - denom += vx0[j] * (x1[i] - x0[j]); - div += vx0[j] * vx0[j]; - } - printf("the calculated delta_t is: %.10f\n", denom/div); - return denom/div; -} int is_lossless_compressed_data(unsigned char* compressedBytes, size_t cmpSize) { @@ -171,11 +35,10 @@ int is_lossless_compressed_data(unsigned char* compressedBytes, size_t cmpSize) return -1; //fast mode (without GZIP or ZSTD) } -unsigned long sz_lossless_compress(int losslessCompressor, int level, unsigned char* data, unsigned long dataLength, unsigned char** compressBytes) +unsigned long sz_lossless_compress(int losslessCompressor, int level, unsigned char* data, unsigned long dataLength, unsigned char* compressBytes) { unsigned long outSize = 0; size_t estimatedCompressedSize = 0; - showme(); switch(losslessCompressor) { case GZIP_COMPRESSOR: @@ -186,8 +49,8 @@ unsigned long sz_lossless_compress(int losslessCompressor, int level, unsigned c estimatedCompressedSize = 200; else estimatedCompressedSize = dataLength*1.2; - *compressBytes = (unsigned char*)malloc(estimatedCompressedSize); - outSize = ZSTD_compress(*compressBytes, estimatedCompressedSize, data, dataLength, level); //default setting of level is 3 + //*compressBytes = (unsigned char*)malloc(estimatedCompressedSize); // comment by tickduan + outSize = ZSTD_compress(compressBytes, estimatedCompressedSize, data, dataLength, level); //default setting of level is 3 break; default: printf("Error: Unrecognized lossless compressor in sz_lossless_compress()\n"); @@ -205,448 +68,10 @@ unsigned long sz_lossless_decompress(int losslessCompressor, unsigned char* comp break; case ZSTD_COMPRESSOR: *oriData = (unsigned char*)malloc(targetOriSize); - ZSTD_decompress(*oriData, targetOriSize, compressBytes, cmpSize); - outSize = targetOriSize; + outSize = ZSTD_decompress(*oriData, targetOriSize, compressBytes, cmpSize); break; default: printf("Error: Unrecognized lossless compressor in sz_lossless_decompress()\n"); } return outSize; } - -unsigned long sz_lossless_decompress65536bytes(int losslessCompressor, unsigned char* compressBytes, unsigned long cmpSize, unsigned char** oriData) -{ - unsigned long outSize = 0; - switch(losslessCompressor) - { - case GZIP_COMPRESSOR: - outSize = zlib_uncompress65536bytes(compressBytes, cmpSize, oriData); - break; - case ZSTD_COMPRESSOR: - *oriData = (unsigned char*)malloc(65536); - memset(*oriData, 0, 65536); - ZSTD_decompress(*oriData, 65536, compressBytes, cmpSize); //the first 32768 bytes should be exact the same. - outSize = 65536; - break; - default: - printf("Error: Unrecognized lossless compressor\n"); - } - return outSize; -} - -void* detransposeData(void* data, int dataType, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1) -{ - size_t len = computeDataLength(r5, r4, r3, r2, r1); - int dim = computeDimension(r5, r4, r3, r2, r1); - if(dataType == SZ_FLOAT) - { - float* ori_data = data; - float* new_data = (float*)malloc(sizeof(float)*len); - if(dim==1) - { - memcpy(new_data, ori_data, sizeof(float)*len); - return new_data; - } - else if(dim==2) - { - size_t i, j, s = 0; - for(i=0;i ---------------- \n", cnt); + printf(" Compress Rate ......... [%.0f%%] \n", rate); + double speed1 = (cnt*sizeof(float)*1000/1024/1024)/use_ms1; + printf(" Compress Time ......... [%.4fms] speed=%.1f MB/s\n", use_ms1, speed1); + double speed2 = (cnt*sizeof(float)*1000/1024/1024)/use_ms2; + printf(" Decompress Time........ [%.4fms] speed=%.1f MB/s\n", use_ms2, speed2); + printf(" Same Rate ............. [%.0f%%] \n\n", same_rate); + + + // free + free(ft2); + free(buff); + free(output); + +} @@ -518,8 +581,9 @@ void test_threadsafe_double(int thread_count){ // ----------------- main ---------------------- // int main(int argc, char *argv[]) { - printf("welcome to use taospack tools v1.1. sizeof(STColumn) = %lu\n", sizeof(STColumn)); + printf("welcome to use taospack tools v1.2 \n"); + gOpenLossy = false; tsLossyInit(); // //tsCompressExit(); @@ -530,9 +594,11 @@ int main(int argc, char *argv[]) { // t if(strcmp(argv[1], "-tone") == 0 || strcmp(argv[1], "-t") == 0 ) { algo = ONE_STAGE_COMP; + gOpenLossy = true; } if(strcmp(argv[1], "-ttwo") == 0) { algo = TWO_STAGE_COMP; + gOpenLossy = false; } if(strcmp(argv[1], "-sf") == 0) { @@ -558,6 +624,9 @@ int main(int argc, char *argv[]) { memTest(); } } + else{ + unitTestFloat(); + } //memTest(); return 0; diff --git a/src/util/inc/tscompression.h b/src/util/inc/tscompression.h index 63da5c9cc5..ea78df6570 100644 --- a/src/util/inc/tscompression.h +++ b/src/util/inc/tscompression.h @@ -35,6 +35,25 @@ extern "C" { #define ONE_STAGE_COMP 1 #define TWO_STAGE_COMP 2 +// +// compressed data first byte foramt +// ------ 7 bit ---- | ---- 1 bit ---- +// algorithm mode +// + +// compression data mode save first byte lower 1 bit +#define MODE_NOCOMPRESS 0 // original data +#define MODE_COMPRESS 1 // compatible old compress + +// compression algorithm save first byte higher 7 bit +#define ALGO_SZ_LOSSY 1 // SZ compress + +#define HEAD_MODE(x) x%2 +#define HEAD_ALGO(x) x/2 + + +extern bool gOpenLossy; + extern int tsCompressINTImp(const char *const input, const int nelements, char *const output, const char type); extern int tsDecompressINTImp(const char *const input, const int nelements, char *const output, const char type); extern int tsCompressBoolImp(const char *const input, const int nelements, char *const output); @@ -48,16 +67,17 @@ extern int tsDecompressDoubleImp(const char *const input, const int nelements, c extern int tsCompressFloatImp(const char *const input, const int nelements, char *const output); extern int tsDecompressFloatImp(const char *const input, const int nelements, char *const output); // lossy -int tsCompressFloatLossyImp(const char * input, const int nelements, const char * output); -int tsDecompressFloatLossyImp(const char * input, int compressedSize, const int nelements, const char * output); -int tsCompressDoubleLossyImp(const char * input, const int nelements, const char * output); -int tsDecompressDoubleLossyImp(const char * input, int compressedSize, const int nelements, const char * output); +int tsCompressFloatLossyImp(const char * input, const int nelements, char *const output); +int tsDecompressFloatLossyImp(const char * input, int compressedSize, const int nelements, char *const output); +int tsCompressDoubleLossyImp(const char * input, const int nelements, char *const output); +int tsDecompressDoubleLossyImp(const char * input, int compressedSize, const int nelements, char *const output); // init bool tsLossyInit(); + static FORCE_INLINE int tsCompressTinyint(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, char algorithm, char *const buffer, int bufferSize) { if (algorithm == ONE_STAGE_COMP) { @@ -200,53 +220,78 @@ static FORCE_INLINE int tsDecompressString(const char *const input, int compress static FORCE_INLINE int tsCompressFloat(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, char algorithm, char *const buffer, int bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsCompressFloatImp(input, nelements, output); - } else if (algorithm == TWO_STAGE_COMP) { - int len = tsCompressFloatImp(input, nelements, buffer); - return tsCompressStringImp(buffer, len, output, outputSize); + // lossy mode + if(gOpenLossy) { + return tsCompressFloatLossyImp(input, nelements, output); + // lossless mode } else { - assert(0); - return -1; + if (algorithm == ONE_STAGE_COMP) { + return tsCompressFloatImp(input, nelements, output); + } else if (algorithm == TWO_STAGE_COMP) { + int len = tsCompressFloatImp(input, nelements, buffer); + return tsCompressStringImp(buffer, len, output, outputSize); + } else { + assert(0); + return -1; + } } } static FORCE_INLINE int tsDecompressFloat(const char *const input, int compressedSize, const int nelements, char *const output, int outputSize, char algorithm, char *const buffer, int bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsDecompressFloatImp(input, nelements, output); - } else if (algorithm == TWO_STAGE_COMP) { - if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; - return tsDecompressFloatImp(buffer, nelements, output); + + if(HEAD_ALGO(input[0]) == ALGO_SZ_LOSSY){ + // decompress lossy + return tsDecompressFloatLossyImp(input, compressedSize, nelements, output); } else { - assert(0); - return -1; + // decompress lossless + if (algorithm == ONE_STAGE_COMP) { + return tsDecompressFloatImp(input, nelements, output); + } else if (algorithm == TWO_STAGE_COMP) { + if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; + return tsDecompressFloatImp(buffer, nelements, output); + } else { + assert(0); + return -1; + } } } static FORCE_INLINE int tsCompressDouble(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, char algorithm, char *const buffer, int bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsCompressDoubleImp(input, nelements, output); - } else if (algorithm == TWO_STAGE_COMP) { - int len = tsCompressDoubleImp(input, nelements, buffer); - return tsCompressStringImp(buffer, len, output, outputSize); + if(gOpenLossy){ + // lossy mode + return tsCompressDoubleLossyImp(input, nelements, output); } else { - assert(0); - return -1; + // lossless mode + if (algorithm == ONE_STAGE_COMP) { + return tsCompressDoubleImp(input, nelements, output); + } else if (algorithm == TWO_STAGE_COMP) { + int len = tsCompressDoubleImp(input, nelements, buffer); + return tsCompressStringImp(buffer, len, output, outputSize); + } else { + assert(0); + return -1; + } } } static FORCE_INLINE int tsDecompressDouble(const char *const input, int compressedSize, const int nelements, char *const output, int outputSize, char algorithm, char *const buffer, int bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsDecompressDoubleImp(input, nelements, output); - } else if (algorithm == TWO_STAGE_COMP) { - if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; - return tsDecompressDoubleImp(buffer, nelements, output); + if(HEAD_ALGO(input[0]) == ALGO_SZ_LOSSY){ + // decompress lossy + return tsDecompressDoubleLossyImp(input, compressedSize, nelements, output); } else { - assert(0); - return -1; + // decompress lossless + if (algorithm == ONE_STAGE_COMP) { + return tsDecompressDoubleImp(input, nelements, output); + } else if (algorithm == TWO_STAGE_COMP) { + if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; + return tsDecompressDoubleImp(buffer, nelements, output); + } else { + assert(0); + return -1; + } } } diff --git a/src/util/src/tcompression.c b/src/util/src/tcompression.c index 300a06e9ac..fb96c55268 100644 --- a/src/util/src/tcompression.c +++ b/src/util/src/tcompression.c @@ -62,6 +62,8 @@ static const int TEST_NUMBER = 1; #define ZIGZAG_ENCODE(T, v) ((u##T)((v) >> (sizeof(T) * 8 - 1))) ^ (((u##T)(v)) << 1) // zigzag encode #define ZIGZAG_DECODE(T, v) ((v) >> 1) ^ -((T)((v)&1)) // zigzag decode +bool gOpenLossy = true; + /* * Compress Integer (Simple8B). */ @@ -411,6 +413,7 @@ int tsCompressStringImp(const char *const input, int inputSize, char *const outp int tsDecompressStringImp(const char *const input, int compressedSize, char *const output, int outputSize) { // compressedSize is the size of data after compression. + if (input[0] == 1) { /* It is compressed by LZ4 algorithm */ const int decompressed_size = LZ4_decompress_safe(input + 1, output, compressedSize - 1, outputSize); @@ -906,18 +909,66 @@ bool tsLossyInit() { // // ---------- float double lossy ----------- // -int tsCompressFloatLossyImp(const char * input, const int nelements, const char * output){ - return tdszCompress(SZ_FLOAT, input, nelements, output); +int tsCompressFloatLossyImp(const char * input, const int nelements, char *const output){ + // compress with sz + int compressedSize = tdszCompress(SZ_FLOAT, input, nelements, output + 1); + unsigned char algo = ALGO_SZ_LOSSY << 1; + if (compressedSize == 0 || compressedSize >= nelements*sizeof(float)){ + // compressed error or large than original + output[0] = MODE_NOCOMPRESS | algo; + memcpy(output + 1, input, nelements * sizeof(float)); + compressedSize = 1 + nelements * sizeof(float); + } else { + // compressed successfully + output[0] = MODE_COMPRESS | algo; + compressedSize += 1; + } + + return compressedSize; } -int tsDecompressFloatLossyImp(const char * input, int compressedSize, const int nelements, const char * output){ - return tdszDecompress(SZ_FLOAT, input, compressedSize, nelements, output); +int tsDecompressFloatLossyImp(const char * input, int compressedSize, const int nelements, char *const output){ + int decompressedSize = 0; + if( HEAD_MODE(input[0]) == MODE_NOCOMPRESS){ + // orginal so memcpy directly + decompressedSize = nelements * sizeof(float); + memcpy(output, input + 1, decompressedSize); + + return decompressedSize; + } + + // decompressed with sz + return tdszDecompress(SZ_FLOAT, input + 1, compressedSize - 1, nelements, output); } -int tsCompressDoubleLossyImp(const char * input, const int nelements, const char * output){ - return tdszCompress(SZ_DOUBLE, input, nelements, output); +int tsCompressDoubleLossyImp(const char * input, const int nelements, char *const output){ + // compress with sz + int compressedSize = tdszCompress(SZ_DOUBLE, input, nelements, output + 1); + unsigned char algo = ALGO_SZ_LOSSY << 1; + if (compressedSize == 0 || compressedSize >= nelements*sizeof(double)) { + // compressed error or large than original + output[0] = MODE_NOCOMPRESS | algo; + memcpy(output + 1, input, nelements * sizeof(double)); + compressedSize = 1 + nelements * sizeof(double); + } else { + // compressed successfully + output[0] = MODE_COMPRESS | algo; + compressedSize += 1; + } + + return compressedSize; } -int tsDecompressDoubleLossyImp(const char * input, int compressedSize, const int nelements, const char * output){ - return tdszDecompress(SZ_DOUBLE, input, compressedSize, nelements, output); +int tsDecompressDoubleLossyImp(const char * input, int compressedSize, const int nelements, char *const output){ + int decompressedSize = 0; + if( HEAD_MODE(input[0]) == MODE_NOCOMPRESS){ + // orginal so memcpy directly + decompressedSize = nelements * sizeof(double); + memcpy(output, input + 1, decompressedSize); + + return decompressedSize; + } + + // decompressed with sz + return tdszDecompress(SZ_DOUBLE, input + 1, compressedSize - 1, nelements, output); } \ No newline at end of file