xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

compare with td finished

This commit is contained in:
tickduan 2021-07-02 10:08:31 +08:00
parent 9ff807755f
commit 9ee3360601
67 changed files with 675 additions and 7656 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
deps/SZ/sz/include/TightDataPointStorageD.h vendored
View File

@ -88,7 +88,7 @@ void new_TightDataPointStorageD2(TightDataPointStorageD **self,
void convertTDPStoBytes_double(TightDataPointStorageD* tdps, unsigned char* bytes, unsigned char* dsLengthBytes, unsigned char sameByte);
void convertTDPStoBytes_double_reserve(TightDataPointStorageD* tdps, unsigned char* bytes, unsigned char* dsLengthBytes, unsigned char sameByte);
void convertTDPStoFlatBytes_double(TightDataPointStorageD *tdps, unsigned char** bytes, size_t *size);
void convertTDPStoFlatBytes_double(TightDataPointStorageD *tdps, unsigned char* bytes, size_t *size);
void convertTDPStoFlatBytes_double_args(TightDataPointStorageD *tdps, unsigned char* bytes, size_t *size);
void free_TightDataPointStorageD(TightDataPointStorageD *tdps);

3
deps/SZ/sz/include/TightDataPointStorageF.h vendored
View File

@ -92,8 +92,7 @@ void new_TightDataPointStorageF2(TightDataPointStorageF **self,
unsigned char* pwrErrBoundBytes, size_t pwrErrBoundBytes_size, unsigned char radExpo);
void convertTDPStoBytes_float(TightDataPointStorageF* tdps, unsigned char* bytes, unsigned char* dsLengthBytes, unsigned char sameByte);
void convertTDPStoBytes_float_reserve(TightDataPointStorageF* tdps, unsigned char* bytes, unsigned char* dsLengthBytes, unsigned char sameByte);
void convertTDPStoFlatBytes_float(TightDataPointStorageF *tdps, unsigned char** bytes, size_t *size);
void convertTDPStoFlatBytes_float(TightDataPointStorageF *tdps, unsigned char* bytes, size_t *size);
void convertTDPStoFlatBytes_float_args(TightDataPointStorageF *tdps, unsigned char* bytes, size_t *size);
void free_TightDataPointStorageF(TightDataPointStorageF *tdps);

66
deps/SZ/sz/include/TightDataPointStorageI.h vendored
View File

@ -1,66 +0,0 @@
/**
* @file TightDataPointStorageI.h
* @author Sheng Di and Dingwen Tao
* @date Aug, 2017
* @brief Header file for the tight data point storage (TDPS).
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _TightDataPointStorageI_H
#define _TightDataPointStorageI_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdio.h>
#include "pub.h"
typedef struct TightDataPointStorageI
{
size_t dataSeriesLength;
int allSameData;
double realPrecision; //it's used as the pwrErrBoundRatio when errBoundMode==PW_REL
size_t exactDataNum;
long minValue;
int exactByteSize;
int dataTypeSize; //the size of data type, e.g., it's 4 when data type is int32_t
int stateNum;
int allNodes;
unsigned char* typeArray; //its size is dataSeriesLength/4 (or xxx/4+1)
size_t typeArray_size;
unsigned char* exactDataBytes;
size_t exactDataBytes_size;
unsigned int intervals; //quantization_intervals
unsigned char isLossless; //a mark to denote whether it's lossless compression (1 is yes, 0 is no)
} TightDataPointStorageI;
int computeRightShiftBits(int exactByteSize, int dataType);
int convertDataTypeSizeCode(int dataTypeSizeCode);
int convertDataTypeSize(int dataTypeSize);
void new_TightDataPointStorageI_Empty(TightDataPointStorageI **self);
int new_TightDataPointStorageI_fromFlatBytes(TightDataPointStorageI **self, unsigned char* flatBytes, size_t flatBytesLength);
void new_TightDataPointStorageI(TightDataPointStorageI **self,
size_t dataSeriesLength, size_t exactDataNum, int byteSize,
int* type, unsigned char* exactDataBytes, size_t exactDataBytes_size,
double realPrecision, long minValue, int intervals, int dataType);
void convertTDPStoBytes_int(TightDataPointStorageI* tdps, unsigned char* bytes, unsigned char sameByte);
void convertTDPStoFlatBytes_int(TightDataPointStorageI *tdps, unsigned char** bytes, size_t *size);
void convertTDPStoFlatBytes_int_args(TightDataPointStorageI *tdps, unsigned char* bytes, size_t *size);
void free_TightDataPointStorageI(TightDataPointStorageI *tdps);
void free_TightDataPointStorageI2(TightDataPointStorageI *tdps);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _TightDataPointStorageI_H ----- */

2
deps/SZ/sz/include/callZlib.h vendored
View File

@ -26,7 +26,7 @@ unsigned long zlib_compress(unsigned char* data, unsigned long dataLength, unsig
unsigned long zlib_compress2(unsigned char* data, unsigned long dataLength, unsigned char** compressBytes, int level);
unsigned long zlib_compress3(unsigned char* data, unsigned long dataLength, unsigned char* compressBytes, int level);
unsigned long zlib_compress4(unsigned char* data, unsigned long dataLength, unsigned char** compressBytes, int level);
unsigned long zlib_compress5(unsigned char* data, unsigned long dataLength, unsigned char** compressBytes, int level);
unsigned long zlib_compress5(unsigned char* data, unsigned long dataLength, unsigned char* compressBytes, int level);
unsigned long zlib_uncompress4(unsigned char* compressBytes, unsigned long cmpSize, unsigned char** oriData, unsigned long targetOriSize);
unsigned long zlib_uncompress5(unsigned char* compressBytes, unsigned long cmpSize, unsigned char** oriData, unsigned long targetOriSize);

7
deps/SZ/sz/include/defines.h vendored
View File

@ -24,7 +24,7 @@
//prediction mode of temporal dimension based compression
#define SZ_PREVIOUS_VALUE_ESTIMATE 0
#define MIN_NUM_OF_ELEMENTS 0 //if the # elements <= 20, skip the compression
#define MIN_NUM_OF_ELEMENTS 50 //if the # elements <= 20, skip the compression
#define SZ_ABS 0
#define REL 1
@ -81,13 +81,14 @@
#define SZ_PERIO_TEMPORAL_COMPRESSION 2
//SUCCESS returning status
#define SZ_SCES 0 //successful
#define SZ_NSCS -1 //Not successful
#define SZ_SUCCESS 0 //successful
#define SZ_FAILED -1 //Not successful
#define SZ_FERR -2 //Failed to open input file
#define SZ_TERR -3 //wrong data type (should be only float or double)
#define SZ_DERR -4 //dimension error
#define SZ_MERR -5 //sz_mode error
#define SZ_BERR -6 //bound-mode error (should be only SZ_ABS, REL, ABS_AND_REL, ABS_OR_REL, or PW_REL)
#define SZ_LITTER_ELEMENT -7
#define SZ_MAINTAIN_VAR_DATA 0
#define SZ_DESTROY_WHOLE_VARSET 1

57
deps/SZ/sz/include/exafelSZ.h vendored
View File

@ -1,57 +0,0 @@
#ifndef EXAFELSZ_H
#define EXAFELSZ_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdint.h>
#include <stdlib.h>
typedef struct exafelSZ_params{
//uint8_t *peaks;
uint16_t *peaksSegs;
uint16_t *peaksRows;
uint16_t *peaksCols;
uint64_t numPeaks;
uint8_t *calibPanel;
uint8_t binSize; //Binning: (pr->binSize x pr->binSize) to (1 x 1)
double tolerance; //SZ pr->tolerance
uint8_t szDim; //1D/2D/3D compression/decompression
//uint8_t szBlockSize; //Currently unused
uint8_t peakSize; //MUST BE ODD AND NOT EVEN! Each peak will have size of: (peakSize x peakSize)
// uint64_t nEvents;
// uint64_t panels;
// uint64_t rows;
// uint64_t cols;
//CALCULATED VARIBALES:
uint64_t binnedRows;
uint64_t binnedCols;
uint8_t peakRadius; //Will be calculated using peakSize
} exafelSZ_params;
void exafelSZ_params_process(exafelSZ_params*pr, size_t panels, size_t rows, size_t cols);
void exafelSZ_params_checkDecomp(exafelSZ_params*pr, size_t panels, size_t rows, size_t cols);
void exafelSZ_params_checkComp(exafelSZ_params*pr, size_t panels, size_t rows, size_t cols);
unsigned char * exafelSZ_Compress(void* _pr,
void* _origData,
size_t events, size_t panels, size_t rows, size_t cols,
size_t *compressedSize);
void* exafelSZ_Decompress(void *_pr,
unsigned char*_compressedBuffer,
size_t events, size_t panels, size_t rows, size_t cols,
size_t compressedSize);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _EXAFELSZ_H ----- */

108
deps/SZ/sz/include/sz.h vendored
View File

@ -22,29 +22,12 @@
#include "Huffman.h"
#include "TightDataPointStorageD.h"
#include "TightDataPointStorageF.h"
#include "TightDataPointStorageI.h"
#include "conf.h"
#include "dataCompression.h"
#include "ByteToolkit.h"
#include "TypeManager.h"
#include "sz_int8.h"
#include "sz_int16.h"
#include "sz_int32.h"
#include "sz_int64.h"
#include "sz_uint8.h"
#include "sz_uint16.h"
#include "sz_uint32.h"
#include "sz_uint64.h"
#include "sz_float.h"
#include "sz_double.h"
#include "szd_int8.h"
#include "szd_int16.h"
#include "szd_int32.h"
#include "szd_int64.h"
#include "szd_uint8.h"
#include "szd_uint16.h"
#include "szd_uint32.h"
#include "szd_uint64.h"
#include "szd_float.h"
#include "szd_double.h"
#include "sz_float_pwr.h"
@ -53,13 +36,10 @@
#include "callZlib.h"
#include "rw.h"
#include "pastri.h"
#include "sz_float_ts.h"
#include "szd_float_ts.h"
#include "utility.h"
#include "CacheTable.h"
#include "MultiLevelCacheTable.h"
#include "MultiLevelCacheTableWideInterval.h"
#include "exafelSZ.h"
#include "sz_stats.h"
#ifdef _WIN32
@ -208,86 +188,26 @@ extern pastri_params pastri_par;
//sz.h
HuffmanTree* SZ_Reset();
int SZ_Init(const char *configFilePath);
int SZ_Init_Params(sz_params *params);
size_t computeDataLength(size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
int computeDimension(size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
int SZ_compress_args_float_subblock(unsigned char* compressedBytes, float *oriData,
size_t r5, size_t r4, size_t r3, size_t r2, size_t r1,
size_t s5, size_t s4, size_t s3, size_t s2, size_t s1,
size_t e5, size_t e4, size_t e3, size_t e2, size_t e1,
size_t *outSize, int errBoundMode, double absErr_Bound, double relBoundRatio);
int SZ_compress_args_double_subblock(unsigned char* compressedBytes, double *oriData,
size_t r5, size_t r4, size_t r3, size_t r2, size_t r1,
size_t s5, size_t s4, size_t s3, size_t s2, size_t s1,
size_t e5, size_t e4, size_t e3, size_t e2, size_t e1,
size_t *outSize, int errBoundMode, double absErr_Bound, double relBoundRatio);
unsigned char *SZ_compress(int dataType, void *data, size_t *outSize, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
unsigned char* SZ_compress_args(int dataType, void *data, size_t *outSize, int errBoundMode, double absErrBound,
double relBoundRatio, double pwrBoundRatio, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
int SZ_compress_args2(int dataType, void *data, unsigned char* compressed_bytes, size_t *outSize,
int errBoundMode, double absErrBound, double relBoundRatio, double pwrBoundRatio,
size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
int SZ_compress_args3(int dataType, void *data, unsigned char* compressed_bytes, size_t *outSize, int errBoundMode, double absErrBound, double relBoundRatio,
size_t r5, size_t r4, size_t r3, size_t r2, size_t r1,
size_t s5, size_t s4, size_t s3, size_t s2, size_t s1,
size_t e5, size_t e4, size_t e3, size_t e2, size_t e1);
unsigned char *SZ_compress_rev_args(int dataType, void *data, void *reservedValue, size_t *outSize, int errBoundMode, double absErrBound, double relBoundRatio,
size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
int SZ_compress_rev_args2(int dataType, void *data, void *reservedValue, unsigned char* compressed_bytes, size_t *outSize, int errBoundMode, double absErrBound, double relBoundRatio,
size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
unsigned char *SZ_compress_rev(int dataType, void *data, void *reservedValue, size_t *outSize, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
void SZ_Create_ParamsExe(sz_params** conf_params, sz_exedata** exe_params);
void *SZ_decompress(int dataType, unsigned char *bytes, size_t byteLength, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
size_t SZ_decompress_args(int dataType, unsigned char *bytes, size_t byteLength, void* decompressed_array, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
sz_metadata* SZ_getMetadata(unsigned char* bytes, sz_exedata* pde_exe);
void SZ_printMetadata(sz_metadata* metadata);
void filloutDimArray(size_t* dim, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
size_t compute_total_batch_size();
void SZ_registerVar(int var_id, char* varName, int dataType, void* data,
int errBoundMode, double absErrBound, double relBoundRatio, double pwRelBoundRatio,
size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
int SZ_deregisterVar_ID(int var_id);
int SZ_deregisterVar(char* varName);
int SZ_deregisterAllVars();
int SZ_compress_ts_select_var(int cmprType, unsigned char* var_ids, unsigned char var_count, unsigned char** newByteData, size_t *outSize);
int SZ_compress_ts(int cmprType, unsigned char** newByteData, size_t *outSize);
void SZ_decompress_ts_select_var(unsigned char* var_ids, unsigned char var_count, unsigned char *bytes, size_t bytesLength);
void SZ_decompress_ts(unsigned char *bytes, size_t byteLength);
void SZ_Finalize();
int SZ_Init(const char *configFilePath);
int SZ_Init_Params(sz_params *params);
//
// compress output data to outData and return outSize
//
size_t SZ_compress_args(int dataType, void *data, size_t r1, unsigned char* outData, sz_params* params);
//
// decompress output data to outData and return outSize
//
size_t SZ_decompress(int dataType, unsigned char *bytes, size_t byteLength, size_t r1, unsigned char* outData);
void convertSZParamsToBytes(sz_params* params, unsigned char* result);
void convertBytesToSZParams(unsigned char* bytes, sz_params* params, sz_exedata* pde_exe);
unsigned char* SZ_compress_customize(const char* appName, void* userPara, int dataType, void* data, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize, int *status);
unsigned char* SZ_compress_customize_threadsafe(const char* cmprName, void* userPara, int dataType, void* data, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize, int *status);
void* SZ_decompress_customize(const char* appName, void* userPara, int dataType, unsigned char* bytes, size_t byteLength, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, int* status);
void* SZ_decompress_customize_threadsafe(const char* cmprName, void* userPara, int dataType, unsigned char* bytes, size_t byteLength, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, int *status);
#ifdef __cplusplus
}
#endif

20
deps/SZ/sz/include/sz_double.h vendored
View File

@ -28,29 +28,17 @@ unsigned int optimize_intervals_double_1D_opt(double *oriData, size_t dataLength
unsigned int optimize_intervals_double_1D_opt_MSST19(double *oriData, size_t dataLength, double realPrecision);
TightDataPointStorageD* SZ_compress_double_1D_MDQ(double *oriData,
size_t dataLength, double realPrecision, double valueRangeSize, double medianValue_d);
void SZ_compress_args_double_StoreOriData(double* oriData, size_t dataLength, unsigned char** newByteData, size_t *outSize);
void SZ_compress_args_double_StoreOriData(double* oriData, size_t dataLength, unsigned char* newByteData, size_t *outSize);
char SZ_compress_args_double_NoCkRngeNoGzip_1D(int cmprType, unsigned char** newByteData, double *oriData, size_t dataLength, double realPrecision, size_t *outSize, double valueRangeSize, double medianValue_d);
char SZ_compress_args_double_NoCkRngeNoGzip_1D( unsigned char* newByteData, double *oriData, size_t dataLength, double realPrecision, size_t *outSize, double valueRangeSize, double medianValue_d);
TightDataPointStorageD* SZ_compress_double_1D_MDQ_MSST19(double *oriData, size_t dataLength, double realPrecision, double valueRangeSize, double medianValue_f);
void SZ_compress_args_double_withinRange(unsigned char** newByteData, double *oriData, size_t dataLength, size_t *outSize);
void SZ_compress_args_double_withinRange(unsigned char* newByteData, double *oriData, size_t dataLength, size_t *outSize);
/*int SZ_compress_args_double_wRngeNoGzip(unsigned char** newByteData, double *oriData,
size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize,
int errBoundMode, double absErr_Bound, double relBoundRatio, double pwrErrRatio);*/
int SZ_compress_args_double(int cmprType, int withRegression, unsigned char** newByteData, double *oriData,
size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize,
int errBoundMode, double absErr_Bound, double relBoundRatio, double pwRelBoundRatio);
int SZ_compress_args_double(double *oriData, size_t r1, unsigned char* newByteData, size_t *outSize, sz_params* params);
void SZ_compress_args_double_NoCkRnge_1D_subblock(unsigned char* compressedBytes, double *oriData, double realPrecision, size_t *outSize, double valueRangeSize, double medianValue_d,
size_t r1, size_t s1, size_t e1);
unsigned int optimize_intervals_double_1D_subblock(double *oriData, double realPrecision, size_t r1, size_t s1, size_t e1);
TightDataPointStorageD* SZ_compress_double_1D_MDQ_subblock(double *oriData, double realPrecision, double valueRangeSize, double medianValue_d,
size_t r1, size_t s1, size_t e1);
#ifdef __cplusplus

8
deps/SZ/sz/include/sz_double_pwr.h vendored
View File

@ -19,17 +19,17 @@ extern "C" {
void compute_segment_precisions_double_1D(double *oriData, size_t dataLength, double* pwrErrBound, unsigned char* pwrErrBoundBytes, double globalPrecision);
unsigned int optimize_intervals_double_1D_pwr(double *oriData, size_t dataLength, double* pwrErrBound);
void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr(unsigned char** newByteData, double *oriData, double globalPrecision, size_t dataLength, size_t *outSize, double min, double max);
void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr(unsigned char* newByteData, double *oriData, double globalPrecision, size_t dataLength, size_t *outSize, double min, double max);
void createRangeGroups_double(double** posGroups, double** negGroups, int** posFlags, int** negFlags);
void compressGroupIDArray_double(char* groupID, TightDataPointStorageD* tdps);
TightDataPointStorageD* SZ_compress_double_1D_MDQ_pwrGroup(double* oriData, size_t dataLength, int errBoundMode,
double absErrBound, double relBoundRatio, double pwrErrRatio, double valueRangeSize, double medianValue_f);
void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwrgroup(unsigned char** newByteData, double *oriData,
void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwrgroup(unsigned char* newByteData, double *oriData,
size_t dataLength, double absErrBound, double relBoundRatio, double pwrErrRatio, double valueRangeSize, double medianValue_f, size_t *outSize);
void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log(unsigned char** newByteData, double *oriData, double globalPrecision, size_t dataLength, size_t *outSize, double min, double max);
void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log(unsigned char* newByteData, double *oriData, double globalPrecision, size_t dataLength, size_t *outSize, double min, double max);
void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log_MSST19(unsigned char** newByteData, double *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, double valueRangeSize, double medianValue_f,
void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log_MSST19(unsigned char* newByteData, double *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, double valueRangeSize, double medianValue_f,
unsigned char* signs, bool* positive, double min, double max, double nearZero);
#ifdef __cplusplus

27
deps/SZ/sz/include/sz_double_ts.h vendored
View File

@ -1,27 +0,0 @@
/**
* @file sz_double_ts.h
* @author Sheng Di
* @date May, 2018
* @brief Header file for the sz_double_ts.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#include "TightDataPointStorageD.h"
#ifndef _SZ_Double_TS_H
#define _SZ_Double_TS_H
#ifdef __cplusplus
extern "C" {
#endif
unsigned int optimize_intervals_double_1D_ts(double *oriData, size_t dataLength, double* preData, double realPrecision);
TightDataPointStorageD* SZ_compress_double_1D_MDQ_ts(double *oriData, size_t dataLength, sz_multisteps* multisteps,
double realPrecision, double valueRangeSize, double medianValue_d);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _SZ_Double_TS_H ----- */

29
deps/SZ/sz/include/sz_float.h vendored
View File

@ -29,9 +29,9 @@ unsigned int optimize_intervals_float_1D_opt_MSST19(float *oriData, size_t dataL
TightDataPointStorageF* SZ_compress_float_1D_MDQ(float *oriData,
size_t dataLength, float realPrecision, float valueRangeSize, float medianValue_f);
void SZ_compress_args_float_StoreOriData(float* oriData, size_t dataLength, unsigned char** newByteData, size_t *outSize);
void SZ_compress_args_float_StoreOriData(float* oriData, size_t dataLength, unsigned char* newByteData, size_t *outSize);
char SZ_compress_args_float_NoCkRngeNoGzip_1D(int cmprType, unsigned char** newByteData, float *oriData,
char SZ_compress_args_float_NoCkRngeNoGzip_1D( unsigned char* newByteData, float *oriData,
size_t dataLength, double realPrecision, size_t *outSize, float valueRangeSize, float medianValue_f);
size_t SZ_compress_float_1D_MDQ_RA_block(float * block_ori_data, float * mean, size_t dim_0, size_t block_dim_0, double realPrecision, int * type, float * unpredictable_data);
@ -40,36 +40,15 @@ size_t SZ_compress_float_1D_MDQ_RA_block_1D_pred(float * block_ori_data, float *
void SZ_blocked_regression(float * block_ori_data, size_t dim_0, size_t dim_1, size_t dim_2, size_t block_dim_0, size_t block_dim_1, size_t block_dim_2, float *params);
unsigned char * SZ_compress_float_1D_MDQ_RA(float *oriData, size_t r1, double realPrecision, size_t * comp_size);
unsigned char * SZ_compress_float_1D_MDQ_decompression_random_access_with_blocked_regression(float *oriData, size_t r1, double realPrecision, size_t * comp_size);
TightDataPointStorageF* SZ_compress_float_1D_MDQ_MSST19(float *oriData,
size_t dataLength, double realPrecision, float valueRangeSize, float medianValue_f);
void SZ_compress_args_float_withinRange(unsigned char** newByteData, float *oriData, size_t dataLength, size_t *outSize);
/*int SZ_compress_args_float_wRngeNoGzip(unsigned char** newByteData, float *oriData,
size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize,
int errBoundMode, double absErr_Bound, double relBoundRatio, double pwrErrRatio);*/
int SZ_compress_args_float(int cmprType, int withRegression, unsigned char** newByteData, float *oriData,
size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize,
int errBoundMode, double absErr_Bound, double relBoundRatio, double pwRelBoundRatio);
int SZ_compress_args_float_subblock(unsigned char* compressedBytes, float *oriData,
size_t r5, size_t r4, size_t r3, size_t r2, size_t r1,
size_t s5, size_t s4, size_t s3, size_t s2, size_t s1,
size_t e5, size_t e4, size_t e3, size_t e2, size_t e1,
size_t *outSize, int errBoundMode, double absErr_Bound, double relBoundRatio);
void SZ_compress_args_float_NoCkRnge_1D_subblock(unsigned char* compressedBytes, float *oriData, double realPrecision, size_t *outSize, float valueRangeSize, float medianValue_f,
size_t r1, size_t s1, size_t e1);
void SZ_compress_args_float_withinRange(unsigned char* newByteData, float *oriData, size_t dataLength, size_t *outSize);
unsigned int optimize_intervals_float_1D_subblock(float *oriData, double realPrecision, size_t r1, size_t s1, size_t e1);
TightDataPointStorageF* SZ_compress_float_1D_MDQ_subblock(float *oriData, double realPrecision, float valueRangeSize, float medianValue_f,
size_t r1, size_t s1, size_t e1);
int SZ_compress_args_float(float *oriData, size_t r1, unsigned char* newByteData, size_t *outSize, sz_params* params);
#ifdef __cplusplus
}

8
deps/SZ/sz/include/sz_float_pwr.h vendored
View File

@ -19,7 +19,7 @@ extern "C" {
void compute_segment_precisions_float_1D(float *oriData, size_t dataLength, float* pwrErrBound, unsigned char* pwrErrBoundBytes, double globalPrecision);
unsigned int optimize_intervals_float_1D_pwr(float *oriData, size_t dataLength, float* pwrErrBound);
void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr(unsigned char** newByteData, float *oriData, double globalPrecision, size_t dataLength, size_t *outSize, float min, float max);
void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr(unsigned char* newByteData, float *oriData, double globalPrecision, size_t dataLength, size_t *outSize, float min, float max);
void createRangeGroups_float(float** posGroups, float** negGroups, int** posFlags, int** negFlags);
@ -28,11 +28,11 @@ int* generateGroupLowerBounds();
TightDataPointStorageF* SZ_compress_float_1D_MDQ_pwrGroup(float* oriData, size_t dataLength, int errBoundMode,
double absErrBound, double relBoundRatio, double pwrErrRatio, float valueRangeSize, float medianValue_f);
void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwrgroup(unsigned char** newByteData, float *oriData,
void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwrgroup(unsigned char* newByteData, float *oriData,
size_t dataLength, double absErrBound, double relBoundRatio, double pwrErrRatio, float valueRangeSize, float medianValue_f, size_t *outSize);
void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_pre_log(unsigned char** newByteData, float *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, float min, float max);
void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_pre_log_MSST19(unsigned char** newByteData, float *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, float valueRangeSize, float medianValue_f,
void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_pre_log(unsigned char* newByteData, float *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, float min, float max);
void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_pre_log_MSST19(unsigned char* newByteData, float *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, float valueRangeSize, float medianValue_f,
unsigned char* signs, bool* positive, float min, float max, float nearZero);
#ifdef __cplusplus

27
deps/SZ/sz/include/sz_float_ts.h vendored
View File

@ -1,27 +0,0 @@
/**
* @file sz_float_ts.h
* @author Sheng Di
* @date May, 2018
* @brief Header file for the sz_float_ts.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#include "TightDataPointStorageF.h"
#ifndef _SZ_Float_TS_H
#define _SZ_Float_TS_H
#ifdef __cplusplus
extern "C" {
#endif
unsigned int optimize_intervals_float_1D_ts(float *oriData, size_t dataLength, float* preData, double realPrecision);
TightDataPointStorageF* SZ_compress_float_1D_MDQ_ts(float *oriData, size_t dataLength, sz_multisteps* multisteps,
double realPrecision, float valueRangeSize, float medianValue_f);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _SZ_Float_TS_H ----- */

48
deps/SZ/sz/include/sz_int16.h vendored
View File

@ -1,48 +0,0 @@
/**
* @file sz_int16.h
* @author Sheng Di
* @date July, 2017
* @brief Header file for the sz_int16.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _SZ_Int16_H
#define _SZ_Int16_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdio.h>
unsigned int optimize_intervals_int16_1D(int16_t *oriData, size_t dataLength, double realPrecision);
unsigned int optimize_intervals_int16_2D(int16_t *oriData, size_t r1, size_t r2, double realPrecision);
unsigned int optimize_intervals_int16_3D(int16_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision);
unsigned int optimize_intervals_int16_4D(int16_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision);
TightDataPointStorageI* SZ_compress_int16_1D_MDQ(int16_t *oriData, size_t dataLength, double realPrecision, int64_t valueRangeSize, int64_t minValue);
void SZ_compress_args_int16_StoreOriData(int16_t* oriData, size_t dataLength, TightDataPointStorageI* tdps, unsigned char** newByteData, size_t *outSize);
void SZ_compress_args_int16_NoCkRngeNoGzip_1D(unsigned char** newByteData, int16_t *oriData,
size_t dataLength, double realPrecision, size_t *outSize, int64_t valueRangeSize, int16_t minValue);
TightDataPointStorageI* SZ_compress_int16_2D_MDQ(int16_t *oriData, size_t r1, size_t r2, double realPrecision, int64_t valueRangeSize, int64_t minValue);
TightDataPointStorageI* SZ_compress_int16_3D_MDQ(int16_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, int64_t valueRangeSize, int64_t minValue);
void SZ_compress_args_int16_NoCkRngeNoGzip_3D(unsigned char** newByteData, int16_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, size_t *outSize, int64_t valueRangeSize, int64_t minValue);
TightDataPointStorageI* SZ_compress_int16_4D_MDQ(int16_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision, int64_t valueRangeSize, int64_t minValue);
void SZ_compress_args_int16_NoCkRngeNoGzip_4D(unsigned char** newByteData, int16_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision,
size_t *outSize, int64_t valueRangeSize, int64_t minValue);
void SZ_compress_args_int16_withinRange(unsigned char** newByteData, int16_t *oriData, size_t dataLength, size_t *outSize);
int SZ_compress_args_int16_wRngeNoGzip(unsigned char** newByteData, int16_t *oriData,
size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize,
int errBoundMode, double absErr_Bound, double relBoundRatio);
int SZ_compress_args_int16(unsigned char** newByteData, int16_t *oriData,
size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize,
int errBoundMode, double absErr_Bound, double relBoundRatio);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _SZ_Int16_H ----- */

48
deps/SZ/sz/include/sz_int32.h vendored
View File

@ -1,48 +0,0 @@
/**
* @file sz_int32.h
* @author Sheng Di
* @date July, 2017
* @brief Header file for the sz_int32.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _SZ_Int32_H
#define _SZ_Int32_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdio.h>
unsigned int optimize_intervals_int32_1D(int32_t *oriData, size_t dataLength, double realPrecision);
unsigned int optimize_intervals_int32_2D(int32_t *oriData, size_t r1, size_t r2, double realPrecision);
unsigned int optimize_intervals_int32_3D(int32_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision);
unsigned int optimize_intervals_int32_4D(int32_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision);
TightDataPointStorageI* SZ_compress_int32_1D_MDQ(int32_t *oriData, size_t dataLength, double realPrecision, int64_t valueRangeSize, int64_t minValue);
void SZ_compress_args_int32_StoreOriData(int32_t* oriData, size_t dataLength, TightDataPointStorageI* tdps, unsigned char** newByteData, size_t *outSize);
void SZ_compress_args_int32_NoCkRngeNoGzip_1D(unsigned char** newByteData, int32_t *oriData,
size_t dataLength, double realPrecision, size_t *outSize, int64_t valueRangeSize, int32_t minValue);
TightDataPointStorageI* SZ_compress_int32_2D_MDQ(int32_t *oriData, size_t r1, size_t r2, double realPrecision, int64_t valueRangeSize, int64_t minValue);
TightDataPointStorageI* SZ_compress_int32_3D_MDQ(int32_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, int64_t valueRangeSize, int64_t minValue);
void SZ_compress_args_int32_NoCkRngeNoGzip_3D(unsigned char** newByteData, int32_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, size_t *outSize, int64_t valueRangeSize, int64_t minValue);
TightDataPointStorageI* SZ_compress_int32_4D_MDQ(int32_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision, int64_t valueRangeSize, int64_t minValue);
void SZ_compress_args_int32_NoCkRngeNoGzip_4D(unsigned char** newByteData, int32_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision,
size_t *outSize, int64_t valueRangeSize, int64_t minValue);
void SZ_compress_args_int32_withinRange(unsigned char** newByteData, int32_t *oriData, size_t dataLength, size_t *outSize);
int SZ_compress_args_int32_wRngeNoGzip(unsigned char** newByteData, int32_t *oriData,
size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize,
int errBoundMode, double absErr_Bound, double relBoundRatio);
int SZ_compress_args_int32(unsigned char** newByteData, int32_t *oriData,
size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize,
int errBoundMode, double absErr_Bound, double relBoundRatio);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _SZ_Int32_H ----- */

48
deps/SZ/sz/include/sz_int64.h vendored
View File

@ -1,48 +0,0 @@
/**
* @file sz_int64.h
* @author Sheng Di
* @date July, 2017
* @brief Header file for the sz_int64.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _SZ_Int64_H
#define _SZ_Int64_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdio.h>
unsigned int optimize_intervals_int64_1D(int64_t *oriData, size_t dataLength, double realPrecision);
unsigned int optimize_intervals_int64_2D(int64_t *oriData, size_t r1, size_t r2, double realPrecision);
unsigned int optimize_intervals_int64_3D(int64_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision);
unsigned int optimize_intervals_int64_4D(int64_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision);
TightDataPointStorageI* SZ_compress_int64_1D_MDQ(int64_t *oriData, size_t dataLength, double realPrecision, int64_t valueRangeSize, int64_t minValue);
void SZ_compress_args_int64_StoreOriData(int64_t* oriData, size_t dataLength, TightDataPointStorageI* tdps, unsigned char** newByteData, size_t *outSize);
void SZ_compress_args_int64_NoCkRngeNoGzip_1D(unsigned char** newByteData, int64_t *oriData,
size_t dataLength, double realPrecision, size_t *outSize, int64_t valueRangeSize, int64_t minValue);
TightDataPointStorageI* SZ_compress_int64_2D_MDQ(int64_t *oriData, size_t r1, size_t r2, double realPrecision, int64_t valueRangeSize, int64_t minValue);
TightDataPointStorageI* SZ_compress_int64_3D_MDQ(int64_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, int64_t valueRangeSize, int64_t minValue);
void SZ_compress_args_int64_NoCkRngeNoGzip_3D(unsigned char** newByteData, int64_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, size_t *outSize, int64_t valueRangeSize, int64_t minValue);
TightDataPointStorageI* SZ_compress_int64_4D_MDQ(int64_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision, int64_t valueRangeSize, int64_t minValue);
void SZ_compress_args_int64_NoCkRngeNoGzip_4D(unsigned char** newByteData, int64_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision,
size_t *outSize, int64_t valueRangeSize, int64_t minValue);
void SZ_compress_args_int64_withinRange(unsigned char** newByteData, int64_t *oriData, size_t dataLength, size_t *outSize);
int SZ_compress_args_int64_wRngeNoGzip(unsigned char** newByteData, int64_t *oriData,
size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize,
int errBoundMode, double absErr_Bound, double relBoundRatio);
int SZ_compress_args_int64(unsigned char** newByteData, int64_t *oriData,
size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize,
int errBoundMode, double absErr_Bound, double relBoundRatio);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _SZ_Int64_H ----- */

48
deps/SZ/sz/include/sz_int8.h vendored
View File

@ -1,48 +0,0 @@
/**
* @file sz_int8.h
* @author Sheng Di
* @date July, 2017
* @brief Header file for the sz_int8.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _SZ_Int8_H
#define _SZ_Int8_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdio.h>
unsigned int optimize_intervals_int8_1D(int8_t *oriData, size_t dataLength, double realPrecision);
unsigned int optimize_intervals_int8_2D(int8_t *oriData, size_t r1, size_t r2, double realPrecision);
unsigned int optimize_intervals_int8_3D(int8_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision);
unsigned int optimize_intervals_int8_4D(int8_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision);
TightDataPointStorageI* SZ_compress_int8_1D_MDQ(int8_t *oriData, size_t dataLength, double realPrecision, int64_t valueRangeSize, int64_t minValue);
void SZ_compress_args_int8_StoreOriData(int8_t* oriData, size_t dataLength, TightDataPointStorageI* tdps, unsigned char** newByteData, size_t *outSize);
void SZ_compress_args_int8_NoCkRngeNoGzip_1D(unsigned char** newByteData, int8_t *oriData,
size_t dataLength, double realPrecision, size_t *outSize, int64_t valueRangeSize, int8_t minValue);
TightDataPointStorageI* SZ_compress_int8_2D_MDQ(int8_t *oriData, size_t r1, size_t r2, double realPrecision, int64_t valueRangeSize, int64_t minValue);
TightDataPointStorageI* SZ_compress_int8_3D_MDQ(int8_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, int64_t valueRangeSize, int64_t minValue);
void SZ_compress_args_int8_NoCkRngeNoGzip_3D(unsigned char** newByteData, int8_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, size_t *outSize, int64_t valueRangeSize, int64_t minValue);
TightDataPointStorageI* SZ_compress_int8_4D_MDQ(int8_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision, int64_t valueRangeSize, int64_t minValue);
void SZ_compress_args_int8_NoCkRngeNoGzip_4D(unsigned char** newByteData, int8_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision,
size_t *outSize, int64_t valueRangeSize, int64_t minValue);
void SZ_compress_args_int8_withinRange(unsigned char** newByteData, int8_t *oriData, size_t dataLength, size_t *outSize);
int SZ_compress_args_int8_wRngeNoGzip(unsigned char** newByteData, int8_t *oriData,
size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize,
int errBoundMode, double absErr_Bound, double relBoundRatio);
int SZ_compress_args_int8(unsigned char** newByteData, int8_t *oriData,
size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize,
int errBoundMode, double absErr_Bound, double relBoundRatio);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _SZ_Int8_H ----- */

47
deps/SZ/sz/include/sz_omp.h vendored
View File

@ -1,47 +0,0 @@
/**
* @file sz_omp.h
* @author Xin Liang
* @date July, 2017
* @brief Header file for the sz_omp.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#include <stdio.h>
#include <stdlib.h>
#ifdef _OPENMP
#include "omp.h"
#endif
#include "sz.h"
#ifndef _SZ_OMP_H
#define _SZ_OMP_H
#ifdef __cplusplus
extern "C" {
#endif
unsigned char * SZ_compress_float_1D_MDQ_openmp(float *oriData, size_t r1, double realPrecision, size_t * comp_size);
unsigned char * SZ_compress_float_2D_MDQ_openmp(float *oriData, size_t r1, size_t r2, double realPrecision, size_t * comp_size);
unsigned char * SZ_compress_float_3D_MDQ_openmp(float *oriData, size_t r1, size_t r2, size_t r3, float realPrecision, size_t * comp_size);
void decompressDataSeries_float_1D_openmp(float** data, size_t r1, unsigned char* comp_data);
void decompressDataSeries_float_3D_openmp(float** data, size_t r1, size_t r2, size_t r3, unsigned char* comp_data);
void decompressDataSeries_float_2D_openmp(float** data, size_t r1, size_t r2, unsigned char* comp_data);
unsigned char * SZ_compress_double_1D_MDQ_openmp(double *oriData, size_t r1, double realPrecision, size_t * comp_size);
unsigned char * SZ_compress_double_2D_MDQ_openmp(double *oriData, size_t r1, size_t r2, double realPrecision, size_t * comp_size);
unsigned char * SZ_compress_double_3D_MDQ_openmp(double *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, size_t * comp_size);
void decompressDataSeries_double_1D_openmp(double** data, size_t r1, unsigned char* comp_data);
void decompressDataSeries_double_2D_openmp(double** data, size_t r1, size_t r2, unsigned char* comp_data);
void decompressDataSeries_double_3D_openmp(double** data, size_t r1, size_t r2, size_t r3, unsigned char* comp_data);
//void Huffman_init_openmp(HuffmanTree* huffmanTree, int *s, size_t length, int thread_num);
void Huffman_init_openmp(HuffmanTree* huffmanTree, int *s, size_t length, int thread_num, size_t * freq);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _SZ_OMP_H ----- */

6
deps/SZ/sz/include/sz_opencl.h vendored
View File

@ -20,7 +20,7 @@ extern "C" {
* sz_opencl_* function.
*
* \param[out] state the sz opencl state
* \return SZ_SCES for success or SZ_NCES on error
* \return SZ_SUCCESS for success or SZ_NCES on error
*/
int sz_opencl_init(struct sz_opencl_state** state);
@ -28,7 +28,7 @@ extern "C" {
* deinitializes an opencl state
*
* \param[in] state the sz opencl state
* \return SZ_SCES
* \return SZ_SUCCESS
*/
int sz_opencl_release(struct sz_opencl_state** state);
@ -53,7 +53,7 @@ extern "C" {
* confirms that the sz opencl state is ready to use by performing a vector addition
*
* \param[in] state the sz opencl state
* \return SZ_SCES if the opencl implementation is functioning
* \return SZ_SUCCESS if the opencl implementation is functioning
*/
int sz_opencl_check(struct sz_opencl_state*);

48
deps/SZ/sz/include/sz_uint16.h vendored
View File

@ -1,48 +0,0 @@
/**
* @file sz_uint16.h
* @author Sheng Di
* @date Nov, 2017
* @brief Header file for the sz_uint16.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _SZ_UInt16_H
#define _SZ_UInt16_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdio.h>
unsigned int optimize_intervals_uint16_1D(uint16_t *oriData, size_t dataLength, double realPrecision);
unsigned int optimize_intervals_uint16_2D(uint16_t *oriData, size_t r1, size_t r2, double realPrecision);
unsigned int optimize_intervals_uint16_3D(uint16_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision);
unsigned int optimize_intervals_uint16_4D(uint16_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision);
TightDataPointStorageI* SZ_compress_uint16_1D_MDQ(uint16_t *oriData, size_t dataLength, double realPrecision, int64_t valueRangeSize, int64_t minValue);
void SZ_compress_args_uint16_StoreOriData(uint16_t* oriData, size_t dataLength, TightDataPointStorageI* tdps, unsigned char** newByteData, size_t *outSize);
void SZ_compress_args_uint16_NoCkRngeNoGzip_1D(unsigned char** newByteData, uint16_t *oriData,
size_t dataLength, double realPrecision, size_t *outSize, int64_t valueRangeSize, uint16_t minValue);
TightDataPointStorageI* SZ_compress_uint16_2D_MDQ(uint16_t *oriData, size_t r1, size_t r2, double realPrecision, int64_t valueRangeSize, int64_t minValue);
TightDataPointStorageI* SZ_compress_uint16_3D_MDQ(uint16_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, int64_t valueRangeSize, int64_t minValue);
void SZ_compress_args_uint16_NoCkRngeNoGzip_3D(unsigned char** newByteData, uint16_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, size_t *outSize, int64_t valueRangeSize, int64_t minValue);
TightDataPointStorageI* SZ_compress_uint16_4D_MDQ(uint16_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision, int64_t valueRangeSize, int64_t minValue);
void SZ_compress_args_uint16_NoCkRngeNoGzip_4D(unsigned char** newByteData, uint16_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision,
size_t *outSize, int64_t valueRangeSize, int64_t minValue);
void SZ_compress_args_uint16_withinRange(unsigned char** newByteData, uint16_t *oriData, size_t dataLength, size_t *outSize);
int SZ_compress_args_uint16_wRngeNoGzip(unsigned char** newByteData, uint16_t *oriData,
size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize,
int errBoundMode, double absErr_Bound, double relBoundRatio);
int SZ_compress_args_uint16(unsigned char** newByteData, uint16_t *oriData,
size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize,
int errBoundMode, double absErr_Bound, double relBoundRatio);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _SZ_UInt16_H ----- */

48
deps/SZ/sz/include/sz_uint32.h vendored
View File

@ -1,48 +0,0 @@
/**
* @file sz_uint32.h
* @author Sheng Di
* @date July, 2017
* @brief Header file for the sz_uint32.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _SZ_UInt32_H
#define _SZ_UInt32_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdio.h>
unsigned int optimize_intervals_uint32_1D(uint32_t *oriData, size_t dataLength, double realPrecision);
unsigned int optimize_intervals_uint32_2D(uint32_t *oriData, size_t r1, size_t r2, double realPrecision);
unsigned int optimize_intervals_uint32_3D(uint32_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision);
unsigned int optimize_intervals_uint32_4D(uint32_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision);
TightDataPointStorageI* SZ_compress_uint32_1D_MDQ(uint32_t *oriData, size_t dataLength, double realPrecision, int64_t valueRangeSize, int64_t minValue);
void SZ_compress_args_uint32_StoreOriData(uint32_t* oriData, size_t dataLength, TightDataPointStorageI* tdps, unsigned char** newByteData, size_t *outSize);
void SZ_compress_args_uint32_NoCkRngeNoGzip_1D(unsigned char** newByteData, uint32_t *oriData,
size_t dataLength, double realPrecision, size_t *outSize, int64_t valueRangeSize, uint32_t minValue);
TightDataPointStorageI* SZ_compress_uint32_2D_MDQ(uint32_t *oriData, size_t r1, size_t r2, double realPrecision, int64_t valueRangeSize, int64_t minValue);
TightDataPointStorageI* SZ_compress_uint32_3D_MDQ(uint32_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, int64_t valueRangeSize, int64_t minValue);
void SZ_compress_args_uint32_NoCkRngeNoGzip_3D(unsigned char** newByteData, uint32_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, size_t *outSize, int64_t valueRangeSize, int64_t minValue);
TightDataPointStorageI* SZ_compress_uint32_4D_MDQ(uint32_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision, int64_t valueRangeSize, int64_t minValue);
void SZ_compress_args_uint32_NoCkRngeNoGzip_4D(unsigned char** newByteData, uint32_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision,
size_t *outSize, int64_t valueRangeSize, int64_t minValue);
void SZ_compress_args_uint32_withinRange(unsigned char** newByteData, uint32_t *oriData, size_t dataLength, size_t *outSize);
int SZ_compress_args_uint32_wRngeNoGzip(unsigned char** newByteData, uint32_t *oriData,
size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize,
int errBoundMode, double absErr_Bound, double relBoundRatio);
int SZ_compress_args_uint32(unsigned char** newByteData, uint32_t *oriData,
size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize,
int errBoundMode, double absErr_Bound, double relBoundRatio);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _SZ_UInt32_H ----- */

48
deps/SZ/sz/include/sz_uint64.h vendored
View File

@ -1,48 +0,0 @@
/**
* @file sz_uint64.h
* @author Sheng Di
* @date July, 2017
* @brief Header file for the sz_uint64.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _SZ_UInt64_H
#define _SZ_UInt64_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdio.h>
unsigned int optimize_intervals_uint64_1D(uint64_t *oriData, size_t dataLength, double realPrecision);
unsigned int optimize_intervals_uint64_2D(uint64_t *oriData, size_t r1, size_t r2, double realPrecision);
unsigned int optimize_intervals_uint64_3D(uint64_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision);
unsigned int optimize_intervals_uint64_4D(uint64_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision);
TightDataPointStorageI* SZ_compress_uint64_1D_MDQ(uint64_t *oriData, size_t dataLength, double realPrecision, uint64_t valueRangeSize, uint64_t minValue);
void SZ_compress_args_uint64_StoreOriData(uint64_t* oriData, size_t dataLength, TightDataPointStorageI* tdps, unsigned char** newByteData, size_t *outSize);
void SZ_compress_args_uint64_NoCkRngeNoGzip_1D(unsigned char** newByteData, uint64_t *oriData,
size_t dataLength, double realPrecision, size_t *outSize, uint64_t valueRangeSize, uint64_t minValue);
TightDataPointStorageI* SZ_compress_uint64_2D_MDQ(uint64_t *oriData, size_t r1, size_t r2, double realPrecision, uint64_t valueRangeSize, uint64_t minValue);
TightDataPointStorageI* SZ_compress_uint64_3D_MDQ(uint64_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, uint64_t valueRangeSize, uint64_t minValue);
void SZ_compress_args_uint64_NoCkRngeNoGzip_3D(unsigned char** newByteData, uint64_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, size_t *outSize, uint64_t valueRangeSize, uint64_t minValue);
TightDataPointStorageI* SZ_compress_uint64_4D_MDQ(uint64_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision, uint64_t valueRangeSize, uint64_t minValue);
void SZ_compress_args_uint64_NoCkRngeNoGzip_4D(unsigned char** newByteData, uint64_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision,
size_t *outSize, uint64_t valueRangeSize, uint64_t minValue);
void SZ_compress_args_uint64_withinRange(unsigned char** newByteData, uint64_t *oriData, size_t dataLength, size_t *outSize);
int SZ_compress_args_uint64_wRngeNoGzip(unsigned char** newByteData, uint64_t *oriData,
size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize,
int errBoundMode, double absErr_Bound, double relBoundRatio);
int SZ_compress_args_uint64(unsigned char** newByteData, uint64_t *oriData,
size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize,
int errBoundMode, double absErr_Bound, double relBoundRatio);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _SZ_UInt64_H ----- */

48
deps/SZ/sz/include/sz_uint8.h vendored
View File

@ -1,48 +0,0 @@
/**
* @file sz_uint8.h
* @author Sheng Di
* @date July, 2017
* @brief Header file for the sz_uint8.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _SZ_UInt8_H
#define _SZ_UInt8_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdio.h>
unsigned int optimize_intervals_uint8_1D(uint8_t *oriData, size_t dataLength, double realPrecision);
unsigned int optimize_intervals_uint8_2D(uint8_t *oriData, size_t r1, size_t r2, double realPrecision);
unsigned int optimize_intervals_uint8_3D(uint8_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision);
unsigned int optimize_intervals_uint8_4D(uint8_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision);
TightDataPointStorageI* SZ_compress_uint8_1D_MDQ(uint8_t *oriData, size_t dataLength, double realPrecision, int64_t valueRangeSize, int64_t minValue);
void SZ_compress_args_uint8_StoreOriData(uint8_t* oriData, size_t dataLength, TightDataPointStorageI* tdps, unsigned char** newByteData, size_t *outSize);
void SZ_compress_args_uint8_NoCkRngeNoGzip_1D(unsigned char** newByteData, uint8_t *oriData,
size_t dataLength, double realPrecision, size_t *outSize, int64_t valueRangeSize, uint8_t minValue);
TightDataPointStorageI* SZ_compress_uint8_2D_MDQ(uint8_t *oriData, size_t r1, size_t r2, double realPrecision, int64_t valueRangeSize, int64_t minValue);
TightDataPointStorageI* SZ_compress_uint8_3D_MDQ(uint8_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, int64_t valueRangeSize, int64_t minValue);
void SZ_compress_args_uint8_NoCkRngeNoGzip_3D(unsigned char** newByteData, uint8_t *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, size_t *outSize, int64_t valueRangeSize, int64_t minValue);
TightDataPointStorageI* SZ_compress_uint8_4D_MDQ(uint8_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision, int64_t valueRangeSize, int64_t minValue);
void SZ_compress_args_uint8_NoCkRngeNoGzip_4D(unsigned char** newByteData, uint8_t *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision,
size_t *outSize, int64_t valueRangeSize, int64_t minValue);
void SZ_compress_args_uint8_withinRange(unsigned char** newByteData, uint8_t *oriData, size_t dataLength, size_t *outSize);
int SZ_compress_args_uint8_wRngeNoGzip(unsigned char** newByteData, uint8_t *oriData,
size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize,
int errBoundMode, double absErr_Bound, double relBoundRatio);
int SZ_compress_args_uint8(unsigned char** newByteData, uint8_t *oriData,
size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize,
int errBoundMode, double absErr_Bound, double relBoundRatio);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _SZ_UInt8_H ----- */

8
deps/SZ/sz/include/szd_double.h vendored
View File

@ -16,13 +16,13 @@ extern "C" {
#include "TightDataPointStorageD.h"
void decompressDataSeries_double_1D(double** data, size_t dataSeriesLength, double* hist_data, TightDataPointStorageD* tdps);
void decompressDataSeries_double_1D(double* data, size_t dataSeriesLength, double* hist_data, TightDataPointStorageD* tdps);
void decompressDataSeries_double_1D_MSST19(double** data, size_t dataSeriesLength, TightDataPointStorageD* tdps);
void decompressDataSeries_double_1D_MSST19(double* data, size_t dataSeriesLength, TightDataPointStorageD* tdps);
void getSnapshotData_double_1D(double** data, size_t dataSeriesLength, TightDataPointStorageD* tdps, int errBoundMode, int compressionType, double* hist_data, sz_params* pde_params);
void getSnapshotData_double_1D(double* data, size_t dataSeriesLength, TightDataPointStorageD* tdps, int errBoundMode, int compressionType, double* hist_data, sz_params* pde_params);
int SZ_decompress_args_double(double** newData, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, unsigned char* cmpBytes, size_t cmpSize, int compressionType, double* hist_data, sz_exedata* pde_exe, sz_params* pde_params);
int SZ_decompress_args_double(double* newData, size_t r1, unsigned char* cmpBytes, size_t cmpSize, int compressionType, double* hist_data, sz_exedata* pde_exe, sz_params* pde_params);
#ifdef __cplusplus
}

16
deps/SZ/sz/include/szd_float.h vendored
View File

@ -16,24 +16,16 @@ extern "C" {
#include "TightDataPointStorageF.h"
void decompressDataSeries_float_1D(float** data, size_t dataSeriesLength, float* hist_data, TightDataPointStorageF* tdps);
void decompressDataSeries_float_1D(float* data, size_t dataSeriesLength, float* hist_data, TightDataPointStorageF* tdps);
void decompressDataSeries_float_1D_MSST19(float** data, size_t dataSeriesLength, TightDataPointStorageF* tdps);
void decompressDataSeries_float_1D_MSST19(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps);
void getSnapshotData_float_1D(float** data, size_t dataSeriesLength, TightDataPointStorageF* tdps, int errBoundMode, int compressionType, float* hist_data, sz_params* pde_params);
void getSnapshotData_float_1D(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps, int errBoundMode, int compressionType, float* hist_data, sz_params* pde_params);
size_t decompressDataSeries_float_1D_RA_block(float * data, float mean, size_t dim_0, size_t block_dim_0, double realPrecision, int * type, float * unpredictable_data);
int SZ_decompress_args_float(float** newData, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, unsigned char* cmpBytes, size_t cmpSize, int compressionType, float* hist_data, sz_exedata* pde_exe, sz_params* pde_params);
int SZ_decompress_args_float(float* newData, size_t r1, unsigned char* cmpBytes, size_t cmpSize, int compressionType, float* hist_data, sz_exedata* pde_exe, sz_params* pde_params);
void decompressDataSeries_float_1D_decompression_given_areas_with_blocked_regression(float** data, size_t r1, size_t s1, size_t e1, unsigned char* comp_data);
int SZ_decompress_args_randomaccess_float(float** newData,
size_t r5, size_t r4, size_t r3, size_t r2, size_t r1,
size_t s5, size_t s4, size_t s3, size_t s2, size_t s1, // start point
size_t e5, size_t e4, size_t e3, size_t e2, size_t e1, // end point
unsigned char* cmpBytes, size_t cmpSize);
#ifdef __cplusplus
}

8
deps/SZ/sz/include/szd_float_pwr.h vendored
View File

@ -14,13 +14,13 @@
extern "C" {
#endif
void decompressDataSeries_float_1D_pwr(float** data, size_t dataSeriesLength, TightDataPointStorageF* tdps);
void decompressDataSeries_float_1D_pwr(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps);
char* decompressGroupIDArray(unsigned char* bytes, size_t dataLength);
void decompressDataSeries_float_1D_pwrgroup(float** data, size_t dataSeriesLength, TightDataPointStorageF* tdps);
void decompressDataSeries_float_1D_pwr_pre_log(float** data, size_t dataSeriesLength, TightDataPointStorageF* tdps);
void decompressDataSeries_float_1D_pwrgroup(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps);
void decompressDataSeries_float_1D_pwr_pre_log(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps);
void decompressDataSeries_float_1D_pwr_pre_log_MSST19(float** data, size_t dataSeriesLength, TightDataPointStorageF* tdps);
void decompressDataSeries_float_1D_pwr_pre_log_MSST19(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps);
#ifdef __cplusplus
}

25
deps/SZ/sz/include/szd_float_ts.h vendored
View File

@ -1,25 +0,0 @@
/**
* @file szd_float_ts.h
* @author Sheng Di
* @date July, 2017
* @brief Header file for the szd_float_ts.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _SZD_Float_TS_H
#define _SZD_Float_TS_H
#ifdef __cplusplus
extern "C" {
#endif
#include "TightDataPointStorageF.h"
void decompressDataSeries_float_1D_ts(float** data, size_t dataSeriesLength, float* hist_data, TightDataPointStorageF* tdps);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _SZD_Float_TS_H ----- */

38
deps/SZ/sz/include/szd_int16.h vendored
View File

@ -1,38 +0,0 @@
/**
* @file szd_int16.h
* @author Sheng Di
* @date July, 2017
* @brief Header file for the szd_int16.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _SZD_Int16_H
#define _SZD_Int16_H
#ifdef __cplusplus
extern "C" {
#endif
#include "TightDataPointStorageI.h"
#define SZ_INT16_MIN -32768
#define SZ_INT16_MAX 32767
void decompressDataSeries_int16_1D(int16_t** data, size_t dataSeriesLength, TightDataPointStorageI* tdps);
void decompressDataSeries_int16_2D(int16_t** data, size_t r1, size_t r2, TightDataPointStorageI* tdps);
void decompressDataSeries_int16_3D(int16_t** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageI* tdps);
void decompressDataSeries_int16_4D(int16_t** data, size_t r1, size_t r2, size_t r3, size_t r4, TightDataPointStorageI* tdps);
void getSnapshotData_int16_1D(int16_t** data, size_t dataSeriesLength, TightDataPointStorageI* tdps, int errBoundMode);
void getSnapshotData_int16_2D(int16_t** data, size_t r1, size_t r2, TightDataPointStorageI* tdps, int errBoundMode);
void getSnapshotData_int16_3D(int16_t** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageI* tdps, int errBoundMode);
void getSnapshotData_int16_4D(int16_t** data, size_t r1, size_t r2, size_t r3, size_t r4, TightDataPointStorageI* tdps, int errBoundMode);
int SZ_decompress_args_int16(int16_t** newData, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, unsigned char* cmpBytes, size_t cmpSize);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _SZD_Int16_H ----- */

38
deps/SZ/sz/include/szd_int32.h vendored
View File

@ -1,38 +0,0 @@
/**
* @file szd_int32.h
* @author Sheng Di
* @date July, 2017
* @brief Header file for the szd_int32.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _SZD_Int32_H
#define _SZD_Int32_H
#ifdef __cplusplus
extern "C" {
#endif
#include "TightDataPointStorageI.h"
#define SZ_INT32_MIN -2147483648
#define SZ_INT32_MAX 2147483647
void decompressDataSeries_int32_1D(int32_t** data, size_t dataSeriesLength, TightDataPointStorageI* tdps);
void decompressDataSeries_int32_2D(int32_t** data, size_t r1, size_t r2, TightDataPointStorageI* tdps);
void decompressDataSeries_int32_3D(int32_t** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageI* tdps);
void decompressDataSeries_int32_4D(int32_t** data, size_t r1, size_t r2, size_t r3, size_t r4, TightDataPointStorageI* tdps);
void getSnapshotData_int32_1D(int32_t** data, size_t dataSeriesLength, TightDataPointStorageI* tdps, int errBoundMode);
void getSnapshotData_int32_2D(int32_t** data, size_t r1, size_t r2, TightDataPointStorageI* tdps, int errBoundMode);
void getSnapshotData_int32_3D(int32_t** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageI* tdps, int errBoundMode);
void getSnapshotData_int32_4D(int32_t** data, size_t r1, size_t r2, size_t r3, size_t r4, TightDataPointStorageI* tdps, int errBoundMode);
int SZ_decompress_args_int32(int32_t** newData, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, unsigned char* cmpBytes, size_t cmpSize);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _SZD_Int32_H ----- */

35
deps/SZ/sz/include/szd_int64.h vendored
View File

@ -1,35 +0,0 @@
/**
* @file szd_int64.h
* @author Sheng Di
* @date July, 2017
* @brief Header file for the szd_int64.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _SZD_Int64_H
#define _SZD_Int64_H
#ifdef __cplusplus
extern "C" {
#endif
#include "TightDataPointStorageI.h"
void decompressDataSeries_int64_1D(int64_t** data, size_t dataSeriesLength, TightDataPointStorageI* tdps);
void decompressDataSeries_int64_2D(int64_t** data, size_t r1, size_t r2, TightDataPointStorageI* tdps);
void decompressDataSeries_int64_3D(int64_t** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageI* tdps);
void decompressDataSeries_int64_4D(int64_t** data, size_t r1, size_t r2, size_t r3, size_t r4, TightDataPointStorageI* tdps);
void getSnapshotData_int64_1D(int64_t** data, size_t dataSeriesLength, TightDataPointStorageI* tdps, int errBoundMode);
void getSnapshotData_int64_2D(int64_t** data, size_t r1, size_t r2, TightDataPointStorageI* tdps, int errBoundMode);
void getSnapshotData_int64_3D(int64_t** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageI* tdps, int errBoundMode);
void getSnapshotData_int64_4D(int64_t** data, size_t r1, size_t r2, size_t r3, size_t r4, TightDataPointStorageI* tdps, int errBoundMode);
int SZ_decompress_args_int64(int64_t** newData, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, unsigned char* cmpBytes, size_t cmpSize);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _SZD_Int64_H ----- */

38
deps/SZ/sz/include/szd_int8.h vendored
View File

@ -1,38 +0,0 @@
/**
* @file szd_int8.h
* @author Sheng Di
* @date July, 2017
* @brief Header file for the szd_int8.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _SZD_Int8_H
#define _SZD_Int8_H
#ifdef __cplusplus
extern "C" {
#endif
#include "TightDataPointStorageI.h"
#define SZ_INT8_MIN -128
#define SZ_INT8_MAX 127
void decompressDataSeries_int8_1D(int8_t** data, size_t dataSeriesLength, TightDataPointStorageI* tdps);
void decompressDataSeries_int8_2D(int8_t** data, size_t r1, size_t r2, TightDataPointStorageI* tdps);
void decompressDataSeries_int8_3D(int8_t** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageI* tdps);
void decompressDataSeries_int8_4D(int8_t** data, size_t r1, size_t r2, size_t r3, size_t r4, TightDataPointStorageI* tdps);
void getSnapshotData_int8_1D(int8_t** data, size_t dataSeriesLength, TightDataPointStorageI* tdps, int errBoundMode);
void getSnapshotData_int8_2D(int8_t** data, size_t r1, size_t r2, TightDataPointStorageI* tdps, int errBoundMode);
void getSnapshotData_int8_3D(int8_t** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageI* tdps, int errBoundMode);
void getSnapshotData_int8_4D(int8_t** data, size_t r1, size_t r2, size_t r3, size_t r4, TightDataPointStorageI* tdps, int errBoundMode);
int SZ_decompress_args_int8(int8_t** newData, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, unsigned char* cmpBytes, size_t cmpSize);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _SZD_Int8_H ----- */

38
deps/SZ/sz/include/szd_uint16.h vendored
View File

@ -1,38 +0,0 @@
/**
* @file szd_uint16.h
* @author Sheng Di
* @date July, 2017
* @brief Header file for the szd_uint16.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _SZD_UInt16_H
#define _SZD_UInt16_H
#ifdef __cplusplus
extern "C" {
#endif
#include "TightDataPointStorageI.h"
#define SZ_UINT16_MIN 0
#define SZ_UINT16_MAX 65535
void decompressDataSeries_uint16_1D(uint16_t** data, size_t dataSeriesLength, TightDataPointStorageI* tdps);
void decompressDataSeries_uint16_2D(uint16_t** data, size_t r1, size_t r2, TightDataPointStorageI* tdps);
void decompressDataSeries_uint16_3D(uint16_t** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageI* tdps);
void decompressDataSeries_uint16_4D(uint16_t** data, size_t r1, size_t r2, size_t r3, size_t r4, TightDataPointStorageI* tdps);
void getSnapshotData_uint16_1D(uint16_t** data, size_t dataSeriesLength, TightDataPointStorageI* tdps, int errBoundMode);
void getSnapshotData_uint16_2D(uint16_t** data, size_t r1, size_t r2, TightDataPointStorageI* tdps, int errBoundMode);
void getSnapshotData_uint16_3D(uint16_t** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageI* tdps, int errBoundMode);
void getSnapshotData_uint16_4D(uint16_t** data, size_t r1, size_t r2, size_t r3, size_t r4, TightDataPointStorageI* tdps, int errBoundMode);
int SZ_decompress_args_uint16(uint16_t** newData, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, unsigned char* cmpBytes, size_t cmpSize);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _SZD_Int16_H ----- */

38
deps/SZ/sz/include/szd_uint32.h vendored
View File

@ -1,38 +0,0 @@
/**
* @file szd_uint32.h
* @author Sheng Di
* @date July, 2017
* @brief Header file for the szd_uint32.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _SZD_UInt32_H
#define _SZD_UInt32_H
#ifdef __cplusplus
extern "C" {
#endif
#include "TightDataPointStorageI.h"
#define SZ_UINT32_MIN 0
#define SZ_UINT32_MAX 4294967295
void decompressDataSeries_uint32_1D(uint32_t** data, size_t dataSeriesLength, TightDataPointStorageI* tdps);
void decompressDataSeries_uint32_2D(uint32_t** data, size_t r1, size_t r2, TightDataPointStorageI* tdps);
void decompressDataSeries_uint32_3D(uint32_t** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageI* tdps);
void decompressDataSeries_uint32_4D(uint32_t** data, size_t r1, size_t r2, size_t r3, size_t r4, TightDataPointStorageI* tdps);
void getSnapshotData_uint32_1D(uint32_t** data, size_t dataSeriesLength, TightDataPointStorageI* tdps, int errBoundMode);
void getSnapshotData_uint32_2D(uint32_t** data, size_t r1, size_t r2, TightDataPointStorageI* tdps, int errBoundMode);
void getSnapshotData_uint32_3D(uint32_t** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageI* tdps, int errBoundMode);
void getSnapshotData_uint32_4D(uint32_t** data, size_t r1, size_t r2, size_t r3, size_t r4, TightDataPointStorageI* tdps, int errBoundMode);
int SZ_decompress_args_uint32(uint32_t** newData, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, unsigned char* cmpBytes, size_t cmpSize);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _SZD_UInt32_H ----- */

35
deps/SZ/sz/include/szd_uint64.h vendored
View File

@ -1,35 +0,0 @@
/**
* @file szd_uint64.h
* @author Sheng Di
* @date July, 2017
* @brief Header file for the szd_uint64.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _SZD_UInt64_H
#define _SZD_UInt64_H
#ifdef __cplusplus
extern "C" {
#endif
#include "TightDataPointStorageI.h"
void decompressDataSeries_uint64_1D(uint64_t** data, size_t dataSeriesLength, TightDataPointStorageI* tdps);
void decompressDataSeries_uint64_2D(uint64_t** data, size_t r1, size_t r2, TightDataPointStorageI* tdps);
void decompressDataSeries_uint64_3D(uint64_t** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageI* tdps);
void decompressDataSeries_uint64_4D(uint64_t** data, size_t r1, size_t r2, size_t r3, size_t r4, TightDataPointStorageI* tdps);
void getSnapshotData_uint64_1D(uint64_t** data, size_t dataSeriesLength, TightDataPointStorageI* tdps, int errBoundMode);
void getSnapshotData_uint64_2D(uint64_t** data, size_t r1, size_t r2, TightDataPointStorageI* tdps, int errBoundMode);
void getSnapshotData_uint64_3D(uint64_t** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageI* tdps, int errBoundMode);
void getSnapshotData_uint64_4D(uint64_t** data, size_t r1, size_t r2, size_t r3, size_t r4, TightDataPointStorageI* tdps, int errBoundMode);
int SZ_decompress_args_uint64(uint64_t** newData, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, unsigned char* cmpBytes, size_t cmpSize);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _SZD_UInt64_H ----- */

38
deps/SZ/sz/include/szd_uint8.h vendored
View File

@ -1,38 +0,0 @@
/**
* @file szd_uint8.h
* @author Sheng Di
* @date July, 2017
* @brief Header file for the szd_uint8.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _SZD_UInt8_H
#define _SZD_UInt8_H
#ifdef __cplusplus
extern "C" {
#endif
#include "TightDataPointStorageI.h"
#define SZ_UINT8_MIN 0
#define SZ_UINT8_MAX 255
void decompressDataSeries_uint8_1D(uint8_t** data, size_t dataSeriesLength, TightDataPointStorageI* tdps);
void decompressDataSeries_uint8_2D(uint8_t** data, size_t r1, size_t r2, TightDataPointStorageI* tdps);
void decompressDataSeries_uint8_3D(uint8_t** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageI* tdps);
void decompressDataSeries_uint8_4D(uint8_t** data, size_t r1, size_t r2, size_t r3, size_t r4, TightDataPointStorageI* tdps);
void getSnapshotData_uint8_1D(uint8_t** data, size_t dataSeriesLength, TightDataPointStorageI* tdps, int errBoundMode);
void getSnapshotData_uint8_2D(uint8_t** data, size_t r1, size_t r2, TightDataPointStorageI* tdps, int errBoundMode);
void getSnapshotData_uint8_3D(uint8_t** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageI* tdps, int errBoundMode);
void getSnapshotData_uint8_4D(uint8_t** data, size_t r1, size_t r2, size_t r3, size_t r4, TightDataPointStorageI* tdps, int errBoundMode);
int SZ_decompress_args_uint8(uint8_t** newData, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, unsigned char* cmpBytes, size_t cmpSize);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _SZD_UInt8_H ----- */

102
deps/SZ/sz/include/szf.h vendored
View File

@ -1,102 +0,0 @@
/**
* @file szf.h
* @author Sheng Di
* @date July, 2017
* @brief Header file for the szf.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _SZF_H
#define _SZF_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdio.h>
//szf.c
void sz_init_c_(char *configFile,int *len,int *ierr);
void sz_finalize_c_();
void SZ_writeData_inBinary_d1_Float_(float* data, char *fileName, int *len);
void sz_compress_d1_float_(float* data, unsigned char *bytes, size_t *outSize, size_t *r1);
void sz_compress_d1_float_rev_(float* data, float *reservedValue, unsigned char *bytes, size_t *outSize, size_t *r1);
void sz_compress_d2_float_(float* data, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2);
void sz_compress_d2_float_rev_(float* data, float *reservedValue, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2);
void sz_compress_d3_float_(float* data, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3);
void sz_compress_d3_float_rev_(float* data, float *reservedValue, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3);
void sz_compress_d4_float_(float* data, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3, size_t *r4);
void sz_compress_d4_float_rev_(float* data, float *reservedValue, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3, size_t *r4);
void sz_compress_d5_float_(float* data, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5);
void sz_compress_d5_float_rev_(float* data, float *reservedValue, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5);
void sz_compress_d1_double_(double* data, unsigned char *bytes, size_t *outSize, size_t *r1);
void sz_compress_d1_double_rev_(double* data, double *reservedValue, unsigned char *bytes, size_t *outSize, size_t *r1);
void sz_compress_d2_double_(double* data, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2);
void sz_compress_d2_double_rev_(double* data, double *reservedValue, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2);
void sz_compress_d3_double_(double* data, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3);
void sz_compress_d3_double_rev_(double* data, double *reservedValue, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3);
void sz_compress_d4_double_(double* data, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3, size_t *r4);
void sz_compress_d4_double_rev_(double* data, double *reservedValue, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3, size_t *r4);
void sz_compress_d5_double_(double* data, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5);
void sz_compress_d5_double_rev_(double* data, double *reservedValue, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5);
void sz_compress_d1_float_args_(float* data, unsigned char *bytes, size_t *outSize, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1);
void sz_compress_d2_float_args_(float* data, unsigned char *bytes, size_t *outSize, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2);
void sz_compress_d3_float_args_(float* data, unsigned char *bytes, size_t *outSize, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2, size_t *r3);
void sz_compress_d4_float_args_(float* data, unsigned char *bytes, size_t *outSize, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4);
void sz_compress_d5_float_args_(float* data, unsigned char *bytes, size_t *outSize, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5);
void sz_compress_d1_double_args_(double* data, unsigned char *bytes, size_t *outSize, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1);
void sz_compress_d2_double_args_(double* data, unsigned char *bytes, size_t *outSize, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2);
void sz_compress_d3_double_args_(double* data, unsigned char *bytes, size_t *outSize, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2, size_t *r3);
void sz_compress_d4_double_args_(double* data, unsigned char *bytes, size_t *outSize, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4);
void sz_compress_d5_double_args_(double* data, unsigned char *bytes, size_t *outSize, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5);
void sz_compress_d1_float_rev_args_(float* data, float *reservedValue, unsigned char *bytes, size_t *outSize, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1);
void sz_compress_d2_float_rev_args_(float* data, float *reservedValue, unsigned char *bytes, size_t *outSize, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2);
void sz_compress_d3_float_rev_args_(float* data, float *reservedValue, unsigned char *bytes, size_t *outSize, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2, size_t *r3);
void sz_compress_d4_float_rev_args_(float* data, float *reservedValue, unsigned char *bytes, size_t *outSize, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4);
void sz_compress_d5_float_rev_args_(float* data, float *reservedValue, unsigned char *bytes, size_t *outSize, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5);
void sz_compress_d1_double_rev_args_(double* data, float *reservedValue, unsigned char *bytes, size_t *outSize, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1);
void sz_compress_d2_double_rev_args_(double* data, float *reservedValue, unsigned char *bytes, size_t *outSize, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2);
void sz_compress_d3_double_rev_args_(double* data, float *reservedValue, unsigned char *bytes, size_t *outSize, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2, size_t *r3);
void sz_compress_d4_double_rev_args_(double* data, double *reservedValue, unsigned char *bytes, size_t *outSize, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4);
void sz_compress_d5_double_rev_args_(double* data, double *reservedValue, unsigned char *bytes, size_t *outSize, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5);
void sz_decompress_d1_float_(unsigned char *bytes, size_t *byteLength, float *data, size_t *r1);
void sz_decompress_d2_float_(unsigned char *bytes, size_t *byteLength, float *data, size_t *r1, size_t *r2);
void sz_decompress_d3_float_(unsigned char *bytes, size_t *byteLength, float *data, size_t *r1, size_t *r2, size_t *r3);
void sz_decompress_d4_float_(unsigned char *bytes, size_t *byteLength, float *data, size_t *r1, size_t *r2, size_t *r3, size_t *r4);
void sz_decompress_d5_float_(unsigned char *bytes, size_t *byteLength, float *data, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5);
void sz_decompress_d1_double_(unsigned char *bytes, size_t *byteLength, double *data, size_t *r1);
void sz_decompress_d2_double_(unsigned char *bytes, size_t *byteLength, double *data, size_t *r1, size_t *r2);
void sz_decompress_d3_double_(unsigned char *bytes, size_t *byteLength, double *data, size_t *r1, size_t *r2, size_t *r3);
void sz_decompress_d4_double_(unsigned char *bytes, size_t *byteLength, double *data, size_t *r1, size_t *r2, size_t *r3, size_t *r4);
void sz_decompress_d5_double_(unsigned char *bytes, size_t *byteLength, double *data, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5);
void sz_batchaddVar_d1_float_(int var_id, char* varName, int *len, float* data, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1);
void sz_batchaddvar_d2_float_(int var_id, char* varName, int *len, float* data, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2);
void sz_batchaddvar_d3_float_(int var_id, char* varName, int *len, float* data, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2, size_t *r3);
void sz_batchaddvar_d4_float_(int var_id, char* varName, int *len, float* data, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4);
void sz_batchaddvar_d5_float_(int var_id, char* varName, int *len, float* data, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5);
void sz_batchaddvar_d1_double_(int var_id, char* varName, int *len, double* data, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1);
void sz_batchaddvar_d2_double_(int var_id, char* varName, int *len, double* data, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2);
void sz_batchaddvar_d3_double_(int var_id, char* varName, int *len, double* data, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2, size_t *r3);
void sz_batchaddvar_d4_double_(int var_id, char* varName, int *len, double* data, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4);
void sz_batchaddvar_d5_double_(int var_id, char* varName, int *len, double* data, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5);
void sz_batchdelvar_c_(char* varName, int *len, int *errState);
void sz_batch_compress_c_(unsigned char* bytes, size_t *outSize);
void sz_batch_decompress_c_(unsigned char* bytes, size_t *byteLength, int *ierr);
void sz_getvardim_c_(char* varName, int *len, int *dim, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5);
void compute_total_batch_size_c_(size_t *totalSize);
void sz_getvardata_float_(char* varName, int *len, float* data);
void sz_getvardata_double_(char* varName, int *len, double* data);
void sz_freevarset_c_(int *mode);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _SZF_H ----- */

1
deps/SZ/sz/include/td_sz.h vendored
View File

@ -3,7 +3,6 @@
#define _TD_SZ_H
#include "defines.h"
#ifdef __cplusplus
extern "C" {
#endif

20
deps/SZ/sz/include/utility.h vendored
View File

@ -16,27 +16,11 @@
extern "C" {
#endif
//sihuan added: use a assistant struct to do sorting and swap that are easy to implement: should
//consider optimizing the performance later.
typedef struct sort_ast_particle{
int64_t id;
float var[6];
} sort_ast_particle;
int compare_struct(const void* obj1, const void* obj2);//sihuan added: the compare function in the qsort parameter for 2 structures
void reorder_vars(SZ_VarSet* vset);//sihuan added: reorder the variables increasingly by their index
size_t intersectAndsort(int64_t* preIndex, size_t preLen, SZ_VarSet* curVar, size_t dataLen, unsigned char* bitarray);
//sihuan added: find intersection and keep new var sorted by id
void write_reordered_tofile(SZ_VarSet* curVar, size_t dataLen);
//sihuan added: write the reordered input to files for further decompression validation
float calculate_delta_t(size_t size);//sihuan added
int is_lossless_compressed_data(unsigned char* compressedBytes, size_t cmpSize);
unsigned long sz_lossless_compress(int losslessCompressor, int level, unsigned char* data, unsigned long dataLength, unsigned char** compressBytes);
unsigned long sz_lossless_compress(int losslessCompressor, int level, unsigned char* data, unsigned long dataLength, unsigned char* compressBytes);
unsigned long sz_lossless_decompress(int losslessCompressor, unsigned char* compressBytes, unsigned long cmpSize, unsigned char** oriData, unsigned long targetOriSize);
unsigned long sz_lossless_decompress65536bytes(int losslessCompressor, unsigned char* compressBytes, unsigned long cmpSize, unsigned char** oriData);
void* detransposeData(void* data, int dataType, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
void* transposeData(void* data, int dataType, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
#ifdef __cplusplus
}

18
deps/SZ/sz/src/TightDataPointStorageD.c vendored
View File

@ -572,7 +572,7 @@ void convertTDPStoBytes_double_reserve(TightDataPointStorageD* tdps, unsigned ch
}
//Convert TightDataPointStorageD to bytes...
void convertTDPStoFlatBytes_double(TightDataPointStorageD *tdps, unsigned char** bytes, size_t *size)
void convertTDPStoFlatBytes_double(TightDataPointStorageD *tdps, unsigned char* bytes, size_t *size)
{
size_t i, k = 0;
unsigned char dsLengthBytes[8];
@ -596,20 +596,20 @@ void convertTDPStoFlatBytes_double(TightDataPointStorageD *tdps, unsigned char**
if(tdps->allSameData==1)
{
size_t totalByteLength = 3 + 1 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + tdps->exactMidBytes_size;
*bytes = (unsigned char *)malloc(sizeof(unsigned char)*totalByteLength);
//bytes = (unsigned char *)malloc(sizeof(unsigned char)*totalByteLength); // comment by tickduan
for (i = 0; i < 3; i++)//3
(*bytes)[k++] = versionNumber[i];
(*bytes)[k++] = sameByte;
bytes[k++] = versionNumber[i];
bytes[k++] = sameByte;
convertSZParamsToBytes(confparams_cpr, &((*bytes)[k]));
convertSZParamsToBytes(confparams_cpr, &(bytes[k]));
k = k + MetaDataByteLength_double;
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
(*bytes)[k++] = dsLengthBytes[i];
bytes[k++] = dsLengthBytes[i];
for (i = 0; i < tdps->exactMidBytes_size; i++)
(*bytes)[k++] = tdps->exactMidBytes[i];
bytes[k++] = tdps->exactMidBytes[i];
*size = totalByteLength;
}
@ -635,9 +635,9 @@ void convertTDPStoFlatBytes_double(TightDataPointStorageD *tdps, unsigned char**
if(confparams_cpr->errorBoundMode == PW_REL && confparams_cpr->accelerate_pw_rel_compression)
totalByteLength += (1+1); // for MSST19
*bytes = (unsigned char *)malloc(sizeof(unsigned char)*totalByteLength);
//*bytes = (unsigned char *)malloc(sizeof(unsigned char)*totalByteLength); comment by tickduan
convertTDPStoBytes_double(tdps, *bytes, dsLengthBytes, sameByte);
convertTDPStoBytes_double(tdps, bytes, dsLengthBytes, sameByte);
*size = totalByteLength;
}

129
deps/SZ/sz/src/TightDataPointStorageF.c vendored
View File

@ -470,116 +470,8 @@ void convertTDPStoBytes_float(TightDataPointStorageF* tdps, unsigned char* bytes
}
}
/*deprecated*/
void convertTDPStoBytes_float_reserve(TightDataPointStorageF* tdps, unsigned char* bytes, unsigned char* dsLengthBytes, unsigned char sameByte)
{
size_t i, k = 0;
unsigned char intervalsBytes[4];
unsigned char typeArrayLengthBytes[8];
unsigned char rTypeLengthBytes[8];
unsigned char exactLengthBytes[8];
unsigned char exactMidBytesLength[8];
unsigned char realPrecisionBytes[8];
unsigned char reservedValueBytes[4];
unsigned char medianValueBytes[4];
unsigned char segment_sizeBytes[8];
unsigned char pwrErrBoundBytes_sizeBytes[4];
unsigned char max_quant_intervals_Bytes[4];
for(i = 0;i<3;i++)//3
bytes[k++] = versionNumber[i];
bytes[k++] = sameByte; //1
convertSZParamsToBytes(confparams_cpr, &(bytes[k]));
k = k + MetaDataByteLength;
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
bytes[k++] = dsLengthBytes[i];
intToBytes_bigEndian(max_quant_intervals_Bytes, confparams_cpr->max_quant_intervals);
for(i = 0;i<4;i++)//4
bytes[k++] = max_quant_intervals_Bytes[i];
if(confparams_cpr->errorBoundMode>=PW_REL)
{
bytes[k++] = tdps->radExpo; //1 byte
sizeToBytes(segment_sizeBytes, confparams_cpr->segment_size);
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
bytes[k++] = segment_sizeBytes[i];
intToBytes_bigEndian(pwrErrBoundBytes_sizeBytes, tdps->pwrErrBoundBytes_size);
for(i = 0;i<4;i++)//4
bytes[k++] = pwrErrBoundBytes_sizeBytes[i];
}
intToBytes_bigEndian(intervalsBytes, tdps->intervals);
for(i = 0;i<4;i++)//4
bytes[k++] = intervalsBytes[i];
floatToBytes(medianValueBytes, tdps->medianValue);
for (i = 0; i < 4; i++)// 4
bytes[k++] = medianValueBytes[i];
bytes[k++] = tdps->reqLength; //1 byte
floatToBytes(realPrecisionBytes, tdps->realPrecision);
for (i = 0; i < 8; i++)// 8
bytes[k++] = realPrecisionBytes[i];
sizeToBytes(typeArrayLengthBytes, tdps->typeArray_size);
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
bytes[k++] = typeArrayLengthBytes[i];
sizeToBytes(rTypeLengthBytes, tdps->rtypeArray_size);
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
bytes[k++] = rTypeLengthBytes[i];
sizeToBytes(exactLengthBytes, tdps->exactDataNum);
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
bytes[k++] = exactLengthBytes[i];
sizeToBytes(exactMidBytesLength, tdps->exactMidBytes_size);
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
bytes[k++] = exactMidBytesLength[i];
floatToBytes(reservedValueBytes, tdps->reservedValue);
for (i = 0; i < 4; i++)// 4
bytes[k++] = reservedValueBytes[i];
memcpy(&(bytes[k]), tdps->rtypeArray, tdps->rtypeArray_size);
k += tdps->rtypeArray_size;
if(confparams_cpr->errorBoundMode>=PW_REL)
{
floatToBytes(exactMidBytesLength, tdps->minLogValue);
for(i=0;i<4;i++)
bytes[k++] = exactMidBytesLength[i];
}
memcpy(&(bytes[k]), tdps->typeArray, tdps->typeArray_size);
k += tdps->typeArray_size;
if(confparams_cpr->errorBoundMode>=PW_REL)
{
memcpy(&(bytes[k]), tdps->pwrErrBoundBytes, tdps->pwrErrBoundBytes_size);
k += tdps->pwrErrBoundBytes_size;
}
memcpy(&(bytes[k]), tdps->leadNumArray, tdps->leadNumArray_size);
k += tdps->leadNumArray_size;
memcpy(&(bytes[k]), tdps->exactMidBytes, tdps->exactMidBytes_size);
k += tdps->exactMidBytes_size;
if(tdps->residualMidBits!=NULL)
{
memcpy(&(bytes[k]), tdps->residualMidBits, tdps->residualMidBits_size);
k += tdps->residualMidBits_size;
}
}
//convert TightDataPointStorageD to bytes...
void convertTDPStoFlatBytes_float(TightDataPointStorageF *tdps, unsigned char** bytes, size_t *size)
void convertTDPStoFlatBytes_float(TightDataPointStorageF *tdps, unsigned char* bytes, size_t *size)
{
size_t i, k = 0;
unsigned char dsLengthBytes[8];
@ -605,20 +497,23 @@ void convertTDPStoFlatBytes_float(TightDataPointStorageF *tdps, unsigned char**
if(tdps->allSameData==1)
{
size_t totalByteLength = 3 + 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + tdps->exactMidBytes_size;
*bytes = (unsigned char *)malloc(sizeof(unsigned char)*totalByteLength);
//*bytes = (unsigned char *)malloc(sizeof(unsigned char)*totalByteLength); // not need malloc comment by tickduan
// check output buffer enough
for (i = 0; i < 3; i++)//3
(*bytes)[k++] = versionNumber[i];
(*bytes)[k++] = sameByte;
bytes[k++] = versionNumber[i];
bytes[k++] = sameByte;
convertSZParamsToBytes(confparams_cpr, &((*bytes)[k]));
convertSZParamsToBytes(confparams_cpr, &(bytes[k]));
k = k + MetaDataByteLength;
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
(*bytes)[k++] = dsLengthBytes[i];
bytes[k++] = dsLengthBytes[i];
for (i = 0; i < tdps->exactMidBytes_size; i++)
(*bytes)[k++] = tdps->exactMidBytes[i];
bytes[k++] = tdps->exactMidBytes[i];
*size = totalByteLength;
}
@ -642,9 +537,9 @@ void convertTDPStoFlatBytes_float(TightDataPointStorageF *tdps, unsigned char**
if(confparams_cpr->errorBoundMode == PW_REL && confparams_cpr->accelerate_pw_rel_compression)
totalByteLength += (1+1); // for MSST19
*bytes = (unsigned char *)malloc(sizeof(unsigned char)*totalByteLength);
//*bytes = (unsigned char *)malloc(sizeof(unsigned char)*totalByteLength); // comment by tickduan
convertTDPStoBytes_float(tdps, *bytes, dsLengthBytes, sameByte);
convertTDPStoBytes_float(tdps, bytes, dsLengthBytes, sameByte);
*size = totalByteLength;
}

8
deps/SZ/sz/src/VarSet.c vendored
View File

@ -119,7 +119,7 @@ int SZ_batchDelVar(char* varName)
int SZ_batchDelVar_ID_vset(SZ_VarSet* vset, int var_id)
{
int delSuccess = SZ_NSCS;
int delSuccess = SZ_FAILED;
SZ_Variable* p = vset->header;
SZ_Variable* q = p->next;
while(q != NULL)
@ -130,7 +130,7 @@ int SZ_batchDelVar_ID_vset(SZ_VarSet* vset, int var_id)
//free_Variable_all(q);
free_Variable_keepOriginalData(q);
vset->count --;
delSuccess = SZ_SCES;
delSuccess = SZ_SUCCESS;
if(q->next==NULL) //means that q is the last variable
vset->lastVar = p;
break;
@ -145,7 +145,7 @@ int SZ_batchDelVar_ID_vset(SZ_VarSet* vset, int var_id)
int SZ_batchDelVar_vset(SZ_VarSet* vset, char* varName)
{
int delSuccess = SZ_NSCS;
int delSuccess = SZ_FAILED;
SZ_Variable* p = vset->header;
SZ_Variable* q = p->next;
while(q != NULL)
@ -157,7 +157,7 @@ int SZ_batchDelVar_vset(SZ_VarSet* vset, char* varName)
//free_Variable_all(q);
free_Variable_keepOriginalData(q);
vset->count --;
delSuccess = SZ_SCES;
delSuccess = SZ_SUCCESS;
break;
}
p = p->next;

10
deps/SZ/sz/src/callZlib.c vendored
View File

@ -23,7 +23,7 @@
#define CHECK_ERR(err, msg) { \
if (err != Z_OK && err != Z_STREAM_END) { \
fprintf(stderr, "%s error: %d\n", msg, err); \
return SZ_NSCS; \
return SZ_FAILED; \
} \
}
@ -202,7 +202,7 @@ unsigned long zlib_compress4(unsigned char* data, unsigned long dataLength, unsi
return c_stream.total_out;
}
unsigned long zlib_compress5(unsigned char* data, unsigned long dataLength, unsigned char** compressBytes, int level)
unsigned long zlib_compress5(unsigned char* data, unsigned long dataLength, unsigned char* compressBytes, int level)
{
int ret, flush;
unsigned have;
@ -222,7 +222,7 @@ unsigned long zlib_compress5(unsigned char* data, unsigned long dataLength, unsi
size_t p_size = 0, av_in = 0;
uLong estCmpLen = deflateBound(&strm, dataLength);
*compressBytes = (unsigned char*)malloc(sizeof(unsigned char)*estCmpLen);
//*compressBytes = (unsigned char*)malloc(sizeof(unsigned char)*estCmpLen); // comment by tickduan no need malloc
unsigned char* out = *compressBytes;
/* compress until end of file */
@ -296,7 +296,7 @@ unsigned long zlib_uncompress2 (unsigned char* compressBytes, unsigned long cmpS
{
printf("Error: zlib_uncompress2: stream.avail_in != cmpSize");
//exit(1);
return SZ_NSCS; //-1
return SZ_FAILED; //-1
}
stream.next_out = *oriData;
@ -309,7 +309,7 @@ unsigned long zlib_uncompress2 (unsigned char* compressBytes, unsigned long cmpS
if (err != Z_OK)
{
printf("Error: zlib_uncompress2: err != Z_OK\n");
return SZ_NSCS;
return SZ_FAILED;
}
err = inflate(&stream, Z_FINISH);

40
deps/SZ/sz/src/conf.c vendored
View File

@ -135,13 +135,13 @@ int SZ_ReadConf(const char* sz_cfgFile) {
confparams_cpr->protectValueRange = 0;
return SZ_SCES;
return SZ_SUCCESS;
}
if (access(sz_cfgFile, F_OK) != 0)
{
printf("[SZ] Configuration file NOT accessible.\n");
return SZ_NSCS;
return SZ_FAILED;
}
//printf("[SZ] Reading SZ configuration file (%s) ...\n", sz_cfgFile);
@ -149,7 +149,7 @@ int SZ_ReadConf(const char* sz_cfgFile) {
if (ini == NULL)
{
printf("[SZ] Iniparser failed to parse the conf. file.\n");
return SZ_NSCS;
return SZ_FAILED;
}
endianTypeString = iniparser_getstring(ini, "ENV:dataEndianType", "LITTLE_ENDIAN_DATA");
@ -161,7 +161,7 @@ int SZ_ReadConf(const char* sz_cfgFile) {
{
printf("Error: Wrong dataEndianType: please set it correctly in sz.config.\n");
iniparser_freedict(ini);
return SZ_NSCS;
return SZ_FAILED;
}
// Reading/setting detection parameters
@ -178,7 +178,7 @@ int SZ_ReadConf(const char* sz_cfgFile) {
else{
printf("[SZ] Error: wrong solution name (please check sz.config file), sol=%s\n", sol_name);
iniparser_freedict(ini);
return SZ_NSCS;
return SZ_FAILED;
}
if(confparams_cpr->sol_ID==SZ || confparams_cpr->sol_ID==SZ_Transpose)
@ -208,7 +208,7 @@ int SZ_ReadConf(const char* sz_cfgFile) {
{
printf("Error: quantization_intervals must be an even number!\n");
iniparser_freedict(ini);
return SZ_NSCS;
return SZ_FAILED;
}
confparams_cpr->predThreshold = (float)iniparser_getdouble(ini, "PARAMETER:predThreshold", 0);
@ -219,7 +219,7 @@ int SZ_ReadConf(const char* sz_cfgFile) {
{
printf("[SZ] Error: Null szMode setting (please check sz.config file)\n");
iniparser_freedict(ini);
return SZ_NSCS;
return SZ_FAILED;
}
else if(strcmp(modeBuf, "SZ_BEST_SPEED")==0)
confparams_cpr->szMode = SZ_BEST_SPEED;
@ -231,7 +231,7 @@ int SZ_ReadConf(const char* sz_cfgFile) {
{
printf("[SZ] Error: Wrong szMode setting (please check sz.config file)\n");
iniparser_freedict(ini);
return SZ_NSCS;
return SZ_FAILED;
}
modeBuf = iniparser_getstring(ini, "PARAMETER:losslessCompressor", "ZSTD_COMPRESSOR");
@ -244,7 +244,7 @@ int SZ_ReadConf(const char* sz_cfgFile) {
printf("[SZ] Error: Wrong losslessCompressor setting (please check sz.config file)\n");\
printf("No Such a lossless compressor: %s\n", modeBuf);
iniparser_freedict(ini);
return SZ_NSCS;
return SZ_FAILED;
}
modeBuf = iniparser_getstring(ini, "PARAMETER:withLinearRegression", "YES");
@ -258,7 +258,7 @@ int SZ_ReadConf(const char* sz_cfgFile) {
{
printf("[SZ] Error: Null Gzip mode setting (please check sz.config file)\n");
iniparser_freedict(ini);
return SZ_NSCS;
return SZ_FAILED;
}
else if(strcmp(modeBuf, "Gzip_NO_COMPRESSION")==0)
confparams_cpr->gzipMode = 0;
@ -271,7 +271,7 @@ int SZ_ReadConf(const char* sz_cfgFile) {
else
{
printf("[SZ] Error: Wrong gzip Mode (please check sz.config file)\n");
return SZ_NSCS;
return SZ_FAILED;
}
modeBuf = iniparser_getstring(ini, "PARAMETER:zstdMode", "Zstd_HIGH_SPEED");
@ -279,7 +279,7 @@ int SZ_ReadConf(const char* sz_cfgFile) {
{
printf("[SZ] Error: Null Zstd mode setting (please check sz.config file)\n");
iniparser_freedict(ini);
return SZ_NSCS;
return SZ_FAILED;
}
else if(strcmp(modeBuf, "Zstd_BEST_SPEED")==0)
confparams_cpr->gzipMode = 1;
@ -294,7 +294,7 @@ int SZ_ReadConf(const char* sz_cfgFile) {
else
{
printf("[SZ] Error: Wrong zstd Mode (please check sz.config file)\n");
return SZ_NSCS;
return SZ_FAILED;
}
modeBuf = iniparser_getstring(ini, "PARAMETER:protectValueRange", "YES");
@ -313,7 +313,7 @@ int SZ_ReadConf(const char* sz_cfgFile) {
{
printf("[SZ] Error: Null error bound setting (please check sz.config file)\n");
iniparser_freedict(ini);
return SZ_NSCS;
return SZ_FAILED;
}
else if(strcmp(errBoundMode,"ABS")==0||strcmp(errBoundMode,"abs")==0)
confparams_cpr->errorBoundMode=SZ_ABS;
@ -343,7 +343,7 @@ int SZ_ReadConf(const char* sz_cfgFile) {
{
printf("[SZ] Error: Wrong error bound mode (please check sz.config file)\n");
iniparser_freedict(ini);
return SZ_NSCS;
return SZ_FAILED;
}
confparams_cpr->absErrBound = (double)iniparser_getdouble(ini, "PARAMETER:absErrBound", 0);
@ -366,7 +366,7 @@ int SZ_ReadConf(const char* sz_cfgFile) {
{
printf("[SZ] Error: Wrong pwr_type setting (please check sz.config file).\n");
iniparser_freedict(ini);
return SZ_NSCS;
return SZ_FAILED;
}
else //by default
confparams_cpr->pwr_type = SZ_PWR_AVG_TYPE;
@ -385,7 +385,7 @@ int SZ_ReadConf(const char* sz_cfgFile) {
}
iniparser_freedict(ini);
return SZ_SCES;
return SZ_SUCCESS;
}
/*-------------------------------------------------------------------------*/
@ -400,12 +400,12 @@ int SZ_ReadConf(const char* sz_cfgFile) {
/*-------------------------------------------------------------------------*/
int SZ_LoadConf(const char* sz_cfgFile) {
int res = SZ_ReadConf(sz_cfgFile);
if (res != SZ_SCES)
if (res != SZ_SUCCESS)
{
printf("[SZ] ERROR: Impossible to read configuration.\n");
return SZ_NSCS;
return SZ_FAILED;
}
return SZ_SCES;
return SZ_SUCCESS;
}
int checkVersion(char* version)

8
deps/SZ/sz/src/dataCompression.c vendored
View File

@ -284,7 +284,7 @@ float max_f(float a, float b)
double getRealPrecision_double(double valueRangeSize, int errBoundMode, double absErrBound, double relBoundRatio, int *status)
{
int state = SZ_SCES;
int state = SZ_SUCCESS;
double precision = 0;
if(errBoundMode==SZ_ABS||errBoundMode==ABS_OR_PW_REL||errBoundMode==ABS_AND_PW_REL)
precision = absErrBound;
@ -307,7 +307,7 @@ double getRealPrecision_double(double valueRangeSize, int errBoundMode, double a
double getRealPrecision_float(float valueRangeSize, int errBoundMode, double absErrBound, double relBoundRatio, int *status)
{
int state = SZ_SCES;
int state = SZ_SUCCESS;
double precision = 0;
if(errBoundMode==SZ_ABS||errBoundMode==ABS_OR_PW_REL||errBoundMode==ABS_AND_PW_REL)
precision = absErrBound;
@ -330,7 +330,7 @@ double getRealPrecision_float(float valueRangeSize, int errBoundMode, double abs
double getRealPrecision_int(long valueRangeSize, int errBoundMode, double absErrBound, double relBoundRatio, int *status)
{
int state = SZ_SCES;
int state = SZ_SUCCESS;
double precision = 0;
if(errBoundMode==SZ_ABS||errBoundMode==ABS_OR_PW_REL||errBoundMode==ABS_AND_PW_REL)
precision = absErrBound;
@ -651,7 +651,7 @@ int getPredictionCoefficients(int layers, int dimension, int **coeff_array, int
printf("Error: dimension must be no greater than 3 in the current version.\n");
*status = SZ_DERR;
}
*status = SZ_SCES;
*status = SZ_SUCCESS;
return size;
}

597
deps/SZ/sz/src/exafelSZ.c vendored
View File

@ -1,597 +0,0 @@
#ifdef __cplusplus
extern "C" {
#endif
#include "sz.h"
void exafelSZ_params_process(exafelSZ_params*pr, size_t panels, size_t rows, size_t cols){
pr->binnedRows=(rows+pr->binSize-1)/pr->binSize;
pr->binnedCols=(cols+pr->binSize-1)/pr->binSize;
pr->peakRadius=(pr->peakSize-1)/2;
}
void exafelSZ_params_checkDecomp(exafelSZ_params*pr, size_t panels, size_t rows, size_t cols){
if(pr->calibPanel==NULL){
printf("ERROR: calibPanel is NULL : calibPanel=%ld\n",(long)pr->calibPanel);
assert(0);
}
if(pr->binSize<1 || pr->tolerance<0 || pr->szDim<1 || pr->szDim>3){
printf("ERROR: Something wrong with the following:\n");
printf("binSize=%d\n",(int)pr->binSize);
printf("tolerance=%d\n",(int)pr->tolerance);
printf("szDim=%d\n",(int)pr->szDim);
assert(0);
}
if(!(pr->peakSize%2)){
printf("ERROR: peakSize = %d cannot be even. It must be odd!\n",(int)pr->peakSize);
assert(0);
}
//if(nEvents<1 || panels<1 || rows<1 || cols<1){
if(panels<1 || rows<1 || cols<1){
printf("ERROR: Something wrong with the following:\n");
printf("panels=%d\n",(int)panels);
printf("rows=%d\n",(int)rows);
printf("cols=%d\n",(int)cols);
assert(0);
}
}
void exafelSZ_params_checkComp(exafelSZ_params*pr, size_t panels, size_t rows, size_t cols){
if(pr->peaksSegs==NULL || pr->peaksRows==NULL || pr->peaksCols==NULL){
printf("ERROR: One or more of the following are NULL : peaksSegs , peaksRows , peaksCols\n");
assert(0);
}
exafelSZ_params_checkDecomp(pr, panels, rows, cols);
}
void exafelSZ_params_print(exafelSZ_params*pr){
printf("Configuration (exafelSZ_params) :\n");
printf("binSize: %d\n",pr->binSize);
printf("tolerance:%e\n",pr->tolerance);
printf("szDim:%d\n",pr->szDim);
printf("peakSize:%d\n",pr->peakSize);
//printf("nEvents:%d\n",pr->nEvents);
//printf("panels:%d\n",pr->panels);
//printf("rows:%d\n",pr->rows);
//printf("cols:%d\n",pr->cols);
printf("\n");
printf("CALCULATED VARIABLES\n");
printf("binnedRows:%ld\n",pr->binnedRows);
printf("binnedCols:%ld\n",pr->binnedCols);
printf("peakRadius:%d\n",pr->peakRadius);
printf("\n");
// outs<<"Configuration (exafelSZ_params) : "<<endl;
// outs<<"SMOOTHING: NO"<<" (ROI and RONI are NOT replaced by local avg values)"<<endl;
// outs<<"binSize:"<<binSize<<endl;
// outs<<"tolerance:"<<tolerance<<endl;
// outs<<"szDim:"<<szDim<<endl;
// outs<<"peakSize:"<<peakSize<<endl;
// outs<<"nEvents:"<<nEvents<<" (# of events per batch)"<<endl;
// outs<<"panels:"<<panels<<" (Panels per event)"<<endl;
// outs<<"rows:"<<rows<<" (Rows per panel)"<<endl;
// outs<<"cols:"<<cols<<" (Columns per panel)"<<endl;
// outs<<endl;
// outs<<"CALCULATED VARIABLES"<<endl;
// outs<<"binnedRows:"<<binnedRows<<" (Rows per panel after binning)"<<endl;
// outs<<"binnedCols:"<<binnedCols<<" (Columns per panel after binning)"<<endl;
// outs<<"peakRadius:"<<peakRadius<<" (Peak radius = (peakSize-1)/2 )"<<endl;
// outs<<endl;
}
//*********************************************************************************
//*********************************************************************************
//*********************************************************************************
//Index Calculator
static inline size_t calcIdx_4D(int i3, int i2, int i1, int i0, int size2, int size1, int size0){
return i0+size0*(i1+size1*(i2+size2*i3));
}
static inline size_t calcIdx_3D(int i2, int i1, int i0, int size1, int size0){
return i0+size0*(i1+size1*i2);
}
static inline size_t calcIdx_2D(int i1, int i0, int size0){
return i0+size0*i1;
}
unsigned char * exafelSZ_Compress(void* _pr,
void* _origData,
size_t r4, size_t r3, size_t r2, size_t r1,
size_t *compressedSize)
{
//printf("COMPRESS\n"); *compressedSize=0; return NULL;
size_t nEvents,panels,rows,cols;
if(r4==0)
nEvents=1;
else
nEvents=r4;
panels=r1;
rows=r2;
cols=r3;
//printf("AMG : exafelSZ_Compress : nEvents,panels,rows,cols = %d , %d , %d , %d\n",nEvents,panels,rows,cols);
float *origData=(float*)_origData;
exafelSZ_params *pr=(exafelSZ_params*)_pr;
exafelSZ_params_process(pr, panels, rows, cols);
exafelSZ_params_checkComp(pr, panels, rows, cols);
//exafelSZ_params_print(pr);
uint8_t *roiM=(uint8_t*)malloc(nEvents*panels*rows*cols) ;
float *roiData=(float*)malloc(nEvents*panels*rows*cols*sizeof(float)) ;
float *binnedData=(float*)malloc(nEvents*panels*pr->binnedRows*pr->binnedCols*sizeof(float)) ;
//float *binnedData=(float*)malloc(nEvents*panels*rows*cols*sizeof(float)) ;
size_t e,p,r,c,pk,ri,ci,br,bc,roii,bi;
/*
printf("AMG : exafelSZ_Compress : pr->numPeaks = %d\n",pr->numPeaks);
printf("S:\n");
for(e=0;e<pr->numPeaks;e++)
printf("%d ",pr->peaksSegs[e]);
printf("\nR:\n");
for(e=0;e<pr->numPeaks;e++)
printf("%d ",pr->peaksRows[e]);
printf("\nC:\n");
for(e=0;e<pr->numPeaks;e++)
printf("%d ",pr->peaksCols[e]);
printf("\n");
*/
//Generate the ROI mask: NOTE: 0 means affirmative in ROI mask! This comes from the python scripts!
//First, initialize with calibration panel:
for(e=0;e<nEvents;e++){ //Event
for(p=0;p<panels;p++){ //Panel
for(r=0;r<rows;r++){ //Row
for(c=0;c<cols;c++){ //Column
//roiM[calcIdx_4D(e,p,r,c,panels,rows,cols)]=pr->calibPanel[calcIdx_2D(r,c,cols)]; //calibPanel is a single segment copied over all the event(image)
roiM[calcIdx_4D(e,p,r,c,panels,rows,cols)]=pr->calibPanel[calcIdx_3D(p,r,c,rows,cols)]; //calibPanel is as big as the event(image) itself
}
}
}
}
//uint64_t peaksBytePos=0; //Position in the peaks buffer
//Now process the peaks and generate the mask:
uint64_t nPeaksTotal=0; //Total number of peaks
for(e=0;e<nEvents;e++){ //Event
//uint64_t nPeaks=*(uint64_t*)(&pr->peaks[peaksBytePos]);
//peaksBytePos+=8;
//peaksBytePos+=8;//Skip the second one! This is due to the problem in Python.
nPeaksTotal+=pr->numPeaks;
for(pk=0;pk<pr->numPeaks;pk++){
//uint16_t p_=*(uint16_t*)(&pr->peaks[peaksBytePos]); //Panel for the current peak
//peaksBytePos+=2;
//uint16_t r_=*(uint16_t*)(&pr->peaks[peaksBytePos]); //Row for the current peak
//peaksBytePos+=2;
//uint16_t c_=*(uint16_t*)(&pr->peaks[peaksBytePos]); //Col for the current peak
//peaksBytePos+=2;
uint16_t p_=pr->peaksSegs[pk];
uint16_t r_=pr->peaksRows[pk];
uint16_t c_=pr->peaksCols[pk];
if(p_>=panels){
printf("ERROR: Peak coordinate out of bounds: Panel=%d, Valid range: 0,%d\n",(int)p_,(int)panels-1);
assert(0);
printf("Skipping this peak...\n");
continue;
}
if(r_>=rows){
printf("ERROR: Peak coordinate out of bounds: Row=%d, Valid range: 0,%d\n",(int)r_,(int)rows-1);
assert(0);
printf("Skipping this peak...\n");
continue;
}
if(c_>=cols){
printf("ERROR: Peak coordinate out of bounds: Col=%d, Valid range: 0,%d\n",(int)c_,(int)cols-1);
assert(0);
printf("Skipping this peak...\n");
continue;
}
for(ri=r_-pr->peakRadius;ri<=r_+pr->peakRadius;ri++){ //ri: row index. Just a temporary variable.
for(ci=c_-pr->peakRadius;ci<=c_+pr->peakRadius;ci++){ //ci: column index. Just a temporary variable.
if(ri<rows && ci<cols){ //Check whether inside the bounds or not
roiM[calcIdx_4D(e,p_,ri,ci,panels,rows,cols)]=0;
}
}
}
}
}
//Save ROI:
uint64_t roiSavedCount=0;
for(e=0;e<nEvents;e++){ //Event
for(p=0;p<panels;p++){ //Panel
for(r=0;r<rows;r++){ //Row
for(c=0;c<cols;c++){ //Column
if(!roiM[calcIdx_4D(e,p,r,c,panels,rows,cols)]){
roiData[roiSavedCount]=origData[calcIdx_4D(e,p,r,c,panels,rows,cols)];
roiSavedCount++;
}
//AMG: Replace ROI and RONI pixels with avg values!
}
}
}
}
//Binning:
for(e=0;e<nEvents;e++){ //Event
for(p=0;p<panels;p++){ //Panel
for(r=0;r<pr->binnedRows;r++){ //Row of the binnedData
for(c=0;c<pr->binnedCols;c++){ //Column of the binnedData
float sum=0;
int nPts=0;
for(br=0;br<pr->binSize;br++) //Bin Row (from origData)
for(bc=0;bc<pr->binSize;bc++) //Bin Column (from origData)
if(r*pr->binSize+br<rows && c*pr->binSize+bc<cols){
// cout<<p<<" "<<r<<" "<<c<<" "<<br<<" "<<bc<<" "<<r*pr->binSize+br<<" "<<c*pr->binSize+bc<<endl;
sum+=origData[calcIdx_4D(e,p,r*pr->binSize+br,c*pr->binSize+bc,panels,rows,cols)];
nPts++;
}
// cout<<"p:"<<p<<" r:"<<r<<" c:"<<c<<" nPts:"<<nPts<<endl;
binnedData[calcIdx_4D(e,p,r,c,panels,pr->binnedRows,pr->binnedCols)]=sum/nPts;
}
}
}
}
//Additional compression using SZ:
size_t szCompressedSize=0;
unsigned char* szComp;
switch(pr->szDim){
case 1:
// szComp=sz_compress_3D(binnedData, 0, 0, nEvents * panels * pr->binnedRows * pr->binnedCols, pr->tolerance, szCompressedSize); //1D
szComp=SZ_compress_args(SZ_FLOAT, binnedData, &szCompressedSize, SZ_ABS, pr->tolerance, 0, 0, 0, 0,0,0, nEvents * panels * pr->binnedRows * pr->binnedCols);
break;
case 2:
// szComp=sz_compress_3D(binnedData, 0, nEvents * panels * pr->binnedRows, pr->binnedCols, pr->tolerance, szCompressedSize); //2D
szComp=SZ_compress_args(SZ_FLOAT, binnedData, &szCompressedSize, SZ_ABS, pr->tolerance, 0, 0, 0, 0,0, nEvents * panels * pr->binnedRows, pr->binnedCols);
break;
case 3:
// szComp=sz_compress_3D(binnedData, nEvents * panels, pr->binnedRows, pr->binnedCols, pr->tolerance, szCompressedSize); //3D
szComp=SZ_compress_args(SZ_FLOAT, binnedData, &szCompressedSize, SZ_ABS, pr->tolerance, 0, 0, 0, 0, nEvents * panels, pr->binnedRows, pr->binnedCols);
break;
default:
printf("ERROR: Wrong szDim : %d It must be 1,2 or 3.\n",(int)pr->szDim);
assert(0);
}
/*
Compressed buffer format: (Types are indicated in parenthesis)
WRITE: nPeaksTotal(uint64_t) (Total number of peaks in this batch)
for(e=0;e<nEvents;e++){ (e for "event")
WRITE: nPeaks[e] (uint64_t) (Number of peaks in this event)
for(p=0;p<nPeaks;p++){ (p for "peak")
nPeaks{
WRITE: peak[e][p] (uint16_t x 3)
}
}
WRITE: roiSavedCount (uint64_t) (How many pixels there are in the ROI data.)
(roiSavedCount is the same # as # of 0's in ROI mask.)
(NOTE:0 means affirmative in ROI mask!)
for(roii=0;roii<roiSavedCount;roii++){ (roii for "ROI data index")
WRITE: ROI_data[roii] (float, 32-bit)
}
WRITE: szCompressedSize (uint64_t) (Compressed data size from SZ.)
WRITE: szComp (unsigned char x SZ_compressed_buffer_size) (Compressed data from SZ.)
NOTE: Calibration panel is not saved. It should be handled by the user.
SUMMARY:
nPeaksTotal : 8 bytes : (1 x uint64_t)
peaks : (8 x nEvents + nPeaksTotal x 3 x 2) bytes : (nEvents x (nPeaks + nPeaks x 3 x uint16_t))
roiSavedCount : 8 Bytes : (1 x uint64_t)
ROI_data : roiSavedCount x 4 : roiSavedCount x float
szCompressedSize : 8 : uint64_t
szComp : szComp x 1 : szComp x (unsigned char)
*/
(*compressedSize)=8+nEvents*8+nPeaksTotal*(2+2+2)+8+roiSavedCount*4+8+szCompressedSize;
//compressedBuffer=new uint8_t[(*compressedSize)];
uint8_t * compressedBuffer=(uint8_t*)malloc(*compressedSize);
uint64_t bytePos;
bytePos=0;
//*(uint64_t*)(&compressedBuffer[bytePos])=nEvents;
//bytePos+=8;
*(uint64_t*)(&compressedBuffer[bytePos])=nPeaksTotal;
bytePos+=8;
// cout<<endl;
// cout<<"COMPRESS:"<<endl;
// cout<<"nPeaksTotal="<<nPeaksTotal<<endl;
// cout<<"bytePos="<<bytePos<<endl;
//printf("\nCOMPRESS:\n");
//printf("nPeaksTotal=%d\n",nPeaksTotal);
//printf("bytePos=%d\n",bytePos);
//peaksBytePos=0;
for(e=0;e<nEvents;e++){
//uint64_t nPeaks=*(uint64_t*)(&pr->peaks[peaksBytePos]);
//peaksBytePos+=8;
////peaksBytePos+=8;//Skip the second one. This is due to the error in Python!
//*(uint64_t*)(&compressedBuffer[bytePos])=nPeaks;
*(uint64_t*)(&compressedBuffer[bytePos])=pr->numPeaks;
bytePos+=8;
//for(pk=0;pk<nPeaks;pk++){
for(pk=0;pk<pr->numPeaks;pk++){
//*(uint16_t*)(&compressedBuffer[bytePos])=*(uint16_t*)(&pr->peaks[peaksBytePos]); //Panel for the current peak
//bytePos+=2;
//peaksBytePos+=2;
//*(uint16_t*)(&compressedBuffer[bytePos])=*(uint16_t*)(&pr->peaks[peaksBytePos]); //Row for the current peak
//bytePos+=2;
//peaksBytePos+=2;
//*(uint16_t*)(&compressedBuffer[bytePos])=*(uint16_t*)(&pr->peaks[peaksBytePos]); //Column for the current peak
//bytePos+=2;
//peaksBytePos+=2;
*(uint16_t*)(&compressedBuffer[bytePos])=pr->peaksSegs[pk]; //Panel for the current peak
bytePos+=2;
*(uint16_t*)(&compressedBuffer[bytePos])=pr->peaksRows[pk]; //Row for the current peak
bytePos+=2;
*(uint16_t*)(&compressedBuffer[bytePos])=pr->peaksCols[pk]; //Column for the current peak
bytePos+=2;
}
}
// cout<<"peaks"<<endl;
// cout<<"bytePos="<<bytePos<<endl;
//printf("peaks\n");
//printf("bytePos=%d\n",bytePos);
*(uint64_t*)(&compressedBuffer[bytePos])=roiSavedCount;
bytePos+=8;
// cout<<"roiSavedCount="<<roiSavedCount<<endl;
// cout<<"bytePos="<<bytePos<<endl;
// cout<<"roiData"<<endl;
//printf("roiSavedCount=%d\n",roiSavedCount);
//printf("bytePos=%d\n",bytePos);
//printf("roiData\n");
for(roii=0;roii<roiSavedCount;roii++){
*(float*)(&compressedBuffer[bytePos])=roiData[roii];
// cout<<roiData[roii]<<",";
bytePos+=4;
}
// cout<<"bytePos="<<bytePos<<endl;
//printf("bytePos=%d\n",bytePos);
*(uint64_t*)(&compressedBuffer[bytePos])=szCompressedSize;
bytePos+=8;
// cout<<"szCompressedSize="<<szCompressedSize<<endl;
// cout<<"bytePos="<<bytePos<<endl;
//printf("szCompressedSize=%d\n",szCompressedSize);
//printf("bytePos=%d\n",bytePos);
for(bi=0;bi<szCompressedSize;bi++){ //bi for "byte index"
*(unsigned char*)(&compressedBuffer[bytePos])=szComp[bi];
bytePos+=1;
}
// cout<<"szComp"<<endl;
// cout<<"bytePos="<<bytePos<<endl;
//printf("szComp\n");
//printf("bytePos=%d\n",bytePos);
if(bytePos!=(*compressedSize)){
printf("ERROR: bytePos = %ld != %ld = compressedSize\n",(long)bytePos,(long)compressedSize);
assert(0);
}
free(szComp);
free(roiM);
free(roiData);
free(binnedData);
// delete [] roiM;
// delete [] roiData;
// delete [] binnedData;
return compressedBuffer;
}
void* exafelSZ_Decompress(void *_pr,
unsigned char*_compressedBuffer,
size_t r4, size_t r3, size_t r2, size_t r1,
size_t compressedSize)
{
size_t nEvents,panels,rows,cols;
if(r4==0)
nEvents=1;
else
nEvents=r4;
panels=r1;
rows=r2;
cols=r3;
//printf("AMG : exafelSZ_Decompress : nEvents,panels,rows,cols = %d , %d , %d , %d\n",nEvents,panels,rows,cols);
//printf("DECOMPRESS\n");return NULL;
uint8_t *compressedBuffer=(uint8_t *)_compressedBuffer;
exafelSZ_params *pr=(exafelSZ_params *)_pr;
exafelSZ_params_process(pr, panels, rows, cols);
exafelSZ_params_checkDecomp(pr, panels, rows, cols);
float *decompressedBuffer=(float*)malloc(nEvents*panels*rows*cols*sizeof(float));
uint8_t *roiM=(uint8_t*)malloc(nEvents*panels*rows*cols);
size_t e,p,r,c,pk,ri,ci,br,bc;
/*
Compressed Data Layout:
nPeaksTotal : 8 bytes : (1 x uint64_t)
peaks : (8 x nEvents + nPeaksTotal x 3 x 2) bytes : (nEvents x (nPeaks + nPeaks x 3 x uint16_t))
roiSavedCount : 8 Bytes : (1 x uint64_t)
ROI_data : roiSavedCount x 4 : roiSavedCount x float
szCompressedSize : 8 : uint64_t
szComp : szComp x 1 : szComp x (unsigned char)
*/
uint64_t bytePos=0;
uint64_t nPeaksTotal=*(uint64_t*)(&compressedBuffer[bytePos]);
bytePos += 8;
// cout<<endl;
// cout<<"DECOMPRESS:"<<endl;
// cout<<"nPeaksTotal="<<nPeaksTotal<<endl;
// cout<<"bytePos="<<bytePos<<endl;
//printf("\nDECOMPRESS:\n");
//printf("nPeaksTotal=%d\n",nPeaksTotal);
//printf("bytePos=%d\n",bytePos);
uint8_t *peaks=(uint8_t*)(&compressedBuffer[bytePos]);
bytePos += (8 * nEvents + nPeaksTotal * 3 * 2);
// cout<<"peaks"<<endl;
// cout<<"bytePos="<<bytePos<<endl;
//printf("peaks\n");
//printf("bytePos=%d\n",bytePos);
uint64_t roiSavedCount=*(uint64_t*)(&compressedBuffer[bytePos]);
bytePos+=8;
// cout<<"roiSavedCount="<<roiSavedCount<<endl;
// cout<<"bytePos="<<bytePos<<endl;
//printf("roiSavedCount=%d\n",roiSavedCount);
//printf("bytePos=%d\n",bytePos);
// cout<<"roiData"<<endl;
float *roiData=(float*)(&compressedBuffer[bytePos]);
bytePos+=(roiSavedCount*4);
// for(uint64_t roii=0;roii<roiSavedCount;roii++){
// cout<<roiData[roii]<<",";
// }
// cout<<"bytePos="<<bytePos<<endl;
//printf("bytePos=%d\n",bytePos);
uint64_t szCompressedSize=*(uint64_t*)(&compressedBuffer[bytePos]);
bytePos+=8;
// cout<<"szCompressedSize="<<szCompressedSize<<endl;
// cout<<"bytePos="<<bytePos<<endl;
//printf("szCompressedSize=%d\n",szCompressedSize);
//printf("bytePos=%d\n",bytePos);
unsigned char *szComp=(unsigned char*)(&compressedBuffer[bytePos]);
bytePos+=szCompressedSize;
// cout<<"szComp"<<endl;
// cout<<"bytePos="<<bytePos<<endl;
// cout<<endl;
//printf("szComp\n");
//printf("bytePos=%d\n\n",bytePos);
//We should have inputs ready by now. Now process them:
//Generate the ROI mask: NOTE: 0 means affirmative in ROI mask! This comes from the python scripts!
//First, initialize with calibration panel:
for(e=0;e<nEvents;e++){ //Event
for(p=0;p<panels;p++){ //Panel
for(r=0;r<rows;r++){ //Row
for(c=0;c<cols;c++){ //Column
if(calcIdx_2D(r,c,cols)<0 ||calcIdx_2D(r,c,cols)>=rows*cols){
printf("ERROR: calcIdx_2D(r,c,cols) = calcIdx_2D(%d,%d,%d) = %d",(int)r,(int)c,(int)cols,(int)calcIdx_2D(r,c,cols));
printf(" is NOT in the correct range: [0,%ld]",(int)rows*cols-1);
assert(0);
}
if(calcIdx_4D(e,p,r,c,panels,rows,cols)<0 ||calcIdx_4D(e,p,r,c,panels,rows,cols)>=nEvents*panels*rows*cols){
printf("ERROR: calcIdx_4D(e,p,r,c,panels,rows,cols) = calcIdx_4D(%d,%d,%d,%d,%d,%d,%d) = %d",(int)e,(int)p,(int)r,(int)c,(int)panels,(int)rows,(int)cols,(int)calcIdx_4D(e,p,r,c,panels,rows,cols));
assert(0);
}
//roiM[calcIdx_4D(e,p,r,c,panels,rows,cols)]=pr->calibPanel[calcIdx_2D(r,c,cols)]; //calibPanel is a single segment copied over all the event(image)
roiM[calcIdx_4D(e,p,r,c,panels,rows,cols)]=pr->calibPanel[calcIdx_3D(p,r,c,rows,cols)]; //calibPanel is as big as the event(image) itself
}
}
}
}
uint64_t peaksBytePos=0; //Position in the peaks buffer
//Now process the peaks and generate the mask:
for(e=0;e<nEvents;e++){ //Event
uint64_t nPeaks=*(uint64_t*)(&peaks[peaksBytePos]);
peaksBytePos+=8;
for(pk=0;pk<nPeaks;pk++){
uint16_t p_=*(uint16_t*)(&peaks[peaksBytePos]); //Panel for the current peak
peaksBytePos+=2;
uint16_t r_=*(uint16_t*)(&peaks[peaksBytePos]); //Row for the current peak
peaksBytePos+=2;
uint16_t c_=*(uint16_t*)(&peaks[peaksBytePos]); //Col for the current peak
peaksBytePos+=2;
if(p_>=panels){
printf("ERROR: Peak coordinate out of bounds: Panel=%d, Valid range: 0,%d\n",(int)p_,(int)panels-1);
assert(0);
printf("Skipping this peak...\n");
continue;
}
if(r_>=rows){
printf("ERROR: Peak coordinate out of bounds: Row=%d, Valid range: 0,%d\n",(int)r_,(int)rows-1);
assert(0);
printf("Skipping this peak...\n");
continue;
}
if(c_>=cols){
printf("ERROR: Peak coordinate out of bounds: Col=%d, Valid range: 0,%d\n",(int)c_,(int)cols-1);
assert(0);
printf("Skipping this peak...\n");
continue;
}
for(ri=r_-pr->peakRadius;ri<=r_+pr->peakRadius;ri++){ //ri: row index. Just a temporary variable.
for(ci=c_-pr->peakRadius;ci<=c_+pr->peakRadius;ci++){ //ci: column index. Just a temporary variable.
if(ri>=0 && ri<rows && ci>=0 && ci<cols){ //Check whether inside bounds or not
roiM[calcIdx_4D(e,p_,ri,ci,panels,rows,cols)]=0;
}
}
}
}
}
//De-compress using SZ:
float* szDecomp;
size_t _szCompressedSize=szCompressedSize;
switch(pr->szDim){
case 1:
szDecomp=SZ_decompress(SZ_FLOAT,szComp,_szCompressedSize,0,0,0,0, nEvents * panels * pr->binnedRows * pr->binnedCols);
break;
case 2:
szDecomp=SZ_decompress(SZ_FLOAT,szComp,_szCompressedSize,0,0,0, nEvents * panels * pr->binnedRows, pr->binnedCols);
break;
case 3:
szDecomp=SZ_decompress(SZ_FLOAT,szComp,_szCompressedSize,0,0,nEvents * panels, pr->binnedRows, pr->binnedCols);
break;
default:
printf("ERROR: Wrong szDim : %d It must be 1,2 or 3.\n",(int)pr->szDim);
assert(0);
}
//szDecomp=(void*)malloc(nEvents*panels*rows*cols*sizeof(float));
// double max_err = 0;
// for(int i=0; i<nEvents * panels * pr->binnedRows * pr->binnedCols; i++){
// double err = fabs(szDecomp[i]-binnedData[i]);
// if(err > max_err) max_err = err;
// }
// cout << "Max err = \t\t\t" << max_err << endl;
//De-binning:
for(e=0;e<nEvents;e++)//Event
for(p=0;p<panels;p++) //Panel
for(r=0;r<pr->binnedRows;r++) //Row of the binnedData
for(c=0;c<pr->binnedCols;c++) //Column of the binnedData
for(br=0;br<pr->binSize;br++) //Bin Row (from origData)
for(bc=0;bc<pr->binSize;bc++) //Bin Column (from origData)
if(r*pr->binSize+br<rows && c*pr->binSize+bc<cols){
decompressedBuffer[calcIdx_4D(e,p,r*pr->binSize+br,c*pr->binSize+bc,panels,rows,cols)] = szDecomp[calcIdx_4D(e,p,r,c,panels,pr->binnedRows,pr->binnedCols)];
}
//Restore ROI:
uint64_t current=0;
for(e=0;e<nEvents;e++)//Event
for(p=0;p<panels;p++) //Panel
for(r=0;r<rows;r++) //Row of the binnedData
for(c=0;c<cols;c++) //Column of the binnedData
if(!roiM[calcIdx_4D(e,p,r,c,panels,rows,cols)]){
decompressedBuffer[calcIdx_4D(e,p,r,c,panels,rows,cols)]=roiData[current];
current++;
}
// delete [] roiM;
free(roiM);
free(szDecomp);
return ((void*)decompressedBuffer);
}
#ifdef __cplusplus
}
#endif

132
deps/SZ/sz/src/rw.c vendored
View File

@ -125,7 +125,7 @@ size_t checkFileSize(char *srcFilePath, int *status)
fseek(pFile, 0, SEEK_END);
filesize = ftell(pFile);
fclose(pFile);
*status = SZ_SCES;
*status = SZ_SUCCESS;
return filesize;
}
@ -153,13 +153,13 @@ unsigned char *readByteData(char *srcFilePath, size_t *byteLength, int *status)
}
fread(byteBuf, 1, *byteLength, pFile);
fclose(pFile);
*status = SZ_SCES;
*status = SZ_SUCCESS;
return byteBuf;
}
double *readDoubleData(char *srcFilePath, size_t *nbEle, int *status)
{
int state = SZ_SCES;
int state = SZ_SUCCESS;
if(dataEndianType==sysEndianType)
{
double *daBuf = readDoubleData_systemEndian(srcFilePath, nbEle,&state);
@ -196,7 +196,7 @@ double *readDoubleData(char *srcFilePath, size_t *nbEle, int *status)
int8_t *readInt8Data(char *srcFilePath, size_t *nbEle, int *status)
{
int state = SZ_SCES;
int state = SZ_SUCCESS;
int8_t *daBuf = readInt8Data_systemEndian(srcFilePath, nbEle, &state);
*status = state;
return daBuf;
@ -204,7 +204,7 @@ int8_t *readInt8Data(char *srcFilePath, size_t *nbEle, int *status)
int16_t *readInt16Data(char *srcFilePath, size_t *nbEle, int *status)
{
int state = SZ_SCES;
int state = SZ_SUCCESS;
if(dataEndianType==sysEndianType)
{
int16_t *daBuf = readInt16Data_systemEndian(srcFilePath, nbEle, &state);
@ -240,7 +240,7 @@ int16_t *readInt16Data(char *srcFilePath, size_t *nbEle, int *status)
uint16_t *readUInt16Data(char *srcFilePath, size_t *nbEle, int *status)
{
int state = SZ_SCES;
int state = SZ_SUCCESS;
if(dataEndianType==sysEndianType)
{
uint16_t *daBuf = readUInt16Data_systemEndian(srcFilePath, nbEle, &state);
@ -276,7 +276,7 @@ uint16_t *readUInt16Data(char *srcFilePath, size_t *nbEle, int *status)
int32_t *readInt32Data(char *srcFilePath, size_t *nbEle, int *status)
{
int state = SZ_SCES;
int state = SZ_SUCCESS;
if(dataEndianType==sysEndianType)
{
int32_t *daBuf = readInt32Data_systemEndian(srcFilePath, nbEle, &state);
@ -312,7 +312,7 @@ int32_t *readInt32Data(char *srcFilePath, size_t *nbEle, int *status)
uint32_t *readUInt32Data(char *srcFilePath, size_t *nbEle, int *status)
{
int state = SZ_SCES;
int state = SZ_SUCCESS;
if(dataEndianType==sysEndianType)
{
uint32_t *daBuf = readUInt32Data_systemEndian(srcFilePath, nbEle, &state);
@ -348,7 +348,7 @@ uint32_t *readUInt32Data(char *srcFilePath, size_t *nbEle, int *status)
int64_t *readInt64Data(char *srcFilePath, size_t *nbEle, int *status)
{
int state = SZ_SCES;
int state = SZ_SUCCESS;
if(dataEndianType==sysEndianType)
{
int64_t *daBuf = readInt64Data_systemEndian(srcFilePath, nbEle, &state);
@ -384,7 +384,7 @@ int64_t *readInt64Data(char *srcFilePath, size_t *nbEle, int *status)
uint64_t *readUInt64Data(char *srcFilePath, size_t *nbEle, int *status)
{
int state = SZ_SCES;
int state = SZ_SUCCESS;
if(dataEndianType==sysEndianType)
{
uint64_t *daBuf = readUInt64Data_systemEndian(srcFilePath, nbEle, &state);
@ -421,7 +421,7 @@ uint64_t *readUInt64Data(char *srcFilePath, size_t *nbEle, int *status)
float *readFloatData(char *srcFilePath, size_t *nbEle, int *status)
{
int state = SZ_SCES;
int state = SZ_SUCCESS;
if(dataEndianType==sysEndianType)
{
float *daBuf = readFloatData_systemEndian(srcFilePath, nbEle, &state);
@ -481,7 +481,7 @@ double *readDoubleData_systemEndian(char *srcFilePath, size_t *nbEle, int *statu
}
fread(daBuf, 8, *nbEle, pFile);
fclose(pFile);
*status = SZ_SCES;
*status = SZ_SUCCESS;
return daBuf;
}
@ -518,7 +518,7 @@ int8_t *readInt8Data_systemEndian(char *srcFilePath, size_t *nbEle, int *status)
}
fread(daBuf, 1, *nbEle, pFile);
fclose(pFile);
*status = SZ_SCES;
*status = SZ_SUCCESS;
return daBuf;
}
@ -555,7 +555,7 @@ int16_t *readInt16Data_systemEndian(char *srcFilePath, size_t *nbEle, int *statu
}
fread(daBuf, 2, *nbEle, pFile);
fclose(pFile);
*status = SZ_SCES;
*status = SZ_SUCCESS;
return daBuf;
}
@ -591,7 +591,7 @@ uint16_t *readUInt16Data_systemEndian(char *srcFilePath, size_t *nbEle, int *sta
}
fread(daBuf, 2, *nbEle, pFile);
fclose(pFile);
*status = SZ_SCES;
*status = SZ_SUCCESS;
return daBuf;
}
@ -627,7 +627,7 @@ int32_t *readInt32Data_systemEndian(char *srcFilePath, size_t *nbEle, int *statu
}
fread(daBuf, 4, *nbEle, pFile);
fclose(pFile);
*status = SZ_SCES;
*status = SZ_SUCCESS;
return daBuf;
}
@ -663,7 +663,7 @@ uint32_t *readUInt32Data_systemEndian(char *srcFilePath, size_t *nbEle, int *sta
}
fread(daBuf, 4, *nbEle, pFile);
fclose(pFile);
*status = SZ_SCES;
*status = SZ_SUCCESS;
return daBuf;
}
@ -699,7 +699,7 @@ int64_t *readInt64Data_systemEndian(char *srcFilePath, size_t *nbEle, int *statu
}
fread(daBuf, 8, *nbEle, pFile);
fclose(pFile);
*status = SZ_SCES;
*status = SZ_SUCCESS;
return daBuf;
}
@ -735,7 +735,7 @@ uint64_t *readUInt64Data_systemEndian(char *srcFilePath, size_t *nbEle, int *sta
}
fread(daBuf, 8, *nbEle, pFile);
fclose(pFile);
*status = SZ_SCES;
*status = SZ_SUCCESS;
return daBuf;
}
@ -771,7 +771,7 @@ float *readFloatData_systemEndian(char *srcFilePath, size_t *nbEle, int *status)
}
fread(daBuf, 4, *nbEle, pFile);
fclose(pFile);
*status = SZ_SCES;
*status = SZ_SUCCESS;
return daBuf;
}
@ -787,7 +787,7 @@ void writeByteData(unsigned char *bytes, size_t byteLength, char *tgtFilePath, i
fwrite(bytes, 1, byteLength, pFile); //write outSize bytes
fclose(pFile);
*status = SZ_SCES;
*status = SZ_SUCCESS;
}
void writeDoubleData(double *data, size_t nbEle, char *tgtFilePath, int *status)
@ -809,7 +809,7 @@ void writeDoubleData(double *data, size_t nbEle, char *tgtFilePath, int *status)
}
fclose(pFile);
*status = SZ_SCES;
*status = SZ_SUCCESS;
}
void writeFloatData(float *data, size_t nbEle, char *tgtFilePath, int *status)
@ -833,12 +833,12 @@ void writeFloatData(float *data, size_t nbEle, char *tgtFilePath, int *status)
}
fclose(pFile);
*status = SZ_SCES;
*status = SZ_SUCCESS;
}
void writeDataSZ(void *data, int dataType, size_t nbEle, char *tgtFilePath, int *status)
{
int state = SZ_SCES;
int state = SZ_SUCCESS;
if(dataType == SZ_FLOAT)
{
float* dataArray = (float *)data;
@ -861,7 +861,7 @@ void writeDataSZ(void *data, int dataType, size_t nbEle, char *tgtFilePath, int
void writeFloatData_inBytes(float *data, size_t nbEle, char* tgtFilePath, int *status)
{
size_t i = 0;
int state = SZ_SCES;
int state = SZ_SUCCESS;
lfloat buf;
unsigned char* bytes = (unsigned char*)malloc(nbEle*sizeof(float));
for(i=0;i<nbEle;i++)
@ -882,7 +882,7 @@ void writeFloatData_inBytes(float *data, size_t nbEle, char* tgtFilePath, int *s
void writeDoubleData_inBytes(double *data, size_t nbEle, char* tgtFilePath, int *status)
{
size_t i = 0, index = 0;
int state = SZ_SCES;
int state = SZ_SUCCESS;
ldouble buf;
unsigned char* bytes = (unsigned char*)malloc(nbEle*sizeof(double));
for(i=0;i<nbEle;i++)
@ -907,7 +907,7 @@ void writeDoubleData_inBytes(double *data, size_t nbEle, char* tgtFilePath, int
void writeShortData_inBytes(short *states, size_t stateLength, char *tgtFilePath, int *status)
{
int state = SZ_SCES;
int state = SZ_SUCCESS;
size_t byteLength = stateLength*2;
unsigned char* bytes = (unsigned char*)malloc(byteLength*sizeof(char));
convertShortArrayToBytes(states, stateLength, bytes);
@ -918,7 +918,7 @@ void writeShortData_inBytes(short *states, size_t stateLength, char *tgtFilePath
void writeUShortData_inBytes(unsigned short *states, size_t stateLength, char *tgtFilePath, int *status)
{
int state = SZ_SCES;
int state = SZ_SUCCESS;
size_t byteLength = stateLength*2;
unsigned char* bytes = (unsigned char*)malloc(byteLength*sizeof(char));
convertUShortArrayToBytes(states, stateLength, bytes);
@ -929,7 +929,7 @@ void writeUShortData_inBytes(unsigned short *states, size_t stateLength, char *t
void writeIntData_inBytes(int *states, size_t stateLength, char *tgtFilePath, int *status)
{
int state = SZ_SCES;
int state = SZ_SUCCESS;
size_t byteLength = stateLength*4;
unsigned char* bytes = (unsigned char*)malloc(byteLength*sizeof(char));
convertIntArrayToBytes(states, stateLength, bytes);
@ -940,7 +940,7 @@ void writeIntData_inBytes(int *states, size_t stateLength, char *tgtFilePath, in
void writeUIntData_inBytes(unsigned int *states, size_t stateLength, char *tgtFilePath, int *status)
{
int state = SZ_SCES;
int state = SZ_SUCCESS;
size_t byteLength = stateLength*4;
unsigned char* bytes = (unsigned char*)malloc(byteLength*sizeof(char));
convertUIntArrayToBytes(states, stateLength, bytes);
@ -951,7 +951,7 @@ void writeUIntData_inBytes(unsigned int *states, size_t stateLength, char *tgtFi
void writeLongData_inBytes(int64_t *states, size_t stateLength, char *tgtFilePath, int *status)
{
int state = SZ_SCES;
int state = SZ_SUCCESS;
size_t byteLength = stateLength*8;
unsigned char* bytes = (unsigned char*)malloc(byteLength*sizeof(char));
convertLongArrayToBytes(states, stateLength, bytes);
@ -962,7 +962,7 @@ void writeLongData_inBytes(int64_t *states, size_t stateLength, char *tgtFilePat
void writeULongData_inBytes(uint64_t *states, size_t stateLength, char *tgtFilePath, int *status)
{
int state = SZ_SCES;
int state = SZ_SUCCESS;
size_t byteLength = stateLength*8;
unsigned char* bytes = (unsigned char*)malloc(byteLength*sizeof(char));
convertULongArrayToBytes(states, stateLength, bytes);
@ -974,7 +974,7 @@ void writeULongData_inBytes(uint64_t *states, size_t stateLength, char *tgtFileP
unsigned short* readShortData(char *srcFilePath, size_t *dataLength, int *status)
{
size_t byteLength = 0;
int state = SZ_SCES;
int state = SZ_SUCCESS;
unsigned char * bytes = readByteData(srcFilePath, &byteLength, &state);
*dataLength = byteLength/2;
unsigned short* states = convertByteDataToUShortArray(bytes, byteLength);
@ -1002,69 +1002,5 @@ void writeStrings(int nbStr, char *str[], char *tgtFilePath, int *status)
}
fclose(pFile);
*status = SZ_SCES;
*status = SZ_SUCCESS;
}
/*
//@deprecated
//binToPFM_float is to convert the floating-point data to PFM supported by Jpeg XT
//But wrong version!
//In order to do the conversion, we need to use https://github.com/thorfdbg/difftest_ng according to Thomas Richter.
void convertToPFM_float(float *data, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, int endianType, char *tgtFilePath, int *status)
{
size_t i, nbEle = computeDataLength(r5, r4, r3, r2, r1);
int dim = computeDimension(r5, r4, r3, r2, r1);
FILE *pFile = fopen(tgtFilePath, "wb");
if (pFile == NULL)
{
printf("Failed to open input file. 3\n");
*status = SZ_NSCS;
return;
}
fputs("PF\n", pFile);
char strBuf[256];
switch(dim)
{
case 1:
sprintf(strBuf, "%zu\n", r1);
break;
case 2:
sprintf(strBuf, "%zu %zu\n", r1, r2);
break;
case 3:
sprintf(strBuf, "%zu %zu %zu\n", r1, r2, r3);
break;
case 4:
sprintf(strBuf, "%zu %zu %zu %zu\n", r1, r2, r3, r4);
break;
case 5:
sprintf(strBuf, "%zu %zu %zu %zu %zu\n", r1, r2, r3, r4, r5);
break;
}
fputs(strBuf, pFile);
if(endianType==LITTLE_ENDIAN)
fputs("-1.0\n", pFile);
else
fputs("1.0\n", pFile);
size_t byteLength = nbEle*sizeof(float);
lfloat buf;
unsigned char* bytes = (unsigned char*)malloc(byteLength);
for(i=0;i<nbEle;i++)
{
buf.value = data[i];
bytes[i*4+0] = buf.byte[0];
bytes[i*4+1] = buf.byte[1];
bytes[i*4+2] = buf.byte[2];
bytes[i*4+3] = buf.byte[3];
}
fwrite(bytes, 1, byteLength, pFile); //write outSize bytes
fclose(pFile);
free(bytes);
*status = SZ_SCES;
}*/

424
deps/SZ/sz/src/sz.c vendored
View File

@ -15,7 +15,6 @@
#include "sz.h"
#include "CompressElement.h"
#include "DynamicByteArray.h"
#include "DynamicIntArray.h"
#include "TightDataPointStorageD.h"
#include "TightDataPointStorageF.h"
#include "zlib.h"
@ -23,14 +22,14 @@
#include "Huffman.h"
#include "conf.h"
#include "utility.h"
#include "exafelSZ.h"
//#include "CurveFillingCompressStorage.h"
int versionNumber[4] = {SZ_VER_MAJOR,SZ_VER_MINOR,SZ_VER_BUILD,SZ_VER_REVISION};
//int SZ_SIZE_TYPE = 8;
int dataEndianType = LITTLE_ENDIAN_DATA; //*endian type of the data read from disk
int sysEndianType; //*sysEndianType is actually set automatically.
int sysEndianType = LITTLE_ENDIAN_SYSTEM ; //*sysEndianType is actually set automatically.
//the confparams should be separate between compression and decopmression, in case of mutual-affection when calling compression/decompression alternatively
sz_params *confparams_cpr = NULL; //used for compression
@ -56,9 +55,17 @@ HuffmanTree* SZ_Reset()
int SZ_Init(const char *configFilePath)
{
// check CPU EndianType
int x = 1;
char *y = (char*)&x;
if(*y==1)
sysEndianType = LITTLE_ENDIAN_SYSTEM;
else //=0
sysEndianType = BIG_ENDIAN_SYSTEM;
int loadFileResult = SZ_LoadConf(configFilePath);
if(loadFileResult==SZ_NSCS)
return SZ_NSCS;
if(loadFileResult==SZ_FAILED)
return SZ_FAILED;
exe_params->SZ_SIZE_TYPE = sizeof(size_t);
@ -66,7 +73,7 @@ int SZ_Init(const char *configFilePath)
{
initSZ_TSC();
}
return SZ_SCES;
return SZ_SUCCESS;
}
int SZ_Init_Params(sz_params *params)
@ -84,71 +91,12 @@ int SZ_Init_Params(sz_params *params)
if(params->quantization_intervals%2!=0)
{
printf("Error: quantization_intervals must be an even number!\n");
return SZ_NSCS;
return SZ_FAILED;
}
return SZ_SCES;
return SZ_SUCCESS;
}
int computeDimension(size_t r5, size_t r4, size_t r3, size_t r2, size_t r1)
{
int dimension;
if(r1==0)
{
dimension = 0;
}
else if(r2==0)
{
dimension = 1;
}
else if(r3==0)
{
dimension = 2;
}
else if(r4==0)
{
dimension = 3;
}
else if(r5==0)
{
dimension = 4;
}
else
{
dimension = 5;
}
return dimension;
}
size_t computeDataLength(size_t r5, size_t r4, size_t r3, size_t r2, size_t r1)
{
size_t dataLength;
if(r1==0)
{
dataLength = 0;
}
else if(r2==0)
{
dataLength = r1;
}
else if(r3==0)
{
dataLength = r1*r2;
}
else if(r4==0)
{
dataLength = r1*r2*r3;
}
else if(r5==0)
{
dataLength = r1*r2*r3*r4;
}
else
{
dataLength = r1*r2*r3*r4*r5;
}
return dataLength;
}
/*-------------------------------------------------------------------------*/
/**
@ -160,72 +108,34 @@ size_t computeDataLength(size_t r5, size_t r4, size_t r3, size_t r2, size_t r1)
**/
/*-------------------------------------------------------------------------*/
unsigned char* SZ_compress_args(int dataType, void *data, size_t *outSize, int errBoundMode, double absErrBound,
double relBoundRatio, double pwrBoundRatio, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1)
//
// compress output data to outData and return outSize
//
size_t SZ_compress_args(int dataType, void *data, size_t r1, unsigned char* outData, sz_params* params)
{
if(confparams_cpr == NULL)
SZ_Init(NULL);
else if(exe_params == NULL)
{
exe_params = (sz_exedata*)malloc(sizeof(sz_exedata));
memset(exe_params, 0, sizeof(sz_exedata));
}
if(exe_params->intvCapacity == 0)
{
exe_params->intvCapacity = confparams_cpr->maxRangeRadius*2;
exe_params->intvRadius = confparams_cpr->maxRangeRadius;
exe_params->optQuantMode = 1;
}
confparams_cpr->dataType = dataType;
size_t outSize = 0;
if(dataType==SZ_FLOAT)
{
unsigned char *newByteData = NULL;
SZ_compress_args_float(-1, confparams_cpr->withRegression, &newByteData, (float *)data, r5, r4, r3, r2, r1,
outSize, errBoundMode, absErrBound, relBoundRatio, pwrBoundRatio);
return newByteData;
SZ_compress_args_float((float *)data, r1, outData, &outSize, params);
}
else if(dataType==SZ_DOUBLE)
{
unsigned char *newByteData;
SZ_compress_args_double(-1, confparams_cpr->withRegression, &newByteData, (double *)data, r5, r4, r3, r2, r1,
outSize, errBoundMode, absErrBound, relBoundRatio, pwrBoundRatio);
return newByteData;
SZ_compress_args_double((double *)data, r1, outData, &outSize, params);
}
else
{
printf("Error: dataType can only be SZ_FLOAT, SZ_DOUBLE, SZ_INT8/16/32/64 or SZ_UINT8/16/32/64.\n");
return NULL;
printf("Error: dataType can only be SZ_FLOAT, SZ_DOUBLE .\n");
return 0;
}
return outSize;
}
unsigned char *SZ_compress(int dataType, void *data, size_t *outSize, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1)
{
unsigned char *newByteData = SZ_compress_args(dataType, data, outSize, confparams_cpr->errorBoundMode, confparams_cpr->absErrBound, confparams_cpr->relBoundRatio,
confparams_cpr->pw_relBoundRatio, r5, r4, r3, r2, r1);
return newByteData;
}
//////////////////
/*-------------------------------------------------------------------------*/
/**
@brief Perform Compression
@param data data to be compressed
@param reservedValue the reserved value
@param outSize the size (in bytes) after compression
@param r5,r4,r3,r2,r1 the sizes of each dimension (supporting only 5 dimensions at most in this version.
@return compressed data (in binary stream)
**/
/*-------------------------------------------------------------------------*/
void *SZ_decompress(int dataType, unsigned char *bytes, size_t byteLength, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1)
//
// decompress output data to outData and return outSize
//
size_t SZ_decompress(int dataType, unsigned char *bytes, size_t byteLength, size_t r1, unsigned char* outData)
{
sz_exedata de_exe;
memset(&de_exe, 0, sizeof(sz_exedata));
@ -234,80 +144,37 @@ void *SZ_decompress(int dataType, unsigned char *bytes, size_t byteLength, size_
sz_params de_params;
memset(&de_params, 0, sizeof(sz_params));
int x = 1;
char *y = (char*)&x;
if(*y==1)
sysEndianType = LITTLE_ENDIAN_SYSTEM;
else //=0
sysEndianType = BIG_ENDIAN_SYSTEM;
size_t outSize = 0;
if(dataType == SZ_FLOAT)
{
float *newFloatData;
SZ_decompress_args_float(&newFloatData, r5, r4, r3, r2, r1, bytes, byteLength, 0, NULL, &de_exe, &de_params);
return newFloatData;
int status = SZ_decompress_args_float((float*)outData, r1, bytes, byteLength, 0, NULL, &de_exe, &de_params);
if(status == SZ_SUCCESS)
{
return r1*sizeof(float);
}
return 0;
}
else if(dataType == SZ_DOUBLE)
{
double *newDoubleData;
SZ_decompress_args_double(&newDoubleData, r5, r4, r3, r2, r1, bytes, byteLength, 0, NULL, &de_exe, &de_params);
return newDoubleData;
{
int status = SZ_decompress_args_double((double*)outData, r1, bytes, byteLength, 0, NULL, &de_exe, &de_params);
if(status == SZ_SUCCESS)
{
return r1*sizeof(double);
}
return 0;
}
else
{
printf("Error: data type cannot be the types other than SZ_FLOAT or SZ_DOUBLE\n");
return NULL;
printf("Error: data type cannot be the types other than SZ_FLOAT or SZ_DOUBLE\n");
}
return outSize;
}
/**
*
*
* return number of elements or -1 if any errors
* */
size_t SZ_decompress_args(int dataType, unsigned char *bytes, size_t byteLength, void* decompressed_array, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1)
{
//size_t i;
size_t nbEle = computeDataLength(r5,r4,r3,r2,r1);
if(dataType == SZ_FLOAT)
{
float* data = (float *)SZ_decompress(dataType, bytes, byteLength, r5, r4, r3, r2, r1);
float* data_array = (float *)decompressed_array;
memcpy(data_array, data, nbEle*sizeof(float));
//for(i=0;i<nbEle;i++)
// data_array[i] = data[i];
free(data); //this free operation seems to not work with BlueG/Q system.
}
else if (dataType == SZ_DOUBLE)
{
double* data = (double *)SZ_decompress(dataType, bytes, byteLength, r5, r4, r3, r2, r1);
double* data_array = (double *)decompressed_array;
memcpy(data_array, data, nbEle*sizeof(double));
//for(i=0;i<nbEle;i++)
// data_array[i] = data[i];
free(data); //this free operation seems to not work with BlueG/Q system.
}
else
{
printf("Error: data type cannot be the types other than SZ_FLOAT or SZ_DOUBLE\n");
return SZ_NSCS; //indicating error
}
return nbEle;
}
void SZ_Finalize()
{
#ifdef HAVE_TIMECMPR
if(sz_varset!=NULL)
SZ_freeVarSet(SZ_MAINTAIN_VAR_DATA);
#endif
if(confparams_dec!=NULL)
{
free(confparams_dec);
@ -323,198 +190,5 @@ void SZ_Finalize()
free(exe_params);
exe_params = NULL;
}
//#ifdef HAVE_TIMECMPR
// if(sz_tsc!=NULL && sz_tsc->metadata_file!=NULL)
// fclose(sz_tsc->metadata_file);
//#endif
}
/**
*
* Inits the compressor for SZ_compress_customize
*
* with SZ_Init(NULL) if not previously initialized and no params passed
* with SZ_InitParam(userPara) otherwise if params are passed
* and doesn't not initialize otherwise
*
* @param sz_params* userPara : the user configuration or null
* @param sz_params* confparams : the current configuration
*/
static void sz_maybe_init_with_user_params(struct sz_params* userPara, struct sz_params* current_params) {
if(userPara==NULL && current_params == NULL)
SZ_Init(NULL);
else if(userPara != NULL)
SZ_Init_Params((sz_params*)userPara);
}
/**
*
* The interface for the user-customized compression method
*
* @param char* comprName : the name of the specific compression approach
* @param void* userPara : the pointer of the user-customized data stracture containing the cusotmized compressors' requried input parameters
* @param int dataType : data type (SZ_FLOAT, SZ_DOUBLE, SZ_INT8, SZ_UINT8, SZ_INT16, SZ_UINT16, ....)
* @param void* data : input dataset
* @param size_t r5 : the size of dimension 5
* @param size_t r4 : the size of dimension 4
* @param size_t r3 : the size of dimension 3
* @param size_t r2 : the size of dimension 2
* @param size_t r1 : the size of dimension 1
* @param size_t outSize : the number of bytes after compression
* @param int *status : the execution status of the compression operation (success: SZ_SCES or fail: SZ_NSCS)
*
* */
unsigned char* SZ_compress_customize(const char* cmprName, void* userPara, int dataType, void* data, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize, int *status)
{
unsigned char* result = NULL;
if(strcmp(cmprName, "SZ2.0")==0 || strcmp(cmprName, "SZ2.1")==0 || strcmp(cmprName, "SZ")==0)
{
sz_maybe_init_with_user_params(userPara, confparams_cpr);
result = SZ_compress(dataType, data, outSize, r5, r4, r3, r2, r1);
*status = SZ_SCES;
}
else if(strcmp(cmprName, "SZ1.4")==0)
{
sz_maybe_init_with_user_params(userPara, confparams_cpr);
confparams_cpr->withRegression = SZ_NO_REGRESSION;
result = SZ_compress(dataType, data, outSize, r5, r4, r3, r2, r1);
*status = SZ_SCES;
}
else if(strcmp(cmprName, "SZ_Transpose")==0)
{
void* transData = transposeData(data, dataType, r5, r4, r3, r2, r1);
sz_maybe_init_with_user_params(userPara, confparams_cpr);
size_t n = computeDataLength(r5, r4, r3, r2, r1);
result = SZ_compress(dataType, transData, outSize, 0, 0, 0, 0, n);
}
else if(strcmp(cmprName, "ExaFEL")==0){
assert(dataType==SZ_FLOAT);
assert(r5==0);
result = exafelSZ_Compress(userPara,data, r4, r3, r2, r1,outSize);
*status = SZ_SCES;
}
else
{
*status = SZ_NSCS;
}
return result;
}
unsigned char* SZ_compress_customize_threadsafe(const char* cmprName, void* userPara, int dataType, void* data, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize, int *status)
{
unsigned char* result = NULL;
if(strcmp(cmprName, "SZ2.0")==0 || strcmp(cmprName, "SZ2.1")==0 || strcmp(cmprName, "SZ")==0)
{
SZ_Init(NULL);
struct sz_params* para = (struct sz_params*)userPara;
if(dataType==SZ_FLOAT)
{
SZ_compress_args_float(-1, SZ_WITH_LINEAR_REGRESSION, &result, (float *)data, r5, r4, r3, r2, r1,
outSize, para->errorBoundMode, para->absErrBound, para->relBoundRatio, para->pw_relBoundRatio);
}
else if(dataType==SZ_DOUBLE)
{
SZ_compress_args_double(-1, SZ_WITH_LINEAR_REGRESSION, &result, (double *)data, r5, r4, r3, r2, r1,
outSize, para->errorBoundMode, para->absErrBound, para->relBoundRatio, para->pw_relBoundRatio);
}
*status = SZ_SCES;
return result;
}
else if(strcmp(cmprName, "SZ1.4")==0)
{
SZ_Init(NULL);
struct sz_params* para = (struct sz_params*)userPara;
if(dataType==SZ_FLOAT)
{
SZ_compress_args_float(-1, SZ_NO_REGRESSION, &result, (float *)data, r5, r4, r3, r2, r1,
outSize, para->errorBoundMode, para->absErrBound, para->relBoundRatio, para->pw_relBoundRatio);
}
else if(dataType==SZ_DOUBLE)
{
SZ_compress_args_double(-1, SZ_NO_REGRESSION, &result, (double *)data, r5, r4, r3, r2, r1,
outSize, para->errorBoundMode, para->absErrBound, para->relBoundRatio, para->pw_relBoundRatio);
}
*status = SZ_SCES;
return result;
}
else if(strcmp(cmprName, "SZ_Transpose")==0)
{
void* transData = transposeData(data, dataType, r5, r4, r3, r2, r1);
struct sz_params* para = (struct sz_params*)userPara;
size_t n = computeDataLength(r5, r4, r3, r2, r1);
result = SZ_compress_args(dataType, transData, outSize, para->errorBoundMode, para->absErrBound, para->relBoundRatio, para->pw_relBoundRatio, 0, 0, 0, 0, n);
*status = SZ_SCES;
}
else if(strcmp(cmprName, "ExaFEL")==0){ //not sure if this part is thread safe!
assert(dataType==SZ_FLOAT);
assert(r5==0);
result = exafelSZ_Compress(userPara,data, r4, r3, r2, r1,outSize);
*status = SZ_SCES;
}
else
{
*status = SZ_NSCS;
}
return result;
}
/**
*
* The interface for the user-customized decompression method
*
* @param char* comprName : the name of the specific compression approach
* @param void* userPara : the pointer of the user-customized data stracture containing the cusotmized compressors' requried input parameters
* @param int dataType : data type (SZ_FLOAT, SZ_DOUBLE, SZ_INT8, SZ_UINT8, SZ_INT16, SZ_UINT16, ....)
* @param unsigned char* bytes : input bytes (the compressed data)
* @param size_t r5 : the size of dimension 5
* @param size_t r4 : the size of dimension 4
* @param size_t r3 : the size of dimension 3
* @param size_t r2 : the size of dimension 2
* @param size_t r1 : the size of dimension 1
* @param int *status : the execution status of the compression operation (success: SZ_SCES or fail: SZ_NSCS)
*
* */
void* SZ_decompress_customize(const char* cmprName, void* userPara, int dataType, unsigned char* bytes, size_t byteLength, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, int *status)
{
void* result = NULL;
if(strcmp(cmprName, "SZ2.0")==0 || strcmp(cmprName, "SZ")==0 || strcmp(cmprName, "SZ1.4")==0)
{
result = SZ_decompress(dataType, bytes, byteLength, r5, r4, r3, r2, r1);
* status = SZ_SCES;
}
else if(strcmp(cmprName, "SZ_Transpose")==0)
{
size_t n = computeDataLength(r5, r4, r3, r2, r1);
void* tmpData = SZ_decompress(dataType, bytes, byteLength, 0, 0, 0, 0, n);
result = detransposeData(tmpData, dataType, r5, r4, r3, r2, r1);
}
else if(strcmp(cmprName, "ExaFEL")==0){
assert(dataType==SZ_FLOAT);
assert(r5==0);
result = exafelSZ_Decompress(userPara,bytes, r4, r3, r2, r1,byteLength);
*status = SZ_SCES;
}
else
{
*status = SZ_NSCS;
}
return result;
}
void* SZ_decompress_customize_threadsafe(const char* cmprName, void* userPara, int dataType, unsigned char* bytes, size_t byteLength, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, int *status)
{
return SZ_decompress_customize(cmprName, userPara, dataType, bytes, byteLength, r5, r4, r3, r2, r1, status);
}

601
deps/SZ/sz/src/sz_double.c vendored
View File

@ -100,169 +100,9 @@ unsigned int optimize_intervals_double_1D(double *oriData, size_t dataLength, do
return powerOf2;
}
unsigned int optimize_intervals_double_2D(double *oriData, size_t r1, size_t r2, double realPrecision)
{
size_t i,j, index;
size_t radiusIndex;
double pred_value = 0, pred_err;
size_t *intervals = (size_t*)malloc(confparams_cpr->maxRangeRadius*sizeof(size_t));
memset(intervals, 0, confparams_cpr->maxRangeRadius*sizeof(size_t));
size_t totalSampleSize = (r1-1)*(r2-1)/confparams_cpr->sampleDistance;
for(i=1;i<r1;i++)
{
for(j=1;j<r2;j++)
{
if((i+j)%confparams_cpr->sampleDistance==0)
{
index = i*r2+j;
pred_value = oriData[index-1] + oriData[index-r2] - oriData[index-r2-1];
pred_err = fabs(pred_value - oriData[index]);
radiusIndex = (unsigned long)((pred_err/realPrecision+1)/2);
if(radiusIndex>=confparams_cpr->maxRangeRadius)
radiusIndex = confparams_cpr->maxRangeRadius - 1;
intervals[radiusIndex]++;
}
}
}
//compute the appropriate number
size_t targetCount = totalSampleSize*confparams_cpr->predThreshold;
size_t sum = 0;
for(i=0;i<confparams_cpr->maxRangeRadius;i++)
{
sum += intervals[i];
if(sum>targetCount)
break;
}
if(i>=confparams_cpr->maxRangeRadius)
i = confparams_cpr->maxRangeRadius-1;
unsigned int accIntervals = 2*(i+1);
unsigned int powerOf2 = roundUpToPowerOf2(accIntervals);
//printf("confparams_cpr->maxRangeRadius = %d, accIntervals=%d, powerOf2=%d\n", confparams_cpr->maxRangeRadius, accIntervals, powerOf2);
if(powerOf2<32)
powerOf2 = 32;
free(intervals);
return powerOf2;
}
unsigned int optimize_intervals_double_3D(double *oriData, size_t r1, size_t r2, size_t r3, double realPrecision)
{
size_t i,j,k, index;
size_t radiusIndex;
size_t r23=r2*r3;
double pred_value = 0, pred_err;
size_t *intervals = (size_t*)malloc(confparams_cpr->maxRangeRadius*sizeof(size_t));
memset(intervals, 0, confparams_cpr->maxRangeRadius*sizeof(size_t));
size_t totalSampleSize = (r1-1)*(r2-1)*(r3-1)/confparams_cpr->sampleDistance;
for(i=1;i<r1;i++)
{
for(j=1;j<r2;j++)
{
for(k=1;k<r3;k++)
{
if((i+j+k)%confparams_cpr->sampleDistance==0)
{
index = i*r23+j*r3+k;
pred_value = oriData[index-1] + oriData[index-r3] + oriData[index-r23]
- oriData[index-1-r23] - oriData[index-r3-1] - oriData[index-r3-r23] + oriData[index-r3-r23-1];
pred_err = fabs(pred_value - oriData[index]);
radiusIndex = (pred_err/realPrecision+1)/2;
if(radiusIndex>=confparams_cpr->maxRangeRadius)
radiusIndex = confparams_cpr->maxRangeRadius - 1;
intervals[radiusIndex]++;
}
}
}
}
//compute the appropriate number
size_t targetCount = totalSampleSize*confparams_cpr->predThreshold;
size_t sum = 0;
for(i=0;i<confparams_cpr->maxRangeRadius;i++)
{
sum += intervals[i];
if(sum>targetCount)
break;
}
if(i>=confparams_cpr->maxRangeRadius)
i = confparams_cpr->maxRangeRadius-1;
unsigned int accIntervals = 2*(i+1);
unsigned int powerOf2 = roundUpToPowerOf2(accIntervals);
if(powerOf2<32)
powerOf2 = 32;
free(intervals);
//printf("confparams_cpr->maxRangeRadius = %d, accIntervals=%d, powerOf2=%d\n", confparams_cpr->maxRangeRadius, accIntervals, powerOf2);
return powerOf2;
}
unsigned int optimize_intervals_double_4D(double *oriData, size_t r1, size_t r2, size_t r3, size_t r4, double realPrecision)
{
size_t i,j,k,l, index;
size_t radiusIndex;
size_t r234=r2*r3*r4;
size_t r34=r3*r4;
double pred_value = 0, pred_err;
size_t *intervals = (size_t*)malloc(confparams_cpr->maxRangeRadius*sizeof(size_t));
memset(intervals, 0, confparams_cpr->maxRangeRadius*sizeof(size_t));
size_t totalSampleSize = (r1-1)*(r2-1)*(r3-1)*(r4-1)/confparams_cpr->sampleDistance;
for(i=1;i<r1;i++)
{
for(j=1;j<r2;j++)
{
for(k=1;k<r3;k++)
{
for (l=1;l<r4;l++)
{
if((i+j+k+l)%confparams_cpr->sampleDistance==0)
{
index = i*r234+j*r34+k*r4+l;
pred_value = oriData[index-1] + oriData[index-r3] + oriData[index-r34]
- oriData[index-1-r34] - oriData[index-r4-1] - oriData[index-r4-r34] + oriData[index-r4-r34-1];
pred_err = fabs(pred_value - oriData[index]);
radiusIndex = (unsigned long)((pred_err/realPrecision+1)/2);
if(radiusIndex>=confparams_cpr->maxRangeRadius)
radiusIndex = confparams_cpr->maxRangeRadius - 1;
intervals[radiusIndex]++;
}
}
}
}
}
//compute the appropriate number
size_t targetCount = totalSampleSize*confparams_cpr->predThreshold;
size_t sum = 0;
for(i=0;i<confparams_cpr->maxRangeRadius;i++)
{
sum += intervals[i];
if(sum>targetCount)
break;
}
if(i>=confparams_cpr->maxRangeRadius)
i = confparams_cpr->maxRangeRadius-1;
unsigned int accIntervals = 2*(i+1);
unsigned int powerOf2 = roundUpToPowerOf2(accIntervals);
if(powerOf2<32)
powerOf2 = 32;
free(intervals);
return powerOf2;
}
TightDataPointStorageD* SZ_compress_double_1D_MDQ(double *oriData,
size_t dataLength, double realPrecision, double valueRangeSize, double medianValue_d)
{
#ifdef HAVE_TIMECMPR
double* decData = NULL;
if(confparams_cpr->szMode == SZ_TEMPORAL_COMPRESSION)
decData = (double*)(multisteps->hist_data);
#endif
unsigned int quantization_intervals;
if(exe_params->optQuantMode==1)
quantization_intervals = optimize_intervals_double_1D_opt(oriData, dataLength, realPrecision);
@ -307,11 +147,7 @@ size_t dataLength, double realPrecision, double valueRangeSize, double medianVal
updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,8);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
listAdd_double(last3CmprsData, vce->data);
#ifdef HAVE_TIMECMPR
if(confparams_cpr->szMode == SZ_TEMPORAL_COMPRESSION)
decData[0] = vce->data;
#endif
listAdd_double(last3CmprsData, vce->data);
//add the second data
type[1] = 0;
@ -320,10 +156,7 @@ size_t dataLength, double realPrecision, double valueRangeSize, double medianVal
memcpy(preDataBytes,vce->curBytes,8);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
listAdd_double(last3CmprsData, vce->data);
#ifdef HAVE_TIMECMPR
if(confparams_cpr->szMode == SZ_TEMPORAL_COMPRESSION)
decData[1] = vce->data;
#endif
int state;
double checkRadius;
double curData;
@ -354,10 +187,6 @@ size_t dataLength, double realPrecision, double valueRangeSize, double medianVal
pred = pred - state*interval;
}
//listAdd_double(last3CmprsData, pred);
#ifdef HAVE_TIMECMPR
if(confparams_cpr->szMode == SZ_TEMPORAL_COMPRESSION)
decData[i] = pred;
#endif
continue;
}
@ -370,11 +199,7 @@ size_t dataLength, double realPrecision, double valueRangeSize, double medianVal
//listAdd_double(last3CmprsData, vce->data);
pred = vce->data;
#ifdef HAVE_TIMECMPR
if(confparams_cpr->szMode == SZ_TEMPORAL_COMPRESSION)
decData[i] = vce->data;
#endif
}//end of for
@ -403,7 +228,7 @@ size_t dataLength, double realPrecision, double valueRangeSize, double medianVal
return tdps;
}
void SZ_compress_args_double_StoreOriData(double* oriData, size_t dataLength, unsigned char** newByteData, size_t *outSize)
void SZ_compress_args_double_StoreOriData(double* oriData, size_t dataLength, unsigned char* newByteData, size_t *outSize)
{
int doubleSize = sizeof(double);
size_t k = 0, i;
@ -413,25 +238,25 @@ void SZ_compress_args_double_StoreOriData(double* oriData, size_t dataLength, un
unsigned char dsLengthBytes[8];
for (i = 0; i < 3; i++)//3
(*newByteData)[k++] = versionNumber[i];
newByteData[k++] = versionNumber[i];
if(exe_params->SZ_SIZE_TYPE==4)//1
(*newByteData)[k++] = 16; //00010000
newByteData[k++] = 16; //00010000
else
(*newByteData)[k++] = 80; //01010000: 01000000 indicates the SZ_SIZE_TYPE=8
newByteData[k++] = 80; //01010000: 01000000 indicates the SZ_SIZE_TYPE=8
convertSZParamsToBytes(confparams_cpr, &((*newByteData)[k]));
convertSZParamsToBytes(confparams_cpr, &(newByteData[k]));
k = k + MetaDataByteLength_double;
sizeToBytes(dsLengthBytes,dataLength);
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)//ST: 4 or 8
(*newByteData)[k++] = dsLengthBytes[i];
newByteData[k++] = dsLengthBytes[i];
if(sysEndianType==BIG_ENDIAN_SYSTEM)
memcpy((*newByteData)+4+MetaDataByteLength_double+exe_params->SZ_SIZE_TYPE, oriData, dataLength*doubleSize);
memcpy(newByteData+4+MetaDataByteLength_double+exe_params->SZ_SIZE_TYPE, oriData, dataLength*doubleSize);
else
{
unsigned char* p = (*newByteData)+4+MetaDataByteLength_double+exe_params->SZ_SIZE_TYPE;
unsigned char* p = newByteData+4+MetaDataByteLength_double+exe_params->SZ_SIZE_TYPE;
for(i=0;i<dataLength;i++,p+=doubleSize)
doubleToBytes(p, oriData[i]);
}
@ -439,45 +264,12 @@ void SZ_compress_args_double_StoreOriData(double* oriData, size_t dataLength, un
}
char SZ_compress_args_double_NoCkRngeNoGzip_1D(int cmprType, unsigned char** newByteData, double *oriData,
char SZ_compress_args_double_NoCkRngeNoGzip_1D(unsigned char* newByteData, double *oriData,
size_t dataLength, double realPrecision, size_t *outSize, double valueRangeSize, double medianValue_d)
{
char compressionType = 0;
TightDataPointStorageD* tdps = NULL;
#ifdef HAVE_TIMECMPR
if(confparams_cpr->szMode == SZ_TEMPORAL_COMPRESSION)
{
int timestep = sz_tsc->currentStep;
if(cmprType == SZ_PERIO_TEMPORAL_COMPRESSION)
{
if(timestep % confparams_cpr->snapshotCmprStep != 0)
{
tdps = SZ_compress_double_1D_MDQ_ts(oriData, dataLength, multisteps, realPrecision, valueRangeSize, medianValue_d);
compressionType = 1; //time-series based compression
}
else
{
tdps = SZ_compress_double_1D_MDQ(oriData, dataLength, realPrecision, valueRangeSize, medianValue_d);
compressionType = 0; //snapshot-based compression
multisteps->lastSnapshotStep = timestep;
}
}
else if(cmprType == SZ_FORCE_SNAPSHOT_COMPRESSION)
{
tdps = SZ_compress_double_1D_MDQ(oriData, dataLength, realPrecision, valueRangeSize, medianValue_d);
compressionType = 0; //snapshot-based compression
multisteps->lastSnapshotStep = timestep;
}
else if(cmprType == SZ_FORCE_TEMPORAL_COMPRESSION)
{
tdps = SZ_compress_double_1D_MDQ_ts(oriData, dataLength, multisteps, realPrecision, valueRangeSize, medianValue_d);
compressionType = 1; //time-series based compression
}
}
else
#endif
tdps = SZ_compress_double_1D_MDQ(oriData, dataLength, realPrecision, valueRangeSize, medianValue_d);
tdps = SZ_compress_double_1D_MDQ(oriData, dataLength, realPrecision, valueRangeSize, medianValue_d);
convertTDPStoFlatBytes_double(tdps, newByteData, outSize);
@ -492,11 +284,6 @@ size_t dataLength, double realPrecision, size_t *outSize, double valueRangeSize,
TightDataPointStorageD* SZ_compress_double_1D_MDQ_MSST19(double *oriData,
size_t dataLength, double realPrecision, double valueRangeSize, double medianValue_f)
{
#ifdef HAVE_TIMECMPR
double* decData = NULL;
if(confparams_cpr->szMode == SZ_TEMPORAL_COMPRESSION)
decData = (double*)(multisteps->hist_data);
#endif
//struct ClockPoint clockPointBuild;
//TimeDurationStart("build", &clockPointBuild);
@ -558,11 +345,7 @@ size_t dataLength, double realPrecision, double valueRangeSize, double medianVal
memcpy(preDataBytes,vce->curBytes,8);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
listAdd_double(last3CmprsData, vce->data);
//miss++;
#ifdef HAVE_TIMECMPR
if(confparams_cpr->szMode == SZ_TEMPORAL_COMPRESSION)
decData[0] = vce->data;
#endif
//miss++;
//add the second data
type[1] = 0;
@ -572,10 +355,7 @@ size_t dataLength, double realPrecision, double valueRangeSize, double medianVal
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
listAdd_double(last3CmprsData, vce->data);
//miss++;
#ifdef HAVE_TIMECMPR
if(confparams_cpr->szMode == SZ_TEMPORAL_COMPRESSION)
decData[1] = vce->data;
#endif
int state;
//double checkRadius;
double curData;
@ -623,10 +403,6 @@ size_t dataLength, double realPrecision, double valueRangeSize, double medianVal
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
pred = vce->data;
//miss++;
#ifdef HAVE_TIMECMPR
if(confparams_cpr->szMode == SZ_TEMPORAL_COMPRESSION)
decData[i] = vce->data;
#endif
}//end of for
@ -657,7 +433,7 @@ size_t dataLength, double realPrecision, double valueRangeSize, double medianVal
}
void SZ_compress_args_double_withinRange(unsigned char** newByteData, double *oriData, size_t dataLength, size_t *outSize)
void SZ_compress_args_double_withinRange(unsigned char* newByteData, double *oriData, size_t dataLength, size_t *outSize)
{
TightDataPointStorageD* tdps = (TightDataPointStorageD*) malloc(sizeof(TightDataPointStorageD));
tdps->rtypeArray = NULL;
@ -685,35 +461,30 @@ void SZ_compress_args_double_withinRange(unsigned char** newByteData, double *or
free_TightDataPointStorageD(tdps);
}
int SZ_compress_args_double(int cmprType, int withRegression, unsigned char** newByteData, double *oriData,
size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize,
int errBoundMode, double absErr_Bound, double relBoundRatio, double pwRelBoundRatio)
{
confparams_cpr->errorBoundMode = errBoundMode;
if(errBoundMode==PW_REL)
{
confparams_cpr->pw_relBoundRatio = pwRelBoundRatio;
}
int status = SZ_SCES;
size_t dataLength = computeDataLength(r5,r4,r3,r2,r1);
if(dataLength <= MIN_NUM_OF_ELEMENTS)
{
*newByteData = SZ_skip_compress_double(oriData, dataLength, outSize);
return status;
}
int SZ_compress_args_double(double *oriData, size_t r1, unsigned char* newByteData, size_t *outSize, sz_params* params)
{
int status = SZ_SUCCESS;
size_t dataLength = r1;
double valueRangeSize = 0, medianValue = 0;
unsigned char * signs = NULL;
bool positive = true;
double nearZero = 0.0;
double min = 0;
if(pwRelBoundRatio < 0.000009999)
confparams_cpr->accelerate_pw_rel_compression = 0;
if(params->pw_relBoundRatio < 0.000009999)
params->accelerate_pw_rel_compression = 0;
// check at least elements count
if(dataLength <= MIN_NUM_OF_ELEMENTS)
{
printf("error, double input elements count=%d less than %d, so need not do compress.\n", dataLength, MIN_NUM_OF_ELEMENTS);
return SZ_LITTER_ELEMENT;
}
if(confparams_cpr->errorBoundMode == PW_REL && confparams_cpr->accelerate_pw_rel_compression == 1)
if(params->errorBoundMode == PW_REL && params->accelerate_pw_rel_compression == 1)
{
signs = (unsigned char *) malloc(dataLength);
memset(signs, 0, dataLength);
@ -722,81 +493,72 @@ int errBoundMode, double absErr_Bound, double relBoundRatio, double pwRelBoundRa
else
min = computeRangeSize_double(oriData, dataLength, &valueRangeSize, &medianValue);
double max = min+valueRangeSize;
confparams_cpr->dmin = min;
confparams_cpr->dmax = max;
params->dmin = min;
params->dmax = max;
double realPrecision = 0;
if(confparams_cpr->errorBoundMode==PSNR)
if(params->errorBoundMode==PSNR)
{
confparams_cpr->errorBoundMode = SZ_ABS;
realPrecision = confparams_cpr->absErrBound = computeABSErrBoundFromPSNR(confparams_cpr->psnr, (double)confparams_cpr->predThreshold, valueRangeSize);
params->errorBoundMode = SZ_ABS;
realPrecision = params->absErrBound = computeABSErrBoundFromPSNR(params->psnr, (double)params->predThreshold, valueRangeSize);
}
else if(confparams_cpr->errorBoundMode==NORM) //norm error = sqrt(sum((xi-xi_)^2))
else if(params->errorBoundMode==NORM) //norm error = sqrt(sum((xi-xi_)^2))
{
confparams_cpr->errorBoundMode = SZ_ABS;
realPrecision = confparams_cpr->absErrBound = computeABSErrBoundFromNORM_ERR(confparams_cpr->normErr, dataLength);
params->errorBoundMode = SZ_ABS;
realPrecision = params->absErrBound = computeABSErrBoundFromNORM_ERR(params->normErr, dataLength);
//printf("realPrecision=%lf\n", realPrecision);
}
else
{
realPrecision = getRealPrecision_double(valueRangeSize, errBoundMode, absErr_Bound, relBoundRatio, &status);
confparams_cpr->absErrBound = realPrecision;
realPrecision = getRealPrecision_double(valueRangeSize, params->errorBoundMode, params->absErrBound, params->relBoundRatio, &status);
params->absErrBound = realPrecision;
}
if(valueRangeSize <= realPrecision)
{
if(confparams_cpr->errorBoundMode>=PW_REL && confparams_cpr->accelerate_pw_rel_compression == 1)
if(params->errorBoundMode>=PW_REL && params->accelerate_pw_rel_compression == 1)
free(signs);
SZ_compress_args_double_withinRange(newByteData, oriData, dataLength, outSize);
}
else
{
size_t tmpOutSize = 0;
unsigned char* tmpByteData;
if (r2==0)
unsigned char* tmpByteData = newByteData;
if(params->szMode != SZ_BEST_SPEED)
{
if(confparams_cpr->errorBoundMode>=PW_REL)
{
if(confparams_cpr->accelerate_pw_rel_compression && confparams_cpr->maxRangeRadius <= 32768)
SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log_MSST19(&tmpByteData, oriData, pwRelBoundRatio, r1, &tmpOutSize, valueRangeSize, medianValue, signs, &positive, min, max, nearZero);
else
SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log(&tmpByteData, oriData, pwRelBoundRatio, r1, &tmpOutSize, min, max);
//SZ_compress_args_double_NoCkRngeNoGzip_1D_pwrgroup(&tmpByteData, oriData, r1, absErr_Bound, relBoundRatio, pwRelBoundRatio, valueRangeSize, medianValue, &tmpOutSize);
}
tmpByteData = (unsigned char*)malloc(r1*sizeof(double)*1.2);
}
if(params->errorBoundMode>=PW_REL)
{
if(params->accelerate_pw_rel_compression && params->maxRangeRadius <= 32768)
SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log_MSST19(tmpByteData, oriData, params->pw_relBoundRatio, r1, &tmpOutSize, valueRangeSize, medianValue, signs, &positive, min, max, nearZero);
else
#ifdef HAVE_TIMECMPR
if(confparams_cpr->szMode == SZ_TEMPORAL_COMPRESSION)
multisteps->compressionType = SZ_compress_args_double_NoCkRngeNoGzip_1D(cmprType, &tmpByteData, oriData, r1, realPrecision, &tmpOutSize, valueRangeSize, medianValue);
else
#endif
{
SZ_compress_args_double_NoCkRngeNoGzip_1D(cmprType, &tmpByteData, oriData, r1, realPrecision, &tmpOutSize, valueRangeSize, medianValue);
if(tmpOutSize>=dataLength*sizeof(double) + 3 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + 1)
SZ_compress_args_double_StoreOriData(oriData, dataLength, &tmpByteData, &tmpOutSize);
}
SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log(tmpByteData, oriData, params->pw_relBoundRatio, r1, &tmpOutSize, min, max);
}
else
{
printf("Error: doesn't support 5 dimensions for now.\n");
status = SZ_DERR;
SZ_compress_args_double_NoCkRngeNoGzip_1D(tmpByteData, oriData, r1, realPrecision, &tmpOutSize, valueRangeSize, medianValue);
if(tmpOutSize>=dataLength*sizeof(double) + 3 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + 1)
SZ_compress_args_double_StoreOriData(oriData, dataLength, tmpByteData, &tmpOutSize);
}
//
//Call Gzip to do the further compression.
//
if(confparams_cpr->szMode==SZ_BEST_SPEED)
if(params->szMode==SZ_BEST_SPEED)
{
*outSize = tmpOutSize;
*newByteData = tmpByteData;
//*newByteData = tmpByteData;
}
else if(confparams_cpr->szMode==SZ_BEST_COMPRESSION || confparams_cpr->szMode==SZ_DEFAULT_COMPRESSION || confparams_cpr->szMode==SZ_TEMPORAL_COMPRESSION)
else if(params->szMode==SZ_BEST_COMPRESSION || params->szMode==SZ_DEFAULT_COMPRESSION || params->szMode==SZ_TEMPORAL_COMPRESSION)
{
*outSize = sz_lossless_compress(confparams_cpr->losslessCompressor, confparams_cpr->gzipMode, tmpByteData, tmpOutSize, newByteData);
*outSize = sz_lossless_compress(params->losslessCompressor, params->gzipMode, tmpByteData, tmpOutSize, newByteData);
free(tmpByteData);
}
else
{
printf("Error: Wrong setting of confparams_cpr->szMode in the double compression.\n");
printf("Error: Wrong setting of params->szMode in the double compression.\n");
status = SZ_MERR;
}
}
@ -804,245 +566,6 @@ int errBoundMode, double absErr_Bound, double relBoundRatio, double pwRelBoundRa
return status;
}
//TODO
int SZ_compress_args_double_subblock(unsigned char* compressedBytes, double *oriData,
size_t r5, size_t r4, size_t r3, size_t r2, size_t r1,
size_t s5, size_t s4, size_t s3, size_t s2, size_t s1,
size_t e5, size_t e4, size_t e3, size_t e2, size_t e1,
size_t *outSize, int errBoundMode, double absErr_Bound, double relBoundRatio)
{
int status = SZ_SCES;
double valueRangeSize = 0, medianValue = 0;
computeRangeSize_double_subblock(oriData, &valueRangeSize, &medianValue, r5, r4, r3, r2, r1, s5, s4, s3, s2, s1, e5, e4, e3, e2, e1);
double realPrecision = getRealPrecision_double(valueRangeSize, errBoundMode, absErr_Bound, relBoundRatio, &status);
if(valueRangeSize <= realPrecision)
{
//TODO
//SZ_compress_args_double_withinRange_subblock();
}
else
{
if (r2==0)
{
//TODO
if(errBoundMode==PW_REL)
{
//TODO
//SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_subblock();
printf ("Current subblock version does not support point-wise relative error bound.\n");
}
else
SZ_compress_args_double_NoCkRnge_1D_subblock(compressedBytes, oriData, realPrecision, outSize, valueRangeSize, medianValue, r1, s1, e1);
}
else
{
printf("Error: doesn't support 5 dimensions for now.\n");
status = SZ_DERR; //dimension error
}
}
return status;
}
void SZ_compress_args_double_NoCkRnge_1D_subblock(unsigned char* compressedBytes, double *oriData, double realPrecision, size_t *outSize, double valueRangeSize, double medianValue_d,
size_t r1, size_t s1, size_t e1)
{
TightDataPointStorageD* tdps = SZ_compress_double_1D_MDQ_subblock(oriData, realPrecision, valueRangeSize, medianValue_d, r1, s1, e1);
if (confparams_cpr->szMode==SZ_BEST_SPEED)
convertTDPStoFlatBytes_double_args(tdps, compressedBytes, outSize);
else if(confparams_cpr->szMode==SZ_BEST_COMPRESSION || confparams_cpr->szMode==SZ_DEFAULT_COMPRESSION)
{
unsigned char *tmpCompBytes;
size_t tmpOutSize;
convertTDPStoFlatBytes_double(tdps, &tmpCompBytes, &tmpOutSize);
*outSize = zlib_compress3(tmpCompBytes, tmpOutSize, compressedBytes, confparams_cpr->gzipMode);
free(tmpCompBytes);
}
else
{
printf ("Error: Wrong setting of confparams_cpr->szMode in the double compression.\n");
}
//TODO
// if(*outSize>dataLength*sizeof(double))
// SZ_compress_args_double_StoreOriData(oriData, dataLength, newByteData, outSize);
free_TightDataPointStorageD(tdps);
}
unsigned int optimize_intervals_double_1D_subblock(double *oriData, double realPrecision, size_t r1, size_t s1, size_t e1)
{
size_t dataLength = e1 - s1 + 1;
oriData = oriData + s1;
size_t i = 0;
unsigned long radiusIndex;
double pred_value = 0, pred_err;
int *intervals = (int*)malloc(confparams_cpr->maxRangeRadius*sizeof(int));
memset(intervals, 0, confparams_cpr->maxRangeRadius*sizeof(int));
size_t totalSampleSize = dataLength/confparams_cpr->sampleDistance;
for(i=2;i<dataLength;i++)
{
if(i%confparams_cpr->sampleDistance==0)
{
pred_value = 2*oriData[i-1] - oriData[i-2];
//pred_value = oriData[i-1];
pred_err = fabs(pred_value - oriData[i]);
radiusIndex = (unsigned long)((pred_err/realPrecision+1)/2);
if(radiusIndex>=confparams_cpr->maxRangeRadius)
radiusIndex = confparams_cpr->maxRangeRadius - 1;
intervals[radiusIndex]++;
}
}
//compute the appropriate number
size_t targetCount = totalSampleSize*confparams_cpr->predThreshold;
size_t sum = 0;
for(i=0;i<confparams_cpr->maxRangeRadius;i++)
{
sum += intervals[i];
if(sum>targetCount)
break;
}
if(i>=confparams_cpr->maxRangeRadius)
i = confparams_cpr->maxRangeRadius-1;
unsigned int accIntervals = 2*(i+1);
unsigned int powerOf2 = roundUpToPowerOf2(accIntervals);
if(powerOf2<32)
powerOf2 = 32;
free(intervals);
return powerOf2;
}
TightDataPointStorageD* SZ_compress_double_1D_MDQ_subblock(double *oriData, double realPrecision, double valueRangeSize, double medianValue_d,
size_t r1, size_t s1, size_t e1)
{
size_t dataLength = e1 - s1 + 1;
unsigned int quantization_intervals;
if(exe_params->optQuantMode==1)
quantization_intervals = optimize_intervals_double_1D_subblock(oriData, realPrecision, r1, s1, e1);
else
quantization_intervals = exe_params->intvCapacity;
//updateQuantizationInfo(quantization_intervals);
int intvRadius = quantization_intervals/2;
size_t i;
int reqLength;
double medianValue = medianValue_d;
short radExpo = getExponent_double(valueRangeSize/2);
computeReqLength_double(realPrecision, radExpo, &reqLength, &medianValue);
int* type = (int*) malloc(dataLength*sizeof(int));
double* spaceFillingValue = oriData + s1; //
DynamicIntArray *exactLeadNumArray;
new_DIA(&exactLeadNumArray, DynArrayInitLen);
DynamicByteArray *exactMidByteArray;
new_DBA(&exactMidByteArray, DynArrayInitLen);
DynamicIntArray *resiBitArray;
new_DIA(&resiBitArray, DynArrayInitLen);
type[0] = 0;
unsigned char preDataBytes[8];
longToBytes_bigEndian(preDataBytes, 0);
int reqBytesLength = reqLength/8;
int resiBitsLength = reqLength%8;
double last3CmprsData[3] = {0};
DoubleValueCompressElement *vce = (DoubleValueCompressElement*)malloc(sizeof(DoubleValueCompressElement));
LossyCompressionElement *lce = (LossyCompressionElement*)malloc(sizeof(LossyCompressionElement));
//add the first data
compressSingleDoubleValue(vce, spaceFillingValue[0], realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,8);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
listAdd_double(last3CmprsData, vce->data);
//add the second data
type[1] = 0;
compressSingleDoubleValue(vce, spaceFillingValue[1], realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,8);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
listAdd_double(last3CmprsData, vce->data);
int state;
double checkRadius;
double curData;
double pred;
double predAbsErr;
checkRadius = (quantization_intervals-1)*realPrecision;
double interval = 2*realPrecision;
for(i=2;i<dataLength;i++)
{
//printf("%.30G\n",last3CmprsData[0]);
curData = spaceFillingValue[i];
pred = 2*last3CmprsData[0] - last3CmprsData[1];
//pred = last3CmprsData[0];
predAbsErr = fabs(curData - pred);
if(predAbsErr<=checkRadius)
{
state = (predAbsErr/realPrecision+1)/2;
if(curData>=pred)
{
type[i] = intvRadius+state;
pred = pred + state*interval;
}
else //curData<pred
{
type[i] = intvRadius-state;
pred = pred - state*interval;
}
listAdd_double(last3CmprsData, pred);
continue;
}
//unpredictable data processing
type[i] = 0;
compressSingleDoubleValue(vce, curData, realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,8);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
listAdd_double(last3CmprsData, vce->data);
}//end of for
size_t exactDataNum = exactLeadNumArray->size;
TightDataPointStorageD* tdps;
new_TightDataPointStorageD(&tdps, dataLength, exactDataNum,
type, exactMidByteArray->array, exactMidByteArray->size,
exactLeadNumArray->array,
resiBitArray->array, resiBitArray->size,
resiBitsLength,
realPrecision, medianValue, (char)reqLength, quantization_intervals, NULL, 0, 0);
//free memory
free_DIA(exactLeadNumArray);
free_DIA(resiBitArray);
free(type);
free(vce);
free(lce);
free(exactMidByteArray); //exactMidByteArray->array has been released in free_TightDataPointStorageF(tdps);
return tdps;
}
unsigned int optimize_intervals_double_1D_opt_MSST19(double *oriData, size_t dataLength, double realPrecision)
{

23
deps/SZ/sz/src/sz_double_pwr.c vendored
View File

@ -17,7 +17,6 @@
#include "sz.h"
#include "CompressElement.h"
#include "DynamicByteArray.h"
#include "DynamicIntArray.h"
#include "TightDataPointStorageD.h"
#include "sz_double.h"
#include "sz_double_pwr.h"
@ -149,7 +148,7 @@ unsigned int optimize_intervals_double_1D_pwr(double *oriData, size_t dataLength
}
void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr(unsigned char** newByteData, double *oriData, double globalPrecision,
void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr(unsigned char* newByteData, double *oriData, double globalPrecision,
size_t dataLength, size_t *outSize, double min, double max)
{
size_t pwrLength = dataLength%confparams_cpr->segment_size==0?dataLength/confparams_cpr->segment_size:dataLength/confparams_cpr->segment_size+1;
@ -327,25 +326,25 @@ size_t dataLength, size_t *outSize, double min, double max)
unsigned char dsLengthBytes[exe_params->SZ_SIZE_TYPE];
intToBytes_bigEndian(dsLengthBytes, dataLength);//4
for (i = 0; i < 3; i++)//3
(*newByteData)[k++] = versionNumber[i];
newByteData[k++] = versionNumber[i];
if(exe_params->SZ_SIZE_TYPE==4)
{
(*newByteData)[k++] = 16; //=00010000
newByteData[k++] = 16; //=00010000
}
else
{
(*newByteData)[k++] = 80;
newByteData[k++] = 80;
}
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)//4 or 8
(*newByteData)[k++] = dsLengthBytes[i];
newByteData[k++] = dsLengthBytes[i];
if(sysEndianType==BIG_ENDIAN_SYSTEM)
memcpy((*newByteData)+4+exe_params->SZ_SIZE_TYPE, oriData, dataLength*doubleSize);
memcpy(newByteData+4+exe_params->SZ_SIZE_TYPE, oriData, dataLength*doubleSize);
else
{
unsigned char* p = (*newByteData)+4+exe_params->SZ_SIZE_TYPE;
unsigned char* p = newByteData+4+exe_params->SZ_SIZE_TYPE;
for(i=0;i<dataLength;i++,p+=doubleSize)
doubleToBytes(p, oriData[i]);
}
@ -404,7 +403,7 @@ void compressGroupIDArray_double(char* groupID, TightDataPointStorageD* tdps)
}
TightDataPointStorageD* SZ_compress_double_1D_MDQ_pwrGroup(double* oriData, size_t dataLength, int errBoundMode,
double absErrBound, double relBoundRatio, double pwrErrRatio, double valueRangeSize, double medianValue_f)
double absErrBound, double relBoundRatio, double pwrErrRatio, double valueRangeSize, double medianValue_f)
{
size_t i;
double *posGroups, *negGroups, *groups;
@ -642,7 +641,7 @@ double absErrBound, double relBoundRatio, double pwrErrRatio, double valueRangeS
return tdps;
}
void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwrgroup(unsigned char** newByteData, double *oriData,
void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwrgroup(unsigned char* newByteData, double *oriData,
size_t dataLength, double absErrBound, double relBoundRatio, double pwrErrRatio, double valueRangeSize, double medianValue_f, size_t *outSize)
{
TightDataPointStorageD* tdps = SZ_compress_double_1D_MDQ_pwrGroup(oriData, dataLength, confparams_cpr->errorBoundMode,
@ -659,7 +658,7 @@ size_t dataLength, double absErrBound, double relBoundRatio, double pwrErrRatio,
#include <stdbool.h>
void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log(unsigned char** newByteData, double *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, double min, double max){
void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log(unsigned char* newByteData, double *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, double min, double max){
double * log_data = (double *) malloc(dataLength * sizeof(double));
@ -719,7 +718,7 @@ void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log(unsigned char** newBy
free_TightDataPointStorageD(tdps);
}
void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log_MSST19(unsigned char** newByteData, double *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, double valueRangeSize, double medianValue_f,
void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log_MSST19(unsigned char* newByteData, double *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, double valueRangeSize, double medianValue_f,
unsigned char* signs, bool* positive, double min, double max, double nearZero){
double multiplier = pow((1+pwrErrRatio), -3.0001);
for(int i=0; i<dataLength; i++){

1416
deps/SZ/sz/src/sz_float.c vendored
View File

File diff suppressed because it is too large Load Diff

33
deps/SZ/sz/src/sz_float_pwr.c vendored
View File

@ -150,7 +150,7 @@ unsigned int optimize_intervals_float_1D_pwr(float *oriData, size_t dataLength,
return powerOf2;
}
void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr(unsigned char** newByteData, float *oriData, double globalPrecision,
void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr(unsigned char* newByteData, float *oriData, double globalPrecision,
size_t dataLength, size_t *outSize, float min, float max)
{
size_t pwrLength = dataLength%confparams_cpr->segment_size==0?dataLength/confparams_cpr->segment_size:dataLength/confparams_cpr->segment_size+1;
@ -284,8 +284,6 @@ size_t dataLength, size_t *outSize, float min, float max)
listAdd_float(last3CmprsData, vce->data);
}//end of for
// char* expSegmentsInBytes;
// int expSegmentsInBytes_size = convertESCToBytes(esc, &expSegmentsInBytes);
int exactDataNum = exactLeadNumArray->size;
TightDataPointStorageF* tdps;
@ -297,17 +295,6 @@ size_t dataLength, size_t *outSize, float min, float max)
resiBitLengthArray->array, resiBitLengthArray->size,
realPrecision, medianValue, (char)reqLength, quantization_intervals, pwrErrBoundBytes, pwrErrBoundBytes_size, radExpo);
//sdi:Debug
/* int sum =0;
for(i=0;i<dataLength;i++)
if(type[i]==0) sum++;
printf("opt_quantizations=%d, exactDataNum=%d, sum=%d\n",quantization_intervals, exactDataNum, sum);
*/
// writeUShortData(type, dataLength, "compressStateBytes.sb");
// unsigned short type_[dataLength];
// SZ_Reset();
// decode_withTree(tdps->typeArray, tdps->typeArray_size, type_);
// printf("tdps->typeArray_size=%d\n", tdps->typeArray_size);
//free memory
free_DBA(resiBitLengthArray);
@ -328,25 +315,25 @@ size_t dataLength, size_t *outSize, float min, float max)
unsigned char dsLengthBytes[exe_params->SZ_SIZE_TYPE];
intToBytes_bigEndian(dsLengthBytes, dataLength);//4
for (i = 0; i < 3; i++)//3
(*newByteData)[k++] = versionNumber[i];
newByteData[k++] = versionNumber[i];
if(exe_params->SZ_SIZE_TYPE==4)
{
(*newByteData)[k++] = 16; //=00010000
newByteData[k++] = 16; //=00010000
}
else
{
(*newByteData)[k++] = 80;
newByteData[k++] = 80;
}
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)//4 or 8
(*newByteData)[k++] = dsLengthBytes[i];
newByteData[k++] = dsLengthBytes[i];
if(sysEndianType==BIG_ENDIAN_SYSTEM)
memcpy((*newByteData)+4+exe_params->SZ_SIZE_TYPE, oriData, dataLength*floatSize);
memcpy(newByteData+4+exe_params->SZ_SIZE_TYPE, oriData, dataLength*floatSize);
else
{
unsigned char* p = (*newByteData)+4+exe_params->SZ_SIZE_TYPE;
unsigned char* p = newByteData+4+exe_params->SZ_SIZE_TYPE;
for(i=0;i<dataLength;i++,p+=floatSize)
floatToBytes(p, oriData[i]);
}
@ -643,7 +630,7 @@ double absErrBound, double relBoundRatio, double pwrErrRatio, float valueRangeSi
return tdps;
}
void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwrgroup(unsigned char** newByteData, float *oriData,
void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwrgroup(unsigned char* newByteData, float *oriData,
size_t dataLength, double absErrBound, double relBoundRatio, double pwrErrRatio, float valueRangeSize, float medianValue_f, size_t *outSize)
{
TightDataPointStorageF* tdps = SZ_compress_float_1D_MDQ_pwrGroup(oriData, dataLength, confparams_cpr->errorBoundMode,
@ -660,7 +647,7 @@ size_t dataLength, double absErrBound, double relBoundRatio, double pwrErrRatio,
#include <stdbool.h>
void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_pre_log(unsigned char** newByteData, float *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, float min, float max){
void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_pre_log(unsigned char* newByteData, float *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, float min, float max){
float * log_data = (float *) malloc(dataLength * sizeof(float));
@ -722,7 +709,7 @@ void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_pre_log(unsigned char** newByt
}
void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_pre_log_MSST19(unsigned char** newByteData, float *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, float valueRangeSize, float medianValue_f,
void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_pre_log_MSST19(unsigned char* newByteData, float *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, float valueRangeSize, float medianValue_f,
unsigned char* signs, bool* positive, float min, float max, float nearZero){
float multiplier = pow((1+pwrErrRatio), -3.0001);
for(int i=0; i<dataLength; i++){

207
deps/SZ/sz/src/sz_float_ts.c vendored
View File

@ -1,207 +0,0 @@
/**
* @file sz_float.c
* @author Sheng Di and Dingwen Tao
* @date Aug, 2016
* @brief SZ_Init, Compression and Decompression functions
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <math.h>
#include "sz.h"
#include "CompressElement.h"
#include "DynamicByteArray.h"
#include "DynamicIntArray.h"
#include "TightDataPointStorageF.h"
#include "zlib.h"
#include "rw.h"
#include "sz_float_ts.h"
unsigned int optimize_intervals_float_1D_ts(float *oriData, size_t dataLength, float* preData, double realPrecision)
{
size_t i = 0, radiusIndex;
float pred_value = 0, pred_err;
size_t *intervals = (size_t*)malloc(confparams_cpr->maxRangeRadius*sizeof(size_t));
memset(intervals, 0, confparams_cpr->maxRangeRadius*sizeof(size_t));
size_t totalSampleSize = dataLength/confparams_cpr->sampleDistance;
for(i=2;i<dataLength;i++)
{
if(i%confparams_cpr->sampleDistance==0)
{
pred_value = preData[i];
pred_err = fabs(pred_value - oriData[i]);
radiusIndex = (unsigned long)((pred_err/realPrecision+1)/2);
if(radiusIndex>=confparams_cpr->maxRangeRadius)
radiusIndex = confparams_cpr->maxRangeRadius - 1;
intervals[radiusIndex]++;
}
}
//compute the appropriate number
size_t targetCount = totalSampleSize*confparams_cpr->predThreshold;
size_t sum = 0;
for(i=0;i<confparams_cpr->maxRangeRadius;i++)
{
sum += intervals[i];
if(sum>targetCount)
break;
}
if(i>=confparams_cpr->maxRangeRadius)
i = confparams_cpr->maxRangeRadius-1;
unsigned int accIntervals = 2*(i+1);
unsigned int powerOf2 = roundUpToPowerOf2(accIntervals);
if(powerOf2<32)
powerOf2 = 32;
free(intervals);
return powerOf2;
}
TightDataPointStorageF* SZ_compress_float_1D_MDQ_ts(float *oriData, size_t dataLength, sz_multisteps* multisteps,
double realPrecision, float valueRangeSize, float medianValue_f)
{
float* preStepData = (float*)(multisteps->hist_data);
//store the decompressed data
//float* decData = (float*)malloc(sizeof(float)*dataLength);
//memset(decData, 0, sizeof(float)*dataLength);
float* decData = preStepData;
unsigned int quantization_intervals;
if(exe_params->optQuantMode==1)
quantization_intervals = optimize_intervals_float_1D_ts(oriData, dataLength, preStepData, realPrecision);
else
quantization_intervals = exe_params->intvCapacity;
updateQuantizationInfo(quantization_intervals);
size_t i;
int reqLength;
float medianValue = medianValue_f;
short radExpo = getExponent_float(valueRangeSize/2);
computeReqLength_float(realPrecision, radExpo, &reqLength, &medianValue);
int* type = (int*) malloc(dataLength*sizeof(int));
float* spaceFillingValue = oriData; //
DynamicIntArray *exactLeadNumArray;
new_DIA(&exactLeadNumArray, DynArrayInitLen);
DynamicByteArray *exactMidByteArray;
new_DBA(&exactMidByteArray, DynArrayInitLen);
DynamicIntArray *resiBitArray;
new_DIA(&resiBitArray, DynArrayInitLen);
unsigned char preDataBytes[4];
intToBytes_bigEndian(preDataBytes, 0);
int reqBytesLength = reqLength/8;
int resiBitsLength = reqLength%8;
FloatValueCompressElement *vce = (FloatValueCompressElement*)malloc(sizeof(FloatValueCompressElement));
LossyCompressionElement *lce = (LossyCompressionElement*)malloc(sizeof(LossyCompressionElement));
//add the first data
type[0] = 0;
compressSingleFloatValue(vce, spaceFillingValue[0], realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Float(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,4);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
decData[0] = vce->data;
//add the second data
type[1] = 0;
compressSingleFloatValue(vce, spaceFillingValue[1], realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Float(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,4);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
decData[1] = vce->data;
int state = 0;
double checkRadius = 0;
float curData = 0;
float pred = 0;
float predAbsErr = 0;
checkRadius = (exe_params->intvCapacity-1)*realPrecision;
double interval = 2*realPrecision;
for(i=2;i<dataLength;i++)
{
curData = spaceFillingValue[i];
pred = preStepData[i];
predAbsErr = fabs(curData - pred);
if(predAbsErr<=checkRadius)
{
state = (predAbsErr/realPrecision+1)/2;
if(curData>=pred)
{
type[i] = exe_params->intvRadius+state;
pred = pred + state*interval;
}
else //curData<pred
{
type[i] = exe_params->intvRadius-state;
pred = pred - state*interval;
}
//double-check the prediction error in case of machine-epsilon impact
if(fabs(curData-pred)>realPrecision)
{
type[i] = 0;
compressSingleFloatValue(vce, curData, realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Float(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,4);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
decData[i] = vce->data;
}
else
{
decData[i] = pred;
}
continue;
}
//unpredictable data processing
type[i] = 0;
compressSingleFloatValue(vce, curData, realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Float(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,4);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
decData[i] = vce->data;
}//end of for
size_t exactDataNum = exactLeadNumArray->size;
TightDataPointStorageF* tdps;
new_TightDataPointStorageF(&tdps, dataLength, exactDataNum,
type, exactMidByteArray->array, exactMidByteArray->size,
exactLeadNumArray->array,
resiBitArray->array, resiBitArray->size,
resiBitsLength,
realPrecision, medianValue, (char)reqLength, quantization_intervals, NULL, 0, 0);
//free memory
free_DIA(exactLeadNumArray);
free_DIA(resiBitArray);
free(type);
free(vce);
free(lce);
free(exactMidByteArray); //exactMidByteArray->array has been released in free_TightDataPointStorageF(tdps);
//memcpy(preStepData, decData, dataLength*sizeof(float)); //update the data
//free(decData);
return tdps;
}

986
deps/SZ/sz/src/sz_omp.c vendored
View File

@ -1,986 +0,0 @@
/**
* @file sz_omp.c
* @author Xin Liang
* @date July, 2017
* @brief the implementation of openMP version
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#include "sz_omp.h"
#include <math.h>
#include <time.h>
double sz_wtime(){
#ifdef _OPENMP
return omp_get_wtime();
#else
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
return (double)ts.tv_sec + (double)ts.tv_nsec / 1000000000.0;
#endif
}
int sz_get_max_threads(){
#ifdef _OPENMP
return omp_get_max_threads();
#else
return 1;
#endif
}
int sz_get_thread_num(){
#ifdef _OPENMP
return omp_get_thread_num();
#else
return 0;
#endif
}
void sz_set_num_threads(int nthreads){
#ifdef _OPENMP
omp_set_num_threads(nthreads);
#endif
}
unsigned char * SZ_compress_float_1D_MDQ_openmp(float *oriData, size_t r1, double realPrecision, size_t * comp_size){
return NULL;
}
unsigned char * SZ_compress_float_2D_MDQ_openmp(float *oriData, size_t r1, size_t r2, double realPrecision, size_t * comp_size){
return NULL;
}
unsigned char * SZ_compress_float_3D_MDQ_openmp(float *oriData, size_t r1, size_t r2, size_t r3, float realPrecision, size_t * comp_size){
float elapsed_time = 0.0;
elapsed_time = -sz_wtime();
unsigned int quantization_intervals;
if(exe_params->optQuantMode==1)
{
// quantization_intervals = optimize_intervals_float_3D(oriData, r1, realPrecision);
quantization_intervals = optimize_intervals_float_3D_opt(oriData, r1, r2, r3, realPrecision);
//quantization_intervals = 32768;
printf("3D number of bins: %d\nerror bound %.20f\n", quantization_intervals, realPrecision);
// exit(0);
updateQuantizationInfo(quantization_intervals);
}
else{
quantization_intervals = exe_params->intvCapacity;
}
elapsed_time += sz_wtime();
printf("opt interval time: %.4f\n", elapsed_time);
elapsed_time = -sz_wtime();
int thread_num = sz_get_max_threads();
int thread_order = (int)log2(thread_num);
size_t num_x = 0, num_y = 0, num_z = 0;
{
int block_thread_order = thread_order / 3;
switch(thread_order % 3){
case 0:{
num_x = 1 << block_thread_order;
num_y = 1 << block_thread_order;
num_z = 1 << block_thread_order;
break;
}
case 1:{
num_x = 1 << (block_thread_order + 1);
num_y = 1 << block_thread_order;
num_z = 1 << block_thread_order;
break;
}
case 2:{
num_x = 1 << (block_thread_order + 1);
num_y = 1 << (block_thread_order + 1);
num_z = 1 << block_thread_order;
break;
}
}
thread_num = num_x * num_y * num_z;
}
sz_set_num_threads(thread_num);
// calculate block dims
printf("number of blocks: %zu %zu %zu\n", num_x, num_y, num_z);
size_t split_index_x, split_index_y, split_index_z;
size_t early_blockcount_x, early_blockcount_y, early_blockcount_z;
size_t late_blockcount_x, late_blockcount_y, late_blockcount_z;
SZ_COMPUTE_BLOCKCOUNT(r1, num_x, split_index_x, early_blockcount_x, late_blockcount_x);
SZ_COMPUTE_BLOCKCOUNT(r2, num_y, split_index_y, early_blockcount_y, late_blockcount_y);
SZ_COMPUTE_BLOCKCOUNT(r3, num_z, split_index_z, early_blockcount_z, late_blockcount_z);
size_t max_num_block_elements = early_blockcount_x * early_blockcount_y * early_blockcount_z;
size_t num_blocks = num_x * num_y * num_z;
size_t num_elements = r1 * r2 * r3;
// printf("max_num_block_elements %d num_blocks %d\n", max_num_block_elements, num_blocks);
size_t dim0_offset = r2 * r3;
size_t dim1_offset = r3;
// printf("malloc blockinfo array start\n");
// fflush(stdout);
size_t buffer_size = early_blockcount_y * early_blockcount_z * sizeof(float);
int * result_type = (int *) malloc(num_elements * sizeof(int));
size_t unpred_data_max_size = max_num_block_elements;
float * result_unpredictable_data = (float *) malloc(unpred_data_max_size * sizeof(float) * num_blocks);
unsigned int * unpredictable_count = (unsigned int *) malloc(num_blocks * sizeof(unsigned int));
float * mean = malloc(num_blocks * sizeof(float));
float * buffer0, * buffer1;
buffer0 = (float *) malloc(buffer_size * thread_num);
buffer1 = (float *) malloc(buffer_size * thread_num);
unsigned char * result = (unsigned char *) malloc(num_elements * (sizeof(int) + sizeof(float)));
size_t * unpred_offset = (size_t *) malloc(num_blocks * sizeof(size_t));
unsigned char * encoding_buffer = (unsigned char *) malloc(max_num_block_elements * sizeof(int) * num_blocks);
size_t * block_offset = (size_t *) malloc(num_blocks * sizeof(size_t));
size_t *freq = (size_t *)malloc(thread_num*quantization_intervals*4*sizeof(size_t));
memset(freq, 0, thread_num*quantization_intervals*4*sizeof(size_t));
size_t stateNum = quantization_intervals*2;
HuffmanTree* huffmanTree = createHuffmanTree(stateNum);
int num_yz = num_y * num_z;
#pragma omp parallel for
for(int t=0; t<thread_num; t++){
int id = sz_get_thread_num();
int i = id/(num_yz);
int j = (id % num_yz) / num_z;
int k = id % num_z;
// printf("%d: %d %d %d\n", sz_get_thread_num(), i, j, k);
size_t offset_x = (i < split_index_x) ? i * early_blockcount_x : i * late_blockcount_x + split_index_x;
size_t offset_y = (j < split_index_y) ? j * early_blockcount_y : j * late_blockcount_y + split_index_y;
size_t offset_z = (k < split_index_z) ? k * early_blockcount_z : k * late_blockcount_z + split_index_z;
float * data_pos = oriData + offset_x * dim0_offset + offset_y * dim1_offset + offset_z;
size_t current_blockcount_x = (i < split_index_x) ? early_blockcount_x : late_blockcount_x;
size_t current_blockcount_y = (j < split_index_y) ? early_blockcount_y : late_blockcount_y;
size_t current_blockcount_z = (k < split_index_z) ? early_blockcount_z : late_blockcount_z;
size_t type_offset = offset_x * dim0_offset + offset_y * current_blockcount_x * dim1_offset + offset_z * current_blockcount_x * current_blockcount_y;
int * type = result_type + type_offset;
float * unpredictable_data = result_unpredictable_data + id * unpred_data_max_size;
float *P0, *P1; // buffer
// P0 = (float *) malloc(buffer_size);
// P1 = (float *) malloc(buffer_size);
P0 = buffer0 + id * early_blockcount_y * early_blockcount_z;
P1 = buffer1 + id * early_blockcount_y * early_blockcount_z;
unpredictable_count[id] = SZ_compress_float_3D_MDQ_RA_block(data_pos, mean + id, r1, r2, r3, current_blockcount_x, current_blockcount_y, current_blockcount_z, realPrecision, P0, P1, type, unpredictable_data);
// free(P0);
// free(P1);
}
elapsed_time += sz_wtime();
printf("compression and quantization time: %.4f\n", elapsed_time);
elapsed_time = -sz_wtime();
// printf("unpred count:\n");
// for(int i=0; i<num_blocks; i++){
// printf("%d ", unpredictable_count[i]);
// }
// printf("\n");
// printf("total_unpred num: %d\n", total_unpred);
// printf("Block wise compression end, num_elements %ld\n", num_elements);
// huffman encode
size_t nodeCount = 0;
Huffman_init_openmp(huffmanTree, result_type, num_elements, thread_num, freq);
elapsed_time += sz_wtime();
printf("Build Huffman: %.4f\n", elapsed_time);
elapsed_time = -sz_wtime();
for (size_t i = 0; i < stateNum; i++)
if (huffmanTree->code[i]) nodeCount++;
nodeCount = nodeCount*2-1;
unsigned char *treeBytes;
unsigned int treeByteSize = convert_HuffTree_to_bytes_anyStates(huffmanTree, nodeCount, &treeBytes);
unsigned int meta_data_offset = 3 + 1 + MetaDataByteLength;
size_t total_unpred = 0;
for(int i=0; i<num_blocks; i++){
total_unpred += unpredictable_count[i];
// printf("%d: %d mean %.2f\n", i, unpredictable_count[i], mean[i]);
}
unsigned char * result_pos = result;
initRandomAccessBytes(result_pos);
result_pos += meta_data_offset;
size_t enCodeSize = 0;
intToBytes_bigEndian(result_pos, thread_num);
result_pos += 4;
floatToBytes(result_pos, realPrecision);
result_pos += sizeof(float);
intToBytes_bigEndian(result_pos, quantization_intervals);
result_pos += 4;
intToBytes_bigEndian(result_pos, treeByteSize);
result_pos += 4;
intToBytes_bigEndian(result_pos, nodeCount);
result_pos += 4;
memcpy(result_pos, treeBytes, treeByteSize);
result_pos += treeByteSize;
memcpy(result_pos, unpredictable_count, num_blocks * sizeof(unsigned int));
result_pos += num_blocks * sizeof(unsigned int);
memcpy(result_pos, mean, num_blocks * sizeof(float));
result_pos += num_blocks * sizeof(float);
// printf("unpred offset: %ld\n", result_pos - result);
// store unpredicable data
// float * unpred_pos = (float *) result_pos;
// for(int t=0; t<thread_num; t++){
// float * unpredictable_data = result_unpredictable_data + t * unpred_data_max_size;
// memcpy(result_pos, unpredictable_data, unpredictable_count[t] * sizeof(float));
// result_pos += unpredictable_count[t]*sizeof(float);
// }
unpred_offset[0] = 0;
for(int t=1; t<thread_num; t++){
unpred_offset[t] = unpredictable_count[t-1] + unpred_offset[t-1];
}
#pragma omp parallel for
for(int t=0; t<thread_num; t++){
int id = sz_get_thread_num();
float * unpredictable_data = result_unpredictable_data + id * unpred_data_max_size;
memcpy(result_pos + unpred_offset[id] * sizeof(float), unpredictable_data, unpredictable_count[id] * sizeof(float));
}
result_pos += total_unpred * sizeof(float);
elapsed_time += sz_wtime();
printf("write misc time: %.4f\n", elapsed_time);
elapsed_time = -sz_wtime();
size_t * block_pos = (size_t *) result_pos;
result_pos += num_blocks * sizeof(size_t);
#pragma omp parallel for
for(int t=0; t<thread_num; t++){
int id = sz_get_thread_num();
int i = id/(num_yz);
int j = (id % num_yz) / num_z;
int k = id % num_z;
unsigned char * encoding_buffer_pos = encoding_buffer + id * max_num_block_elements * sizeof(int);
size_t enCodeSize = 0;
size_t offset_x = (i < split_index_x) ? i * early_blockcount_x : i * late_blockcount_x + split_index_x;
size_t offset_y = (j < split_index_y) ? j * early_blockcount_y : j * late_blockcount_y + split_index_y;
size_t offset_z = (k < split_index_z) ? k * early_blockcount_z : k * late_blockcount_z + split_index_z;
size_t current_blockcount_x = (i < split_index_x) ? early_blockcount_x : late_blockcount_x;
size_t current_blockcount_y = (j < split_index_y) ? early_blockcount_y : late_blockcount_y;
size_t current_blockcount_z = (k < split_index_z) ? early_blockcount_z : late_blockcount_z;
size_t current_block_elements = current_blockcount_x * current_blockcount_y * current_blockcount_z;
size_t type_offset = offset_x * dim0_offset + offset_y * current_blockcount_x * dim1_offset + offset_z * current_blockcount_x * current_blockcount_y;
int * type = result_type + type_offset;
encode(huffmanTree, type, current_block_elements, encoding_buffer_pos, &enCodeSize);
block_pos[id] = enCodeSize;
}
elapsed_time += sz_wtime();
printf("Parallel Huffman encoding elapsed time: %.4f\n", elapsed_time);
elapsed_time = -sz_wtime();
// for(int t=0; t<thread_num; t++){
// memcpy(result_pos, encoding_buffer + t * max_num_block_elements * sizeof(int), block_pos[t]);
// result_pos += block_pos[t];
// }
block_offset[0] = 0;
for(int t=1; t<thread_num; t++){
block_offset[t] = block_pos[t-1] + block_offset[t-1];
}
#pragma omp parallel for
for(int t=0; t<thread_num; t++){
int id = sz_get_thread_num();
memcpy(result_pos + block_offset[id], encoding_buffer + t * max_num_block_elements * sizeof(int), block_pos[t]);
}
result_pos += block_offset[thread_num - 1] + block_pos[thread_num - 1];
elapsed_time += sz_wtime();
printf("Final copy elapsed time: %.4f\n", elapsed_time);
// {
// int status;
// writeIntData_inBytes(result_type, num_elements, "/Users/LiangXin/github/SZ-develop/example/openmp/comp001_type.dat", &status);
// }
// int status;
// writeIntData_inBytes(result_type, num_elements, "/Users/LiangXin/github/SZ-develop/example/openmp/omp_type.dat", &status);
// printf("type array size: %ld\n", enCodeSize);
result_pos += enCodeSize;
size_t totalEncodeSize = 0;
totalEncodeSize = result_pos - result;
// printf("Total size %ld\n", totalEncodeSize);
free(freq);
free(buffer0);
free(buffer1);
free(treeBytes);
free(unpred_offset);
free(block_offset);
free(encoding_buffer);
free(mean);
free(result_unpredictable_data);
free(unpredictable_count);
free(result_type);
SZ_ReleaseHuffman(huffmanTree);
*comp_size = totalEncodeSize;
return result;
}
void decompressDataSeries_float_1D_openmp(float** data, size_t r1, unsigned char* comp_data){
}
void decompressDataSeries_float_2D_openmp(float** data, size_t r1, size_t r2, unsigned char* comp_data){
}
void decompressDataSeries_float_3D_openmp(float** data, size_t r1, size_t r2, size_t r3, unsigned char* comp_data){
if(confparams_dec==NULL)
confparams_dec = (sz_params*)malloc(sizeof(sz_params));
memset(confparams_dec, 0, sizeof(sz_params));
if(exe_params==NULL)
exe_params = (sz_exedata*)malloc(sizeof(sz_exedata));
memset(exe_params, 0, sizeof(sz_exedata));
// printf("num_block_elements %d num_blocks %d\n", max_num_block_elements, num_blocks);
// fflush(stdout);
double elapsed_time = 0.0;
elapsed_time = -sz_wtime();
size_t dim0_offset = r2 * r3;
size_t dim1_offset = r3;
size_t num_elements = r1 * r2 * r3;
unsigned char * comp_data_pos = comp_data;
//int meta_data_offset = 3 + 1 + MetaDataByteLength;
//comp_data_pos += meta_data_offset;
int thread_num = bytesToInt_bigEndian(comp_data_pos);
comp_data_pos += 4;
int thread_order = (int)log2(thread_num);
size_t num_x = 0, num_y = 0, num_z = 0;
{
int block_thread_order = thread_order / 3;
switch(thread_order % 3){
case 0:{
num_x = 1 << block_thread_order;
num_y = 1 << block_thread_order;
num_z = 1 << block_thread_order;
break;
}
case 1:{
num_x = 1 << (block_thread_order + 1);
num_y = 1 << block_thread_order;
num_z = 1 << block_thread_order;
break;
}
case 2:{
num_x = 1 << (block_thread_order + 1);
num_y = 1 << (block_thread_order + 1);
num_z = 1 << block_thread_order;
break;
}
}
}
printf("number of blocks: %zu %zu %zu, thread_num %d\n", num_x, num_y, num_z, thread_num);
sz_set_num_threads(thread_num);
size_t split_index_x, split_index_y, split_index_z;
size_t early_blockcount_x, early_blockcount_y, early_blockcount_z;
size_t late_blockcount_x, late_blockcount_y, late_blockcount_z;
SZ_COMPUTE_BLOCKCOUNT(r1, num_x, split_index_x, early_blockcount_x, late_blockcount_x);
SZ_COMPUTE_BLOCKCOUNT(r2, num_y, split_index_y, early_blockcount_y, late_blockcount_y);
SZ_COMPUTE_BLOCKCOUNT(r3, num_z, split_index_z, early_blockcount_z, late_blockcount_z);
size_t num_blocks = num_x * num_y * num_z;
size_t * unpred_offset = (size_t *) malloc(num_blocks * sizeof(size_t));
*data = (float*)malloc(sizeof(float)*num_elements);
int * result_type = (int *) malloc(num_elements * sizeof(int));
size_t * block_offset = (size_t *) malloc(num_blocks * sizeof(size_t));
float realPrecision = bytesToFloat(comp_data_pos);
comp_data_pos += sizeof(float);
unsigned int intervals = bytesToInt_bigEndian(comp_data_pos);
comp_data_pos += sizeof(float);
size_t stateNum = intervals*2;
HuffmanTree* huffmanTree = createHuffmanTree(stateNum);
updateQuantizationInfo(intervals);
// exe_params->intvRadius = (int)((tdps->intervals - 1)/ 2);
unsigned int tree_size = bytesToInt_bigEndian(comp_data_pos);
comp_data_pos += sizeof(unsigned int);
size_t huffman_nodes = bytesToInt_bigEndian(comp_data_pos);
huffmanTree->allNodes = huffman_nodes;
// printf("Reconstruct huffman tree with node count %ld\n", nodeCount);
// fflush(stdout);
node root = reconstruct_HuffTree_from_bytes_anyStates(huffmanTree, comp_data_pos+4, huffmanTree->allNodes);
comp_data_pos += 4 + tree_size;
unsigned int * unpred_count = (unsigned int *) comp_data_pos;
comp_data_pos += num_blocks * sizeof(unsigned int);
float * mean_pos = (float *) comp_data_pos;
comp_data_pos += num_blocks * sizeof(float);
float * result_unpredictable_data = (float *) comp_data_pos;
size_t total_unpred = 0;
for(int i=0; i<num_blocks; i++){
unpred_offset[i] = total_unpred;
total_unpred += unpred_count[i];
}
comp_data_pos += total_unpred * sizeof(float);
// printf("unpred count:\n");
// for(int i=0; i<num_blocks; i++){
// printf("%d ", unpred_count[i]);
// }
// printf("\n");
// for(int i=0; i<1000; i++){
// printf("%.2f ", result_unpredictable_data[i]);
// }
// printf("\ntotal_unpred num: %d\n", total_unpred);
// for(int i=0; i<num_blocks; i++){
// printf("%d unpred offset %ld\n", i, unpred_offset[i]);
// for(int tmp=0; tmp<10; tmp++){
// printf("%.2f ", (result_unpredictable_data + unpred_offset[i])[tmp]);
// }
// printf("\n");
// }
// exit(0);
// printf("Block wise decompression start: %d %d %d\n", early_blockcount_x, early_blockcount_y, early_blockcount_z);
// fflush(stdout);
// decode(comp_data_pos, num_elements, root, result_type);
size_t * block_pos = (size_t *) comp_data_pos;
comp_data_pos += num_blocks * sizeof(size_t);
block_offset[0] = 0;
for(int t=1; t<thread_num; t++){
block_offset[t] = block_pos[t-1] + block_offset[t-1];
}
int num_yz = num_y * num_z;
elapsed_time += sz_wtime();
printf("Read data info elapsed time: %.4f\n", elapsed_time);
elapsed_time = -sz_wtime();
#pragma omp parallel for
for(int t=0; t<thread_num; t++){
int id = sz_get_thread_num();
int i = id/(num_yz);
int j = (id % num_yz) / num_z;
int k = id % num_z;
size_t offset_x = (i < split_index_x) ? i * early_blockcount_x : i * late_blockcount_x + split_index_x;
size_t offset_y = (j < split_index_y) ? j * early_blockcount_y : j * late_blockcount_y + split_index_y;
size_t offset_z = (k < split_index_z) ? k * early_blockcount_z : k * late_blockcount_z + split_index_z;
size_t current_blockcount_x = (i < split_index_x) ? early_blockcount_x : late_blockcount_x;
size_t current_blockcount_y = (j < split_index_y) ? early_blockcount_y : late_blockcount_y;
size_t current_blockcount_z = (k < split_index_z) ? early_blockcount_z : late_blockcount_z;
size_t type_offset = offset_x * dim0_offset + offset_y * current_blockcount_x * dim1_offset + offset_z * current_blockcount_x * current_blockcount_y;
int * type = result_type + type_offset;
decode(comp_data_pos + block_offset[id], current_blockcount_x*current_blockcount_y*current_blockcount_z, root, type);
}
elapsed_time += sz_wtime();
printf("Parallel Huffman decoding elapsed time: %.4f\n", elapsed_time);
elapsed_time = -sz_wtime();
#pragma omp parallel for
for(int t=0; t<thread_num; t++){
int id = sz_get_thread_num();
int i = id/(num_yz);
int j = (id % num_yz) / num_z;
int k = id % num_z;
// printf("%d: %d %d %d\n", sz_get_thread_num(), i, j, k);
size_t offset_x = (i < split_index_x) ? i * early_blockcount_x : i * late_blockcount_x + split_index_x;
size_t offset_y = (j < split_index_y) ? j * early_blockcount_y : j * late_blockcount_y + split_index_y;
size_t offset_z = (k < split_index_z) ? k * early_blockcount_z : k * late_blockcount_z + split_index_z;
float * data_pos = *data + offset_x * dim0_offset + offset_y * dim1_offset + offset_z;
size_t current_blockcount_x = (i < split_index_x) ? early_blockcount_x : late_blockcount_x;
size_t current_blockcount_y = (j < split_index_y) ? early_blockcount_y : late_blockcount_y;
size_t current_blockcount_z = (k < split_index_z) ? early_blockcount_z : late_blockcount_z;
size_t type_offset = offset_x * dim0_offset + offset_y * current_blockcount_x * dim1_offset + offset_z * current_blockcount_x * current_blockcount_y;
int * type = result_type + type_offset;
float * unpredictable_data = result_unpredictable_data + unpred_offset[id];
float mean = mean_pos[id];
// printf("\n%d\ndata_offset: %ld\n", t, offset_x * dim0_offset + offset_y * dim1_offset + offset_z);
// printf("mean: %.2f\n", mean);
// for(int tmp=0; tmp<10; tmp++){
// printf("%.2f ", unpredictable_data[tmp]);
// }
// printf("\n\n");
decompressDataSeries_float_3D_RA_block(data_pos, mean, r1, r2, r3, current_blockcount_x, current_blockcount_y, current_blockcount_z, realPrecision, type, unpredictable_data);
}
elapsed_time += sz_wtime();
printf("Parallel decompress elapsed time: %.4f\n", elapsed_time);
free(block_offset);
free(result_type);
free(unpred_offset);
SZ_ReleaseHuffman(huffmanTree);
}
//Double Precision
unsigned char * SZ_compress_double_1D_MDQ_openmp(double *oriData, size_t r1, double realPrecision, size_t * comp_size){
return NULL;
}
unsigned char * SZ_compress_double_2D_MDQ_openmp(double *oriData, size_t r1, size_t r2, double realPrecision, size_t * comp_size){
return NULL;
}
unsigned char * SZ_compress_double_3D_MDQ_openmp(double *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, size_t * comp_size){
float elapsed_time = 0.0;
elapsed_time = -sz_wtime();
unsigned int quantization_intervals;
if(exe_params->optQuantMode==1)
{
// quantization_intervals = optimize_intervals_float_3D(oriData, r1, realPrecision);
quantization_intervals = optimize_intervals_double_3D_opt(oriData, r1, r2, r3, realPrecision);
//quantization_intervals = 32768;
printf("3D number of bins: %d\nerror bound %.20f\n", quantization_intervals, realPrecision);
// exit(0);
updateQuantizationInfo(quantization_intervals);
}
else{
quantization_intervals = exe_params->intvCapacity;
}
elapsed_time += sz_wtime();
printf("opt interval time: %.4f\n", elapsed_time);
elapsed_time = -sz_wtime();
int thread_num = sz_get_max_threads();
int thread_order = (int)log2(thread_num);
size_t num_x = 0, num_y = 0, num_z = 0;
{
int block_thread_order = thread_order / 3;
switch(thread_order % 3){
case 0:{
num_x = 1 << block_thread_order;
num_y = 1 << block_thread_order;
num_z = 1 << block_thread_order;
break;
}
case 1:{
num_x = 1 << (block_thread_order + 1);
num_y = 1 << block_thread_order;
num_z = 1 << block_thread_order;
break;
}
case 2:{
num_x = 1 << (block_thread_order + 1);
num_y = 1 << (block_thread_order + 1);
num_z = 1 << block_thread_order;
break;
}
}
thread_num = num_x * num_y * num_z;
}
sz_set_num_threads(thread_num);
// calculate block dims
printf("number of blocks: %zu %zu %zu\n", num_x, num_y, num_z);
size_t split_index_x, split_index_y, split_index_z;
size_t early_blockcount_x, early_blockcount_y, early_blockcount_z;
size_t late_blockcount_x, late_blockcount_y, late_blockcount_z;
SZ_COMPUTE_BLOCKCOUNT(r1, num_x, split_index_x, early_blockcount_x, late_blockcount_x);
SZ_COMPUTE_BLOCKCOUNT(r2, num_y, split_index_y, early_blockcount_y, late_blockcount_y);
SZ_COMPUTE_BLOCKCOUNT(r3, num_z, split_index_z, early_blockcount_z, late_blockcount_z);
size_t max_num_block_elements = early_blockcount_x * early_blockcount_y * early_blockcount_z;
size_t num_blocks = num_x * num_y * num_z;
size_t num_elements = r1 * r2 * r3;
// printf("max_num_block_elements %d num_blocks %d\n", max_num_block_elements, num_blocks);
size_t dim0_offset = r2 * r3;
size_t dim1_offset = r3;
// printf("malloc blockinfo array start\n");
// fflush(stdout);
size_t buffer_size = early_blockcount_y * early_blockcount_z * sizeof(double);
int * result_type = (int *) malloc(num_elements * sizeof(int));
size_t unpred_data_max_size = max_num_block_elements;
double * result_unpredictable_data = (double *) malloc(unpred_data_max_size * sizeof(double) * num_blocks);
unsigned int * unpredictable_count = (unsigned int *) malloc(num_blocks * sizeof(unsigned int));
double * mean = malloc(num_blocks * sizeof(double));
double * buffer0, * buffer1;
buffer0 = (double *) malloc(buffer_size * thread_num);
buffer1 = (double *) malloc(buffer_size * thread_num);
unsigned char * result = (unsigned char *) malloc(num_elements * (sizeof(int) + sizeof(double)));
size_t * unpred_offset = (size_t *) malloc(num_blocks * sizeof(size_t));
unsigned char * encoding_buffer = (unsigned char *) malloc(max_num_block_elements * sizeof(int) * num_blocks);
size_t * block_offset = (size_t *) malloc(num_blocks * sizeof(size_t));
size_t *freq = (size_t *)malloc(thread_num*quantization_intervals*4*sizeof(size_t));
memset(freq, 0, thread_num*quantization_intervals*4*sizeof(size_t));
size_t stateNum = quantization_intervals*2;
HuffmanTree* huffmanTree = createHuffmanTree(stateNum);
int num_yz = num_y * num_z;
#pragma omp parallel for
for(int t=0; t<thread_num; t++){
int id = sz_get_thread_num();
int i = id/(num_yz);
int j = (id % num_yz) / num_z;
int k = id % num_z;
// printf("%d: %d %d %d\n", sz_get_thread_num(), i, j, k);
size_t offset_x = (i < split_index_x) ? i * early_blockcount_x : i * late_blockcount_x + split_index_x;
size_t offset_y = (j < split_index_y) ? j * early_blockcount_y : j * late_blockcount_y + split_index_y;
size_t offset_z = (k < split_index_z) ? k * early_blockcount_z : k * late_blockcount_z + split_index_z;
double * data_pos = oriData + offset_x * dim0_offset + offset_y * dim1_offset + offset_z;
size_t current_blockcount_x = (i < split_index_x) ? early_blockcount_x : late_blockcount_x;
size_t current_blockcount_y = (j < split_index_y) ? early_blockcount_y : late_blockcount_y;
size_t current_blockcount_z = (k < split_index_z) ? early_blockcount_z : late_blockcount_z;
size_t type_offset = offset_x * dim0_offset + offset_y * current_blockcount_x * dim1_offset + offset_z * current_blockcount_x * current_blockcount_y;
int * type = result_type + type_offset;
double * unpredictable_data = result_unpredictable_data + id * unpred_data_max_size;
double *P0, *P1; // buffer
P0 = buffer0 + id * early_blockcount_y * early_blockcount_z;
P1 = buffer1 + id * early_blockcount_y * early_blockcount_z;
unpredictable_count[id] = SZ_compress_double_3D_MDQ_RA_block(data_pos, mean + id, r1, r2, r3, current_blockcount_x, current_blockcount_y, current_blockcount_z, realPrecision, P0, P1, type, unpredictable_data);
}
elapsed_time += sz_wtime();
printf("compression and quantization time: %.4f\n", elapsed_time);
elapsed_time = -sz_wtime();
// printf("unpred count:\n");
// for(int i=0; i<num_blocks; i++){
// printf("%d ", unpredictable_count[i]);
// }
// printf("\n");
// printf("total_unpred num: %d\n", total_unpred);
// printf("Block wise compression end, num_elements %ld\n", num_elements);
// huffman encode
size_t nodeCount = 0;
Huffman_init_openmp(huffmanTree, result_type, num_elements, thread_num, freq);
elapsed_time += sz_wtime();
printf("Build Huffman: %.4f\n", elapsed_time);
elapsed_time = -sz_wtime();
for (size_t i = 0; i < stateNum; i++)
if (huffmanTree->code[i]) nodeCount++;
nodeCount = nodeCount*2-1;
unsigned char *treeBytes;
unsigned int treeByteSize = convert_HuffTree_to_bytes_anyStates(huffmanTree, nodeCount, &treeBytes);
unsigned int meta_data_offset = 3 + 1 + MetaDataByteLength;
size_t total_unpred = 0;
for(int i=0; i<num_blocks; i++){
total_unpred += unpredictable_count[i];
// printf("%d: %d mean %.2f\n", i, unpredictable_count[i], mean[i]);
}
unsigned char * result_pos = result;
initRandomAccessBytes(result_pos);
result_pos += meta_data_offset;
size_t enCodeSize = 0;
intToBytes_bigEndian(result_pos, thread_num);
result_pos += sizeof(int);
doubleToBytes(result_pos, realPrecision);
result_pos += sizeof(double);
intToBytes_bigEndian(result_pos, quantization_intervals);
result_pos += 4;
intToBytes_bigEndian(result_pos, treeByteSize);
result_pos += 4;
intToBytes_bigEndian(result_pos, nodeCount);
result_pos += 4;
memcpy(result_pos, treeBytes, treeByteSize);
result_pos += treeByteSize;
memcpy(result_pos, unpredictable_count, num_blocks * sizeof(unsigned int));
result_pos += num_blocks * sizeof(unsigned int);
memcpy(result_pos, mean, num_blocks * sizeof(double));
result_pos += num_blocks * sizeof(double);
unpred_offset[0] = 0;
for(int t=1; t<thread_num; t++){
unpred_offset[t] = unpredictable_count[t-1] + unpred_offset[t-1];
}
#pragma omp parallel for
for(int t=0; t<thread_num; t++){
int id = sz_get_thread_num();
double * unpredictable_data = result_unpredictable_data + id * unpred_data_max_size;
memcpy(result_pos + unpred_offset[id] * sizeof(double), unpredictable_data, unpredictable_count[id] * sizeof(double));
}
result_pos += total_unpred * sizeof(double);
elapsed_time += sz_wtime();
printf("write misc time: %.4f\n", elapsed_time);
elapsed_time = -sz_wtime();
size_t * block_pos = (size_t *) result_pos;
result_pos += num_blocks * sizeof(size_t);
#pragma omp parallel for
for(int t=0; t<thread_num; t++){
int id = sz_get_thread_num();
int i = id/(num_yz);
int j = (id % num_yz) / num_z;
int k = id % num_z;
unsigned char * encoding_buffer_pos = encoding_buffer + id * max_num_block_elements * sizeof(int);
size_t enCodeSize = 0;
size_t offset_x = (i < split_index_x) ? i * early_blockcount_x : i * late_blockcount_x + split_index_x;
size_t offset_y = (j < split_index_y) ? j * early_blockcount_y : j * late_blockcount_y + split_index_y;
size_t offset_z = (k < split_index_z) ? k * early_blockcount_z : k * late_blockcount_z + split_index_z;
size_t current_blockcount_x = (i < split_index_x) ? early_blockcount_x : late_blockcount_x;
size_t current_blockcount_y = (j < split_index_y) ? early_blockcount_y : late_blockcount_y;
size_t current_blockcount_z = (k < split_index_z) ? early_blockcount_z : late_blockcount_z;
size_t current_block_elements = current_blockcount_x * current_blockcount_y * current_blockcount_z;
size_t type_offset = offset_x * dim0_offset + offset_y * current_blockcount_x * dim1_offset + offset_z * current_blockcount_x * current_blockcount_y;
int * type = result_type + type_offset;
encode(huffmanTree, type, current_block_elements, encoding_buffer_pos, &enCodeSize);
block_pos[id] = enCodeSize;
}
elapsed_time += sz_wtime();
printf("Parallel Huffman encoding elapsed time: %.4f\n", elapsed_time);
elapsed_time = -sz_wtime();
// for(int t=0; t<thread_num; t++){
// memcpy(result_pos, encoding_buffer + t * max_num_block_elements * sizeof(int), block_pos[t]);
// result_pos += block_pos[t];
// }
block_offset[0] = 0;
for(int t=1; t<thread_num; t++){
block_offset[t] = block_pos[t-1] + block_offset[t-1];
}
#pragma omp parallel for
for(int t=0; t<thread_num; t++){
int id = sz_get_thread_num();
memcpy(result_pos + block_offset[id], encoding_buffer + t * max_num_block_elements * sizeof(int), block_pos[t]);
}
result_pos += block_offset[thread_num - 1] + block_pos[thread_num - 1];
elapsed_time += sz_wtime();
printf("Final copy elapsed time: %.4f\n", elapsed_time);
// {
// int status;
// writeIntData_inBytes(result_type, num_elements, "/Users/LiangXin/github/SZ-develop/example/openmp/comp001_type.dat", &status);
// }
// int status;
// writeIntData_inBytes(result_type, num_elements, "/Users/LiangXin/github/SZ-develop/example/openmp/omp_type.dat", &status);
// printf("type array size: %ld\n", enCodeSize);
result_pos += enCodeSize;
size_t totalEncodeSize = 0;
totalEncodeSize = result_pos - result;
// printf("Total size %ld\n", totalEncodeSize);
free(freq);
free(buffer0);
free(buffer1);
free(treeBytes);
free(unpred_offset);
free(block_offset);
free(encoding_buffer);
free(mean);
free(result_unpredictable_data);
free(unpredictable_count);
free(result_type);
SZ_ReleaseHuffman(huffmanTree);
*comp_size = totalEncodeSize;
return result;
}
void decompressDataSeries_double_1D_openmp(double** data, size_t r1, unsigned char* comp_data){
}
void decompressDataSeries_double_2D_openmp(double** data, size_t r1, size_t r2, unsigned char* comp_data){
}
void decompressDataSeries_double_3D_openmp(double** data, size_t r1, size_t r2, size_t r3, unsigned char* comp_data)
{
if(confparams_dec==NULL)
confparams_dec = (sz_params*)malloc(sizeof(sz_params));
memset(confparams_dec, 0, sizeof(sz_params));
if(exe_params==NULL)
exe_params = (sz_exedata*)malloc(sizeof(sz_exedata));
memset(exe_params, 0, sizeof(sz_exedata));
// printf("num_block_elements %d num_blocks %d\n", max_num_block_elements, num_blocks);
// fflush(stdout);
double elapsed_time = 0.0;
elapsed_time = -sz_wtime();
size_t dim0_offset = r2 * r3;
size_t dim1_offset = r3;
size_t num_elements = r1 * r2 * r3;
unsigned char * comp_data_pos = comp_data;
//int meta_data_offset = 3 + 1 + MetaDataByteLength;
//comp_data_pos += meta_data_offset;
int thread_num = bytesToInt_bigEndian(comp_data_pos);
comp_data_pos += sizeof(int);
int thread_order = (int)log2(thread_num);
size_t num_x = 0, num_y = 0, num_z = 0;
{
int block_thread_order = thread_order / 3;
switch(thread_order % 3){
case 0:{
num_x = 1 << block_thread_order;
num_y = 1 << block_thread_order;
num_z = 1 << block_thread_order;
break;
}
case 1:{
num_x = 1 << (block_thread_order + 1);
num_y = 1 << block_thread_order;
num_z = 1 << block_thread_order;
break;
}
case 2:{
num_x = 1 << (block_thread_order + 1);
num_y = 1 << (block_thread_order + 1);
num_z = 1 << block_thread_order;
break;
}
}
}
printf("number of blocks: %zu %zu %zu, thread_num %d\n", num_x, num_y, num_z, thread_num);
sz_set_num_threads(thread_num);
size_t split_index_x, split_index_y, split_index_z;
size_t early_blockcount_x, early_blockcount_y, early_blockcount_z;
size_t late_blockcount_x, late_blockcount_y, late_blockcount_z;
SZ_COMPUTE_BLOCKCOUNT(r1, num_x, split_index_x, early_blockcount_x, late_blockcount_x);
SZ_COMPUTE_BLOCKCOUNT(r2, num_y, split_index_y, early_blockcount_y, late_blockcount_y);
SZ_COMPUTE_BLOCKCOUNT(r3, num_z, split_index_z, early_blockcount_z, late_blockcount_z);
size_t num_blocks = num_x * num_y * num_z;
size_t * unpred_offset = (size_t *) malloc(num_blocks * sizeof(size_t));
*data = (double*)malloc(sizeof(double)*num_elements);
int * result_type = (int *) malloc(num_elements * sizeof(int));
size_t * block_offset = (size_t *) malloc(num_blocks * sizeof(size_t));
double realPrecision = bytesToDouble(comp_data_pos);
comp_data_pos += sizeof(double);
unsigned int intervals = bytesToInt_bigEndian(comp_data_pos);
comp_data_pos += sizeof(double);
size_t stateNum = intervals*2;
HuffmanTree* huffmanTree = createHuffmanTree(stateNum);
updateQuantizationInfo(intervals);
// exe_params->intvRadius = (int)((tdps->intervals - 1)/ 2);
unsigned int tree_size = bytesToInt_bigEndian(comp_data_pos);
comp_data_pos += sizeof(unsigned int);
size_t huffman_nodes = bytesToInt_bigEndian(comp_data_pos);
huffmanTree->allNodes = huffman_nodes;
// printf("Reconstruct huffman tree with node count %ld\n", nodeCount);
// fflush(stdout);
node root = reconstruct_HuffTree_from_bytes_anyStates(huffmanTree, comp_data_pos+4, huffmanTree->allNodes);
comp_data_pos += 4 + tree_size;
unsigned int * unpred_count = (unsigned int *) comp_data_pos;
comp_data_pos += num_blocks * sizeof(unsigned int);
double * mean_pos = (double *) comp_data_pos;
comp_data_pos += num_blocks * sizeof(double);
double * result_unpredictable_data = (double *) comp_data_pos;
size_t total_unpred = 0;
for(int i=0; i<num_blocks; i++){
unpred_offset[i] = total_unpred;
total_unpred += unpred_count[i];
}
comp_data_pos += total_unpred * sizeof(double);
size_t * block_pos = (size_t *) comp_data_pos;
comp_data_pos += num_blocks * sizeof(size_t);
block_offset[0] = 0;
for(int t=1; t<thread_num; t++){
block_offset[t] = block_pos[t-1] + block_offset[t-1];
}
int num_yz = num_y * num_z;
elapsed_time += sz_wtime();
printf("Read data info elapsed time: %.4f\n", elapsed_time);
elapsed_time = -sz_wtime();
#pragma omp parallel for
for(int t=0; t<thread_num; t++){
int id = sz_get_thread_num();
int i = id/(num_yz);
int j = (id % num_yz) / num_z;
int k = id % num_z;
size_t offset_x = (i < split_index_x) ? i * early_blockcount_x : i * late_blockcount_x + split_index_x;
size_t offset_y = (j < split_index_y) ? j * early_blockcount_y : j * late_blockcount_y + split_index_y;
size_t offset_z = (k < split_index_z) ? k * early_blockcount_z : k * late_blockcount_z + split_index_z;
size_t current_blockcount_x = (i < split_index_x) ? early_blockcount_x : late_blockcount_x;
size_t current_blockcount_y = (j < split_index_y) ? early_blockcount_y : late_blockcount_y;
size_t current_blockcount_z = (k < split_index_z) ? early_blockcount_z : late_blockcount_z;
size_t type_offset = offset_x * dim0_offset + offset_y * current_blockcount_x * dim1_offset + offset_z * current_blockcount_x * current_blockcount_y;
int * type = result_type + type_offset;
decode(comp_data_pos + block_offset[id], current_blockcount_x*current_blockcount_y*current_blockcount_z, root, type);
}
elapsed_time += sz_wtime();
printf("Parallel Huffman decoding elapsed time: %.4f\n", elapsed_time);
elapsed_time = -sz_wtime();
#pragma omp parallel for
for(int t=0; t<thread_num; t++){
int id = sz_get_thread_num();
int i = id/(num_yz);
int j = (id % num_yz) / num_z;
int k = id % num_z;
// printf("%d: %d %d %d\n", sz_get_thread_num(), i, j, k);
size_t offset_x = (i < split_index_x) ? i * early_blockcount_x : i * late_blockcount_x + split_index_x;
size_t offset_y = (j < split_index_y) ? j * early_blockcount_y : j * late_blockcount_y + split_index_y;
size_t offset_z = (k < split_index_z) ? k * early_blockcount_z : k * late_blockcount_z + split_index_z;
double * data_pos = *data + offset_x * dim0_offset + offset_y * dim1_offset + offset_z;
size_t current_blockcount_x = (i < split_index_x) ? early_blockcount_x : late_blockcount_x;
size_t current_blockcount_y = (j < split_index_y) ? early_blockcount_y : late_blockcount_y;
size_t current_blockcount_z = (k < split_index_z) ? early_blockcount_z : late_blockcount_z;
size_t type_offset = offset_x * dim0_offset + offset_y * current_blockcount_x * dim1_offset + offset_z * current_blockcount_x * current_blockcount_y;
int * type = result_type + type_offset;
double * unpredictable_data = result_unpredictable_data + unpred_offset[id];
double mean = mean_pos[id];
decompressDataSeries_double_3D_RA_block(data_pos, mean, r1, r2, r3, current_blockcount_x, current_blockcount_y, current_blockcount_z, realPrecision, type, unpredictable_data);
}
elapsed_time += sz_wtime();
printf("Parallel decompress elapsed time: %.4f\n", elapsed_time);
free(block_offset);
free(result_type);
free(unpred_offset);
SZ_ReleaseHuffman(huffmanTree);
}
void Huffman_init_openmp(HuffmanTree* huffmanTree, int *s, size_t length, int thread_num, size_t * freq){
size_t i;
// size_t *freq = (size_t *)malloc(thread_num*huffmanTree->allNodes*sizeof(size_t));
// memset(freq, 0, thread_num*huffmanTree->allNodes*sizeof(size_t));
size_t block_size = (length - 1)/ thread_num + 1;
size_t block_residue = length - (thread_num - 1) * block_size;
#pragma omp parallel for
for(int t=0; t<thread_num; t++){
int id = sz_get_thread_num();
int * s_pos = s + id * block_size;
size_t * freq_pos = freq + id * huffmanTree->allNodes;
if(id < thread_num - 1){
for(size_t i=0; i<block_size; i++){
freq_pos[s_pos[i]] ++;
}
}
else{
for(size_t i=0; i<block_residue; i++){
freq_pos[s_pos[i]] ++;
}
}
}
size_t * freq_pos = freq + huffmanTree->allNodes;
for(int t=1; t<thread_num; t++){
for(i = 0; i<huffmanTree->allNodes; i++){
freq[i] += freq_pos[i];
}
freq_pos += huffmanTree->allNodes;
}
for (i = 0; i < huffmanTree->allNodes; i++)
if (freq[i])
qinsert(huffmanTree, new_node(huffmanTree, freq[i], i, 0, 0));
while (huffmanTree->qend > 2)
qinsert(huffmanTree, new_node(huffmanTree, 0, 0, qremove(huffmanTree), qremove(huffmanTree)));
build_code(huffmanTree, huffmanTree->qq[1], 0, 0, 0);
// free(freq);
}

92
deps/SZ/sz/src/szd_double.c vendored
View File

@ -18,11 +18,11 @@
#include "szd_double_ts.h"
#include "utility.h"
int SZ_decompress_args_double(double** newData, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, unsigned char* cmpBytes,
size_t cmpSize, int compressionType, double* hist_data, sz_exedata* pde_exe, sz_params* pde_params)
int SZ_decompress_args_double(double* newData, size_t r1, unsigned char* cmpBytes,
size_t cmpSize, int compressionType, double* hist_data, sz_exedata* pde_exe, sz_params* pde_params)
{
int status = SZ_SCES;
size_t dataLength = computeDataLength(r5,r4,r3,r2,r1);
int status = SZ_SUCCESS;
size_t dataLength = r1;
//unsigned char* tmpBytes;
size_t targetUncompressSize = dataLength <<3; //i.e., *8
@ -39,6 +39,8 @@ size_t cmpSize, int compressionType, double* hist_data, sz_exedata* pde_exe, sz_
else
pde_params->szMode = SZ_BEST_SPEED;
}
if(pde_params->szMode==SZ_BEST_SPEED)
{
tmpSize = cmpSize;
@ -68,20 +70,20 @@ size_t cmpSize, int compressionType, double* hist_data, sz_exedata* pde_exe, sz_
TightDataPointStorageD* tdps;
int errBoundMode = new_TightDataPointStorageD_fromFlatBytes(&tdps, szTmpBytes, tmpSize, pde_exe, pde_params);
int dim = computeDimension(r5,r4,r3,r2,r1);
int dim = r1;
int doubleSize = sizeof(double);
if(tdps->isLossless)
{
*newData = (double*)malloc(doubleSize*dataLength);
// *newData = (double*)malloc(doubleSize*dataLength); comment by tickduan
if(sysEndianType==BIG_ENDIAN_SYSTEM)
{
memcpy(*newData, szTmpBytes+4+MetaDataByteLength_double+exe_params->SZ_SIZE_TYPE, dataLength*doubleSize);
memcpy(newData, szTmpBytes+4+MetaDataByteLength_double+exe_params->SZ_SIZE_TYPE, dataLength*doubleSize);
}
else
{
unsigned char* p = szTmpBytes+4+MetaDataByteLength_double+exe_params->SZ_SIZE_TYPE;
for(i=0;i<dataLength;i++,p+=doubleSize)
(*newData)[i] = bytesToDouble(p);
newData[i] = bytesToDouble(p);
}
}
else if(pde_params->sol_ID==SZ_Transpose)
@ -92,29 +94,17 @@ size_t cmpSize, int compressionType, double* hist_data, sz_exedata* pde_exe, sz_
{
if(tdps->raBytes_size > 0) //v2.0
{
if (dim == 1)
getSnapshotData_double_1D(newData,r1,tdps, errBoundMode, 0, hist_data, pde_params);
else
{
printf("Error: currently support only at most 4 dimensions!\n");
status = SZ_DERR;
}
getSnapshotData_double_1D(newData,r1,tdps, errBoundMode, 0, hist_data, pde_params);
}
else //1.4.13 or time-based compression
{
if (dim == 1)
getSnapshotData_double_1D(newData,r1,tdps, errBoundMode, compressionType, hist_data, pde_params);
else
{
printf("Error: currently support only at most 4 dimensions!\n");
status = SZ_DERR;
}
getSnapshotData_double_1D(newData,r1,tdps, errBoundMode, compressionType, hist_data, pde_params);
}
}
if(pde_params->protectValueRange)
{
double* nd = *newData;
double* nd = newData;
double min = pde_params->dmin;
double max = pde_params->dmax;
for(i=0;i<dataLength;i++)
@ -135,7 +125,7 @@ size_t cmpSize, int compressionType, double* hist_data, sz_exedata* pde_exe, sz_
return status;
}
void decompressDataSeries_double_1D(double** data, size_t dataSeriesLength, double* hist_data, TightDataPointStorageD* tdps)
void decompressDataSeries_double_1D(double* data, size_t dataSeriesLength, double* hist_data, TightDataPointStorageD* tdps)
{
//updateQuantizationInfo(tdps->intervals);
int intvRadius = tdps->intervals/2;
@ -147,7 +137,7 @@ void decompressDataSeries_double_1D(double** data, size_t dataSeriesLength, doub
double interval = tdps->realPrecision*2;
convertByteArray2IntArray_fast_2b(tdps->exactDataNum, tdps->leadNumArray, tdps->leadNumArray_size, &leadNum);
*data = (double*)malloc(sizeof(double)*dataSeriesLength);
//*data = (double*)malloc(sizeof(double)*dataSeriesLength); comment by tickduan
int* type = (int*)malloc(dataSeriesLength*sizeof(int));
@ -212,30 +202,25 @@ void decompressDataSeries_double_1D(double** data, size_t dataSeriesLength, doub
}
exactData = bytesToDouble(curBytes);
(*data)[i] = exactData + medianValue;
data[i] = exactData + medianValue;
memcpy(preBytes,curBytes,8);
break;
default:
//predValue = 2 * (*data)[i-1] - (*data)[i-2];
predValue = (*data)[i-1];
(*data)[i] = predValue + (type_-intvRadius)*interval;
//predValue = 2 * data[i-1] - data[i-2];
predValue = data[i-1];
data[i] = predValue + (type_-intvRadius)*interval;
break;
}
//printf("%.30G\n",(*data)[i]);
//printf("%.30G\n",data[i]);
}
#ifdef HAVE_TIMECMPR
if(confparams_dec->szMode == SZ_TEMPORAL_COMPRESSION)
memcpy(hist_data, (*data), dataSeriesLength*sizeof(double));
#endif
free(leadNum);
free(type);
return;
}
/*MSST19*/
void decompressDataSeries_double_1D_MSST19(double** data, size_t dataSeriesLength, TightDataPointStorageD* tdps)
void decompressDataSeries_double_1D_MSST19(double* data, size_t dataSeriesLength, TightDataPointStorageD* tdps)
{
//updateQuantizationInfo(tdps->intervals);
int intvRadius = tdps->intervals/2;
@ -248,7 +233,7 @@ void decompressDataSeries_double_1D_MSST19(double** data, size_t dataSeriesLengt
//double interval = tdps->realPrecision*2;
convertByteArray2IntArray_fast_2b(tdps->exactDataNum, tdps->leadNumArray, tdps->leadNumArray_size, &leadNum);
*data = (double*)malloc(sizeof(double)*dataSeriesLength);
//*data = (double*)malloc(sizeof(double)*dataSeriesLength); comment by tickduan
int* type = (int*)malloc(dataSeriesLength*sizeof(int));
@ -318,52 +303,39 @@ void decompressDataSeries_double_1D_MSST19(double** data, size_t dataSeriesLengt
}
exactData = bytesToDouble(curBytes);
(*data)[i] = exactData;
data[i] = exactData;
memcpy(preBytes,curBytes,8);
predValue = (*data)[i];
predValue = data[i];
break;
default:
//predValue = 2 * (*data)[i-1] - (*data)[i-2];
//predValue = (*data)[i-1];
//predValue = 2 * data[i-1] - data[i-2];
//predValue = data[i-1];
predValue = fabs(predValue) * precisionTable[type_];
(*data)[i] = predValue;
data[i] = predValue;
break;
}
}
#ifdef HAVE_TIMECMPR
if(confparams_dec->szMode == SZ_TEMPORAL_COMPRESSION)
memcpy(multisteps->hist_data, (*data), dataSeriesLength*sizeof(double));
#endif
free(precisionTable);
free(leadNum);
free(type);
return;
}
void getSnapshotData_double_1D(double** data, size_t dataSeriesLength, TightDataPointStorageD* tdps, int errBoundMode, int compressionType, double* hist_data, sz_params* pde_params)
void getSnapshotData_double_1D(double* data, size_t dataSeriesLength, TightDataPointStorageD* tdps, int errBoundMode, int compressionType, double* hist_data, sz_params* pde_params)
{
size_t i;
if (tdps->allSameData) {
double value = bytesToDouble(tdps->exactMidBytes);
*data = (double*)malloc(sizeof(double)*dataSeriesLength);
//*data = (double*)malloc(sizeof(double)*dataSeriesLength); comment by tickduan
for (i = 0; i < dataSeriesLength; i++)
(*data)[i] = value;
data[i] = value;
} else {
if (tdps->rtypeArray == NULL) {
if(errBoundMode < PW_REL)
{
#ifdef HAVE_TIMECMPR
if(confparams_dec->szMode == SZ_TEMPORAL_COMPRESSION)
{
if(multisteps->compressionType == 0) //snapshot
decompressDataSeries_double_1D(data, dataSeriesLength, hist_data, tdps);
else
decompressDataSeries_double_1D_ts(data, dataSeriesLength, hist_data, tdps);
}
else
#endif
decompressDataSeries_double_1D(data, dataSeriesLength, hist_data, tdps);
decompressDataSeries_double_1D(data, dataSeriesLength, hist_data, tdps);
}
else
{

44
deps/SZ/sz/src/szd_double_pwr.c vendored
View File

@ -22,7 +22,7 @@
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wchar-subscripts"
void decompressDataSeries_double_1D_pwr(double** data, size_t dataSeriesLength, TightDataPointStorageD* tdps)
void decompressDataSeries_double_1D_pwr(double* data, size_t dataSeriesLength, TightDataPointStorageD* tdps)
{
updateQuantizationInfo(tdps->intervals);
unsigned char tmpPrecBytes[8] = {0}; //used when needing to convert bytes to double values
@ -36,7 +36,7 @@ void decompressDataSeries_double_1D_pwr(double** data, size_t dataSeriesLength,
double interval = 0;// = (double)tdps->realPrecision*2;
convertByteArray2IntArray_fast_2b(tdps->exactDataNum, tdps->leadNumArray, tdps->leadNumArray_size, &leadNum);
*data = (double*)malloc(sizeof(double)*dataSeriesLength);
//*data = (double*)malloc(sizeof(double)*dataSeriesLength); comment by tickduan
int* type = (int*)malloc(dataSeriesLength*sizeof(int));
@ -117,16 +117,16 @@ void decompressDataSeries_double_1D_pwr(double** data, size_t dataSeriesLength,
}
exactData = bytesToDouble(curBytes);
(*data)[i] = exactData + medianValue;
data[i] = exactData + medianValue;
memcpy(preBytes,curBytes,8);
break;
default:
//predValue = 2 * (*data)[i-1] - (*data)[i-2];
predValue = (*data)[i-1];
(*data)[i] = predValue + (type_-exe_params->intvRadius)*interval;
//predValue = 2 * data[i-1] - data[i-2];
predValue = data[i-1];
data[i] = predValue + (type_-exe_params->intvRadius)*interval;
break;
}
//printf("%.30G\n",(*data)[i]);
//printf("%.30G\n",data[i]);
}
free(leadNum);
free(type);
@ -135,7 +135,7 @@ void decompressDataSeries_double_1D_pwr(double** data, size_t dataSeriesLength,
void decompressDataSeries_double_1D_pwrgroup(double** data, size_t dataSeriesLength, TightDataPointStorageD* tdps)
void decompressDataSeries_double_1D_pwrgroup(double* data, size_t dataSeriesLength, TightDataPointStorageD* tdps)
{
double *posGroups, *negGroups, *groups;
double pos_01_group, neg_01_group;
@ -148,7 +148,7 @@ void decompressDataSeries_double_1D_pwrgroup(double** data, size_t dataSeriesLen
convertByteArray2IntArray_fast_2b(tdps->exactDataNum, tdps->leadNumArray, tdps->leadNumArray_size, &leadNum);
*data = (double*)malloc(sizeof(double)*dataSeriesLength);
//*data = (double*)malloc(sizeof(double)*dataSeriesLength); comment by tickduan
int* type = (int*)malloc(dataSeriesLength*sizeof(int));
@ -266,7 +266,7 @@ void decompressDataSeries_double_1D_pwrgroup(double** data, size_t dataSeriesLen
exactData = bytesToDouble(curBytes);
exactData = exactData + medianValue;
(*data)[i] = exactData;
data[i] = exactData;
memcpy(preBytes,curBytes,8);
groups[indexGrpID] = exactData;
@ -292,7 +292,7 @@ void decompressDataSeries_double_1D_pwrgroup(double** data, size_t dataSeriesLen
// curValue = rawGrpID>0?pow(2,groupNum):-pow(2,groupNum);
//}
(*data)[i] = curValue;
data[i] = curValue;
groups[indexGrpID] = curValue;
break;
}
@ -309,7 +309,7 @@ void decompressDataSeries_double_1D_pwrgroup(double** data, size_t dataSeriesLen
free(groupID);
}
void decompressDataSeries_double_1D_pwr_pre_log(double** data, size_t dataSeriesLength, TightDataPointStorageD* tdps) {
void decompressDataSeries_double_1D_pwr_pre_log(double* data, size_t dataSeriesLength, TightDataPointStorageD* tdps) {
decompressDataSeries_double_1D(data, dataSeriesLength, NULL, tdps);
double threshold = tdps->minLogValue;
@ -318,22 +318,22 @@ void decompressDataSeries_double_1D_pwr_pre_log(double** data, size_t dataSeries
sz_lossless_decompress(ZSTD_COMPRESSOR, tdps->pwrErrBoundBytes, tdps->pwrErrBoundBytes_size, &signs, dataSeriesLength);
for(size_t i=0; i<dataSeriesLength; i++){
if((*data)[i] < threshold) (*data)[i] = 0;
else (*data)[i] = exp2((*data)[i]);
if(signs[i]) (*data)[i] = -((*data)[i]);
if(data[i] < threshold) data[i] = 0;
else data[i] = exp2(data[i]);
if(signs[i]) data[i] = -(data[i]);
}
free(signs);
}
else{
for(size_t i=0; i<dataSeriesLength; i++){
if((*data)[i] < threshold) (*data)[i] = 0;
else (*data)[i] = exp2((*data)[i]);
if(data[i] < threshold) data[i] = 0;
else data[i] = exp2(data[i]);
}
}
}
void decompressDataSeries_double_1D_pwr_pre_log_MSST19(double** data, size_t dataSeriesLength, TightDataPointStorageD* tdps)
void decompressDataSeries_double_1D_pwr_pre_log_MSST19(double* data, size_t dataSeriesLength, TightDataPointStorageD* tdps)
{
decompressDataSeries_double_1D_MSST19(data, dataSeriesLength, tdps);
double threshold = tdps->minLogValue;
@ -349,12 +349,12 @@ void decompressDataSeries_double_1D_pwr_pre_log_MSST19(double** data, size_t dat
else
sz_lossless_decompress(ZSTD_COMPRESSOR, tdps->pwrErrBoundBytes, tdps->pwrErrBoundBytes_size, &signs, dataSeriesLength);
for(size_t i=0; i<dataSeriesLength; i++){
if((*data)[i] < threshold && (*data)[i] >= 0){
(*data)[i] = 0;
if(data[i] < threshold && data[i] >= 0){
data[i] = 0;
continue;
}
if(signs[i]){
ptr = (uint64_t*)(*data) + i;
ptr = (uint64_t*)data + i;
*ptr |= 0x8000000000000000;
}
}
@ -362,7 +362,7 @@ void decompressDataSeries_double_1D_pwr_pre_log_MSST19(double** data, size_t dat
}
else{
for(size_t i=0; i<dataSeriesLength; i++){
if((*data)[i] < threshold) (*data)[i] = 0;
if(data[i] < threshold) data[i] = 0;
}
}
}

114
deps/SZ/sz/src/szd_double_ts.c vendored
View File

@ -1,114 +0,0 @@
/**
* @file szd_double_ts.c
* @author Sheng Di and Dingwen Tao
* @date Aug, 2016
* @brief
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include "szd_double.h"
#include "TightDataPointStorageD.h"
#include "sz.h"
#include "Huffman.h"
#include "szd_double_ts.h"
void decompressDataSeries_double_1D_ts(double** data, size_t dataSeriesLength, double* hist_data, TightDataPointStorageD* tdps)
{
double* lastSnapshotData = hist_data;
updateQuantizationInfo(tdps->intervals);
size_t i, j, k = 0, p = 0, l = 0; // k is to track the location of residual_bit
// in resiMidBits, p is to track the
// byte_index of resiMidBits, l is for
// leadNum
unsigned char* leadNum;
double interval = tdps->realPrecision*2;
convertByteArray2IntArray_fast_2b(tdps->exactDataNum, tdps->leadNumArray, tdps->leadNumArray_size, &leadNum);
*data = (double*)malloc(sizeof(double)*dataSeriesLength);
int* type = (int*)malloc(dataSeriesLength*sizeof(int));
HuffmanTree* huffmanTree = createHuffmanTree(tdps->stateNum);
decode_withTree(huffmanTree, tdps->typeArray, dataSeriesLength, type);
SZ_ReleaseHuffman(huffmanTree);
unsigned char preBytes[8];
unsigned char curBytes[8];
memset(preBytes, 0, 8);
size_t curByteIndex = 0;
int reqBytesLength, resiBitsLength, resiBits;
unsigned char leadingNum;
double medianValue, exactData, predValue = 0;
reqBytesLength = tdps->reqLength/8;
resiBitsLength = tdps->reqLength%8;
medianValue = tdps->medianValue;
int type_;
for (i = 0; i < dataSeriesLength; i++) {
type_ = type[i];
switch (type_) {
case 0:
// compute resiBits
resiBits = 0;
if (resiBitsLength != 0) {
int kMod8 = k % 8;
int rightMovSteps = getRightMovingSteps(kMod8, resiBitsLength);
if (rightMovSteps > 0) {
int code = getRightMovingCode(kMod8, resiBitsLength);
resiBits = (tdps->residualMidBits[p] & code) >> rightMovSteps;
} else if (rightMovSteps < 0) {
int code1 = getLeftMovingCode(kMod8);
int code2 = getRightMovingCode(kMod8, resiBitsLength);
int leftMovSteps = -rightMovSteps;
rightMovSteps = 8 - leftMovSteps;
resiBits = (tdps->residualMidBits[p] & code1) << leftMovSteps;
p++;
resiBits = resiBits
| ((tdps->residualMidBits[p] & code2) >> rightMovSteps);
} else // rightMovSteps == 0
{
int code = getRightMovingCode(kMod8, resiBitsLength);
resiBits = (tdps->residualMidBits[p] & code);
p++;
}
k += resiBitsLength;
}
// recover the exact data
memset(curBytes, 0, 8);
leadingNum = leadNum[l++];
memcpy(curBytes, preBytes, leadingNum);
for (j = leadingNum; j < reqBytesLength; j++)
curBytes[j] = tdps->exactMidBytes[curByteIndex++];
if (resiBitsLength != 0) {
unsigned char resiByte = (unsigned char) (resiBits << (8 - resiBitsLength));
curBytes[reqBytesLength] = resiByte;
}
exactData = bytesToDouble(curBytes);
(*data)[i] = exactData + medianValue;
memcpy(preBytes,curBytes,8);
break;
default:
//predValue = (*data)[i-1];
if(confparams_dec->szMode == SZ_TEMPORAL_COMPRESSION)
predValue = lastSnapshotData[i];
(*data)[i] = predValue + (type_-exe_params->intvRadius)*interval;
break;
}
//printf("%.30G\n",(*data)[i]);
}
memcpy(hist_data, (*data), dataSeriesLength*sizeof(double));
free(leadNum);
free(type);
return;
}

542
deps/SZ/sz/src/szd_float.c vendored
View File

@ -15,7 +15,6 @@
#include "sz.h"
#include "Huffman.h"
#include "szd_float_pwr.h"
#include "szd_float_ts.h"
#include "utility.h"
@ -45,13 +44,13 @@ void cost_end_()
* int compressionType: 1 (time-based compression) ; 0 (space-based compression)
* hist_data: only valid when compressionType==1, hist_data is the historical dataset such as the data in previous time step
*
* @return status SUCCESSFUL (SZ_SCES) or not (other error codes) f
* @return status SUCCESSFUL (SZ_SUCCESS) or not (other error codes) f
* */
int SZ_decompress_args_float(float** newData, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, unsigned char* cmpBytes,
size_t cmpSize, int compressionType, float* hist_data, sz_exedata* pde_exe, sz_params* pde_params)
int SZ_decompress_args_float(float* newData, size_t r1, unsigned char* cmpBytes,
size_t cmpSize, int compressionType, float* hist_data, sz_exedata* pde_exe, sz_params* pde_params)
{
int status = SZ_SCES;
size_t dataLength = computeDataLength(r5,r4,r3,r2,r1);
int status = SZ_SUCCESS;
size_t dataLength = r1;
//unsigned char* tmpBytes;
size_t targetUncompressSize = dataLength <<2; //i.e., *4
@ -80,9 +79,7 @@ size_t cmpSize, int compressionType, float* hist_data, sz_exedata* pde_exe, sz_p
if(targetUncompressSize<MIN_ZLIB_DEC_ALLOMEM_BYTES) //Considering the minimum size
targetUncompressSize = MIN_ZLIB_DEC_ALLOMEM_BYTES;
tmpSize = sz_lossless_decompress(pde_params->losslessCompressor, cmpBytes, (unsigned long)cmpSize, &szTmpBytes, (unsigned long)targetUncompressSize+4+MetaDataByteLength+exe_params->SZ_SIZE_TYPE);// (unsigned long)targetUncompressSize+8: consider the total length under lossless compression mode is actually 3+4+1+targetUncompressSize
//szTmpBytes = (unsigned char*)malloc(sizeof(unsigned char)*tmpSize);
//memcpy(szTmpBytes, tmpBytes, tmpSize);
//free(tmpBytes); //release useless memory
}
else
{
@ -101,20 +98,19 @@ size_t cmpSize, int compressionType, float* hist_data, sz_exedata* pde_exe, sz_p
int errBoundMode = new_TightDataPointStorageF_fromFlatBytes(&tdps, szTmpBytes, tmpSize, pde_exe, pde_params);
//writeByteData(tdps->typeArray, tdps->typeArray_size, "decompress-typebytes.tbt");
int dim = computeDimension(r5,r4,r3,r2,r1);
int floatSize = sizeof(float);
if(tdps->isLossless)
{
*newData = (float*)malloc(floatSize*dataLength);
//*newData = (float*)malloc(floatSize*dataLength); comment by tickduan
if(sysEndianType==BIG_ENDIAN_SYSTEM)
{
memcpy(*newData, szTmpBytes+4+MetaDataByteLength+exe_params->SZ_SIZE_TYPE, dataLength*floatSize);
memcpy(newData, szTmpBytes+4+MetaDataByteLength+exe_params->SZ_SIZE_TYPE, dataLength*floatSize);
}
else
{
unsigned char* p = szTmpBytes+4+MetaDataByteLength+exe_params->SZ_SIZE_TYPE;
for(i=0;i<dataLength;i++,p+=floatSize)
(*newData)[i] = bytesToFloat(p);
newData[i] = bytesToFloat(p);
}
}
else if(pde_params->sol_ID==SZ_Transpose)
@ -125,30 +121,18 @@ size_t cmpSize, int compressionType, float* hist_data, sz_exedata* pde_exe, sz_p
{
if(tdps->raBytes_size > 0) //v2.0
{
if (dim == 1)
getSnapshotData_float_1D(newData,r1,tdps, errBoundMode, 0, hist_data, pde_params);
else
{
printf("Error: currently support only at most 4 dimensions!\n");
status = SZ_DERR;
}
getSnapshotData_float_1D(newData,r1,tdps, errBoundMode, 0, hist_data, pde_params);
}
else //1.4.13 or time-based compression
{
if (dim == 1)
getSnapshotData_float_1D(newData,r1,tdps, errBoundMode, compressionType, hist_data, pde_params);
else
{
printf("Error: currently support only at most 4 dimensions!\n");
status = SZ_DERR;
}
getSnapshotData_float_1D(newData,r1,tdps, errBoundMode, compressionType, hist_data, pde_params);
}
}
//cost_start_();
if(pde_params->protectValueRange)
{
float* nd = *newData;
float* nd = newData;
float min = pde_params->fmin;
float max = pde_params->fmax;
for(i=0;i<dataLength;i++)
@ -170,7 +154,7 @@ size_t cmpSize, int compressionType, float* hist_data, sz_exedata* pde_exe, sz_p
return status;
}
void decompressDataSeries_float_1D(float** data, size_t dataSeriesLength, float* hist_data, TightDataPointStorageF* tdps)
void decompressDataSeries_float_1D(float* data, size_t dataSeriesLength, float* hist_data, TightDataPointStorageF* tdps)
{
//updateQuantizationInfo(tdps->intervals);
int intvRadius = tdps->intervals/2;
@ -182,7 +166,7 @@ void decompressDataSeries_float_1D(float** data, size_t dataSeriesLength, float*
float interval = tdps->realPrecision*2;
convertByteArray2IntArray_fast_2b(tdps->exactDataNum, tdps->leadNumArray, tdps->leadNumArray_size, &leadNum);
*data = (float*)malloc(sizeof(float)*dataSeriesLength);
//data = (float*)malloc(sizeof(float)*dataSeriesLength); // comment by tickduan
int* type = (int*)malloc(dataSeriesLength*sizeof(int));
@ -247,30 +231,25 @@ void decompressDataSeries_float_1D(float** data, size_t dataSeriesLength, float*
}
exactData = bytesToFloat(curBytes);
(*data)[i] = exactData + medianValue;
data[i] = exactData + medianValue;
memcpy(preBytes,curBytes,4);
break;
default:
//predValue = 2 * (*data)[i-1] - (*data)[i-2];
predValue = (*data)[i-1];
(*data)[i] = predValue + (float)(type_-intvRadius)*interval;
//predValue = 2 * data[i-1] - data[i-2];
predValue = data[i-1];
data[i] = predValue + (float)(type_-intvRadius)*interval;
break;
}
//printf("%.30G\n",(*data)[i]);
//printf("%.30G\n",data[i]);
}
#ifdef HAVE_TIMECMPR
if(pde_params->szMode == SZ_TEMPORAL_COMPRESSION)
memcpy(hist_data, (*data), dataSeriesLength*sizeof(float));
#endif
free(leadNum);
free(type);
return;
}
/*MSST19*/
void decompressDataSeries_float_1D_MSST19(float** data, size_t dataSeriesLength, TightDataPointStorageF* tdps)
void decompressDataSeries_float_1D_MSST19(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps)
{
//updateQuantizationInfo(tdps->intervals);
int intvRadius = tdps->intervals/2;
@ -283,7 +262,7 @@ void decompressDataSeries_float_1D_MSST19(float** data, size_t dataSeriesLength,
//double interval = tdps->realPrecision*2;
convertByteArray2IntArray_fast_2b(tdps->exactDataNum, tdps->leadNumArray, tdps->leadNumArray_size, &leadNum);
*data = (float*)malloc(sizeof(float)*dataSeriesLength);
// *data = (float*)malloc(sizeof(float)*dataSeriesLength); comment by tickduan
int* type = (int*)malloc(dataSeriesLength*sizeof(int));
@ -352,24 +331,20 @@ void decompressDataSeries_float_1D_MSST19(float** data, size_t dataSeriesLength,
}
exactData = bytesToFloat(curBytes);
(*data)[i] = exactData;
data[i] = exactData;
memcpy(preBytes,curBytes,4);
predValue = (*data)[i];
predValue = data[i];
break;
default:
//predValue = 2 * (*data)[i-1] - (*data)[i-2];
//predValue = (*data)[i-1];
//predValue = 2 * data[i-1] - data[i-2];
//predValue = data[i-1];
predValue = fabs(predValue) * precisionTable[type_];
(*data)[i] = predValue;
data[i] = predValue;
break;
}
//printf("%.30G\n",(*data)[i]);
//printf("%.30G\n",data[i]);
}
#ifdef HAVE_TIMECMPR
if(confparams_dec->szMode == SZ_TEMPORAL_COMPRESSION)
memcpy(multisteps->hist_data, (*data), dataSeriesLength*sizeof(float));
#endif
free(precisionTable);
free(leadNum);
free(type);
@ -377,30 +352,20 @@ void decompressDataSeries_float_1D_MSST19(float** data, size_t dataSeriesLength,
}
void getSnapshotData_float_1D(float** data, size_t dataSeriesLength, TightDataPointStorageF* tdps, int errBoundMode, int compressionType, float* hist_data, sz_params* pde_params)
void getSnapshotData_float_1D(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps, int errBoundMode, int compressionType, float* hist_data, sz_params* pde_params)
{
size_t i;
if (tdps->allSameData) {
float value = bytesToFloat(tdps->exactMidBytes);
*data = (float*)malloc(sizeof(float)*dataSeriesLength);
//*data = (float*)malloc(sizeof(float)*dataSeriesLength); commnet by tickduan
for (i = 0; i < dataSeriesLength; i++)
(*data)[i] = value;
data[i] = value;
} else {
if (tdps->rtypeArray == NULL) {
if(errBoundMode < PW_REL)
{
#ifdef HAVE_TIMECMPR
if(pde_params->szMode == SZ_TEMPORAL_COMPRESSION)
{
if(compressionType == 0) //snapshot
decompressDataSeries_float_1D(data, dataSeriesLength, hist_data, tdps);
else
decompressDataSeries_float_1D_ts(data, dataSeriesLength, hist_data, tdps);
}
else
#endif
decompressDataSeries_float_1D(data, dataSeriesLength, hist_data, tdps);
{
decompressDataSeries_float_1D(data, dataSeriesLength, hist_data, tdps);
}
else
{
@ -416,446 +381,3 @@ void getSnapshotData_float_1D(float** data, size_t dataSeriesLength, TightDataPo
}
}
}
size_t decompressDataSeries_float_1D_RA_block(float * data, float mean, size_t dim_0, size_t block_dim_0, double realPrecision, int * type, float * unpredictable_data){
int intvRadius = exe_params->intvRadius;
size_t unpredictable_count = 0;
float * cur_data_pos = data;
size_t type_index = 0;
int type_;
float last_over_thres = mean;
for(size_t i=0; i<block_dim_0; i++){
type_ = type[type_index];
if(type_ == 0){
cur_data_pos[0] = unpredictable_data[unpredictable_count ++];
last_over_thres = cur_data_pos[0];
}
else{
cur_data_pos[0] = last_over_thres + 2 * (type_ - intvRadius) * realPrecision;
last_over_thres = cur_data_pos[0];
}
type_index ++;
cur_data_pos ++;
}
return unpredictable_count;
}
#ifdef HAVE_RANDOMACCESS
void decompressDataSeries_float_1D_decompression_given_areas_with_blocked_regression(float** data, size_t r1, size_t s1, size_t e1, unsigned char* comp_data){
unsigned char * comp_data_pos = comp_data;
size_t block_size = bytesToInt_bigEndian(comp_data_pos);
comp_data_pos += sizeof(int);
// calculate block dims
size_t num_x;
num_x = (r1 - 1) / block_size + 1;
size_t max_num_block_elements = block_size;
size_t num_blocks = num_x;
double realPrecision = bytesToDouble(comp_data_pos);
comp_data_pos += sizeof(double);
unsigned int intervals = bytesToInt_bigEndian(comp_data_pos);
comp_data_pos += sizeof(int);
//updateQuantizationInfo(intervals);
unsigned int tree_size = bytesToInt_bigEndian(comp_data_pos);
comp_data_pos += sizeof(int);
int stateNum = 2*intervals;
HuffmanTree* huffmanTree = createHuffmanTree(stateNum);
int nodeCount = bytesToInt_bigEndian(comp_data_pos);
node root = reconstruct_HuffTree_from_bytes_anyStates(huffmanTree,comp_data_pos+sizeof(int), nodeCount);
comp_data_pos += sizeof(int) + tree_size;
float mean;
unsigned char use_mean;
memcpy(&use_mean, comp_data_pos, sizeof(unsigned char));
comp_data_pos += sizeof(unsigned char);
memcpy(&mean, comp_data_pos, sizeof(float));
comp_data_pos += sizeof(float);
size_t reg_count = 0;
unsigned char * indicator;
size_t indicator_bitlength = (num_blocks - 1)/8 + 1;
convertByteArray2IntArray_fast_1b(num_blocks, comp_data_pos, indicator_bitlength, &indicator);
comp_data_pos += indicator_bitlength;
for(size_t i=0; i<num_blocks; i++){
if(!indicator[i]) reg_count ++;
}
int coeff_intvRadius[2];
int * coeff_result_type = (int *) malloc(num_blocks*2*sizeof(int));
int * coeff_type[2];
double precision[2];
float * coeff_unpred_data[2];
if(reg_count > 0){
for(int i=0; i<2; i++){
precision[i] = bytesToDouble(comp_data_pos);
comp_data_pos += sizeof(double);
coeff_intvRadius[i] = bytesToInt_bigEndian(comp_data_pos);
comp_data_pos += sizeof(int);
unsigned int tree_size = bytesToInt_bigEndian(comp_data_pos);
comp_data_pos += sizeof(int);
int stateNum = 2*coeff_intvRadius[i]*2;
HuffmanTree* huffmanTree = createHuffmanTree(stateNum);
int nodeCount = bytesToInt_bigEndian(comp_data_pos);
node root = reconstruct_HuffTree_from_bytes_anyStates(huffmanTree, comp_data_pos+sizeof(int), nodeCount);
comp_data_pos += sizeof(int) + tree_size;
coeff_type[i] = coeff_result_type + i * num_blocks;
size_t typeArray_size = bytesToSize(comp_data_pos);
decode(comp_data_pos + sizeof(size_t), reg_count, root, coeff_type[i]);
comp_data_pos += sizeof(size_t) + typeArray_size;
int coeff_unpred_count = bytesToInt_bigEndian(comp_data_pos);
comp_data_pos += sizeof(int);
coeff_unpred_data[i] = (float *) comp_data_pos;
comp_data_pos += coeff_unpred_count * sizeof(float);
SZ_ReleaseHuffman(huffmanTree);
}
}
float last_coefficients[2] = {0.0};
int coeff_unpred_data_count[2] = {0};
// decompress coeffcients
float * reg_params = (float *) malloc(2*num_blocks*sizeof(float));
memset(reg_params, 0, 2*num_blocks*sizeof(float));
float * reg_params_pos = reg_params;
size_t coeff_index = 0;
for(size_t i=0; i<num_blocks; i++){
if(!indicator[i]){
float pred;
int type_;
for(int e=0; e<2; e++){
type_ = coeff_type[e][coeff_index];
if (type_ != 0){
pred = last_coefficients[e];
last_coefficients[e] = pred + 2 * (type_ - coeff_intvRadius[e]) * precision[e];
}
else{
last_coefficients[e] = coeff_unpred_data[e][coeff_unpred_data_count[e]];
coeff_unpred_data_count[e] ++;
}
reg_params_pos[e] = last_coefficients[e];
}
coeff_index ++;
}
reg_params_pos += 2;
}
//updateQuantizationInfo(intervals);
int intvRadius = intervals/2;
size_t total_unpred;
memcpy(&total_unpred, comp_data_pos, sizeof(size_t));
comp_data_pos += sizeof(size_t);
size_t compressed_blockwise_unpred_count_size;
memcpy(&compressed_blockwise_unpred_count_size, comp_data_pos, sizeof(size_t));
comp_data_pos += sizeof(size_t);
int * blockwise_unpred_count = NULL;
SZ_decompress_args_int32(&blockwise_unpred_count, 0, 0, 0, 0, num_blocks, comp_data_pos, compressed_blockwise_unpred_count_size);
comp_data_pos += compressed_blockwise_unpred_count_size;
size_t * unpred_offset = (size_t *) malloc(num_blocks * sizeof(size_t));
size_t cur_offset = 0;
for(size_t i=0; i<num_blocks; i++){
unpred_offset[i] = cur_offset;
cur_offset += blockwise_unpred_count[i];
}
float * unpred_data = (float *) comp_data_pos;
comp_data_pos += total_unpred * sizeof(float);
size_t compressed_type_array_block_size;
memcpy(&compressed_type_array_block_size, comp_data_pos, sizeof(size_t));
comp_data_pos += sizeof(size_t);
unsigned short * type_array_block_size = NULL;
SZ_decompress_args_uint16(&type_array_block_size, 0, 0, 0, 0, num_blocks, comp_data_pos, compressed_type_array_block_size);
comp_data_pos += compressed_type_array_block_size;
// compute given area
size_t sx = s1 / block_size;
size_t ex = (e1 - 1) / block_size + 1;
unsigned short * type_array_block_size_pos = type_array_block_size;
size_t * type_array_offset = (size_t *) malloc(num_blocks * sizeof(size_t));
size_t * type_array_offset_pos = type_array_offset;
size_t cur_type_array_offset = 0;
for(size_t i=0; i<num_x; i++){
*(type_array_offset_pos++) = cur_type_array_offset;
cur_type_array_offset += *(type_array_block_size_pos++);
}
free(type_array_block_size);
int * result_type = (int *) malloc((ex - sx)*block_size*sizeof(int));
int * block_type = result_type;
for(size_t i=sx; i<ex; i++){
size_t index = i;
decode(comp_data_pos + type_array_offset[index], max_num_block_elements, root, block_type);
block_type += max_num_block_elements;
}
SZ_ReleaseHuffman(huffmanTree);
free(type_array_offset);
int * type = NULL;
float * data_pos = *data;
int dec_buffer_size = block_size + 1;
float * dec_buffer = (float *) malloc(dec_buffer_size*sizeof(float));
memset(dec_buffer, 0, dec_buffer_size*sizeof(float));
float * block_data_pos_x = NULL;
// printf("decompression start, %d %d %d, %d %d %d, total unpred %ld\n", sx, sy, sz, ex, ey, ez, total_unpred);
// fflush(stdout);
float * dec_block_data = (float *) malloc((ex - sx)*block_size*sizeof(float));
memset(dec_block_data, 0, (ex - sx)*block_size*sizeof(float));
if(use_mean){
for(size_t i=sx; i<ex; i++){
data_pos = dec_buffer + 1;
type = result_type + (i-sx) * block_size;
coeff_index = i;
float * block_unpred = unpred_data + unpred_offset[coeff_index];
if(indicator[coeff_index]){
// decompress by SZ
float * block_data_pos;
float pred;
size_t index = 0;
int type_;
size_t unpredictable_count = 0;
for(size_t ii=0; ii<block_size; ii++){
block_data_pos = data_pos + ii;
type_ = type[index];
if(type_ == 1){
*block_data_pos = mean;
}
else if(type_ == 0){
*block_data_pos = block_unpred[unpredictable_count ++];
}
else{
pred = block_data_pos[-1];
*block_data_pos = pred + 2 * (type_ - intvRadius) * realPrecision;
}
index ++;
}
}
else{
// decompress by regression
reg_params_pos = reg_params + 2*coeff_index;
{
float pred;
int type_;
size_t index = 0;
size_t unpredictable_count = 0;
for(size_t ii=0; ii<block_size; ii++){
type_ = type[index];
if (type_ != 0){
pred = reg_params_pos[0] * ii + reg_params_pos[1];
data_pos[ii] = pred + 2 * (type_ - intvRadius) * realPrecision;
}
else{
data_pos[ii] = block_unpred[unpredictable_count ++];
}
index ++;
}
}
}
// mv data back
block_data_pos_x = dec_block_data + (i-sx)*block_size;
for(int ii=0; ii<block_size; ii++){
if(i*block_size + ii >= r1) break;
*block_data_pos_x = data_pos[ii];
block_data_pos_x ++;
}
}
}
else{
for(size_t i=sx; i<ex; i++){
data_pos = dec_buffer + 1;
type = result_type + (i-sx) * block_size;
coeff_index = i;
float * block_unpred = unpred_data + unpred_offset[coeff_index];
if(indicator[coeff_index]){
// decompress by SZ
float * block_data_pos;
float pred;
size_t index = 0;
int type_;
size_t unpredictable_count = 0;
for(size_t ii=0; ii<block_size; ii++){
block_data_pos = data_pos + ii;
type_ = type[index];
if(type_ == 0){
*block_data_pos = block_unpred[unpredictable_count ++];
}
else{
pred = block_data_pos[-1];
*block_data_pos = pred + 2 * (type_ - intvRadius) * realPrecision;
}
index ++;
}
}
else{
// decompress by regression
reg_params_pos = reg_params + 2*coeff_index;
{
float pred;
int type_;
size_t index = 0;
size_t unpredictable_count = 0;
for(size_t ii=0; ii<block_size; ii++){
type_ = type[index];
if (type_ != 0){
pred = reg_params_pos[0] * ii + reg_params_pos[1];
data_pos[ii] = pred + 2 * (type_ - intvRadius) * realPrecision;
}
else{
data_pos[ii] = block_unpred[unpredictable_count ++];
}
index ++;
}
}
}
// mv data back
block_data_pos_x = dec_block_data + (i-sx)*block_size;
for(int ii=0; ii<block_size; ii++){
if(i*block_size + ii >= r1) break;
*block_data_pos_x = data_pos[ii];
block_data_pos_x ++;
}
}
}
free(unpred_offset);
free(reg_params);
free(blockwise_unpred_count);
free(dec_buffer);
free(coeff_result_type);
free(indicator);
free(result_type);
// extract data
int resi_x = s1 % block_size;
*data = (float*) malloc(sizeof(float)*(e1 - s1));
float * final_data_pos = *data;
float * block_data_pos = dec_block_data + resi_x;
for(int i=0; i<(e1 - s1); i++){
*(final_data_pos++) = *(block_data_pos++);
}
free(dec_block_data);
}
int SZ_decompress_args_randomaccess_float(float** newData,
size_t r5, size_t r4, size_t r3, size_t r2, size_t r1,
size_t s5, size_t s4, size_t s3, size_t s2, size_t s1, // start point
size_t e5, size_t e4, size_t e3, size_t e2, size_t e1, // end point
unsigned char* cmpBytes, size_t cmpSize)
{
if(confparams_dec==NULL)
confparams_dec = (sz_params*)malloc(sizeof(sz_params));
memset(confparams_dec, 0, sizeof(sz_params));
if(exe_params==NULL)
exe_params = (sz_exedata*)malloc(sizeof(sz_exedata));
memset(exe_params, 0, sizeof(sz_exedata));
int x = 1;
char *y = (char*)&x;
if(*y==1)
sysEndianType = LITTLE_ENDIAN_SYSTEM;
else //=0
sysEndianType = BIG_ENDIAN_SYSTEM;
confparams_dec->randomAccess = 1;
int status = SZ_SCES;
size_t dataLength = computeDataLength(r5,r4,r3,r2,r1);
//unsigned char* tmpBytes;
size_t targetUncompressSize = dataLength <<2; //i.e., *4
//tmpSize must be "much" smaller than dataLength
size_t i, tmpSize = 8+MetaDataByteLength+exe_params->SZ_SIZE_TYPE;
unsigned char* szTmpBytes;
if(cmpSize!=8+4+MetaDataByteLength && cmpSize!=8+8+MetaDataByteLength) //4,8 means two posibilities of SZ_SIZE_TYPE
{
confparams_dec->losslessCompressor = is_lossless_compressed_data(cmpBytes, cmpSize);
if(confparams_dec->szMode!=SZ_TEMPORAL_COMPRESSION)
{
if(confparams_dec->losslessCompressor!=-1)
confparams_dec->szMode = SZ_BEST_COMPRESSION;
else
confparams_dec->szMode = SZ_BEST_SPEED;
}
if(confparams_dec->szMode==SZ_BEST_SPEED)
{
tmpSize = cmpSize;
szTmpBytes = cmpBytes;
}
else if(confparams_dec->szMode==SZ_BEST_COMPRESSION || confparams_dec->szMode==SZ_DEFAULT_COMPRESSION || confparams_dec->szMode==SZ_TEMPORAL_COMPRESSION)
{
if(targetUncompressSize<MIN_ZLIB_DEC_ALLOMEM_BYTES) //Considering the minimum size
targetUncompressSize = MIN_ZLIB_DEC_ALLOMEM_BYTES;
tmpSize = sz_lossless_decompress(confparams_dec->losslessCompressor, cmpBytes, (unsigned long)cmpSize, &szTmpBytes, (unsigned long)targetUncompressSize+4+MetaDataByteLength+exe_params->SZ_SIZE_TYPE);// (unsigned long)targetUncompressSize+8: consider the total length under lossless compression mode is actually 3+4+1+targetUncompressSize
}
else
{
printf("Wrong value of confparams_dec->szMode in the double compressed bytes.\n");
status = SZ_MERR;
return status;
}
}
else
szTmpBytes = cmpBytes;
TightDataPointStorageF* tdps;
new_TightDataPointStorageF_fromFlatBytes(&tdps, szTmpBytes, tmpSize);
int dim = computeDimension(r5,r4,r3,r2,r1);
int floatSize = sizeof(float);
if(tdps->isLossless)
{
*newData = (float*)malloc(floatSize*dataLength);
if(sysEndianType==BIG_ENDIAN_SYSTEM)
{
memcpy(*newData, szTmpBytes+4+MetaDataByteLength+exe_params->SZ_SIZE_TYPE, dataLength*floatSize);
}
else
{
unsigned char* p = szTmpBytes+4+MetaDataByteLength+exe_params->SZ_SIZE_TYPE;
for(i=0;i<dataLength;i++,p+=floatSize)
(*newData)[i] = bytesToFloat(p);
}
}
else
{
if(confparams_dec->randomAccess == 0 && (s1+s2+s3+s4+s5>0 || (r5-e5+r4-e4+r3-e3+r2-e2+r1-e1 > 0)))
{
printf("Error: you specified the random access mode for decompression, but the compressed data were generate in the non-random-access way.!\n");
status = SZ_DERR;
}
else if (dim == 1)
{
//printf("Error: random access mode doesn't support 1D yet, but only 3D.\n");
decompressDataSeries_float_1D_decompression_given_areas_with_blocked_regression(newData, r1, s1, e1, tdps->raBytes);
//status = SZ_DERR;
}
else
{
printf("Error: currently support only at most 4 dimensions!\n");
status = SZ_DERR;
}
}
free_TightDataPointStorageF2(tdps);
if(confparams_dec->szMode!=SZ_BEST_SPEED && cmpSize!=8+MetaDataByteLength+exe_params->SZ_SIZE_TYPE)
free(szTmpBytes);
return status;
}
#endif

44
deps/SZ/sz/src/szd_float_pwr.c vendored
View File

@ -23,7 +23,7 @@
#pragma GCC diagnostic ignored "-Wchar-subscripts"
void decompressDataSeries_float_1D_pwr(float** data, size_t dataSeriesLength, TightDataPointStorageF* tdps)
void decompressDataSeries_float_1D_pwr(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps)
{
updateQuantizationInfo(tdps->intervals);
unsigned char tmpPrecBytes[4] = {0}; //used when needing to convert bytes to float values
@ -37,7 +37,7 @@ void decompressDataSeries_float_1D_pwr(float** data, size_t dataSeriesLength, Ti
convertByteArray2IntArray_fast_2b(tdps->exactDataNum, tdps->leadNumArray, tdps->leadNumArray_size, &leadNum);
*data = (float*)malloc(sizeof(float)*dataSeriesLength);
//*data = (float*)malloc(sizeof(float)*dataSeriesLength); // comment by tickduan
int* type = (int*)malloc(dataSeriesLength*sizeof(int));
@ -122,16 +122,16 @@ void decompressDataSeries_float_1D_pwr(float** data, size_t dataSeriesLength, Ti
}
exactData = bytesToFloat(curBytes);
(*data)[i] = exactData + medianValue;
data[i] = exactData + medianValue;
memcpy(preBytes,curBytes,4);
break;
default:
//predValue = 2 * (*data)[i-1] - (*data)[i-2];
predValue = (*data)[i-1];
(*data)[i] = predValue + (type_-exe_params->intvRadius)*interval;
//predValue = 2 * data[i-1] - data[i-2];
predValue = data[i-1];
data[i] = predValue + (type_-exe_params->intvRadius)*interval;
break;
}
//printf("%.30G\n",(*data)[i]);
//printf("%.30G\n",data[i]);
}
free(leadNum);
free(type);
@ -139,7 +139,7 @@ void decompressDataSeries_float_1D_pwr(float** data, size_t dataSeriesLength, Ti
}
void decompressDataSeries_float_1D_pwrgroup(float** data, size_t dataSeriesLength, TightDataPointStorageF* tdps)
void decompressDataSeries_float_1D_pwrgroup(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps)
{
float *posGroups, *negGroups, *groups;
float pos_01_group, neg_01_group;
@ -152,7 +152,7 @@ void decompressDataSeries_float_1D_pwrgroup(float** data, size_t dataSeriesLengt
convertByteArray2IntArray_fast_2b(tdps->exactDataNum, tdps->leadNumArray, tdps->leadNumArray_size, &leadNum);
*data = (float*)malloc(sizeof(float)*dataSeriesLength);
//*data = (float*)malloc(sizeof(float)*dataSeriesLength); // comment by tickduan
int* type = (int*)malloc(dataSeriesLength*sizeof(int));
@ -268,7 +268,7 @@ void decompressDataSeries_float_1D_pwrgroup(float** data, size_t dataSeriesLengt
exactData = bytesToFloat(curBytes);
exactData = exactData + medianValue;
(*data)[i] = exactData;
data[i] = exactData;
memcpy(preBytes,curBytes,4);
groups[indexGrpID] = exactData;
@ -294,7 +294,7 @@ void decompressDataSeries_float_1D_pwrgroup(float** data, size_t dataSeriesLengt
// curValue = rawGrpID>0?pow(2,groupNum):-pow(2,groupNum);
//}
(*data)[i] = curValue;
data[i] = curValue;
groups[indexGrpID] = curValue;
break;
}
@ -311,7 +311,7 @@ void decompressDataSeries_float_1D_pwrgroup(float** data, size_t dataSeriesLengt
free(groupID);
}
void decompressDataSeries_float_1D_pwr_pre_log(float** data, size_t dataSeriesLength, TightDataPointStorageF* tdps) {
void decompressDataSeries_float_1D_pwr_pre_log(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps) {
decompressDataSeries_float_1D(data, dataSeriesLength, NULL, tdps);
float threshold = tdps->minLogValue;
@ -319,22 +319,22 @@ void decompressDataSeries_float_1D_pwr_pre_log(float** data, size_t dataSeriesLe
unsigned char * signs;
sz_lossless_decompress(ZSTD_COMPRESSOR, tdps->pwrErrBoundBytes, tdps->pwrErrBoundBytes_size, &signs, dataSeriesLength);
for(size_t i=0; i<dataSeriesLength; i++){
if((*data)[i] < threshold) (*data)[i] = 0;
else (*data)[i] = exp2((*data)[i]);
if(signs[i]) (*data)[i] = -((*data)[i]);
if(data[i] < threshold) data[i] = 0;
else data[i] = exp2(data[i]);
if(signs[i]) data[i] = -(data[i]);
}
free(signs);
}
else{
for(size_t i=0; i<dataSeriesLength; i++){
if((*data)[i] < threshold) (*data)[i] = 0;
else (*data)[i] = exp2((*data)[i]);
if(data[i] < threshold) data[i] = 0;
else data[i] = exp2(data[i]);
}
}
}
void decompressDataSeries_float_1D_pwr_pre_log_MSST19(float** data, size_t dataSeriesLength, TightDataPointStorageF* tdps)
void decompressDataSeries_float_1D_pwr_pre_log_MSST19(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps)
{
decompressDataSeries_float_1D_MSST19(data, dataSeriesLength, tdps);
float threshold = tdps->minLogValue;
@ -350,12 +350,12 @@ void decompressDataSeries_float_1D_pwr_pre_log_MSST19(float** data, size_t dataS
else
sz_lossless_decompress(ZSTD_COMPRESSOR, tdps->pwrErrBoundBytes, tdps->pwrErrBoundBytes_size, &signs, dataSeriesLength);
for(size_t i=0; i<dataSeriesLength; i++){
if((*data)[i] < threshold && (*data)[i] >= 0){
(*data)[i] = 0;
if(data[i] < threshold && data[i] >= 0){
data[i] = 0;
continue;
}
if(signs[i]){
ptr = (uint32_t*)(*data) + i;
ptr = (uint32_t*)data + i;
*ptr |= 0x80000000;
}
}
@ -363,7 +363,7 @@ void decompressDataSeries_float_1D_pwr_pre_log_MSST19(float** data, size_t dataS
}
else{
for(size_t i=0; i<dataSeriesLength; i++){
if((*data)[i] < threshold) (*data)[i] = 0;
if(data[i] < threshold) data[i] = 0;
}
}
}

115
deps/SZ/sz/src/szd_float_ts.c vendored
View File

@ -1,115 +0,0 @@
/**
* @file szd_float_ts.c
* @author Sheng Di and Dingwen Tao
* @date Aug, 2016
* @brief
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include "szd_float.h"
#include "TightDataPointStorageF.h"
#include "sz.h"
#include "Huffman.h"
#include "szd_float_ts.h"
void decompressDataSeries_float_1D_ts(float** data, size_t dataSeriesLength, float* hist_data, TightDataPointStorageF* tdps)
{
float* lastSnapshotData = hist_data;
updateQuantizationInfo(tdps->intervals);
size_t i, j, k = 0, p = 0, l = 0; // k is to track the location of residual_bit
// in resiMidBits, p is to track the
// byte_index of resiMidBits, l is for
// leadNum
unsigned char* leadNum;
double interval = tdps->realPrecision*2;
convertByteArray2IntArray_fast_2b(tdps->exactDataNum, tdps->leadNumArray, tdps->leadNumArray_size, &leadNum);
*data = (float*)malloc(sizeof(float)*dataSeriesLength);
int* type = (int*)malloc(dataSeriesLength*sizeof(int));
HuffmanTree* huffmanTree = createHuffmanTree(tdps->stateNum);
decode_withTree(huffmanTree, tdps->typeArray, dataSeriesLength, type);
SZ_ReleaseHuffman(huffmanTree);
unsigned char preBytes[4];
unsigned char curBytes[4];
memset(preBytes, 0, 4);
size_t curByteIndex = 0;
int reqBytesLength, resiBitsLength, resiBits;
unsigned char leadingNum;
float medianValue, exactData, predValue = 0;
reqBytesLength = tdps->reqLength/8;
resiBitsLength = tdps->reqLength%8;
medianValue = tdps->medianValue;
int type_;
for (i = 0; i < dataSeriesLength; i++) {
type_ = type[i];
switch (type_) {
case 0:
// compute resiBits
resiBits = 0;
if (resiBitsLength != 0) {
int kMod8 = k % 8;
int rightMovSteps = getRightMovingSteps(kMod8, resiBitsLength);
if (rightMovSteps > 0) {
int code = getRightMovingCode(kMod8, resiBitsLength);
resiBits = (tdps->residualMidBits[p] & code) >> rightMovSteps;
} else if (rightMovSteps < 0) {
int code1 = getLeftMovingCode(kMod8);
int code2 = getRightMovingCode(kMod8, resiBitsLength);
int leftMovSteps = -rightMovSteps;
rightMovSteps = 8 - leftMovSteps;
resiBits = (tdps->residualMidBits[p] & code1) << leftMovSteps;
p++;
resiBits = resiBits
| ((tdps->residualMidBits[p] & code2) >> rightMovSteps);
} else // rightMovSteps == 0
{
int code = getRightMovingCode(kMod8, resiBitsLength);
resiBits = (tdps->residualMidBits[p] & code);
p++;
}
k += resiBitsLength;
}
// recover the exact data
memset(curBytes, 0, 4);
leadingNum = leadNum[l++];
memcpy(curBytes, preBytes, leadingNum);
for (j = leadingNum; j < reqBytesLength; j++)
curBytes[j] = tdps->exactMidBytes[curByteIndex++];
if (resiBitsLength != 0) {
unsigned char resiByte = (unsigned char) (resiBits << (8 - resiBitsLength));
curBytes[reqBytesLength] = resiByte;
}
exactData = bytesToFloat(curBytes);
(*data)[i] = exactData + medianValue;
memcpy(preBytes,curBytes,4);
break;
default:
//predValue = (*data)[i-1];
if(confparams_dec->szMode == SZ_TEMPORAL_COMPRESSION)
predValue = lastSnapshotData[i];
(*data)[i] = predValue + (type_-exe_params->intvRadius)*interval;
break;
}
//printf("%.30G\n",(*data)[i]);
}
memcpy(hist_data, (*data), dataSeriesLength*sizeof(float));
free(leadNum);
free(type);
return;
}

570
deps/SZ/sz/src/szf.c vendored
View File

@ -1,570 +0,0 @@
/**
* @file szf.c
* @author Sheng Di
* @date April, 2015
* @brief the key C binding file to connect Fortran and C
* (C) 2015 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#include <string.h>
#include <stdio.h>
#include <stdlib.h>
#include "sz.h"
#include "szf.h"
//special notice: all the function names in this file must be lower-cases!!
void sz_init_c_(char *configFile,int *len,int *ierr)
{
int i;
char s2[*len+1];
for(i=0;i<*len;i++)
s2[i]=configFile[i];
s2[*len]='\0';
// printf("sconfigFile=%s\n",configFile);
*ierr = SZ_Init(s2);
}
void sz_finalize_c_()
{
SZ_Finalize();
}
//compress with config (without args in function)
void sz_compress_d1_float_(float* data, unsigned char *bytes, size_t *outSize, size_t *r1)
{
unsigned char *tmp_bytes = SZ_compress(SZ_FLOAT, data, outSize, 0, 0, 0, 0, *r1);
memcpy(bytes, tmp_bytes, *outSize);
free(tmp_bytes);
}
void sz_compress_d1_float_rev_(float* data, float *reservedValue, unsigned char *bytes, size_t *outSize, size_t *r1)
{
unsigned char *tmp_bytes = SZ_compress_rev(SZ_FLOAT, data, reservedValue, outSize, 0, 0, 0, 0, *r1);
memcpy(bytes, tmp_bytes, *outSize);
free(tmp_bytes);
}
void sz_compress_d2_float_(float* data, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2)
{
unsigned char *tmp_bytes = SZ_compress(SZ_FLOAT, data, outSize, 0, 0, 0, *r2, *r1);
memcpy(bytes, tmp_bytes, *outSize);
free(tmp_bytes);
}
void sz_compress_d2_float_rev_(float* data, float *reservedValue, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2)
{
unsigned char *tmp_bytes = SZ_compress_rev(SZ_FLOAT, data, reservedValue, outSize, 0, 0, 0, *r2, *r1);
memcpy(bytes, tmp_bytes, *outSize);
free(tmp_bytes);
}
void sz_compress_d3_float_(float* data, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3)
{
unsigned char *tmp_bytes = SZ_compress(SZ_FLOAT, data, outSize, 0, 0, *r3, *r2, *r1);
memcpy(bytes, tmp_bytes, *outSize);
free(tmp_bytes);
}
void sz_compress_d3_float_rev_(float* data, float *reservedValue, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3)
{
unsigned char *tmp_bytes = SZ_compress_rev(SZ_FLOAT, data, reservedValue, outSize, 0, 0, *r3, *r2, *r1);
memcpy(bytes, tmp_bytes, *outSize);
free(tmp_bytes);
}
void sz_compress_d4_float_(float* data, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3, size_t *r4)
{
unsigned char *tmp_bytes = SZ_compress(SZ_FLOAT, data, outSize, 0, *r4, *r3, *r2, *r1);
memcpy(bytes, tmp_bytes, *outSize);
free(tmp_bytes);
}
void sz_compress_d4_float_rev_(float* data, float *reservedValue, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3, size_t *r4)
{
unsigned char *tmp_bytes = SZ_compress_rev(SZ_FLOAT, data, reservedValue, outSize, 0, *r4, *r3, *r2, *r1);
memcpy(bytes, tmp_bytes, *outSize);
free(tmp_bytes);
}
void sz_compress_d5_float_(float* data, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5)
{
unsigned char *tmp_bytes = SZ_compress(SZ_FLOAT, data, outSize, *r5, *r4, *r3, *r2, *r1);
memcpy(bytes, tmp_bytes, *outSize);
free(tmp_bytes);
}
void sz_compress_d5_float_rev_(float* data, float *reservedValue, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5)
{
unsigned char *tmp_bytes = SZ_compress_rev(SZ_FLOAT, data, reservedValue, outSize, *r5, *r4, *r3, *r2, *r1);
memcpy(bytes, tmp_bytes, *outSize);
free(tmp_bytes);
}
void sz_compress_d1_double_(double* data, unsigned char *bytes, size_t *outSize, size_t *r1)
{
unsigned char *tmp_bytes = SZ_compress(SZ_DOUBLE, data, outSize, 0, 0, 0, 0, *r1);
memcpy(bytes, tmp_bytes, *outSize);
free(tmp_bytes);
}
void sz_compress_d1_double_rev_(double* data, double *reservedValue, unsigned char *bytes, size_t *outSize, size_t *r1)
{
unsigned char *tmp_bytes = SZ_compress_rev(SZ_DOUBLE, data, reservedValue, outSize, 0, 0, 0, 0, *r1);
memcpy(bytes, tmp_bytes, *outSize);
free(tmp_bytes);
}
void sz_compress_d2_double_(double* data, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2)
{
unsigned char *tmp_bytes = SZ_compress(SZ_DOUBLE, data, outSize, 0, 0, 0, *r2, *r1);
memcpy(bytes, tmp_bytes, *outSize);
free(tmp_bytes);
}
void sz_compress_d2_double_rev_(double* data, double *reservedValue, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2)
{
unsigned char *tmp_bytes = SZ_compress_rev(SZ_DOUBLE, data, reservedValue, outSize, 0, 0, 0, *r2, *r1);
memcpy(bytes, tmp_bytes, *outSize);
free(tmp_bytes);
}
void sz_compress_d3_double_(double* data, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3)
{
unsigned char *tmp_bytes = SZ_compress(SZ_DOUBLE, data, outSize, 0, 0, *r3, *r2, *r1);
memcpy(bytes, tmp_bytes, *outSize);
free(tmp_bytes);
}
void sz_compress_d3_double_rev_(double* data, double *reservedValue, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3)
{
unsigned char *tmp_bytes = SZ_compress_rev(SZ_DOUBLE, data, reservedValue, outSize, 0, 0, *r3, *r2, *r1);
memcpy(bytes, tmp_bytes, *outSize);
free(tmp_bytes);
}
void sz_compress_d4_double_(double* data, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3, size_t *r4)
{
unsigned char *tmp_bytes = SZ_compress(SZ_DOUBLE, data, outSize, 0, *r4, *r3, *r2, *r1);
memcpy(bytes, tmp_bytes, *outSize);
free(tmp_bytes);
}
void sz_compress_d4_double_rev_(double* data, double *reservedValue, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3, size_t *r4)
{
unsigned char *tmp_bytes = SZ_compress_rev(SZ_DOUBLE, data, reservedValue, outSize, 0, *r4, *r3, *r2, *r1);
memcpy(bytes, tmp_bytes, *outSize);
free(tmp_bytes);
}
void sz_compress_d5_double_(double* data, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5)
{
unsigned char *tmp_bytes = SZ_compress(SZ_DOUBLE, data, outSize, *r5, *r4, *r3, *r2, *r1);
memcpy(bytes, tmp_bytes, *outSize);
free(tmp_bytes);
}
void sz_compress_d5_double_rev_(double* data, double *reservedValue, unsigned char *bytes, size_t *outSize, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5)
{
unsigned char *tmp_bytes = SZ_compress_rev(SZ_DOUBLE, data, reservedValue, outSize, *r5, *r4, *r3, *r2, *r1);
memcpy(bytes, tmp_bytes, *outSize);
free(tmp_bytes);
}
//compress with args
void sz_compress_d1_float_args_(float* data, unsigned char *bytes, size_t *outSize, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1)
{
unsigned char *tmp_bytes = SZ_compress_args(SZ_FLOAT, data, outSize, *errBoundMode, *absErrBound, *relBoundRatio, 0.1, 0, 0, 0, 0, *r1);
memcpy(bytes, tmp_bytes, *outSize);
free(tmp_bytes);
}
void sz_compress_d2_float_args_(float* data, unsigned char *bytes, size_t *outSize, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2)
{
unsigned char *tmp_bytes = SZ_compress_args(SZ_FLOAT, data, outSize, *errBoundMode, *absErrBound, *relBoundRatio, 0.1, 0, 0, 0, *r2, *r1);
memcpy(bytes, tmp_bytes, *outSize);
free(tmp_bytes);
}
void sz_compress_d3_float_args_(float* data, unsigned char *bytes, size_t *outSize, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2, size_t *r3)
{
unsigned char *tmp_bytes = SZ_compress_args(SZ_FLOAT, data, outSize, *errBoundMode, *absErrBound, *relBoundRatio, 0.1, 0, 0, *r3, *r2, *r1);
memcpy(bytes, tmp_bytes, *outSize);
free(tmp_bytes);
}
void sz_compress_d4_float_args_(float* data, unsigned char *bytes, size_t *outSize, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4)
{
unsigned char *tmp_bytes = SZ_compress_args(SZ_FLOAT, data, outSize, *errBoundMode, *absErrBound, *relBoundRatio, 0.1, 0, *r4, *r3, *r2, *r1);
memcpy(bytes, tmp_bytes, *outSize);
free(tmp_bytes);
}
void sz_compress_d5_float_args_(float* data, unsigned char *bytes, size_t *outSize, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5)
{
unsigned char *tmp_bytes = SZ_compress_args(SZ_FLOAT, data, outSize, *errBoundMode, *absErrBound, *relBoundRatio, 0.1, *r5, *r4, *r3, *r2, *r1);
memcpy(bytes, tmp_bytes, *outSize);
free(tmp_bytes);
}
void sz_compress_d1_double_args_(double* data, unsigned char *bytes, size_t *outSize, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1)
{
unsigned char *tmp_bytes = SZ_compress_args(SZ_DOUBLE, data, outSize, *errBoundMode, *absErrBound, *relBoundRatio, 0.1, 0, 0, 0, 0, *r1);
memcpy(bytes, tmp_bytes, *outSize);
free(tmp_bytes);
}
void sz_compress_d2_double_args_(double* data, unsigned char *bytes, size_t *outSize, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2)
{
unsigned char *tmp_bytes = SZ_compress_args(SZ_DOUBLE, data, outSize, *errBoundMode, *absErrBound, *relBoundRatio, 0.1, 0, 0, 0, *r2, *r1);
memcpy(bytes, tmp_bytes, *outSize);
free(tmp_bytes);
}
void sz_compress_d3_double_args_(double* data, unsigned char *bytes, size_t *outSize, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2, size_t *r3)
{
unsigned char *tmp_bytes = SZ_compress_args(SZ_DOUBLE, data, outSize, *errBoundMode, *absErrBound, *relBoundRatio, 0.1, 0, 0, *r3, *r2, *r1);
memcpy(bytes, tmp_bytes, *outSize);
free(tmp_bytes);
}
void sz_compress_d4_double_args_(double* data, unsigned char *bytes, size_t *outSize, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4)
{
unsigned char *tmp_bytes = SZ_compress_args(SZ_DOUBLE, data, outSize, *errBoundMode, *absErrBound, *relBoundRatio, 0.1, 0, *r4, *r3, *r2, *r1);
memcpy(bytes, tmp_bytes, *outSize);
free(tmp_bytes);
}
void sz_compress_d5_double_args_(double* data, unsigned char *bytes, size_t *outSize, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5)
{
unsigned char *tmp_bytes = SZ_compress_args(SZ_DOUBLE, data, outSize, *errBoundMode, *absErrBound, *relBoundRatio, 0.1, *r5, *r4, *r3, *r2, *r1);
memcpy(bytes, tmp_bytes, *outSize);
free(tmp_bytes);
}
//--------------
void sz_compress_d1_float_rev_args_(float* data, float *reservedValue, unsigned char *bytes, size_t *outSize, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1)
{
unsigned char *tmp_bytes = SZ_compress_rev_args(SZ_FLOAT, data, reservedValue, outSize, *errBoundMode, *absErrBound, *relBoundRatio, 0, 0, 0, 0, *r1);
memcpy(bytes, tmp_bytes, *outSize);
free(tmp_bytes);
}
void sz_compress_d2_float_rev_args_(float* data, float *reservedValue, unsigned char *bytes, size_t *outSize, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2)
{
unsigned char *tmp_bytes = SZ_compress_rev_args(SZ_FLOAT, data, reservedValue, outSize, *errBoundMode, *absErrBound, *relBoundRatio, 0, 0, 0, *r2, *r1);
memcpy(bytes, tmp_bytes, *outSize);
free(tmp_bytes);
}
void sz_compress_d3_float_rev_args_(float* data, float *reservedValue, unsigned char *bytes, size_t *outSize, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2, size_t *r3)
{
unsigned char *tmp_bytes = SZ_compress_rev_args(SZ_FLOAT, data, reservedValue, outSize, *errBoundMode, *absErrBound, *relBoundRatio, 0, 0, *r3, *r2, *r1);
memcpy(bytes, tmp_bytes, *outSize);
free(tmp_bytes);
}
void sz_compress_d4_float_rev_args_(float* data, float *reservedValue, unsigned char *bytes, size_t *outSize, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4)
{
unsigned char *tmp_bytes = SZ_compress_rev_args(SZ_FLOAT, data, reservedValue, outSize, *errBoundMode, *absErrBound, *relBoundRatio, 0, *r4, *r3, *r2, *r1);
memcpy(bytes, tmp_bytes, *outSize);
free(tmp_bytes);
}
void sz_compress_d5_float_rev_args_(float* data, float *reservedValue, unsigned char *bytes, size_t *outSize, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5)
{
unsigned char *tmp_bytes = SZ_compress_rev_args(SZ_FLOAT, data, reservedValue, outSize, *errBoundMode, *absErrBound, *relBoundRatio, *r5, *r4, *r3, *r2, *r1);
memcpy(bytes, tmp_bytes, *outSize);
free(tmp_bytes);
}
void sz_compress_d1_double_rev_args_(double* data, float *reservedValue, unsigned char *bytes, size_t *outSize, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1)
{
unsigned char *tmp_bytes = SZ_compress_rev_args(SZ_DOUBLE, data, reservedValue, outSize, *errBoundMode, *absErrBound, *relBoundRatio, 0, 0, 0, 0, *r1);
memcpy(bytes, tmp_bytes, *outSize);
free(tmp_bytes);
}
void sz_compress_d2_double_rev_args_(double* data, float *reservedValue, unsigned char *bytes, size_t *outSize, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2)
{
unsigned char *tmp_bytes = SZ_compress_rev_args(SZ_DOUBLE, data, reservedValue, outSize, *errBoundMode, *absErrBound, *relBoundRatio, 0, 0, 0, *r2, *r1);
memcpy(bytes, tmp_bytes, *outSize);
free(tmp_bytes);
}
void sz_compress_d3_double_rev_args_(double* data, float *reservedValue, unsigned char *bytes, size_t *outSize, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2, size_t *r3)
{
unsigned char *tmp_bytes = SZ_compress_rev_args(SZ_DOUBLE, data, reservedValue, outSize, *errBoundMode, *absErrBound, *relBoundRatio, 0, 0, *r3, *r2, *r1);
memcpy(bytes, tmp_bytes, *outSize);
}
void sz_compress_d4_double_rev_args_(double* data, double *reservedValue, unsigned char *bytes, size_t *outSize, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4)
{
unsigned char *tmp_bytes = SZ_compress_rev_args(SZ_DOUBLE, data, reservedValue, outSize, *errBoundMode, *absErrBound, *relBoundRatio, 0, *r4, *r3, *r2, *r1);
memcpy(bytes, tmp_bytes, *outSize);
free(tmp_bytes);
}
void sz_compress_d5_double_rev_args_(double* data, double *reservedValue, unsigned char *bytes, size_t *outSize, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5)
{
unsigned char *tmp_bytes = SZ_compress_rev_args(SZ_DOUBLE, data, reservedValue, outSize, *errBoundMode, *absErrBound, *relBoundRatio, *r5, *r4, *r3, *r2, *r1);
memcpy(bytes, tmp_bytes, *outSize);
free(tmp_bytes);
}
//decompress
void sz_decompress_d1_float_(unsigned char *bytes, size_t *byteLength, float *data, size_t *r1)
{
float *tmp_data = SZ_decompress(SZ_FLOAT, bytes, *byteLength, 0, 0, 0, 0, *r1);
memcpy(data, tmp_data, (*r1)*sizeof(float));
free(tmp_data);
}
void sz_decompress_d2_float_(unsigned char *bytes, size_t *byteLength, float *data, size_t *r1, size_t *r2)
{
size_t r;
float *tmp_data = SZ_decompress(SZ_FLOAT, bytes, *byteLength, 0, 0, 0, *r2, *r1);
r=(*r1)*(*r2);
memcpy(data, tmp_data, r*sizeof(float));
free(tmp_data);
}
void sz_decompress_d3_float_(unsigned char *bytes, size_t *byteLength, float *data, size_t *r1, size_t *r2, size_t *r3)
{
size_t r;
float *tmp_data = SZ_decompress(SZ_FLOAT, bytes, *byteLength, 0, 0, *r3, *r2, *r1);
r=(*r1)*(*r2)*(*r3);
memcpy(data, tmp_data, r*sizeof(float));
free(tmp_data);
}
void sz_decompress_d4_float_(unsigned char *bytes, size_t *byteLength, float *data, size_t *r1, size_t *r2, size_t *r3, size_t *r4)
{
size_t r;
float *tmp_data = SZ_decompress(SZ_FLOAT, bytes, *byteLength, 0, *r4, *r3, *r2, *r1);
r=(*r1)*(*r2)*(*r3)*(*r4);
memcpy(data, tmp_data, r*sizeof(float));
free(tmp_data);
}
void sz_decompress_d5_float_(unsigned char *bytes, size_t *byteLength, float *data, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5)
{
size_t r;
float *tmp_data = SZ_decompress(SZ_FLOAT, bytes, *byteLength, *r5, *r4, *r3, *r2, *r1);
r=(*r1)*(*r2)*(*r3)*(*r4)*(*r5);
memcpy(data, tmp_data, r*sizeof(float));
free(tmp_data);
}
void sz_decompress_d1_double_(unsigned char *bytes, size_t *byteLength, double *data, size_t *r1)
{
double *tmp_data = SZ_decompress(SZ_DOUBLE, bytes, *byteLength, 0, 0, 0, 0, *r1);
memcpy(data, tmp_data, (*r1)*sizeof(double));
free(tmp_data);
}
void sz_decompress_d2_double_(unsigned char *bytes, size_t *byteLength, double *data, size_t *r1, size_t *r2)
{
size_t r;
double *tmp_data = SZ_decompress(SZ_DOUBLE, bytes, *byteLength, 0, 0, 0, *r2, *r1);
r=(*r1)*(*r2);
memcpy(data, tmp_data, r*sizeof(double));
free(tmp_data);
}
void sz_decompress_d3_double_(unsigned char *bytes, size_t *byteLength, double *data, size_t *r1, size_t *r2, size_t *r3)
{
size_t r;
double *tmp_data = SZ_decompress(SZ_DOUBLE, bytes, *byteLength, 0, 0, *r3, *r2, *r1);
r=(*r1)*(*r2)*(*r3);
memcpy(data, tmp_data, r*sizeof(double));
free(tmp_data);
}
void sz_decompress_d4_double_(unsigned char *bytes, size_t *byteLength, double *data, size_t *r1, size_t *r2, size_t *r3, size_t *r4)
{
size_t r;
double *tmp_data = SZ_decompress(SZ_DOUBLE, bytes, *byteLength, 0, *r4, *r3, *r2, *r1);
r=(*r1)*(*r2)*(*r3)*(*r4);
memcpy(data, tmp_data, r*sizeof(double));
free(tmp_data);
}
void sz_decompress_d5_double_(unsigned char *bytes, size_t *byteLength, double *data, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5)
{
size_t r;
double *tmp_data = SZ_decompress(SZ_DOUBLE, bytes, *byteLength, *r5, *r4, *r3, *r2, *r1);
r=(*r1)*(*r2)*(*r3)*(*r4)*(*r5);
memcpy(data, tmp_data, r*sizeof(double));
free(tmp_data);
}
//-----------------TODO: batch mode-----------
void sz_batchaddvar_d1_float_(int var_id, char* varName, int *len, float* data, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1)
{
int i;
char s2[*len+1];
for(i=0;i<*len;i++)
s2[i]=varName[i];
s2[*len]='\0';
SZ_batchAddVar(var_id, s2, SZ_FLOAT, data, *errBoundMode, *absErrBound, *relBoundRatio, 0.1, 0, 0, 0, 0, *r1);
}
void sz_batchaddvar_d2_float_(int var_id, char* varName, int *len, float* data, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2)
{
int i;
char s2[*len+1];
for(i=0;i<*len;i++)
s2[i]=varName[i];
s2[*len]='\0';
SZ_batchAddVar(var_id, s2, SZ_FLOAT, data, *errBoundMode, *absErrBound, *relBoundRatio, 0.1, 0, 0, 0, *r2, *r1);
}
void sz_batchaddvar_d3_float_(int var_id, char* varName, int *len, float* data, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2, size_t *r3)
{
int i;
char s2[*len+1];
for(i=0;i<*len;i++)
s2[i]=varName[i];
s2[*len]='\0';
SZ_batchAddVar(var_id, s2, SZ_FLOAT, data, *errBoundMode, *absErrBound, *relBoundRatio, 0.1, 0, 0, *r3, *r2, *r1);
}
void sz_batchaddvar_d4_float_(int var_id, char* varName, int *len, float* data, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4)
{
int i;
char s2[*len+1];
for(i=0;i<*len;i++)
s2[i]=varName[i];
s2[*len]='\0';
SZ_batchAddVar(var_id, s2, SZ_FLOAT, data, *errBoundMode, *absErrBound, *relBoundRatio, 0.1, 0, *r4, *r3, *r2, *r1);
}
void sz_batchaddvar_d5_float_(int var_id, char* varName, int *len, float* data, int *errBoundMode, float *absErrBound, float *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5)
{
int i;
char s2[*len+1];
for(i=0;i<*len;i++)
s2[i]=varName[i];
s2[*len]='\0';
SZ_batchAddVar(var_id, s2, SZ_FLOAT, data, *errBoundMode, *absErrBound, *relBoundRatio, 0.1, *r5, *r4, *r3, *r2, *r1);
}
void sz_batchaddvar_d1_double_(int var_id, char* varName, int *len, double* data, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1)
{
int i;
char s2[*len+1];
for(i=0;i<*len;i++)
s2[i]=varName[i];
s2[*len]='\0';
SZ_batchAddVar(var_id, s2, SZ_DOUBLE, data, *errBoundMode, *absErrBound, *relBoundRatio, 0.1, 0, 0, 0, 0, *r1);
}
void sz_batchaddvar_d2_double_(int var_id, char* varName, int *len, double* data, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2)
{
int i;
char s2[*len+1];
for(i=0;i<*len;i++)
s2[i]=varName[i];
s2[*len]='\0';
SZ_batchAddVar(var_id, s2, SZ_DOUBLE, data, *errBoundMode, *absErrBound, *relBoundRatio, 0.1, 0, 0, 0, *r2, *r1);
}
void sz_batchaddvar_d3_double_(int var_id, char* varName, int *len, double* data, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2, size_t *r3)
{
int i;
char s2[*len+1];
for(i=0;i<*len;i++)
s2[i]=varName[i];
s2[*len]='\0';
SZ_batchAddVar(var_id, s2, SZ_DOUBLE, data, *errBoundMode, *absErrBound, *relBoundRatio, 0.1, 0, 0, *r3, *r2, *r1);
}
void sz_batchaddvar_d4_double_(int var_id, char* varName, int *len, double* data, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4)
{
int i;
char s2[*len+1];
for(i=0;i<*len;i++)
s2[i]=varName[i];
s2[*len]='\0';
SZ_batchAddVar(var_id, s2, SZ_DOUBLE, data, *errBoundMode, *absErrBound, *relBoundRatio, 0.1, 0, *r4, *r3, *r2, *r1);
}
void sz_batchaddvar_d5_double_(int var_id, char* varName, int *len, double* data, int *errBoundMode, double *absErrBound, double *relBoundRatio, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5)
{
int i;
char s2[*len+1];
for(i=0;i<*len;i++)
s2[i]=varName[i];
s2[*len]='\0';
SZ_batchAddVar(var_id, s2, SZ_DOUBLE, data, *errBoundMode, *absErrBound, *relBoundRatio, 0.1, *r5, *r4, *r3, *r2, *r1);
}
void sz_batchdelvar_c_(char* varName, int *len, int *errState)
{
int i;
char s2[*len+1];
for(i=0;i<*len;i++)
s2[i]=varName[i];
s2[*len]='\0';
*errState = SZ_batchDelVar(s2);
}
/*@deprecated*/
void sz_batch_compress_c_(unsigned char* bytes, size_t *outSize)
{
//unsigned char* tmp_bytes = SZ_batch_compress(outSize);
//memcpy(bytes, tmp_bytes, *outSize);
//free(tmp_bytes);
}
/*@deprecated*/
void sz_batch_decompress_c_(unsigned char* bytes, size_t *byteLength, int *ierr)
{
//SZ_batch_decompress(bytes, *byteLength, ierr);
}
void sz_getvardim_c_(char* varName, int *len, int *dim, size_t *r1, size_t *r2, size_t *r3, size_t *r4, size_t *r5)
{
int i;
char s2[*len+1];
for(i=0;i<*len;i++)
s2[i]=varName[i];
s2[*len]='\0';
SZ_getVarData(s2, r5, r4, r3, r2, r1);
*dim = computeDimension(*r5, *r4, *r3, *r2, *r1);
}
void compute_total_batch_size_c_(size_t *totalSize)
{
*totalSize = compute_total_batch_size();
}
void sz_getvardata_float_(char* varName, int *len, float* data)
{
int i;
size_t r1, r2, r3, r4, r5;
char s2[*len+1];
for(i=0;i<*len;i++)
s2[i]=varName[i];
s2[*len]='\0';
float* tmp_data = (float*)SZ_getVarData(s2, &r5, &r4, &r3, &r2, &r1);
int size = computeDataLength(r5, r4, r3, r2, r1);
memcpy(data, tmp_data, size*sizeof(float));
free(tmp_data);
}
void sz_getvardata_double_(char* varName, int *len, double* data)
{
int i;
size_t r1, r2, r3, r4, r5;
char s2[*len+1];
for(i=0;i<*len;i++)
s2[i]=varName[i];
s2[*len]='\0';
double* tmp_data = (double*)SZ_getVarData(s2, &r5, &r4, &r3, &r2, &r1);
int size = computeDataLength(r5, r4, r3, r2, r1);
memcpy(data, tmp_data, size*sizeof(double));
//free(tmp_data);
}
void sz_freevarset_c_(int *mode)
{
SZ_freeVarSet(*mode);
}

49
deps/SZ/sz/src/td_sz.c vendored
View File

@ -2,21 +2,10 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include "td_sz.h"
#include "CompressElement.h"
#include "DynamicByteArray.h"
#include "DynamicIntArray.h"
#include "TightDataPointStorageD.h"
#include "TightDataPointStorageF.h"
#include "zlib.h"
#include "rw.h"
#include "Huffman.h"
#include "conf.h"
#include "utility.h"
#include "exafelSZ.h"
#include "sz.h"
#include "defines.h"
//
@ -24,15 +13,11 @@
//
int tdszCompress(int type, const char * input, const int nelements, const char * output)
{
size_t outSize = 0;
void* pOut = SZ_compress(type, (void*)input, &outSize, 0, 0, 0, 0, (size_t)nelements);
if(pOut == NULL)
return 0;
// check valid
sz_params comp_params = *confparams_cpr;
// copy to dest
memcpy(output, pOut, outSize);
free(pOut);
return outSize;
size_t outSize = SZ_compress_args(type, input, (size_t)nelements, (unsigned char*)output, &comp_params);
return (int)outSize;
}
//
@ -40,24 +25,6 @@ int tdszCompress(int type, const char * input, const int nelements, const char *
//
int tdszDecompress(int type, const char * input, int compressedSize, const int nelements, const char * output)
{
int width = 0;
if(type == SZ_FLOAT)
width = sizeof(float);
else if(type == SZ_DOUBLE)
width = sizeof(double);
else
return 0;
void* pOut = SZ_decompress(type, (void*)input, compressedSize, 0, 0, 0, 0, (size_t)nelements);
if(pOut == NULL)
return 0;
size_t outSize = nelements * width;
// copy to dest
memcpy(output, pOut, outSize);
free(pOut);
return outSize;
size_t outSize = SZ_decompress(type, (void*)input, compressedSize, (size_t)nelements, (unsigned char*)output);
return (int)outSize;
}

583
deps/SZ/sz/src/utility.c vendored
View File

@ -16,142 +16,6 @@
#include "callZlib.h"
#include "zstd.h"
int compare_struct(const void* obj1, const void* obj2){
struct sort_ast_particle * srt1 = (struct sort_ast_particle*)obj1;
struct sort_ast_particle * srt2 = (struct sort_ast_particle*)obj2;
return srt1->id - srt2->id;
}
void reorder_vars(SZ_VarSet* vset){
SZ_Variable* v[7];
SZ_Variable* v_tmp;
int i, j;
//v[0]
for (v_tmp = vset->header->next, i = 0; i < 7; i++){
v[i] = v_tmp;
v_tmp = v_tmp->next;
}
//printf("here");
size_t dataLen = computeDataLength(v[0]->r5, v[0]->r4, v[0]->r3, v[0]->r2, v[0]->r1);
//sihuan debug
//printf("the data length is (in sorting): %u", dataLen);
struct sort_ast_particle* particle = (struct sort_ast_particle*) malloc(sizeof(struct sort_ast_particle)*dataLen);
for (i = 0; i < dataLen; i++){
particle[i].id = ((int64_t*)v[6]->data)[i];
// printf("%llu ", particle[i].id);
for (j = 0; j < 6; j++)
particle[i].var[j] = ((float*)v[j]->data)[i];
}
//sihuan debug
#if 0
printf("index before sorting: \n");
for (i = 0; i < 5; i++){
printf("%llu ", particle[i].id);
printf("%.5f ", ((float*)v[0]->data)[i]);
}
#endif
//printf("\n");
//sihuan debug
//for (i = 0; i < 5; i++)//{
//for (j = 0; j < 6; j++)
// printf("%.5f ", particle[i].var[j]);
// printf("%llu ", particle[i].id );
///}
//printf("\n\n");
qsort(particle, dataLen, sizeof(struct sort_ast_particle), compare_struct);
for (i = 0; i < dataLen; i++){
((int64_t*)v[6]->data)[i] = particle[i].id;
for (j = 0; j < 6; j++)
((float*)v[j]->data)[i] = particle[i].var[j];
}
free(particle);
//sihuan debug
#if 0
for (i = 0; i < 5; i++){
printf("%llu ", particle[i].id);
printf("%.5f ", ((float*)v[0]->data)[i]);
}
printf("\n");
#endif
}
size_t intersectAndsort(int64_t* preIndex, size_t preLen, SZ_VarSet* curVar, size_t dataLen, unsigned char* bitarray){
size_t i, j, k, m, cnt;
i = j = k = m = cnt = 0;
SZ_Variable* v[7];
SZ_Variable* v_tmp;
//v[0]
for (v_tmp = curVar->header->next, i = 0; i < 7; i++){
v[i] = v_tmp;
v_tmp = v_tmp->next;
}
for (i = 0; i < preLen; i++)
bitarray[i] = '0';
i = 0;
while(i < preLen && j < dataLen){
if (preIndex[i] == ((int64_t*)v[6]->data)[j]){
cnt++;
int64_t tmp;
tmp = ((int64_t*)v[6]->data)[k];
((int64_t*)v[6]->data)[k] = ((int64_t*)v[6]->data)[j];
((int64_t*)v[6]->data)[j] = tmp;
float data_tmp;
for (m = 0; m < 6; m++){
data_tmp = ((float*)v[m]->data)[k];
((float*)v[m]->data)[k] = ((float*)v[m]->data)[j];
((float*)v[m]->data)[j] = data_tmp;
}
k++; i++; j++;
}
else if (preIndex[i] < ((int64_t*)v[6]->data)[j]){
bitarray[i] = '1';
i++;
}
else j++;
}
printf("intersect count is: %zu, i j k pre curlen is: %zu, %zu, %zu, %zu, %zu\n\n", cnt, i, j, k, preLen, dataLen);
return cnt;
}
void write_reordered_tofile(SZ_VarSet* curVar, size_t dataLen){
int var_index; //0 for x, 1 for y...,3 for vx...5 for vz
int i;
char outputfile_name[256];
SZ_Variable* v[7]; SZ_Variable* v_tmp;
for (v_tmp = curVar->header->next, i = 0; i < 6; i++){
v[i] = v_tmp;
v_tmp = v_tmp->next;
}
for (var_index = 0; var_index < 6; var_index++){
sprintf(outputfile_name, "reordered_input_%d_%d.in", sz_tsc->currentStep, var_index);
int status_tmp;
writeFloatData_inBytes((float*)v[var_index]->data, dataLen, outputfile_name, &status_tmp);
}
}
float calculate_delta_t(size_t size){
SZ_Variable* v_tmp = sz_varset->header->next;
while(strcmp(v_tmp->varName, "x")) v_tmp = v_tmp->next;
float* x1 = (float*) v_tmp->data;
float* x0 = (float*) v_tmp->multisteps->hist_data;
while(strcmp(v_tmp->varName, "vx")) v_tmp = v_tmp->next;
float* vx0 = (float*) v_tmp->multisteps->hist_data;
int i, j;
double denom = 0.0;
double div = 0.0;
for (i = 0, j = 0; i < size; i++, j++){
while(sz_tsc->bit_array[j] == '1') j++;
denom += vx0[j] * (x1[i] - x0[j]);
div += vx0[j] * vx0[j];
}
printf("the calculated delta_t is: %.10f\n", denom/div);
return denom/div;
}
int is_lossless_compressed_data(unsigned char* compressedBytes, size_t cmpSize)
{
@ -171,11 +35,10 @@ int is_lossless_compressed_data(unsigned char* compressedBytes, size_t cmpSize)
return -1; //fast mode (without GZIP or ZSTD)
}
unsigned long sz_lossless_compress(int losslessCompressor, int level, unsigned char* data, unsigned long dataLength, unsigned char** compressBytes)
unsigned long sz_lossless_compress(int losslessCompressor, int level, unsigned char* data, unsigned long dataLength, unsigned char* compressBytes)
{
unsigned long outSize = 0;
size_t estimatedCompressedSize = 0;
showme();
switch(losslessCompressor)
{
case GZIP_COMPRESSOR:
@ -186,8 +49,8 @@ unsigned long sz_lossless_compress(int losslessCompressor, int level, unsigned c
estimatedCompressedSize = 200;
else
estimatedCompressedSize = dataLength*1.2;
*compressBytes = (unsigned char*)malloc(estimatedCompressedSize);
outSize = ZSTD_compress(*compressBytes, estimatedCompressedSize, data, dataLength, level); //default setting of level is 3
//*compressBytes = (unsigned char*)malloc(estimatedCompressedSize); // comment by tickduan
outSize = ZSTD_compress(compressBytes, estimatedCompressedSize, data, dataLength, level); //default setting of level is 3
break;
default:
printf("Error: Unrecognized lossless compressor in sz_lossless_compress()\n");
@ -205,448 +68,10 @@ unsigned long sz_lossless_decompress(int losslessCompressor, unsigned char* comp
break;
case ZSTD_COMPRESSOR:
*oriData = (unsigned char*)malloc(targetOriSize);
ZSTD_decompress(*oriData, targetOriSize, compressBytes, cmpSize);
outSize = targetOriSize;
outSize = ZSTD_decompress(*oriData, targetOriSize, compressBytes, cmpSize);
break;
default:
printf("Error: Unrecognized lossless compressor in sz_lossless_decompress()\n");
}
return outSize;
}
unsigned long sz_lossless_decompress65536bytes(int losslessCompressor, unsigned char* compressBytes, unsigned long cmpSize, unsigned char** oriData)
{
unsigned long outSize = 0;
switch(losslessCompressor)
{
case GZIP_COMPRESSOR:
outSize = zlib_uncompress65536bytes(compressBytes, cmpSize, oriData);
break;
case ZSTD_COMPRESSOR:
*oriData = (unsigned char*)malloc(65536);
memset(*oriData, 0, 65536);
ZSTD_decompress(*oriData, 65536, compressBytes, cmpSize); //the first 32768 bytes should be exact the same.
outSize = 65536;
break;
default:
printf("Error: Unrecognized lossless compressor\n");
}
return outSize;
}
void* detransposeData(void* data, int dataType, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1)
{
size_t len = computeDataLength(r5, r4, r3, r2, r1);
int dim = computeDimension(r5, r4, r3, r2, r1);
if(dataType == SZ_FLOAT)
{
float* ori_data = data;
float* new_data = (float*)malloc(sizeof(float)*len);
if(dim==1)
{
memcpy(new_data, ori_data, sizeof(float)*len);
return new_data;
}
else if(dim==2)
{
size_t i, j, s = 0;
for(i=0;i<r2;i++)
for(j=0;j<r1;j++)
{
//size_t s = i*r1+j;
size_t t = j*r2+i;
new_data[t] = ori_data[s++];
}
}
else if(dim==3)
{
size_t i, j, k, s = 0;
size_t B = r1*r2;
for(i=0;i<r2;i++)
for(j=0;j<r1;j++)
for(k=0;k<r3;k++)
{
size_t t = k*B+i*r1+j;
new_data[t] = ori_data[s++];
}
}
else if(dim==4)
{
size_t i, j, k, w, s = 0;
size_t C = r2*r1;
size_t B = r3*C;
for(i=0;i<r3;i++)
for(j=0;j<r2;j++)
for(k=0;k<r1;k++)
for(w=0;w<r4;w++)
{
size_t t = w*B+i*C+j*r1+k;
new_data[t] = ori_data[s++];
}
}
return new_data;
}
else if(dataType == SZ_DOUBLE)
{
double* ori_data = data;
double* new_data = (double*)malloc(sizeof(double)*len);
if(dim==1)
{
memcpy(new_data, ori_data, sizeof(double)*len);
return new_data;
}
else if(dim==2)
{
size_t i, j, s = 0;
for(i=0;i<r2;i++)
for(j=0;j<r1;j++)
{
//size_t s = i*r1+j;
size_t t = j*r2+i;
new_data[t] = ori_data[s++];
}
}
else if(dim==3)
{
size_t i, j, k, s = 0;
size_t B = r1*r2;
for(i=0;i<r2;i++)
for(j=0;j<r1;j++)
for(k=0;k<r3;k++)
{
size_t t = k*B+i*r1+j;
new_data[t] = ori_data[s++];
}
}
else if(dim==4)
{
size_t i, j, k, w, s = 0;
size_t C = r2*r1;
size_t B = r3*C;
for(i=0;i<r3;i++)
for(j=0;j<r2;j++)
for(k=0;k<r1;k++)
for(w=0;w<r4;w++)
{
size_t t = w*B+i*C+j*r1+k;
new_data[t] = ori_data[s++];
}
}
return new_data;
}
else if(dataType == SZ_UINT16)
{
uint16_t* ori_data = data;
uint16_t* new_data = (uint16_t*)malloc(sizeof(uint16_t)*len);
if(dim==1)
{
memcpy(new_data, ori_data, sizeof(uint16_t)*len);
return new_data;
}
else if(dim==2)
{
size_t i, j, s = 0;
for(i=0;i<r2;i++)
for(j=0;j<r1;j++)
{
//size_t s = i*r1+j;
size_t t = j*r2+i;
new_data[t] = ori_data[s++];
}
}
else if(dim==3)
{
size_t i, j, k, s = 0;
size_t B = r1*r2;
for(i=0;i<r2;i++)
for(j=0;j<r1;j++)
for(k=0;k<r3;k++)
{
size_t t = k*B+i*r1+j;
new_data[t] = ori_data[s++];
}
}
else if(dim==4)
{
size_t i, j, k, w, s = 0;
size_t C = r2*r1;
size_t B = r3*C;
for(i=0;i<r3;i++)
for(j=0;j<r2;j++)
for(k=0;k<r1;k++)
for(w=0;w<r4;w++)
{
size_t t = w*B+i*C+j*r1+k;
new_data[t] = ori_data[s++];
}
}
return new_data;
}
else if(dataType == SZ_INT16)
{
int16_t* ori_data = data;
int16_t* new_data = (int16_t*)malloc(sizeof(int16_t)*len);
if(dim==1)
{
memcpy(new_data, ori_data, sizeof(int16_t)*len);
return new_data;
}
else if(dim==2)
{
size_t i, j, s = 0;
for(i=0;i<r2;i++)
for(j=0;j<r1;j++)
{
//size_t s = i*r1+j;
size_t t = j*r2+i;
new_data[t] = ori_data[s++];
}
}
else if(dim==3)
{
size_t i, j, k, s = 0;
size_t B = r1*r2;
for(i=0;i<r2;i++)
for(j=0;j<r1;j++)
for(k=0;k<r3;k++)
{
size_t t = k*B+i*r1+j;
new_data[t] = ori_data[s++];
}
}
else if(dim==4)
{
size_t i, j, k, w, s = 0;
size_t C = r2*r1;
size_t B = r3*C;
for(i=0;i<r3;i++)
for(j=0;j<r2;j++)
for(k=0;k<r1;k++)
for(w=0;w<r4;w++)
{
size_t t = w*B+i*C+j*r1+k;
new_data[t] = ori_data[s++];
}
}
return new_data;
}
else
{
return NULL;
}
}
void* transposeData(void* data, int dataType, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1)
{
size_t len = computeDataLength(r5, r4, r3, r2, r1);
int dim = computeDimension(r5, r4, r3, r2, r1);
if(dataType == SZ_FLOAT)
{
float* ori_data = data;
float* new_data = (float*)malloc(sizeof(float)*len);
if(dim==1)
{
memcpy(new_data, ori_data, sizeof(float)*len);
}
else if(dim==2)
{
size_t i, j, s = 0;
for(i=0;i<r2;i++)
for(j=0;j<r1;j++)
{
size_t t = j*r2+i;
new_data[t] = ori_data[s++];
}
}
else if(dim==3)
{
size_t i, j, k, s = 0;
//size_t B = r2*r1;
for(i=0;i<r3;i++)
for(j=0;j<r2;j++)
for(k=0;k<r1;k++)
{
size_t jk = j*r1+k;
//size_t s = i*B+jk;
size_t t = r3*jk+i;
new_data[t] = ori_data[s++];
}
}
else if(dim==4)
{
size_t C = r2*r1;
//size_t B = r3*C;
size_t D = C*r4;
size_t i, j, k, w, s = 0;
for(i=0;i<r4;i++)
for(j=0;j<r3;j++)
for(k=0;k<r2;k++)
for(w=0;w<r1;w++)
{
size_t kw = k*r1+w;
//size_t s = i*B+j*C+kw;
size_t t = j*D+r4*kw+i;
new_data[t] = ori_data[s++];
}
}
return new_data;
}
else if(dataType == SZ_DOUBLE)
{
double* ori_data = data;
double* new_data = (double*)malloc(sizeof(double)*len);
if(dim==1)
{
memcpy(new_data, ori_data, sizeof(double)*len);
}
else if(dim==2)
{
size_t i, j, s = 0;
for(i=0;i<r2;i++)
for(j=0;j<r1;j++)
{
size_t t = j*r2+i;
new_data[t] = ori_data[s++];
}
}
else if(dim==3)
{
size_t i, j, k, s = 0;
//size_t B = r2*r1;
for(i=0;i<r3;i++)
for(j=0;j<r2;j++)
for(k=0;k<r1;k++)
{
size_t jk = j*r1+k;
//size_t s = i*B+jk;
size_t t = r3*jk+i;
new_data[t] = ori_data[s++];
}
}
else if(dim==4)
{
size_t C = r2*r1;
//size_t B = r3*C;
size_t D = C*r4;
size_t i, j, k, w, s = 0;
for(i=0;i<r4;i++)
for(j=0;j<r3;j++)
for(k=0;k<r2;k++)
for(w=0;w<r1;w++)
{
size_t kw = k*r1+w;
//size_t s = i*B+j*C+kw;
size_t t = j*D+r4*kw+i;
new_data[t] = ori_data[s++];
}
}
return new_data;
}
else if(dataType == SZ_UINT16)
{
uint16_t* ori_data = data;
uint16_t* new_data = (uint16_t*)malloc(sizeof(uint16_t)*len);
if(dim==1)
{
memcpy(new_data, ori_data, sizeof(uint16_t)*len);
}
else if(dim==2)
{
size_t i, j, s = 0;
for(i=0;i<r2;i++)
for(j=0;j<r1;j++)
{
size_t t = j*r2+i;
new_data[t] = ori_data[s++];
}
}
else if(dim==3)
{
size_t i, j, k, s = 0;
//size_t B = r2*r1;
for(i=0;i<r3;i++)
for(j=0;j<r2;j++)
for(k=0;k<r1;k++)
{
size_t jk = j*r1+k;
//size_t s = i*B+jk;
size_t t = r3*jk+i;
new_data[t] = ori_data[s++];
}
}
else if(dim==4)
{
size_t C = r2*r1;
//size_t B = r3*C;
size_t D = C*r4;
size_t i, j, k, w, s = 0;
for(i=0;i<r4;i++)
for(j=0;j<r3;j++)
for(k=0;k<r2;k++)
for(w=0;w<r1;w++)
{
size_t kw = k*r1+w;
//size_t s = i*B+j*C+kw;
size_t t = j*D+r4*kw+i;
new_data[t] = ori_data[s++];
}
}
return new_data;
}
else if(dataType == SZ_INT16)
{
int16_t* ori_data = data;
int16_t* new_data = (int16_t*)malloc(sizeof(int16_t)*len);
if(dim==1)
{
memcpy(new_data, ori_data, sizeof(int16_t)*len);
}
else if(dim==2)
{
size_t i, j, s = 0;
for(i=0;i<r2;i++)
for(j=0;j<r1;j++)
{
size_t t = j*r2+i;
new_data[t] = ori_data[s++];
}
}
else if(dim==3)
{
size_t i, j, k, s = 0;
//size_t B = r2*r1;
for(i=0;i<r3;i++)
for(j=0;j<r2;j++)
for(k=0;k<r1;k++)
{
size_t jk = j*r1+k;
//size_t s = i*B+jk;
size_t t = r3*jk+i;
new_data[t] = ori_data[s++];
}
}
else if(dim==4)
{
size_t C = r2*r1;
//size_t B = r3*C;
size_t D = C*r4;
size_t i, j, k, w, s = 0;
for(i=0;i<r4;i++)
for(j=0;j<r3;j++)
for(k=0;k<r2;k++)
for(w=0;w<r1;w++)
{
size_t kw = k*r1+w;
//size_t s = i*B+j*C+kw;
size_t t = j*D+r4*kw+i;
new_data[t] = ori_data[s++];
}
}
return new_data;
}
else
{
printf("Error. transpose data doesn't support data type %d\n", dataType);
return NULL;
}
}

3
deps/SZ/zstd/compress/zstd_compress.c vendored
View File

@ -26,9 +26,6 @@
#include "zstd_opt.h"
#include "zstd_ldm.h"
void showme(){
printf(" hello show me.");
}
/*-*************************************
* Helper functions

11
src/common/src/ttypes.c
View File

@ -368,22 +368,21 @@ static void getStatics_nchr(const void *pData, int32_t numOfRow, int64_t *min, i
}
tDataTypeDescriptor tDataTypes[15] = {
{TSDB_DATA_TYPE_NULL, 6,1, "NOTYPE", NULL, NULL, NULL},
{TSDB_DATA_TYPE_NULL, 6, 1, "NOTYPE", NULL, NULL, NULL},
{TSDB_DATA_TYPE_BOOL, 4, CHAR_BYTES, "BOOL", tsCompressBool, tsDecompressBool, getStatics_bool},
{TSDB_DATA_TYPE_TINYINT, 7, CHAR_BYTES, "TINYINT", tsCompressTinyint, tsDecompressTinyint, getStatics_i8},
{TSDB_DATA_TYPE_SMALLINT, 8, SHORT_BYTES, "SMALLINT", tsCompressSmallint, tsDecompressSmallint, getStatics_i16},
{TSDB_DATA_TYPE_INT, 3, INT_BYTES, "INT", tsCompressInt, tsDecompressInt, getStatics_i32},
{TSDB_DATA_TYPE_BIGINT, 6, LONG_BYTES, "BIGINT", tsCompressBigint, tsDecompressBigint, getStatics_i64},
{TSDB_DATA_TYPE_FLOAT, 5, FLOAT_BYTES, "FLOAT", tsCompressFloatLossy, tsDecompressFloatLossy, getStatics_f},
{TSDB_DATA_TYPE_DOUBLE, 6, DOUBLE_BYTES, "DOUBLE", tsCompressDoubleLossy, tsDecompressDoubleLossy, getStatics_d},
{TSDB_DATA_TYPE_BINARY, 6, 0, "BINARY", tsCompressString, tsDecompressString, getStatics_bin},
{TSDB_DATA_TYPE_FLOAT, 5, FLOAT_BYTES, "FLOAT", tsCompressFloat, tsDecompressFloat, getStatics_f},
{TSDB_DATA_TYPE_DOUBLE, 6, DOUBLE_BYTES, "DOUBLE", tsCompressDouble, tsDecompressDouble, getStatics_d},
{TSDB_DATA_TYPE_BINARY, 6, 0, "BINARY", tsCompressString, tsDecompressString, getStatics_bin},
{TSDB_DATA_TYPE_TIMESTAMP, 9, LONG_BYTES, "TIMESTAMP", tsCompressTimestamp, tsDecompressTimestamp, getStatics_i64},
{TSDB_DATA_TYPE_NCHAR, 5, 8, "NCHAR", tsCompressString, tsDecompressString, getStatics_nchr},
{TSDB_DATA_TYPE_NCHAR, 5, 8, "NCHAR", tsCompressString, tsDecompressString, getStatics_nchr},
{TSDB_DATA_TYPE_UTINYINT, 16, CHAR_BYTES, "TINYINT UNSIGNED", tsCompressTinyint, tsDecompressTinyint, getStatics_u8},
{TSDB_DATA_TYPE_USMALLINT, 17, SHORT_BYTES, "SMALLINT UNSIGNED", tsCompressSmallint, tsDecompressSmallint, getStatics_u16},
{TSDB_DATA_TYPE_UINT, 12, INT_BYTES, "INT UNSIGNED", tsCompressInt, tsDecompressInt, getStatics_u32},
{TSDB_DATA_TYPE_UBIGINT, 15, LONG_BYTES, "BIGINT UNSIGNED", tsCompressBigint, tsDecompressBigint, getStatics_u64},
};
char tTokenTypeSwitcher[13] = {

77
src/kit/taospack/taospack.c
View File

@ -58,6 +58,7 @@ float toFloat(char* buf){
return (float)atoll(buf);
}
//
// read float
//
@ -120,6 +121,7 @@ float check_same(float* ft1, float* ft2, int count){
//
// test compress and decompress
//
extern bool gOpenLossy;
bool testFile(const char* inFile, char algorithm){
// check valid
if(inFile == NULL || inFile[0] == 0 ){
@ -405,7 +407,7 @@ int memTest() {
cost_end(" tscompress");
printf(" compress return len=%d input len=%d\n", ret_len, input_len);
printf(" compress rate=%.1f an-rate=%.0f%%\n", (float)input_len/(float)ret_len, 100*(float)ret_len/(float)input_len);
printf(" compress sz rate=%.1f an-rate=%.2f%%\n", (float)input_len/(float)ret_len, 100*(float)ret_len/(float)input_len);
//
// decompress
@ -447,7 +449,7 @@ int memTest() {
void* memTestThread(void* lparam) {
//memTest();
printf(" enter thread ....\n");
for(int i=0; i< 1000000; i++)
for(int i=0; i< 5; i++)
{
memTest();
printf(" start i=%d .... \n", i);
@ -480,7 +482,7 @@ void test_threadsafe(int thread_count){
void* memTestThreadDouble(void* lparam) {
//memTest();
printf(" enter thread ....\n");
for(int i=0; i< 50000; i++)
for(int i=0; i< 10; i++)
{
memTest();
printf(" double start i=%d .... \n", i);
@ -510,7 +512,68 @@ void test_threadsafe_double(int thread_count){
}
void unitTestFloat() {
float ft1 [] = {1.11, 2.22, 3.333};
int cnt = sizeof(ft1)/sizeof(float);
float* floats = ft1;
int algorithm = 2;
// compress
const char* input = (const char*)floats;
int input_len = cnt * sizeof(float);
int output_len = input_len + 1024;
char* output = (char*) malloc(output_len);
char* buff = (char*) malloc(input_len);
int buff_len = input_len;
printf(" ft1 have count=%d \n", cnt);
strcpy(output, "abcde");
cost_start();
int ret_len = 0;
ret_len = tsCompressFloatLossy(input, input_len, cnt, output, output_len, algorithm, buff, buff_len);
if(ret_len == 0) {
printf(" compress error.\n");
return ;
}
double use_ms1 = cost_end("compress");
printf(" compress len=%d input len=%d\n", ret_len, input_len);
float rate=100*(float)ret_len/(float)input_len;
printf(" compress rate=%.1f an-rate=%.4f%%\n", (float)input_len/(float)ret_len, rate);
//
// decompress
//
float* ft2 = (float*)malloc(input_len);
cost_start();
int code = 0;
code = tsDecompressFloatLossy(output, ret_len, cnt, (char*)ft2, input_len, algorithm, buff, buff_len);
double use_ms2 = cost_end("Decompress");
printf(" Decompress return length=%d \n", code);
// compare same
float same_rate = check_same(floats, ft2, cnt);
printf("\n ------------------ count:%d TD <SZ> ---------------- \n", cnt);
printf(" Compress Rate ......... [%.0f%%] \n", rate);
double speed1 = (cnt*sizeof(float)*1000/1024/1024)/use_ms1;
printf(" Compress Time ......... [%.4fms] speed=%.1f MB/s\n", use_ms1, speed1);
double speed2 = (cnt*sizeof(float)*1000/1024/1024)/use_ms2;
printf(" Decompress Time........ [%.4fms] speed=%.1f MB/s\n", use_ms2, speed2);
printf(" Same Rate ............. [%.0f%%] \n\n", same_rate);
// free
free(ft2);
free(buff);
free(output);
}
@ -518,8 +581,9 @@ void test_threadsafe_double(int thread_count){
// ----------------- main ----------------------
//
int main(int argc, char *argv[]) {
printf("welcome to use taospack tools v1.1. sizeof(STColumn) = %lu\n", sizeof(STColumn));
printf("welcome to use taospack tools v1.2 \n");
gOpenLossy = false;
tsLossyInit();
//
//tsCompressExit();
@ -530,9 +594,11 @@ int main(int argc, char *argv[]) {
// t
if(strcmp(argv[1], "-tone") == 0 || strcmp(argv[1], "-t") == 0 ) {
algo = ONE_STAGE_COMP;
gOpenLossy = true;
}
if(strcmp(argv[1], "-ttwo") == 0) {
algo = TWO_STAGE_COMP;
gOpenLossy = false;
}
if(strcmp(argv[1], "-sf") == 0) {
@ -558,6 +624,9 @@ int main(int argc, char *argv[]) {
memTest();
}
}
else{
unitTestFloat();
}
//memTest();
return 0;

109
src/util/inc/tscompression.h
View File

@ -35,6 +35,25 @@ extern "C" {
#define ONE_STAGE_COMP 1
#define TWO_STAGE_COMP 2
//
// compressed data first byte foramt
// ------ 7 bit ---- | ---- 1 bit ----
// algorithm mode
//
// compression data mode save first byte lower 1 bit
#define MODE_NOCOMPRESS 0 // original data
#define MODE_COMPRESS 1 // compatible old compress
// compression algorithm save first byte higher 7 bit
#define ALGO_SZ_LOSSY 1 // SZ compress
#define HEAD_MODE(x) x%2
#define HEAD_ALGO(x) x/2
extern bool gOpenLossy;
extern int tsCompressINTImp(const char *const input, const int nelements, char *const output, const char type);
extern int tsDecompressINTImp(const char *const input, const int nelements, char *const output, const char type);
extern int tsCompressBoolImp(const char *const input, const int nelements, char *const output);
@ -48,16 +67,17 @@ extern int tsDecompressDoubleImp(const char *const input, const int nelements, c
extern int tsCompressFloatImp(const char *const input, const int nelements, char *const output);
extern int tsDecompressFloatImp(const char *const input, const int nelements, char *const output);
// lossy
int tsCompressFloatLossyImp(const char * input, const int nelements, const char * output);
int tsDecompressFloatLossyImp(const char * input, int compressedSize, const int nelements, const char * output);
int tsCompressDoubleLossyImp(const char * input, const int nelements, const char * output);
int tsDecompressDoubleLossyImp(const char * input, int compressedSize, const int nelements, const char * output);
int tsCompressFloatLossyImp(const char * input, const int nelements, char *const output);
int tsDecompressFloatLossyImp(const char * input, int compressedSize, const int nelements, char *const output);
int tsCompressDoubleLossyImp(const char * input, const int nelements, char *const output);
int tsDecompressDoubleLossyImp(const char * input, int compressedSize, const int nelements, char *const output);
// init
bool tsLossyInit();
static FORCE_INLINE int tsCompressTinyint(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, char algorithm,
char *const buffer, int bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
@ -200,53 +220,78 @@ static FORCE_INLINE int tsDecompressString(const char *const input, int compress
static FORCE_INLINE int tsCompressFloat(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
char algorithm, char *const buffer, int bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsCompressFloatImp(input, nelements, output);
} else if (algorithm == TWO_STAGE_COMP) {
int len = tsCompressFloatImp(input, nelements, buffer);
return tsCompressStringImp(buffer, len, output, outputSize);
// lossy mode
if(gOpenLossy) {
return tsCompressFloatLossyImp(input, nelements, output);
// lossless mode
} else {
assert(0);
return -1;
if (algorithm == ONE_STAGE_COMP) {
return tsCompressFloatImp(input, nelements, output);
} else if (algorithm == TWO_STAGE_COMP) {
int len = tsCompressFloatImp(input, nelements, buffer);
return tsCompressStringImp(buffer, len, output, outputSize);
} else {
assert(0);
return -1;
}
}
}
static FORCE_INLINE int tsDecompressFloat(const char *const input, int compressedSize, const int nelements, char *const output,
int outputSize, char algorithm, char *const buffer, int bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsDecompressFloatImp(input, nelements, output);
} else if (algorithm == TWO_STAGE_COMP) {
if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1;
return tsDecompressFloatImp(buffer, nelements, output);
if(HEAD_ALGO(input[0]) == ALGO_SZ_LOSSY){
// decompress lossy
return tsDecompressFloatLossyImp(input, compressedSize, nelements, output);
} else {
assert(0);
return -1;
// decompress lossless
if (algorithm == ONE_STAGE_COMP) {
return tsDecompressFloatImp(input, nelements, output);
} else if (algorithm == TWO_STAGE_COMP) {
if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1;
return tsDecompressFloatImp(buffer, nelements, output);
} else {
assert(0);
return -1;
}
}
}
static FORCE_INLINE int tsCompressDouble(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
char algorithm, char *const buffer, int bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsCompressDoubleImp(input, nelements, output);
} else if (algorithm == TWO_STAGE_COMP) {
int len = tsCompressDoubleImp(input, nelements, buffer);
return tsCompressStringImp(buffer, len, output, outputSize);
if(gOpenLossy){
// lossy mode
return tsCompressDoubleLossyImp(input, nelements, output);
} else {
assert(0);
return -1;
// lossless mode
if (algorithm == ONE_STAGE_COMP) {
return tsCompressDoubleImp(input, nelements, output);
} else if (algorithm == TWO_STAGE_COMP) {
int len = tsCompressDoubleImp(input, nelements, buffer);
return tsCompressStringImp(buffer, len, output, outputSize);
} else {
assert(0);
return -1;
}
}
}
static FORCE_INLINE int tsDecompressDouble(const char *const input, int compressedSize, const int nelements, char *const output,
int outputSize, char algorithm, char *const buffer, int bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsDecompressDoubleImp(input, nelements, output);
} else if (algorithm == TWO_STAGE_COMP) {
if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1;
return tsDecompressDoubleImp(buffer, nelements, output);
if(HEAD_ALGO(input[0]) == ALGO_SZ_LOSSY){
// decompress lossy
return tsDecompressDoubleLossyImp(input, compressedSize, nelements, output);
} else {
assert(0);
return -1;
// decompress lossless
if (algorithm == ONE_STAGE_COMP) {
return tsDecompressDoubleImp(input, nelements, output);
} else if (algorithm == TWO_STAGE_COMP) {
if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1;
return tsDecompressDoubleImp(buffer, nelements, output);
} else {
assert(0);
return -1;
}
}
}

67
src/util/src/tcompression.c
View File

@ -62,6 +62,8 @@ static const int TEST_NUMBER = 1;
#define ZIGZAG_ENCODE(T, v) ((u##T)((v) >> (sizeof(T) * 8 - 1))) ^ (((u##T)(v)) << 1) // zigzag encode
#define ZIGZAG_DECODE(T, v) ((v) >> 1) ^ -((T)((v)&1)) // zigzag decode
bool gOpenLossy = true;
/*
* Compress Integer (Simple8B).
*/
@ -411,6 +413,7 @@ int tsCompressStringImp(const char *const input, int inputSize, char *const outp
int tsDecompressStringImp(const char *const input, int compressedSize, char *const output, int outputSize) {
// compressedSize is the size of data after compression.
if (input[0] == 1) {
/* It is compressed by LZ4 algorithm */
const int decompressed_size = LZ4_decompress_safe(input + 1, output, compressedSize - 1, outputSize);
@ -906,18 +909,66 @@ bool tsLossyInit() {
//
// ---------- float double lossy -----------
//
int tsCompressFloatLossyImp(const char * input, const int nelements, const char * output){
return tdszCompress(SZ_FLOAT, input, nelements, output);
int tsCompressFloatLossyImp(const char * input, const int nelements, char *const output){
// compress with sz
int compressedSize = tdszCompress(SZ_FLOAT, input, nelements, output + 1);
unsigned char algo = ALGO_SZ_LOSSY << 1;
if (compressedSize == 0 || compressedSize >= nelements*sizeof(float)){
// compressed error or large than original
output[0] = MODE_NOCOMPRESS | algo;
memcpy(output + 1, input, nelements * sizeof(float));
compressedSize = 1 + nelements * sizeof(float);
} else {
// compressed successfully
output[0] = MODE_COMPRESS | algo;
compressedSize += 1;
}
return compressedSize;
}
int tsDecompressFloatLossyImp(const char * input, int compressedSize, const int nelements, const char * output){
return tdszDecompress(SZ_FLOAT, input, compressedSize, nelements, output);
int tsDecompressFloatLossyImp(const char * input, int compressedSize, const int nelements, char *const output){
int decompressedSize = 0;
if( HEAD_MODE(input[0]) == MODE_NOCOMPRESS){
// orginal so memcpy directly
decompressedSize = nelements * sizeof(float);
memcpy(output, input + 1, decompressedSize);
return decompressedSize;
}
// decompressed with sz
return tdszDecompress(SZ_FLOAT, input + 1, compressedSize - 1, nelements, output);
}
int tsCompressDoubleLossyImp(const char * input, const int nelements, const char * output){
return tdszCompress(SZ_DOUBLE, input, nelements, output);
int tsCompressDoubleLossyImp(const char * input, const int nelements, char *const output){
// compress with sz
int compressedSize = tdszCompress(SZ_DOUBLE, input, nelements, output + 1);
unsigned char algo = ALGO_SZ_LOSSY << 1;
if (compressedSize == 0 || compressedSize >= nelements*sizeof(double)) {
// compressed error or large than original
output[0] = MODE_NOCOMPRESS | algo;
memcpy(output + 1, input, nelements * sizeof(double));
compressedSize = 1 + nelements * sizeof(double);
} else {
// compressed successfully
output[0] = MODE_COMPRESS | algo;
compressedSize += 1;
}
return compressedSize;
}
int tsDecompressDoubleLossyImp(const char * input, int compressedSize, const int nelements, const char * output){
return tdszDecompress(SZ_DOUBLE, input, compressedSize, nelements, output);
int tsDecompressDoubleLossyImp(const char * input, int compressedSize, const int nelements, char *const output){
int decompressedSize = 0;
if( HEAD_MODE(input[0]) == MODE_NOCOMPRESS){
// orginal so memcpy directly
decompressedSize = nelements * sizeof(double);
memcpy(output, input + 1, decompressedSize);
return decompressedSize;
}
// decompressed with sz
return tdszDecompress(SZ_DOUBLE, input + 1, compressedSize - 1, nelements, output);
}