xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

remove pwr_ reduce to 41s

This commit is contained in:
tickduan 2021-07-06 09:23:08 +08:00
parent 3e8cfacdae
commit 3e24cb15ce
31 changed files with 214 additions and 4545 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
deps/SZ/sz/include/CompressElement.h vendored
View File

@ -45,7 +45,6 @@ typedef struct LossyCompressionElement
int residualMidBits;
} LossyCompressionElement;
char* decompressGroupIDArray(unsigned char* bytes, size_t dataLength);
short computeGroupNum_float(float value);
short computeGroupNum_double(double value);

23
deps/SZ/sz/include/TightDataPointStorageD.h vendored
View File

@ -33,10 +33,7 @@ typedef struct TightDataPointStorageD
size_t exactDataNum;
double reservedValue;
unsigned char* rtypeArray;
size_t rtypeArray_size;
unsigned char* typeArray; //its size is dataSeriesLength/4 (or xxx/4+1)
size_t typeArray_size;
@ -54,9 +51,6 @@ typedef struct TightDataPointStorageD
unsigned char isLossless; //a mark to denote whether it's lossless compression (1 is yes, 0 is no)
size_t segment_size;
unsigned char* pwrErrBoundBytes;
int pwrErrBoundBytes_size;
unsigned char* raBytes;
size_t raBytes_size;
@ -75,22 +69,11 @@ void new_TightDataPointStorageD(TightDataPointStorageD **self,
unsigned char* leadNumIntArray, //leadNumIntArray contains readable numbers....
unsigned char* resiMidBits, size_t resiMidBits_size,
unsigned char resiBitLength,
double realPrecision, double medianValue, char reqLength, unsigned int intervals,
unsigned char* pwrErrBoundBytes, size_t pwrErrBoundBytes_size, unsigned char radExpo);
void new_TightDataPointStorageD2(TightDataPointStorageD **self,
size_t dataSeriesLength, size_t exactDataNum,
int* type, unsigned char* exactMidBytes, size_t exactMidBytes_size,
unsigned char* leadNumIntArray, //leadNumIntArray contains readable numbers....
unsigned char* resiMidBits, size_t resiMidBits_size,
unsigned char* resiBitLength, size_t resiBitLengthSize,
double realPrecision, double medianValue, char reqLength, unsigned int intervals,
unsigned char* pwrErrBoundBytes, size_t pwrErrBoundBytes_size, unsigned char radExpo);
double realPrecision, double medianValue, char reqLength,
unsigned int intervals, unsigned char radExpo);
void convertTDPStoBytes_double(TightDataPointStorageD* tdps, unsigned char* bytes, unsigned char* dsLengthBytes, unsigned char sameByte);
void convertTDPStoBytes_double_reserve(TightDataPointStorageD* tdps, unsigned char* bytes, unsigned char* dsLengthBytes, unsigned char sameByte);
bool convertTDPStoFlatBytes_double(TightDataPointStorageD *tdps, unsigned char* bytes, size_t *size);
void convertTDPStoFlatBytes_double_args(TightDataPointStorageD *tdps, unsigned char* bytes, size_t *size);
void free_TightDataPointStorageD(TightDataPointStorageD *tdps);
void free_TightDataPointStorageD2(TightDataPointStorageD *tdps);

23
deps/SZ/sz/include/TightDataPointStorageF.h vendored
View File

@ -33,13 +33,10 @@ typedef struct TightDataPointStorageF
size_t exactDataNum;
float reservedValue;
unsigned char* rtypeArray;
size_t rtypeArray_size;
float minLogValue;
unsigned char* typeArray; //its size is dataSeriesLength/4 (or xxx/4+1)
size_t typeArray_size;
size_t typeArray_size;
unsigned char* leadNumArray; //its size is exactDataNum/4 (or exactDataNum/4+1)
size_t leadNumArray_size;
@ -56,8 +53,6 @@ typedef struct TightDataPointStorageF
size_t segment_size;
unsigned char* pwrErrBoundBytes;
int pwrErrBoundBytes_size;
unsigned char* raBytes;
size_t raBytes_size;
@ -77,24 +72,10 @@ void new_TightDataPointStorageF(TightDataPointStorageF **self,
unsigned char* resiMidBits, size_t resiMidBits_size,
unsigned char resiBitLength,
double realPrecision, float medianValue, char reqLength, unsigned int intervals,
unsigned char* pwrErrBoundBytes, size_t pwrErrBoundBytes_size, unsigned char radExpo);
/**
* This function is designed for first-version of the point-wise relative error bound (developed by Sheng Di for TPDS18 paper)
*
* */
void new_TightDataPointStorageF2(TightDataPointStorageF **self,
size_t dataSeriesLength, size_t exactDataNum,
int* type, unsigned char* exactMidBytes, size_t exactMidBytes_size,
unsigned char* leadNumIntArray, //leadNumIntArray contains readable numbers....
unsigned char* resiMidBits, size_t resiMidBits_size,
unsigned char* resiBitLength, size_t resiBitLengthSize,
double realPrecision, float medianValue, char reqLength, unsigned int intervals,
unsigned char* pwrErrBoundBytes, size_t pwrErrBoundBytes_size, unsigned char radExpo);
unsigned char radExpo);
void convertTDPStoBytes_float(TightDataPointStorageF* tdps, unsigned char* bytes, unsigned char* dsLengthBytes, unsigned char sameByte);
bool convertTDPStoFlatBytes_float(TightDataPointStorageF *tdps, unsigned char* bytes, size_t *size);
void convertTDPStoFlatBytes_float_args(TightDataPointStorageF *tdps, unsigned char* bytes, size_t *size);
void free_TightDataPointStorageF(TightDataPointStorageF *tdps);
void free_TightDataPointStorageF2(TightDataPointStorageF *tdps);

24
deps/SZ/sz/include/dataCompression.h vendored
View File

@ -30,21 +30,9 @@ extern "C" {
//dataCompression.c
int computeByteSizePerIntValue(long valueRangeSize);
long computeRangeSize_int(void* oriData, int dataType, size_t size, int64_t* valueRangeSize);
double computeRangeSize_double(double* oriData, size_t size, double* valueRangeSize, double* medianValue);
float computeRangeSize_float(float* oriData, size_t size, float* valueRangeSize, float* medianValue);
float computeRangeSize_float_MSST19(float* oriData, size_t size, float* valueRangeSize, float* medianValue, unsigned char * signs, bool* positive, float* nearZero);
double computeRangeSize_double_MSST19(double* oriData, size_t size, double* valueRangeSize, double* medianValue, unsigned char * signs, bool* positive, double* nearZero);
double computeRangeSize_double_subblock(double* oriData, double* valueRangeSize, double* medianValue,
size_t r5, size_t r4, size_t r3, size_t r2, size_t r1,
size_t s5, size_t s4, size_t s3, size_t s2, size_t s1,
size_t e5, size_t e4, size_t e3, size_t e2, size_t e1);
float computeRangeSize_float_subblock(float* oriData, float* valueRangeSize, float* medianValue,
size_t r5, size_t r4, size_t r3, size_t r2, size_t r1,
size_t s5, size_t s4, size_t s3, size_t s2, size_t s1,
size_t e5, size_t e4, size_t e3, size_t e2, size_t e1);
double min_d(double a, double b);
double max_d(double a, double b);
float min_f(float a, float b);
@ -56,22 +44,10 @@ void symTransform_8bytes(unsigned char data[8]);
void symTransform_2bytes(unsigned char data[2]);
void symTransform_4bytes(unsigned char data[4]);
void compressInt8Value(int8_t tgtValue, int8_t minValue, int byteSize, unsigned char* bytes);
void compressInt16Value(int16_t tgtValue, int16_t minValue, int byteSize, unsigned char* bytes);
void compressInt32Value(int32_t tgtValue, int32_t minValue, int byteSize, unsigned char* bytes);
void compressInt64Value(int64_t tgtValue, int64_t minValue, int byteSize, unsigned char* bytes);
void compressUInt8Value(uint8_t tgtValue, uint8_t minValue, int byteSize, unsigned char* bytes);
void compressUInt16Value(uint16_t tgtValue, uint16_t minValue, int byteSize, unsigned char* bytes);
void compressUInt32Value(uint32_t tgtValue, uint32_t minValue, int byteSize, unsigned char* bytes);
void compressUInt64Value(uint64_t tgtValue, uint64_t minValue, int byteSize, unsigned char* bytes);
void compressSingleFloatValue(FloatValueCompressElement *vce, float tgtValue, float precision, float medianValue,
int reqLength, int reqBytesLength, int resiBitsLength);
void compressSingleFloatValue_MSST19(FloatValueCompressElement *vce, float tgtValue, float precision, int reqLength, int reqBytesLength, int resiBitsLength);
void compressSingleDoubleValue(DoubleValueCompressElement *vce, double tgtValue, double precision, double medianValue,
int reqLength, int reqBytesLength, int resiBitsLength);
void compressSingleDoubleValue_MSST19(DoubleValueCompressElement *vce, double tgtValue, double precision, int reqLength, int reqBytesLength, int resiBitsLength);
int compIdenticalLeadingBytesCount_double(unsigned char* preBytes, unsigned char* curBytes);
int compIdenticalLeadingBytesCount_float(unsigned char* preBytes, unsigned char* curBytes);

5
deps/SZ/sz/include/defines.h vendored
View File

@ -10,8 +10,8 @@
#ifndef _SZ_DEFINES_H
#define _SZ_DEFINES_H
#define SZ_VERNUM 0x0200
#define SZ_VER_MAJOR 1 // curretn version
// define data format version
#define DATA_FROMAT_VER1 1 // curretn version
#define PASTRI 103
@ -88,6 +88,7 @@
#define SZ_BERR -6 //bound-mode error (should be only SZ_ABS, REL, ABS_AND_REL, ABS_OR_REL, or PW_REL)
#define SZ_LITTER_ELEMENT -7
#define SZ_ALGORITHM_ERR -8
#define SZ_FORMAT_ERR -9
#define SZ_MAINTAIN_VAR_DATA 0
#define SZ_DESTROY_WHOLE_VARSET 1

6
deps/SZ/sz/include/sz.h vendored
View File

@ -19,7 +19,6 @@
#include "DynamicByteArray.h"
#include "DynamicIntArray.h"
#include "VarSet.h"
#include "Huffman.h"
#include "TightDataPointStorageD.h"
#include "TightDataPointStorageF.h"
#include "conf.h"
@ -30,8 +29,6 @@
#include "sz_double.h"
#include "szd_float.h"
#include "szd_double.h"
#include "sz_float_pwr.h"
#include "sz_double_pwr.h"
#include "sz_opencl.h"
#include "callZlib.h"
#include "rw.h"
@ -185,9 +182,6 @@ extern sz_tsc_metadata *sz_tsc;
extern pastri_params pastri_par;
#endif
//sz.h
HuffmanTree* SZ_Reset();
void SZ_Finalize();
int SZ_Init(const char *configFilePath);

4
deps/SZ/sz/include/sz_double.h vendored
View File

@ -20,22 +20,18 @@ extern "C" {
unsigned char* SZ_skip_compress_double(double* data, size_t dataLength, size_t* outSize);
void computeReqLength_double(double realPrecision, short radExpo, int* reqLength, double* medianValue);
short computeReqLength_double_MSST19(double realPrecision);
unsigned int optimize_intervals_double_1D(double *oriData, size_t dataLength, double realPrecision);
unsigned int optimize_intervals_double_1D_opt(double *oriData, size_t dataLength, double realPrecision);
unsigned int optimize_intervals_double_1D_opt_MSST19(double *oriData, size_t dataLength, double realPrecision);
TightDataPointStorageD* SZ_compress_double_1D_MDQ(double *oriData,
size_t dataLength, double realPrecision, double valueRangeSize, double medianValue_d);
void SZ_compress_args_double_StoreOriData(double* oriData, size_t dataLength, unsigned char* newByteData, size_t *outSize);
bool SZ_compress_args_double_NoCkRngeNoGzip_1D( unsigned char* newByteData, double *oriData, size_t dataLength, double realPrecision, size_t *outSize, double valueRangeSize, double medianValue_d);
TightDataPointStorageD* SZ_compress_double_1D_MDQ_MSST19(double *oriData, size_t dataLength, double realPrecision, double valueRangeSize, double medianValue_f);
void SZ_compress_args_double_withinRange(unsigned char* newByteData, double *oriData, size_t dataLength, size_t *outSize);

40
deps/SZ/sz/include/sz_double_pwr.h vendored
View File

@ -1,40 +0,0 @@
/**
* @file sz_double.h
* @author Sheng Di
* @date July, 2017
* @brief Header file for the sz_double.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _SZ_Double_PWR_H
#define _SZ_Double_PWR_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdio.h>
#include <stdbool.h>
void compute_segment_precisions_double_1D(double *oriData, size_t dataLength, double* pwrErrBound, unsigned char* pwrErrBoundBytes, double globalPrecision);
unsigned int optimize_intervals_double_1D_pwr(double *oriData, size_t dataLength, double* pwrErrBound);
void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr(unsigned char* newByteData, double *oriData, double globalPrecision, size_t dataLength, size_t *outSize, double min, double max);
void createRangeGroups_double(double** posGroups, double** negGroups, int** posFlags, int** negFlags);
void compressGroupIDArray_double(char* groupID, TightDataPointStorageD* tdps);
TightDataPointStorageD* SZ_compress_double_1D_MDQ_pwrGroup(double* oriData, size_t dataLength, int errBoundMode,
double absErrBound, double relBoundRatio, double pwrErrRatio, double valueRangeSize, double medianValue_f);
void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwrgroup(unsigned char* newByteData, double *oriData,
size_t dataLength, double absErrBound, double relBoundRatio, double pwrErrRatio, double valueRangeSize, double medianValue_f, size_t *outSize);
void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log(unsigned char* newByteData, double *oriData, double globalPrecision, size_t dataLength, size_t *outSize, double min, double max);
void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log_MSST19(unsigned char* newByteData, double *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, double valueRangeSize, double medianValue_f,
unsigned char* signs, bool* positive, double min, double max, double nearZero);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _SZ_Double_PWR_H ----- */

6
deps/SZ/sz/include/sz_float.h vendored
View File

@ -17,15 +17,12 @@ extern "C" {
unsigned char* SZ_skip_compress_float(float* data, size_t dataLength, size_t* outSize);
void computeReqLength_float(double realPrecision, short radExpo, int* reqLength, float* medianValue);
short computeReqLength_float_MSST19(double realPrecision);
unsigned int optimize_intervals_float_1D(float *oriData, size_t dataLength, double realPrecision);
unsigned int optimize_intervals_and_compute_dense_position_float_1D(float *oriData, size_t dataLength, double realPrecision, float * dense_pos);
unsigned int optimize_intervals_float_1D_opt(float *oriData, size_t dataLength, double realPrecision);
unsigned int optimize_intervals_float_1D_opt_MSST19(float *oriData, size_t dataLength, double realPrecision);
TightDataPointStorageF* SZ_compress_float_1D_MDQ(float *oriData,
size_t dataLength, float realPrecision, float valueRangeSize, float medianValue_f);
@ -40,9 +37,6 @@ void SZ_blocked_regression(float * block_ori_data, size_t dim_0, size_t dim_1, s
unsigned char * SZ_compress_float_1D_MDQ_RA(float *oriData, size_t r1, double realPrecision, size_t * comp_size);
TightDataPointStorageF* SZ_compress_float_1D_MDQ_MSST19(float *oriData,
size_t dataLength, double realPrecision, float valueRangeSize, float medianValue_f);
void SZ_compress_args_float_withinRange(unsigned char* newByteData, float *oriData, size_t dataLength, size_t *outSize);

43
deps/SZ/sz/include/sz_float_pwr.h vendored
View File

@ -1,43 +0,0 @@
/**
* @file sz_float.h
* @author Sheng Di
* @date July, 2017
* @brief Header file for the sz_float.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _SZ_Float_PWR_H
#define _SZ_Float_PWR_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdio.h>
#include <stdbool.h>
void compute_segment_precisions_float_1D(float *oriData, size_t dataLength, float* pwrErrBound, unsigned char* pwrErrBoundBytes, double globalPrecision);
unsigned int optimize_intervals_float_1D_pwr(float *oriData, size_t dataLength, float* pwrErrBound);
void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr(unsigned char* newByteData, float *oriData, double globalPrecision, size_t dataLength, size_t *outSize, float min, float max);
void createRangeGroups_float(float** posGroups, float** negGroups, int** posFlags, int** negFlags);
void compressGroupIDArray_float(char* groupID, TightDataPointStorageF* tdps);
int* generateGroupLowerBounds();
TightDataPointStorageF* SZ_compress_float_1D_MDQ_pwrGroup(float* oriData, size_t dataLength, int errBoundMode,
double absErrBound, double relBoundRatio, double pwrErrRatio, float valueRangeSize, float medianValue_f);
void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwrgroup(unsigned char* newByteData, float *oriData,
size_t dataLength, double absErrBound, double relBoundRatio, double pwrErrRatio, float valueRangeSize, float medianValue_f, size_t *outSize);
void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_pre_log(unsigned char* newByteData, float *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, float min, float max);
void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_pre_log_MSST19(unsigned char* newByteData, float *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, float valueRangeSize, float medianValue_f,
unsigned char* signs, bool* positive, float min, float max, float nearZero);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _SZ_Float_PWR_H ----- */

2
deps/SZ/sz/include/szd_double.h vendored
View File

@ -18,8 +18,6 @@ extern "C" {
void decompressDataSeries_double_1D(double* data, size_t dataSeriesLength, double* hist_data, TightDataPointStorageD* tdps);
void decompressDataSeries_double_1D_MSST19(double* data, size_t dataSeriesLength, TightDataPointStorageD* tdps);
void getSnapshotData_double_1D(double* data, size_t dataSeriesLength, TightDataPointStorageD* tdps, int errBoundMode, int compressionType, double* hist_data, sz_params* pde_params);
int SZ_decompress_args_double(double* newData, size_t r1, unsigned char* cmpBytes, size_t cmpSize, int compressionType, double* hist_data, sz_exedata* pde_exe, sz_params* pde_params);

28
deps/SZ/sz/include/szd_double_pwr.h vendored
View File

@ -1,28 +0,0 @@
/**
* @file szd_double_pwr.h
* @author Sheng Di
* @date July, 2017
* @brief Header file for the szd_double_pwr.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _SZD_Double_PWR_H
#define _SZD_Double_PWR_H
#ifdef __cplusplus
extern "C" {
#endif
void decompressDataSeries_double_1D_pwr(double** data, size_t dataSeriesLength, TightDataPointStorageD* tdps);
void decompressDataSeries_double_1D_pwrgroup(double** data, size_t dataSeriesLength, TightDataPointStorageD* tdps);
void decompressDataSeries_double_1D_pwr_pre_log(double** data, size_t dataSeriesLength, TightDataPointStorageD* tdps);
void decompressDataSeries_double_1D_pwr_pre_log_MSST19(double** data, size_t dataSeriesLength, TightDataPointStorageD* tdps);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _SZD_Double_PWR_H ----- */

25
deps/SZ/sz/include/szd_double_ts.h vendored
View File

@ -1,25 +0,0 @@
/**
* @file szd_double_ts.h
* @author Sheng Di
* @date July, 2017
* @brief Header file for the szd_double_ts.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _SZD_Double_TS_H
#define _SZD_Double_TS_H
#ifdef __cplusplus
extern "C" {
#endif
#include "TightDataPointStorageD.h"
void decompressDataSeries_double_1D_ts(double** data, size_t dataSeriesLength, double* hist_data, TightDataPointStorageD* tdps);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _SZD_Double_TS_H ----- */

2
deps/SZ/sz/include/szd_float.h vendored
View File

@ -18,8 +18,6 @@ extern "C" {
void decompressDataSeries_float_1D(float* data, size_t dataSeriesLength, float* hist_data, TightDataPointStorageF* tdps);
void decompressDataSeries_float_1D_MSST19(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps);
void getSnapshotData_float_1D(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps, int errBoundMode, int compressionType, float* hist_data, sz_params* pde_params);
int SZ_decompress_args_float(float* newData, size_t r1, unsigned char* cmpBytes, size_t cmpSize, int compressionType, float* hist_data, sz_exedata* pde_exe, sz_params* pde_params);

30
deps/SZ/sz/include/szd_float_pwr.h vendored
View File

@ -1,30 +0,0 @@
/**
* @file szd_float_pwr.h
* @author Sheng Di
* @date July, 2017
* @brief Header file for the szd_float_pwr.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _SZD_Float_PWR_H
#define _SZD_Float_PWR_H
#ifdef __cplusplus
extern "C" {
#endif
void decompressDataSeries_float_1D_pwr(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps);
char* decompressGroupIDArray(unsigned char* bytes, size_t dataLength);
void decompressDataSeries_float_1D_pwrgroup(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps);
void decompressDataSeries_float_1D_pwr_pre_log(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps);
void decompressDataSeries_float_1D_pwr_pre_log_MSST19(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _SZD_Float_PWR_H ----- */

6
deps/SZ/sz/src/ByteToolkit.c vendored
View File

@ -906,10 +906,10 @@ void convertSZParamsToBytes(sz_params* params, unsigned char* result)
//int16ToBytes_bigEndian(&result[3], tmp2);
//errorBoundMode; //4bits(0.5 byte)
result[1] = params->errorBoundMode;
//result[1] = params->errorBoundMode;
//data type (float, double, int8, int16, ....) //10 choices, so 4 bits
result[1] = (result[1] << 4) | (params->dataType & 0x17);
//result[1] = (result[1] << 4) | (params->dataType & 0x17);
//result[5]: abs_err_bound or psnr //4 bytes
//result[9]: rel_bound_ratio or pwr_err_bound//4 bytes
@ -1001,10 +1001,10 @@ void convertBytesToSZParams(unsigned char* bytes, sz_params* params, sz_exedata*
//params->sampleDistance = bytesToInt16_bigEndian(&bytes[1]);
//params->predThreshold = 1.0*bytesToInt16_bigEndian(&bytes[3])/10000.0;
/*
params->dataType = bytes[1] & 0x07;
params->errorBoundMode = (bytes[1] & 0xf0) >> 4;
/*
switch(params->errorBoundMode)
{
case SZ_ABS:

25
deps/SZ/sz/src/CompressElement.c vendored
View File

@ -16,31 +16,6 @@
#include <sz.h>
#include <CompressElement.h>
char* decompressGroupIDArray(unsigned char* bytes, size_t dataLength)
{
HuffmanTree* huffmanTree = SZ_Reset(); //create a default huffman tree
int* standGroupID = (int*)malloc(dataLength*sizeof(int));
decode_withTree(huffmanTree, bytes, dataLength, standGroupID);
SZ_ReleaseHuffman(huffmanTree);
char* groupID = (char*)malloc(dataLength*sizeof(char));
size_t i = 0;
int lastGroupIDValue = 0, curStandIDValue = 0, curGroupIDValue = 0;
int offset = 2*(GROUP_COUNT + 2);
curGroupIDValue = groupID[0] = standGroupID[0] - GROUP_COUNT;
lastGroupIDValue = curGroupIDValue;
for(i=1;i<dataLength;i++)
{
curStandIDValue = standGroupID[i];
curGroupIDValue = curStandIDValue + lastGroupIDValue - offset;
lastGroupIDValue = curGroupIDValue;
groupID[i] = curGroupIDValue;
}
free(standGroupID);
return groupID;
}
inline short computeGroupNum_float(float value)
{

431
deps/SZ/sz/src/TightDataPointStorageD.c vendored
View File

@ -12,8 +12,8 @@
#include <string.h>
#include "TightDataPointStorageD.h"
#include "sz.h"
#include "defines.h"
#include "Huffman.h"
//#include "rw.h"
void new_TightDataPointStorageD_Empty(TightDataPointStorageD **this)
{
@ -25,12 +25,6 @@ void new_TightDataPointStorageD_Empty(TightDataPointStorageD **this)
(*this)->reqLength = 0;
(*this)->radExpo = 0;
(*this)->rtypeArray = NULL;
(*this)->rtypeArray_size = 0;
(*this)->typeArray = NULL; //its size is dataSeriesLength/4 (or xxx/4+1)
(*this)->typeArray_size = 0;
(*this)->leadNumArray = NULL; //its size is exactDataNum/4 (or exactDataNum/4+1)
(*this)->leadNumArray_size = 0;
@ -44,8 +38,6 @@ void new_TightDataPointStorageD_Empty(TightDataPointStorageD **this)
(*this)->isLossless = 0;
(*this)->segment_size = 0;
(*this)->pwrErrBoundBytes = NULL;
(*this)->pwrErrBoundBytes_size = 0;
(*this)->raBytes = NULL;
(*this)->raBytes_size = 0;
@ -56,29 +48,25 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi
{
new_TightDataPointStorageD_Empty(this);
size_t i, index = 0;
size_t pwrErrBoundBytes_size = 0, segmentL = 0, radExpoL = 0, pwrErrBoundBytesL = 0;
unsigned char version = flatBytes[index++]; //3
unsigned char sameRByte = flatBytes[index++]; //1
if(checkVersion(version)!=1)
// parse data format
switch (version)
{
//wrong version
printf("Wrong version: \nCompressed-data %s . program version=%d\n", version, versionNumber);
return 0;
case DATA_FROMAT_VER1:
break;
default:
printf(" error, compressed data format can not be recognised. ver=%d\n ", version);
return SZ_ABS;
}
int same = sameRByte & 0x01;
(*this)->isLossless = (sameRByte & 0x10)>>4;
int isPW_REL = (sameRByte & 0x20)>>5;
exe_params->SZ_SIZE_TYPE = ((sameRByte & 0x40)>>6)==1?8:4;
//pde_params->protectValueRange = (sameRByte & 0x04)>>2;
pde_params->accelerate_pw_rel_compression = (sameRByte & 0x08) >> 3;
int errorBoundMode = SZ_ABS;
if(isPW_REL)
{
errorBoundMode = PW_REL;
segmentL = exe_params->SZ_SIZE_TYPE;
pwrErrBoundBytesL = 4;
}
convertBytesToSZParams(&(flatBytes[index]), pde_params, pde_exe);
@ -111,9 +99,7 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi
(*this)->raBytes_size = flatBytesLength - 3 - 1 - MetaDataByteLength_double - exe_params->SZ_SIZE_TYPE;
(*this)->raBytes = &(flatBytes[index]);
return errorBoundMode;
}
int rtype_ = 0;//sameRByte & 0x08; //1000
}
unsigned char byteBuf[8];
@ -123,24 +109,6 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi
pde_params->maxRangeRadius = max_quant_intervals/2;
if(errorBoundMode>=PW_REL)
{
(*this)->radExpo = flatBytes[index++];//1
radExpoL = 1;
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
byteBuf[i] = flatBytes[index++];
pde_params->segment_size = (*this)->segment_size = bytesToSize(byteBuf);// exe_params->SZ_SIZE_TYPE
for (i = 0; i < 4; i++)
byteBuf[i] = flatBytes[index++];
pwrErrBoundBytes_size = (*this)->pwrErrBoundBytes_size = bytesToInt_bigEndian(byteBuf);// 4
}
else
{
pwrErrBoundBytes_size = 0;
(*this)->pwrErrBoundBytes = NULL;
}
for (i = 0; i < 4; i++)
byteBuf[i] = flatBytes[index++];
(*this)->intervals = bytesToInt_bigEndian(byteBuf);// 4
@ -151,29 +119,14 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi
(*this)->reqLength = flatBytes[index++]; //1
if(isPW_REL && pde_params->accelerate_pw_rel_compression)
{
(*this)->plus_bits = flatBytes[index++];
(*this)->max_bits = flatBytes[index++];
}
for (i = 0; i < 8; i++)
byteBuf[i] = flatBytes[index++];
(*this)->realPrecision = bytesToDouble(byteBuf);//8
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
byteBuf[i] = flatBytes[index++];
byteBuf[i] = flatBytes[index++];
(*this)->typeArray_size = bytesToSize(byteBuf);// exe_params->SZ_SIZE_TYPE
if(rtype_!=0)
{
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)
byteBuf[i] = flatBytes[index++];
(*this)->rtypeArray_size = bytesToSize(byteBuf);//ST
}
else
(*this)->rtypeArray_size = 0;
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
byteBuf[i] = flatBytes[index++];
(*this)->exactDataNum = bytesToSize(byteBuf);// ST
@ -182,17 +135,6 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi
byteBuf[i] = flatBytes[index++];
(*this)->exactMidBytes_size = bytesToSize(byteBuf);// ST
if (rtype_ != 0) {
if((*this)->rtypeArray_size>0)
(*this)->rtypeArray = (unsigned char*)malloc(sizeof(unsigned char)*(*this)->rtypeArray_size);
else
(*this)->rtypeArray = NULL;
for (i = 0; i < 8; i++)
byteBuf[i] = flatBytes[index++];
(*this)->reservedValue = bytesToDouble(byteBuf);//8
}
size_t logicLeadNumBitsNum = (*this)->exactDataNum * 2;
if (logicLeadNumBitsNum % 8 == 0)
{
@ -203,43 +145,19 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi
(*this)->leadNumArray_size = (logicLeadNumBitsNum >> 3) + 1;
}
int minLogValueSize = 0;
if(errorBoundMode>=PW_REL)
minLogValueSize = 8;
if ((*this)->rtypeArray != NULL)
{
// -3 -2 modify to -1 -1
(*this)->residualMidBits_size = flatBytesLength - 1 - 1 - MetaDataByteLength_double - exe_params->SZ_SIZE_TYPE - 4 - radExpoL - segmentL - pwrErrBoundBytesL - 4 - 8 - 1 - 8
- exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - minLogValueSize - exe_params->SZ_SIZE_TYPE - 8 - (*this)->rtypeArray_size
- minLogValueSize - (*this)->typeArray_size - (*this)->leadNumArray_size
- (*this)->exactMidBytes_size - pwrErrBoundBytes_size - 1 - 1;
for (i = 0; i < (*this)->rtypeArray_size; i++)
(*this)->rtypeArray[i] = flatBytes[index++];
}
else
{
(*this)->residualMidBits_size = flatBytesLength - 1 - 1 - MetaDataByteLength_double - exe_params->SZ_SIZE_TYPE - 4 - radExpoL - segmentL - pwrErrBoundBytesL - 4 - 8 - 1 - 8
- exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - minLogValueSize - (*this)->typeArray_size
- (*this)->leadNumArray_size - (*this)->exactMidBytes_size - pwrErrBoundBytes_size - 1 - 1;
}
if(errorBoundMode >= PW_REL){
(*this)->minLogValue = bytesToDouble(&flatBytes[index]);
index+=8;
}
(*this)->typeArray = &flatBytes[index];
//retrieve the number of states (i.e., stateNum)
(*this)->allNodes = bytesToInt_bigEndian((*this)->typeArray); //the first 4 bytes store the stateNum
(*this)->stateNum = ((*this)->allNodes+1)/2;
index+=(*this)->typeArray_size;
(*this)->pwrErrBoundBytes = &flatBytes[index];
index+=pwrErrBoundBytes_size;
// todo need check length
(*this)->residualMidBits_size = flatBytesLength - 1 - 1 - MetaDataByteLength - exe_params->SZ_SIZE_TYPE - 4 - 4 - 4 - 1 - 8
- exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE
- (*this)->leadNumArray_size - (*this)->exactMidBytes_size - (*this)->typeArray_size;
(*this)->leadNumArray = &flatBytes[index];
index+=(*this)->leadNumArray_size;
@ -250,7 +168,6 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi
(*this)->residualMidBits = &flatBytes[index];
//index+=(*this)->residualMidBits_size;
return errorBoundMode;
}
@ -270,7 +187,8 @@ void new_TightDataPointStorageD(TightDataPointStorageD **this,
unsigned char* resiMidBits, size_t resiMidBits_size,
unsigned char resiBitLength,
double realPrecision, double medianValue, char reqLength, unsigned int intervals,
unsigned char* pwrErrBoundBytes, size_t pwrErrBoundBytes_size, unsigned char radExpo) {
unsigned char radExpo)
{
//int i = 0;
*this = (TightDataPointStorageD *)malloc(sizeof(TightDataPointStorageD));
(*this)->allSameData = 0;
@ -281,15 +199,9 @@ void new_TightDataPointStorageD(TightDataPointStorageD **this,
(*this)->dataSeriesLength = dataSeriesLength;
(*this)->exactDataNum = exactDataNum;
(*this)->rtypeArray = NULL;
(*this)->rtypeArray_size = 0;
int stateNum = 2*intervals;
HuffmanTree* huffmanTree = createHuffmanTree(stateNum);
if(confparams_cpr->errorBoundMode == PW_REL && confparams_cpr->accelerate_pw_rel_compression)
(*this)->max_bits = encode_withTree_MSST19(huffmanTree, type, dataSeriesLength, &(*this)->typeArray, &(*this)->typeArray_size);
else
encode_withTree(huffmanTree, type, dataSeriesLength, &(*this)->typeArray, &(*this)->typeArray_size);
encode_withTree(huffmanTree, type, dataSeriesLength, &(*this)->typeArray, &(*this)->typeArray_size);
SZ_ReleaseHuffman(huffmanTree);
(*this)->exactMidBytes = exactMidBytes;
@ -302,65 +214,8 @@ void new_TightDataPointStorageD(TightDataPointStorageD **this,
(*this)->intervals = intervals;
(*this)->isLossless = 0;
if(confparams_cpr->errorBoundMode>=PW_REL)
(*this)->pwrErrBoundBytes = pwrErrBoundBytes;
else
(*this)->pwrErrBoundBytes = NULL;
(*this)->radExpo = radExpo;
(*this)->pwrErrBoundBytes_size = pwrErrBoundBytes_size;
}
void new_TightDataPointStorageD2(TightDataPointStorageD **this,
size_t dataSeriesLength, size_t exactDataNum,
int* type, unsigned char* exactMidBytes, size_t exactMidBytes_size,
unsigned char* leadNumIntArray, //leadNumIntArray contains readable numbers....
unsigned char* resiMidBits, size_t resiMidBits_size,
unsigned char* resiBitLength, size_t resiBitLengthSize,
double realPrecision, double medianValue, char reqLength, unsigned int intervals,
unsigned char* pwrErrBoundBytes, size_t pwrErrBoundBytes_size, unsigned char radExpo) {
//int i = 0;
*this = (TightDataPointStorageD *)malloc(sizeof(TightDataPointStorageD));
(*this)->allSameData = 0;
(*this)->realPrecision = realPrecision;
(*this)->medianValue = medianValue;
(*this)->reqLength = reqLength;
(*this)->dataSeriesLength = dataSeriesLength;
(*this)->exactDataNum = exactDataNum;
(*this)->rtypeArray = NULL;
(*this)->rtypeArray_size = 0;
int stateNum = 2*intervals;
HuffmanTree* huffmanTree = createHuffmanTree(stateNum);
encode_withTree(huffmanTree, type, dataSeriesLength, &(*this)->typeArray, &(*this)->typeArray_size);
SZ_ReleaseHuffman(huffmanTree);
(*this)->exactMidBytes = exactMidBytes;
(*this)->exactMidBytes_size = exactMidBytes_size;
(*this)->leadNumArray_size = convertIntArray2ByteArray_fast_2b(leadNumIntArray, exactDataNum, &((*this)->leadNumArray));
//(*this)->residualMidBits = resiMidBits;
//(*this)->residualMidBits_size = resiMidBits_size;
(*this)->residualMidBits_size = convertIntArray2ByteArray_fast_dynamic2(resiMidBits, resiBitLength, resiBitLengthSize, &((*this)->residualMidBits));
(*this)->intervals = intervals;
(*this)->isLossless = 0;
if(confparams_cpr->errorBoundMode>=PW_REL)
(*this)->pwrErrBoundBytes = pwrErrBoundBytes;
else
(*this)->pwrErrBoundBytes = NULL;
(*this)->radExpo = radExpo;
(*this)->pwrErrBoundBytes_size = pwrErrBoundBytes_size;
}
void convertTDPStoBytes_double(TightDataPointStorageD* tdps, unsigned char* bytes, unsigned char* dsLengthBytes, unsigned char sameByte)
@ -373,9 +228,6 @@ void convertTDPStoBytes_double(TightDataPointStorageD* tdps, unsigned char* byte
unsigned char realPrecisionBytes[8];
unsigned char medianValueBytes[8];
unsigned char segment_sizeBytes[8];
unsigned char pwrErrBoundBytes_sizeBytes[4];
unsigned char max_quant_intervals_Bytes[4];
bytes[k++] = versionNumber;
@ -390,19 +242,6 @@ void convertTDPStoBytes_double(TightDataPointStorageD* tdps, unsigned char* byte
for(i = 0;i<4;i++)//4
bytes[k++] = max_quant_intervals_Bytes[i];
if(confparams_cpr->errorBoundMode>=PW_REL)
{
bytes[k++] = tdps->radExpo; //1 byte
sizeToBytes(segment_sizeBytes, confparams_cpr->segment_size);
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
bytes[k++] = segment_sizeBytes[i];
intToBytes_bigEndian(pwrErrBoundBytes_sizeBytes, tdps->pwrErrBoundBytes_size);
for(i = 0;i<4;i++)//4
bytes[k++] = pwrErrBoundBytes_sizeBytes[i];
}
intToBytes_bigEndian(intervalsBytes, tdps->intervals);
for(i = 0;i<4;i++)//4
bytes[k++] = intervalsBytes[i];
@ -413,16 +252,10 @@ void convertTDPStoBytes_double(TightDataPointStorageD* tdps, unsigned char* byte
bytes[k++] = tdps->reqLength; //1 byte
if(confparams_cpr->errorBoundMode == PW_REL && confparams_cpr->accelerate_pw_rel_compression==1)
{
bytes[k++] = tdps->plus_bits;
bytes[k++] = tdps->max_bits;
}
doubleToBytes(realPrecisionBytes, tdps->realPrecision);
for (i = 0; i < 8; i++)// 8
bytes[k++] = realPrecisionBytes[i];
bytes[k++] = realPrecisionBytes[i];
sizeToBytes(typeArrayLengthBytes, tdps->typeArray_size);
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
bytes[k++] = typeArrayLengthBytes[i];
@ -441,15 +274,10 @@ void convertTDPStoBytes_double(TightDataPointStorageD* tdps, unsigned char* byte
for(i = 0;i < 8; i++)
bytes[k++] = exactMidBytesLength[i];
}
// copy data
memcpy(&(bytes[k]), tdps->typeArray, tdps->typeArray_size);
k += tdps->typeArray_size;
if(confparams_cpr->errorBoundMode>=PW_REL)
{
memcpy(&(bytes[k]), tdps->pwrErrBoundBytes, tdps->pwrErrBoundBytes_size);
k += tdps->pwrErrBoundBytes_size;
}
memcpy(&(bytes[k]), tdps->leadNumArray, tdps->leadNumArray_size);
k += tdps->leadNumArray_size;
memcpy(&(bytes[k]), tdps->exactMidBytes, tdps->exactMidBytes_size);
@ -462,110 +290,6 @@ void convertTDPStoBytes_double(TightDataPointStorageD* tdps, unsigned char* byte
}
}
void convertTDPStoBytes_double_reserve(TightDataPointStorageD* tdps, unsigned char* bytes, unsigned char* dsLengthBytes, unsigned char sameByte)
{
size_t i, k = 0;
unsigned char intervalsBytes[4];
unsigned char typeArrayLengthBytes[8];
unsigned char rTypeLengthBytes[8];
unsigned char exactLengthBytes[8];
unsigned char exactMidBytesLength[8];
unsigned char reservedValueBytes[8];
unsigned char realPrecisionBytes[8];
unsigned char medianValueBytes[8];
unsigned char segment_sizeBytes[8];
unsigned char pwrErrBoundBytes_sizeBytes[4];
unsigned char max_quant_intervals_Bytes[4];
bytes[k++] = versionNumber;
bytes[k++] = sameByte; //1
convertSZParamsToBytes(confparams_cpr, &(bytes[k]));
k = k + MetaDataByteLength_double;
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
bytes[k++] = dsLengthBytes[i];
intToBytes_bigEndian(max_quant_intervals_Bytes, confparams_cpr->max_quant_intervals);
for(i = 0;i<4;i++)//4
bytes[k++] = max_quant_intervals_Bytes[i];
if(confparams_cpr->errorBoundMode>=PW_REL)
{
bytes[k++] = tdps->radExpo; //1 byte
sizeToBytes(segment_sizeBytes, confparams_cpr->segment_size);
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//4
bytes[k++] = segment_sizeBytes[i];
intToBytes_bigEndian(pwrErrBoundBytes_sizeBytes, tdps->pwrErrBoundBytes_size);
for(i = 0;i<4;i++)//4
bytes[k++] = pwrErrBoundBytes_sizeBytes[i];
}
intToBytes_bigEndian(intervalsBytes, tdps->intervals);
for(i = 0;i<4;i++)//4
bytes[k++] = intervalsBytes[i];
doubleToBytes(medianValueBytes, tdps->medianValue);
for (i = 0; i < 8; i++)// 8
bytes[k++] = medianValueBytes[i];
bytes[k++] = tdps->reqLength; //1 byte
doubleToBytes(realPrecisionBytes, tdps->realPrecision);
for (i = 0; i < 8; i++)// 8
bytes[k++] = realPrecisionBytes[i];
sizeToBytes(typeArrayLengthBytes, tdps->typeArray_size);
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
bytes[k++] = typeArrayLengthBytes[i];
sizeToBytes(rTypeLengthBytes, tdps->rtypeArray_size);
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
bytes[k++] = rTypeLengthBytes[i];
sizeToBytes(exactLengthBytes, tdps->exactDataNum);
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
bytes[k++] = exactLengthBytes[i];
sizeToBytes(exactMidBytesLength, tdps->exactMidBytes_size);
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
bytes[k++] = exactMidBytesLength[i];
doubleToBytes(reservedValueBytes, tdps->reservedValue);
for (i = 0; i < 8; i++)// 8
bytes[k++] = reservedValueBytes[i];
memcpy(&(bytes[k]), tdps->rtypeArray, tdps->rtypeArray_size);
k += tdps->rtypeArray_size;
if(confparams_cpr->errorBoundMode>=PW_REL)
{
doubleToBytes(exactMidBytesLength, tdps->minLogValue);
for(i = 0;i < 8; i++)
bytes[k++] = exactMidBytesLength[i];
}
memcpy(&(bytes[k]), tdps->typeArray, tdps->typeArray_size);
k += tdps->typeArray_size;
if(confparams_cpr->errorBoundMode>=PW_REL)
{
memcpy(&(bytes[k]), tdps->pwrErrBoundBytes, tdps->pwrErrBoundBytes_size);
k += tdps->pwrErrBoundBytes_size;
}
memcpy(&(bytes[k]), tdps->leadNumArray, tdps->leadNumArray_size);
k += tdps->leadNumArray_size;
memcpy(&(bytes[k]), tdps->exactMidBytes, tdps->exactMidBytes_size);
k += tdps->exactMidBytes_size;
if(tdps->residualMidBits!=NULL)
{
memcpy(&(bytes[k]), tdps->residualMidBits, tdps->residualMidBits_size);
k += tdps->residualMidBits_size;
}
}
//Convert TightDataPointStorageD to bytes...
bool convertTDPStoFlatBytes_double(TightDataPointStorageD *tdps, unsigned char* bytes, size_t *size)
{
@ -611,27 +335,15 @@ bool convertTDPStoFlatBytes_double(TightDataPointStorageD *tdps, unsigned char*
*size = totalByteLength;
}
else if (tdps->rtypeArray == NULL)
else
{
size_t residualMidBitsLength = tdps->residualMidBits == NULL ? 0 : tdps->residualMidBits_size;
size_t segmentL = 0, radExpoL = 0, pwrBoundArrayL = 0;
int minLogValueSize = 0;
if(confparams_cpr->errorBoundMode>=PW_REL)
{
segmentL = exe_params->SZ_SIZE_TYPE;
radExpoL = 1;
pwrBoundArrayL = 4;
minLogValueSize = 8;
}
size_t totalByteLength = 3 + 1 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + 4 + radExpoL + segmentL + pwrBoundArrayL + 4 + 8 + 1 + 8
+ exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE
+ minLogValueSize /*max absolute log value*/
+ tdps->typeArray_size + tdps->leadNumArray_size
+ tdps->exactMidBytes_size + residualMidBitsLength + tdps->pwrErrBoundBytes_size;
if(confparams_cpr->errorBoundMode == PW_REL && confparams_cpr->accelerate_pw_rel_compression)
totalByteLength += (1+1); // for MSST19
size_t totalByteLength = 1 + 1 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + 4 + 4 + 8 + 1 + 8
+ exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE
+ tdps->typeArray_size
+ tdps->leadNumArray_size
+ tdps->exactMidBytes_size
+ residualMidBitsLength;
//*bytes = (unsigned char *)malloc(sizeof(unsigned char)*totalByteLength); comment by tickduan
if(totalByteLength >= tdps->dataSeriesLength * sizeof(double))
@ -642,96 +354,19 @@ bool convertTDPStoFlatBytes_double(TightDataPointStorageD *tdps, unsigned char*
convertTDPStoBytes_double(tdps, bytes, dsLengthBytes, sameByte);
*size = totalByteLength;
}
else //the case with reserved value
{
//TODO
return false;
}
return true;
}
void convertTDPStoFlatBytes_double_args(TightDataPointStorageD *tdps, unsigned char* bytes, size_t *size)
{
size_t i, k = 0;
unsigned char dsLengthBytes[8];
if(exe_params->SZ_SIZE_TYPE==4)
intToBytes_bigEndian(dsLengthBytes, tdps->dataSeriesLength);//4
else
longToBytes_bigEndian(dsLengthBytes, tdps->dataSeriesLength);//8
unsigned char sameByte = tdps->allSameData==1?(unsigned char)1:(unsigned char)0; //0000,0001
sameByte = sameByte | (confparams_cpr->szMode << 1); //0000,0110
if(tdps->isLossless)
sameByte = (unsigned char) (sameByte | 0x10); // 0001,0000
if(confparams_cpr->errorBoundMode>=PW_REL)
sameByte = (unsigned char) (sameByte | 0x20); // 0010,0000, the 5th bit
if(exe_params->SZ_SIZE_TYPE==8)
sameByte = (unsigned char) (sameByte | 0x40); //0100,0000, the 6th bit
if(confparams_cpr->errorBoundMode == PW_REL && confparams_cpr->accelerate_pw_rel_compression)
sameByte = (unsigned char) (sameByte | 0x08); //0000,1000, the 7th bit
//if(confparams_cpr->protectValueRange)
// sameByte = (unsigned char) (sameByte | 0x04); //0000,0100
if(tdps->allSameData==1)
{
size_t totalByteLength = 1 + 1 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + tdps->exactMidBytes_size;
bytes[k++] = versionNumber;
bytes[k++] = sameByte;
convertSZParamsToBytes(confparams_cpr, &(bytes[k]));
k = k + MetaDataByteLength_double;
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
bytes[k++] = dsLengthBytes[i];
for (i = 0; i < tdps->exactMidBytes_size; i++)
bytes[k++] = tdps->exactMidBytes[i];
*size = totalByteLength;
}
else if (tdps->rtypeArray == NULL)
{
size_t residualMidBitsLength = tdps->residualMidBits == NULL ? 0 : tdps->residualMidBits_size;
size_t segmentL = 0, radExpoL = 0, pwrBoundArrayL = 0;
if(confparams_cpr->errorBoundMode>=PW_REL)
{
segmentL = exe_params->SZ_SIZE_TYPE;
radExpoL = 1;
pwrBoundArrayL = 4;
}
size_t totalByteLength = 3 + 1 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE+ 4 + radExpoL + segmentL + pwrBoundArrayL + 4 + 8 + 1 + 8
+ exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE
+ tdps->typeArray_size + tdps->leadNumArray_size
+ tdps->exactMidBytes_size + residualMidBitsLength + tdps->pwrErrBoundBytes_size;
if(confparams_cpr->errorBoundMode == PW_REL && confparams_cpr->accelerate_pw_rel_compression)
totalByteLength += (1+1); // for MSST19
convertTDPStoBytes_double(tdps, bytes, dsLengthBytes, sameByte);
*size = totalByteLength;
}
else //the case with reserved value
{
//TODO
}
}
void free_TightDataPointStorageD(TightDataPointStorageD *tdps)
{
if(tdps->rtypeArray!=NULL)
free(tdps->rtypeArray);
if(tdps->typeArray!=NULL)
free(tdps->typeArray);
if(tdps->leadNumArray!=NULL)
free(tdps->leadNumArray);
if(tdps->exactMidBytes!=NULL)
free(tdps->exactMidBytes);
if(tdps->residualMidBits!=NULL)
free(tdps->residualMidBits);
if(tdps->pwrErrBoundBytes!=NULL)
free(tdps->pwrErrBoundBytes);
free(tdps);
}

417
deps/SZ/sz/src/TightDataPointStorageF.c vendored
View File

@ -12,8 +12,8 @@
#include <string.h>
#include "TightDataPointStorageF.h"
#include "sz.h"
#include "defines.h"
#include "Huffman.h"
//#include "rw.h"
void new_TightDataPointStorageF_Empty(TightDataPointStorageF **this)
{
@ -25,12 +25,6 @@ void new_TightDataPointStorageF_Empty(TightDataPointStorageF **this)
(*this)->reqLength = 0;
(*this)->radExpo = 0;
(*this)->rtypeArray = NULL;
(*this)->rtypeArray_size = 0;
(*this)->typeArray = NULL; //its size is dataSeriesLength/4 (or xxx/4+1)
(*this)->typeArray_size = 0;
(*this)->leadNumArray = NULL; //its size is exactDataNum/4 (or exactDataNum/4+1)
(*this)->leadNumArray_size = 0;
@ -44,8 +38,6 @@ void new_TightDataPointStorageF_Empty(TightDataPointStorageF **this)
(*this)->isLossless = 0;
(*this)->segment_size = 0;
(*this)->pwrErrBoundBytes = NULL;
(*this)->pwrErrBoundBytes_size = 0;
(*this)->raBytes = NULL;
(*this)->raBytes_size = 0;
@ -55,47 +47,38 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi
{
new_TightDataPointStorageF_Empty(this);
size_t i, index = 0;
size_t pwrErrBoundBytes_size = 0, segmentL = 0, radExpoL = 0, pwrErrBoundBytesL = 0;
//
// parse tdps
//
// 1 version(1)
unsigned char version = flatBytes[index++]; //1
unsigned char sameRByte = flatBytes[index++]; //1
if(checkVersion(version) != 1)
// parse data format
switch (version)
{
//wrong version
printf("Wrong version: \nCompressed-data version =%d , program version=%d \n", version, versionNumber);
return 0;
}
//note that 1000,0000 is reserved for regression tag.
case DATA_FROMAT_VER1:
break;
default:
printf(" error, float compressed data format can not be recognised. ver=%d\n ", version);
return SZ_ABS;
}
// 2 same(1) //note that 1000,0000 is reserved for regression tag.
int same = sameRByte & 0x01; //0000,0001
(*this)->isLossless = (sameRByte & 0x10)>>4; //0001,0000
int isPW_REL = (sameRByte & 0x20)>>5; //0010,0000
exe_params->SZ_SIZE_TYPE = ((sameRByte & 0x40)>>6)==1?8:4; //0100,0000
//pde_params->randomAccess = (sameRByte & 0x02) >> 1;
//pde_params->szMode = (sameRByte & 0x06) >> 1; //0000,0110 (in fact, this szMode could be removed because convertSZParamsToBytes will overwrite it)
//pde_params->protectValueRange = (sameRByte & 0x04)>>2;
pde_params->accelerate_pw_rel_compression = (sameRByte & 0x08) >> 3;//0000,1000
(*this)->isLossless = (sameRByte & 0x10)>>4; //0001,0000 //0010,0000
exe_params->SZ_SIZE_TYPE = ((sameRByte & 0x40)>>6)==1?8:4; //0100,0000
int errorBoundMode = SZ_ABS;
if(isPW_REL)
{
errorBoundMode = PW_REL;
segmentL = exe_params->SZ_SIZE_TYPE;
pwrErrBoundBytesL = 4;
}
// 3 meta(2)
convertBytesToSZParams(&(flatBytes[index]), pde_params, pde_exe);
index += MetaDataByteLength;
int isRegression = (sameRByte >> 7) & 0x01;
// 4 element count(4)
unsigned char dsLengthBytes[8];
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
dsLengthBytes[i] = flatBytes[index++];
(*this)->dataSeriesLength = bytesToSize(dsLengthBytes);// 4 or 8
(*this)->dataSeriesLength = bytesToSize(dsLengthBytes);// 4 or 8
if((*this)->isLossless==1)
{
//(*this)->exactMidBytes = flatBytes+8;
@ -110,90 +93,48 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi
}
else
(*this)->allSameData = 0;
// regression
int isRegression = (sameRByte >> 7) & 0x01;
if(isRegression == 1)
{
(*this)->raBytes_size = flatBytesLength - 1 - 1 - MetaDataByteLength - exe_params->SZ_SIZE_TYPE;
(*this)->raBytes = &(flatBytes[index]);
return errorBoundMode;
}
int rtype_ = 0;//sameRByte & 0x08; //=00001000
// 5 quant intervals(4)
unsigned char byteBuf[8];
for (i = 0; i < 4; i++)
byteBuf[i] = flatBytes[index++];
int max_quant_intervals = bytesToInt_bigEndian(byteBuf);// 4
pde_params->maxRangeRadius = max_quant_intervals/2;
if(errorBoundMode>=PW_REL)
{
(*this)->radExpo = flatBytes[index++];//1
radExpoL = 1;
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
byteBuf[i] = flatBytes[index++];
pde_params->segment_size = (*this)->segment_size = bytesToSize(byteBuf);// exe_params->SZ_SIZE_TYPE
for (i = 0; i < 4; i++)
byteBuf[i] = flatBytes[index++];
pwrErrBoundBytes_size = (*this)->pwrErrBoundBytes_size = bytesToInt_bigEndian(byteBuf);// 4
}
else
{
pwrErrBoundBytes_size = 0;
(*this)->pwrErrBoundBytes = NULL;
}
// 6 intervals
for (i = 0; i < 4; i++)
byteBuf[i] = flatBytes[index++];
(*this)->intervals = bytesToInt_bigEndian(byteBuf);// 4
// 7 median
for (i = 0; i < 4; i++)
byteBuf[i] = flatBytes[index++];
(*this)->medianValue = bytesToFloat(byteBuf); //4
// 8 reqLength
(*this)->reqLength = flatBytes[index++]; //1
if(isPW_REL && pde_params->accelerate_pw_rel_compression)
{
(*this)->plus_bits = flatBytes[index++];
(*this)->max_bits = flatBytes[index++];
}
// 9 realPrecision(8)
for (i = 0; i < 8; i++)
byteBuf[i] = flatBytes[index++];
(*this)->realPrecision = bytesToDouble(byteBuf);//8
// 10 typeArray_size
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
byteBuf[i] = flatBytes[index++];
(*this)->typeArray_size = bytesToSize(byteBuf);// 4
if(rtype_!=0)
{
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)
byteBuf[i] = flatBytes[index++];
(*this)->rtypeArray_size = bytesToSize(byteBuf);//(ST)
}
else
(*this)->rtypeArray_size = 0;
// 11 exactNum
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
byteBuf[i] = flatBytes[index++];
byteBuf[i] = flatBytes[index++];
(*this)->exactDataNum = bytesToSize(byteBuf);// ST
// 12 mid size
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
byteBuf[i] = flatBytes[index++];
(*this)->exactMidBytes_size = bytesToSize(byteBuf);// ST
if (rtype_ != 0) {
if((*this)->rtypeArray_size>0)
(*this)->rtypeArray = (unsigned char*)malloc(sizeof(unsigned char)*(*this)->rtypeArray_size);
else
(*this)->rtypeArray = NULL;
for (i = 0; i < 4; i++)
byteBuf[i] = flatBytes[index++];
(*this)->reservedValue = bytesToFloat(byteBuf);//4
}
// leadNum size calc from exactDataNum
// calc leadNumArray_size
size_t logicLeadNumBitsNum = (*this)->exactDataNum * 2;
if (logicLeadNumBitsNum % 8 == 0)
{
@ -202,57 +143,29 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi
else
{
(*this)->leadNumArray_size = (logicLeadNumBitsNum >> 3) + 1;
}
int minLogValueSize = 0;
if(errorBoundMode>=PW_REL)
minLogValueSize = 4;
if ((*this)->rtypeArray != NULL)
{
// modify from -3 -1 to -1 -1 tickduan
(*this)->residualMidBits_size = flatBytesLength - 1 - 1 - MetaDataByteLength - exe_params->SZ_SIZE_TYPE - 4 - radExpoL - segmentL - pwrErrBoundBytesL - 4 - 4 - 1 - 8
- exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - minLogValueSize - exe_params->SZ_SIZE_TYPE - 4 - (*this)->rtypeArray_size
- minLogValueSize - (*this)->typeArray_size - (*this)->leadNumArray_size
- (*this)->exactMidBytes_size - pwrErrBoundBytes_size - 1 - 1;
for (i = 0; i < (*this)->rtypeArray_size; i++)
(*this)->rtypeArray[i] = flatBytes[index++];
}
else
{
(*this)->residualMidBits_size = flatBytesLength - 1 - 1 - MetaDataByteLength - exe_params->SZ_SIZE_TYPE - 4 - radExpoL - segmentL - pwrErrBoundBytesL - 4 - 4 - 1 - 8
- exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - minLogValueSize - (*this)->typeArray_size
- (*this)->leadNumArray_size - (*this)->exactMidBytes_size - pwrErrBoundBytes_size - 1 - 1;
}
if(errorBoundMode>=PW_REL)
{
(*this)->minLogValue = bytesToFloat(&flatBytes[index]);
index+=4;
}
}
// 13 typeArray
(*this)->typeArray = &flatBytes[index];
//retrieve the number of states (i.e., stateNum)
(*this)->allNodes = bytesToInt_bigEndian((*this)->typeArray); //the first 4 bytes store the stateNum
(*this)->stateNum = ((*this)->allNodes+1)/2;
index+=(*this)->typeArray_size;
(*this)->pwrErrBoundBytes = &flatBytes[index];
index+=pwrErrBoundBytes_size;
// 14 leadNumArray
(*this)->leadNumArray = &flatBytes[index];
index+=(*this)->leadNumArray_size;
index += (*this)->leadNumArray_size;
// 15 exactMidBytes
(*this)->exactMidBytes = &flatBytes[index];
index+=(*this)->exactMidBytes_size;
// 16 residualMidBits
(*this)->residualMidBits = &flatBytes[index];
// calc residualMidBits_size
(*this)->residualMidBits_size = flatBytesLength - 1 - 1 - MetaDataByteLength - exe_params->SZ_SIZE_TYPE - 4 - 4 - 4 - 1 - 8
- exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE
- (*this)->leadNumArray_size - (*this)->exactMidBytes_size - (*this)->typeArray_size;
//index+=(*this)->residualMidBits_size;
return errorBoundMode;
}
@ -272,7 +185,7 @@ void new_TightDataPointStorageF(TightDataPointStorageF **this,
unsigned char* resiMidBits, size_t resiMidBits_size,
unsigned char resiBitLength,
double realPrecision, float medianValue, char reqLength, unsigned int intervals,
unsigned char* pwrErrBoundBytes, size_t pwrErrBoundBytes_size, unsigned char radExpo) {
unsigned char radExpo) {
*this = (TightDataPointStorageF *)malloc(sizeof(TightDataPointStorageF));
(*this)->allSameData = 0;
@ -283,9 +196,7 @@ void new_TightDataPointStorageF(TightDataPointStorageF **this,
(*this)->dataSeriesLength = dataSeriesLength;
(*this)->exactDataNum = exactDataNum;
(*this)->rtypeArray = NULL;
(*this)->rtypeArray_size = 0;
// encode type with huffMan
int stateNum = 2*intervals;
HuffmanTree* huffmanTree = createHuffmanTree(stateNum);
if(confparams_cpr->errorBoundMode == PW_REL && confparams_cpr->accelerate_pw_rel_compression)
@ -293,7 +204,7 @@ void new_TightDataPointStorageF(TightDataPointStorageF **this,
else
encode_withTree(huffmanTree, type, dataSeriesLength, &(*this)->typeArray, &(*this)->typeArray_size);
SZ_ReleaseHuffman(huffmanTree);
(*this)->exactMidBytes = exactMidBytes;
(*this)->exactMidBytes_size = exactMidBytes_size;
@ -305,64 +216,7 @@ void new_TightDataPointStorageF(TightDataPointStorageF **this,
(*this)->isLossless = 0;
if(confparams_cpr->errorBoundMode>=PW_REL)
(*this)->pwrErrBoundBytes = pwrErrBoundBytes;
else
(*this)->pwrErrBoundBytes = NULL;
(*this)->radExpo = radExpo;
(*this)->pwrErrBoundBytes_size = pwrErrBoundBytes_size;
}
void new_TightDataPointStorageF2(TightDataPointStorageF **this,
size_t dataSeriesLength, size_t exactDataNum,
int* type, unsigned char* exactMidBytes, size_t exactMidBytes_size,
unsigned char* leadNumIntArray, //leadNumIntArray contains readable numbers....
unsigned char* resiMidBits, size_t resiMidBits_size,
unsigned char* resiBitLength, size_t resiBitLengthSize,
double realPrecision, float medianValue, char reqLength, unsigned int intervals,
unsigned char* pwrErrBoundBytes, size_t pwrErrBoundBytes_size, unsigned char radExpo) {
//int i = 0;
*this = (TightDataPointStorageF *)malloc(sizeof(TightDataPointStorageF));
(*this)->allSameData = 0;
(*this)->realPrecision = realPrecision;
(*this)->medianValue = medianValue;
(*this)->reqLength = reqLength;
(*this)->dataSeriesLength = dataSeriesLength;
(*this)->exactDataNum = exactDataNum;
(*this)->rtypeArray = NULL;
(*this)->rtypeArray_size = 0;
int stateNum = 2*intervals;
HuffmanTree* huffmanTree = createHuffmanTree(stateNum);
encode_withTree(huffmanTree, type, dataSeriesLength, &(*this)->typeArray, &(*this)->typeArray_size);
SZ_ReleaseHuffman(huffmanTree);
(*this)->exactMidBytes = exactMidBytes;
(*this)->exactMidBytes_size = exactMidBytes_size;
(*this)->leadNumArray_size = convertIntArray2ByteArray_fast_2b(leadNumIntArray, exactDataNum, &((*this)->leadNumArray));
//(*this)->residualMidBits = resiMidBits;
//(*this)->residualMidBits_size = resiMidBits_size;
(*this)->residualMidBits_size = convertIntArray2ByteArray_fast_dynamic2(resiMidBits, resiBitLength, resiBitLengthSize, &((*this)->residualMidBits));
(*this)->intervals = intervals;
(*this)->isLossless = 0;
if(confparams_cpr->errorBoundMode>=PW_REL)
(*this)->pwrErrBoundBytes = pwrErrBoundBytes;
else
(*this)->pwrErrBoundBytes = NULL;
(*this)->radExpo = radExpo;
(*this)->pwrErrBoundBytes_size = pwrErrBoundBytes_size;
}
void convertTDPStoBytes_float(TightDataPointStorageF* tdps, unsigned char* bytes, unsigned char* dsLengthBytes, unsigned char sameByte)
@ -377,7 +231,6 @@ void convertTDPStoBytes_float(TightDataPointStorageF* tdps, unsigned char* bytes
unsigned char medianValueBytes[4];
unsigned char segment_sizeBytes[8];
unsigned char pwrErrBoundBytes_sizeBytes[4];
unsigned char max_quant_intervals_Bytes[4];
// 1 version
@ -393,76 +246,43 @@ void convertTDPStoBytes_float(TightDataPointStorageF* tdps, unsigned char* bytes
intToBytes_bigEndian(max_quant_intervals_Bytes, confparams_cpr->max_quant_intervals);
// 5 max_quant_intervals length
for(i = 0;i<4;i++)//4
bytes[k++] = max_quant_intervals_Bytes[i];
if(confparams_cpr->errorBoundMode>=PW_REL)
{
// 6 range exponent
bytes[k++] = tdps->radExpo; //1 byte
// 7 segmetn size
sizeToBytes(segment_sizeBytes, confparams_cpr->segment_size);
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
bytes[k++] = segment_sizeBytes[i];
// 8 pwrErrBoundBytes_size
intToBytes_bigEndian(pwrErrBoundBytes_sizeBytes, tdps->pwrErrBoundBytes_size);
for(i = 0;i<4;i++)//4
bytes[k++] = pwrErrBoundBytes_sizeBytes[i];
}
// 9 intervals
bytes[k++] = max_quant_intervals_Bytes[i];
// 6 intervals
intToBytes_bigEndian(intervalsBytes, tdps->intervals);
for(i = 0;i<4;i++)//4
bytes[k++] = intervalsBytes[i];
// 10 median
bytes[k++] = intervalsBytes[i];
// 7 median
floatToBytes(medianValueBytes, tdps->medianValue);
for (i = 0; i < 4; i++)// 4
bytes[k++] = medianValueBytes[i];
// 11 reqLength
// 8 reqLength
bytes[k++] = tdps->reqLength; //1 byte
// 12 plus max
if(confparams_cpr->errorBoundMode == PW_REL && confparams_cpr->accelerate_pw_rel_compression)
{
bytes[k++] = tdps->plus_bits;
bytes[k++] = tdps->max_bits;
}
// 13 realPrecision
// 9 realPrecision
doubleToBytes(realPrecisionBytes, tdps->realPrecision);
for (i = 0; i < 8; i++)// 8
bytes[k++] = realPrecisionBytes[i];
// 14 typeArray size
bytes[k++] = realPrecisionBytes[i];
// 10 typeArray size
sizeToBytes(typeArrayLengthBytes, tdps->typeArray_size);
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
bytes[k++] = typeArrayLengthBytes[i];
// 15 exactDataNum leadNum calc by this , so not save leadNum
bytes[k++] = typeArrayLengthBytes[i];
// 11 exactDataNum leadNum calc by this , so not save leadNum
sizeToBytes(exactLengthBytes, tdps->exactDataNum);
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
bytes[k++] = exactLengthBytes[i];
// 16 Mid Length
// 12 Mid size
sizeToBytes(exactMidBytesLength, tdps->exactMidBytes_size);
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
bytes[k++] = exactMidBytesLength[i];
if(confparams_cpr->errorBoundMode>=PW_REL)
{
floatToBytes(exactMidBytesLength, tdps->minLogValue);
for(i=0;i<4;i++)
bytes[k++] = exactMidBytesLength[i];
}
// 17 type data
// 13 typeArray
memcpy(&(bytes[k]), tdps->typeArray, tdps->typeArray_size);
k += tdps->typeArray_size;
if(confparams_cpr->errorBoundMode>=PW_REL)
{
memcpy(&(bytes[k]), tdps->pwrErrBoundBytes, tdps->pwrErrBoundBytes_size);
k += tdps->pwrErrBoundBytes_size;
}
//18 leadNum data
k += tdps->typeArray_size;
// 14 leadNumArray_size
memcpy(&(bytes[k]), tdps->leadNumArray, tdps->leadNumArray_size);
k += tdps->leadNumArray_size;
// 19 mid data
// 15 mid data
memcpy(&(bytes[k]), tdps->exactMidBytes, tdps->exactMidBytes_size);
k += tdps->exactMidBytes_size;
// 20 residual
k += tdps->exactMidBytes_size;
// 16 residualMidBits
if(tdps->residualMidBits!=NULL)
{
memcpy(&(bytes[k]), tdps->residualMidBits, tdps->residualMidBits_size);
@ -524,29 +344,20 @@ bool convertTDPStoFlatBytes_float(TightDataPointStorageF *tdps, unsigned char* b
*size = totalByteLength;
}
else if (tdps->rtypeArray == NULL)
else
{
//
// not same format
//
size_t residualMidBitsLength = tdps->residualMidBits == NULL ? 0 : tdps->residualMidBits_size;
size_t segmentL = 0, radExpoL = 0, pwrBoundArrayL = 0;
int minLogValueSize = 0;
if(confparams_cpr->errorBoundMode>=PW_REL)
{
segmentL = exe_params->SZ_SIZE_TYPE;
radExpoL = 1;
pwrBoundArrayL = 4;
minLogValueSize = 4;
}
// version(1) + samebyte(1)
size_t totalByteLength = 1 + 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 4 + radExpoL + segmentL + pwrBoundArrayL + 4 + 4 + 1 + 8
+ exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE + minLogValueSize
+ tdps->typeArray_size + tdps->leadNumArray_size
+ tdps->exactMidBytes_size + residualMidBitsLength + tdps->pwrErrBoundBytes_size;
if(confparams_cpr->errorBoundMode == PW_REL && confparams_cpr->accelerate_pw_rel_compression)
totalByteLength += (1+1); // for MSST19
size_t totalByteLength = 1 + 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 4 + 4 + 4 + 1 + 8
+ exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE
+ tdps->typeArray_size
+ tdps->leadNumArray_size
+ tdps->exactMidBytes_size
+ residualMidBitsLength;
//*bytes = (unsigned char *)malloc(sizeof(unsigned char)*totalByteLength); // comment by tickduan
if(totalByteLength >= tdps->dataSeriesLength * sizeof(float))
@ -559,99 +370,21 @@ bool convertTDPStoFlatBytes_float(TightDataPointStorageF *tdps, unsigned char* b
*size = totalByteLength;
return true;
}
else //the case with reserved value
{
//TODO
*size = 0;
return false;
}
return true;
}
void convertTDPStoFlatBytes_float_args(TightDataPointStorageF *tdps, unsigned char* bytes, size_t *size)
{
size_t i, k = 0;
unsigned char dsLengthBytes[8];
if(exe_params->SZ_SIZE_TYPE==4)
intToBytes_bigEndian(dsLengthBytes, tdps->dataSeriesLength);//4
else
longToBytes_bigEndian(dsLengthBytes, tdps->dataSeriesLength);//8
unsigned char sameByte = tdps->allSameData==1?(unsigned char)1:(unsigned char)0;
sameByte = sameByte | (confparams_cpr->szMode << 1);
if(tdps->isLossless)
sameByte = (unsigned char) (sameByte | 0x10);
if(confparams_cpr->errorBoundMode>=PW_REL)
sameByte = (unsigned char) (sameByte | 0x20); // 00100000, the 5th bit
if(exe_params->SZ_SIZE_TYPE==8)
sameByte = (unsigned char) (sameByte | 0x40); // 01000000, the 6th bit
if(confparams_cpr->errorBoundMode == PW_REL && confparams_cpr->accelerate_pw_rel_compression)
sameByte = (unsigned char) (sameByte | 0x08);
if(tdps->allSameData==1)
{
size_t totalByteLength = 1 + 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + tdps->exactMidBytes_size;
//*bytes = (unsigned char *)malloc(sizeof(unsigned char)*totalByteLength);
bytes[k++] = versionNumber;
bytes[k++] = sameByte;
convertSZParamsToBytes(confparams_cpr, &(bytes[k]));
k = k + MetaDataByteLength;
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
bytes[k++] = dsLengthBytes[i];
for (i = 0; i < tdps->exactMidBytes_size; i++)
bytes[k++] = tdps->exactMidBytes[i];
*size = totalByteLength;
}
else if (tdps->rtypeArray == NULL)
{
size_t residualMidBitsLength = tdps->residualMidBits == NULL ? 0 : tdps->residualMidBits_size;
size_t segmentL = 0, radExpoL = 0, pwrBoundArrayL = 0;
if(confparams_cpr->errorBoundMode>=PW_REL)
{
segmentL = exe_params->SZ_SIZE_TYPE;
radExpoL = 1;
pwrBoundArrayL = 4;
}
size_t totalByteLength = 1 + 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 4 + radExpoL + segmentL + pwrBoundArrayL + 4 + 4 + 1 + 8
+ exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE
+ tdps->typeArray_size + tdps->leadNumArray_size
+ tdps->exactMidBytes_size + residualMidBitsLength + tdps->pwrErrBoundBytes_size;
if(confparams_cpr->errorBoundMode == PW_REL && confparams_cpr->accelerate_pw_rel_compression)
totalByteLength += (1+1); // for MSST19
convertTDPStoBytes_float(tdps, bytes, dsLengthBytes, sameByte);
*size = totalByteLength;
}
else //the case with reserved value
{
//TODO
}
}
/**
* to free the memory used in the compression
* */
void free_TightDataPointStorageF(TightDataPointStorageF *tdps)
{
if(tdps->rtypeArray!=NULL)
free(tdps->rtypeArray);
if(tdps->typeArray!=NULL)
free(tdps->typeArray);
if(tdps->leadNumArray!=NULL)
free(tdps->leadNumArray);
if(tdps->exactMidBytes!=NULL)
free(tdps->exactMidBytes);
if(tdps->residualMidBits!=NULL)
free(tdps->residualMidBits);
if(tdps->pwrErrBoundBytes!=NULL)
free(tdps->pwrErrBoundBytes);
free(tdps);
}

298
deps/SZ/sz/src/dataCompression.c vendored
View File

@ -18,83 +18,7 @@
#include "CompressElement.h"
#include "dataCompression.h"
int computeByteSizePerIntValue(long valueRangeSize)
{
if(valueRangeSize<=256)
return 1;
else if(valueRangeSize<=65536)
return 2;
else if(valueRangeSize<=4294967296) //2^32
return 4;
else
return 8;
}
long computeRangeSize_int(void* oriData, int dataType, size_t size, int64_t* valueRangeSize)
{
size_t i = 0;
long max = 0, min = 0;
if(dataType==SZ_UINT8)
{
unsigned char* data = (unsigned char*)oriData;
unsigned char data_;
min = data[0], max = min;
computeMinMax(data);
}
else if(dataType == SZ_INT8)
{
char* data = (char*)oriData;
char data_;
min = data[0], max = min;
computeMinMax(data);
}
else if(dataType == SZ_UINT16)
{
unsigned short* data = (unsigned short*)oriData;
unsigned short data_;
min = data[0], max = min;
computeMinMax(data);
}
else if(dataType == SZ_INT16)
{
short* data = (short*)oriData;
short data_;
min = data[0], max = min;
computeMinMax(data);
}
else if(dataType == SZ_UINT32)
{
unsigned int* data = (unsigned int*)oriData;
unsigned int data_;
min = data[0], max = min;
computeMinMax(data);
}
else if(dataType == SZ_INT32)
{
int* data = (int*)oriData;
int data_;
min = data[0], max = min;
computeMinMax(data);
}
else if(dataType == SZ_UINT64)
{
unsigned long* data = (unsigned long*)oriData;
unsigned long data_;
min = data[0], max = min;
computeMinMax(data);
}
else if(dataType == SZ_INT64)
{
long* data = (long *)oriData;
long data_;
min = data[0], max = min;
computeMinMax(data);
}
*valueRangeSize = max - min;
return min;
}
float computeRangeSize_float(float* oriData, size_t size, float* valueRangeSize, float* medianValue)
{
@ -115,34 +39,6 @@ float computeRangeSize_float(float* oriData, size_t size, float* valueRangeSize,
return min;
}
float computeRangeSize_float_MSST19(float* oriData, size_t size, float* valueRangeSize, float* medianValue, unsigned char * signs, bool* positive, float* nearZero)
{
size_t i = 0;
float min = oriData[0];
float max = min;
*nearZero = min;
for(i=1;i<size;i++)
{
float data = oriData[i];
if(data <0){
signs[i] = 1;
*positive = false;
}
if(oriData[i] != 0 && fabsf(oriData[i]) < fabsf(*nearZero)){
*nearZero = oriData[i];
}
if(min>data)
min = data;
else if(max<data)
max = data;
}
*valueRangeSize = max - min;
*medianValue = min + *valueRangeSize/2;
return min;
}
double computeRangeSize_double(double* oriData, size_t size, double* valueRangeSize, double* medianValue)
{
size_t i = 0;
@ -162,94 +58,6 @@ double computeRangeSize_double(double* oriData, size_t size, double* valueRangeS
return min;
}
double computeRangeSize_double_MSST19(double* oriData, size_t size, double* valueRangeSize, double* medianValue, unsigned char * signs, bool* positive, double* nearZero)
{
size_t i = 0;
double min = oriData[0];
double max = min;
*nearZero = min;
for(i=1;i<size;i++)
{
double data = oriData[i];
if(data <0){
signs[i] = 1;
*positive = false;
}
if(oriData[i] != 0 && fabs(oriData[i]) < fabs(*nearZero)){
*nearZero = oriData[i];
}
if(min>data)
min = data;
else if(max<data)
max = data;
}
*valueRangeSize = max - min;
*medianValue = min + *valueRangeSize/2;
return min;
}
float computeRangeSize_float_subblock(float* oriData, float* valueRangeSize, float* medianValue,
size_t r5, size_t r4, size_t r3, size_t r2, size_t r1,
size_t s5, size_t s4, size_t s3, size_t s2, size_t s1,
size_t e5, size_t e4, size_t e3, size_t e2, size_t e1)
{
size_t i1, i2, i3, i4, i5;
size_t index_start = s5*(r4*r3*r2*r1) + s4*(r3*r2*r1) + s3*(r2*r1) + s2*r1 + s1;
float min = oriData[index_start];
float max = min;
for (i5 = s5; i5 <= e5; i5++)
for (i4 = s4; i4 <= e4; i4++)
for (i3 = s3; i3 <= e3; i3++)
for (i2 = s2; i2 <= e2; i2++)
for (i1 = s1; i1 <= e1; i1++)
{
size_t index = i5*(r4*r3*r2*r1) + i4*(r3*r2*r1) + i3*(r2*r1) + i2*r1 + i1;
float data = oriData[index];
if (min>data)
min = data;
else if(max<data)
max = data;
}
*valueRangeSize = max - min;
*medianValue = min + *valueRangeSize/2;
return min;
}
double computeRangeSize_double_subblock(double* oriData, double* valueRangeSize, double* medianValue,
size_t r5, size_t r4, size_t r3, size_t r2, size_t r1,
size_t s5, size_t s4, size_t s3, size_t s2, size_t s1,
size_t e5, size_t e4, size_t e3, size_t e2, size_t e1)
{
size_t i1, i2, i3, i4, i5;
size_t index_start = s5*(r4*r3*r2*r1) + s4*(r3*r2*r1) + s3*(r2*r1) + s2*r1 + s1;
double min = oriData[index_start];
double max = min;
for (i5 = s5; i5 <= e5; i5++)
for (i4 = s4; i4 <= e4; i4++)
for (i3 = s3; i3 <= e3; i3++)
for (i2 = s2; i2 <= e2; i2++)
for (i1 = s1; i1 <= e1; i1++)
{
size_t index = i5*(r4*r3*r2*r1) + i4*(r3*r2*r1) + i3*(r2*r1) + i2*r1 + i1;
double data = oriData[index];
if (min>data)
min = data;
else if(max<data)
max = data;
}
*valueRangeSize = max - min;
*medianValue = min + *valueRangeSize/2;
return min;
}
double min_d(double a, double b)
{
if(a<b)
@ -388,66 +196,6 @@ inline void symTransform_4bytes(unsigned char data[4])
data[2] = tmp;
}
inline void compressInt8Value(int8_t tgtValue, int8_t minValue, int byteSize, unsigned char* bytes)
{
uint8_t data = tgtValue - minValue;
memcpy(bytes, &data, byteSize); //byteSize==1
}
inline void compressInt16Value(int16_t tgtValue, int16_t minValue, int byteSize, unsigned char* bytes)
{
uint16_t data = tgtValue - minValue;
unsigned char tmpBytes[2];
int16ToBytes_bigEndian(tmpBytes, data);
memcpy(bytes, tmpBytes + 2 - byteSize, byteSize);
}
inline void compressInt32Value(int32_t tgtValue, int32_t minValue, int byteSize, unsigned char* bytes)
{
uint32_t data = tgtValue - minValue;
unsigned char tmpBytes[4];
int32ToBytes_bigEndian(tmpBytes, data);
memcpy(bytes, tmpBytes + 4 - byteSize, byteSize);
}
inline void compressInt64Value(int64_t tgtValue, int64_t minValue, int byteSize, unsigned char* bytes)
{
uint64_t data = tgtValue - minValue;
unsigned char tmpBytes[8];
int64ToBytes_bigEndian(tmpBytes, data);
memcpy(bytes, tmpBytes + 8 - byteSize, byteSize);
}
inline void compressUInt8Value(uint8_t tgtValue, uint8_t minValue, int byteSize, unsigned char* bytes)
{
uint8_t data = tgtValue - minValue;
memcpy(bytes, &data, byteSize); //byteSize==1
}
inline void compressUInt16Value(uint16_t tgtValue, uint16_t minValue, int byteSize, unsigned char* bytes)
{
uint16_t data = tgtValue - minValue;
unsigned char tmpBytes[2];
int16ToBytes_bigEndian(tmpBytes, data);
memcpy(bytes, tmpBytes + 2 - byteSize, byteSize);
}
inline void compressUInt32Value(uint32_t tgtValue, uint32_t minValue, int byteSize, unsigned char* bytes)
{
uint32_t data = tgtValue - minValue;
unsigned char tmpBytes[4];
int32ToBytes_bigEndian(tmpBytes, data);
memcpy(bytes, tmpBytes + 4 - byteSize, byteSize);
}
inline void compressUInt64Value(uint64_t tgtValue, uint64_t minValue, int byteSize, unsigned char* bytes)
{
uint64_t data = tgtValue - minValue;
unsigned char tmpBytes[8];
int64ToBytes_bigEndian(tmpBytes, data);
memcpy(bytes, tmpBytes + 8 - byteSize, byteSize);
}
inline void compressSingleFloatValue(FloatValueCompressElement *vce, float oriValue, float precision, float medianValue,
int reqLength, int reqBytesLength, int resiBitsLength)
{
@ -472,52 +220,6 @@ inline void compressSingleFloatValue(FloatValueCompressElement *vce, float oriVa
vce->resiBitsLength = resiBitsLength;
}
void compressSingleFloatValue_MSST19(FloatValueCompressElement *vce, float tgtValue, float precision, int reqLength, int reqBytesLength, int resiBitsLength)
{
float normValue = tgtValue;
lfloat lfBuf;
lfBuf.value = normValue;
int ignBytesLength = 32 - reqLength;
if(ignBytesLength<0)
ignBytesLength = 0;
int tmp_int = lfBuf.ivalue;
intToBytes_bigEndian(vce->curBytes, tmp_int);
lfBuf.ivalue = (lfBuf.ivalue >> ignBytesLength) << ignBytesLength;
//float tmpValue = lfBuf.value;
vce->data = lfBuf.value;
vce->curValue = tmp_int;
vce->reqBytesLength = reqBytesLength;
vce->resiBitsLength = resiBitsLength;
}
void compressSingleDoubleValue_MSST19(DoubleValueCompressElement *vce, double tgtValue, double precision, int reqLength, int reqBytesLength, int resiBitsLength)
{
ldouble lfBuf;
lfBuf.value = tgtValue;
int ignBytesLength = 64 - reqLength;
if(ignBytesLength<0)
ignBytesLength = 0;
long tmp_long = lfBuf.lvalue;
longToBytes_bigEndian(vce->curBytes, tmp_long);
lfBuf.lvalue = (lfBuf.lvalue >> ignBytesLength) << ignBytesLength;
//float tmpValue = lfBuf.value;
vce->data = lfBuf.value;
vce->curValue = tmp_long;
vce->reqBytesLength = reqBytesLength;
vce->resiBitsLength = resiBitsLength;
}
void compressSingleDoubleValue(DoubleValueCompressElement *vce, double tgtValue, double precision, double medianValue,
int reqLength, int reqBytesLength, int resiBitsLength)
{

19
deps/SZ/sz/src/sz.c vendored
View File

@ -19,13 +19,12 @@
#include "TightDataPointStorageF.h"
#include "zlib.h"
#include "rw.h"
#include "Huffman.h"
#include "conf.h"
#include "utility.h"
//#include "CurveFillingCompressStorage.h"
unsigned char versionNumber = SZ_VER_MAJOR;
unsigned char versionNumber = DATA_FROMAT_VER1;
int SZ_SIZE_TYPE_DEFUALT = 4;
int dataEndianType = LITTLE_ENDIAN_DATA; //*endian type of the data read from disk
@ -48,10 +47,7 @@ sz_tsc_metadata *sz_tsc = NULL;
pastri_params pastri_par;
#endif
HuffmanTree* SZ_Reset()
{
return createDefaultHuffmanTree();
}
int SZ_Init(const char *configFilePath)
{
@ -191,7 +187,15 @@ void SZ_Finalize()
}
}
void modulePath(char *buf, int size)
{
char path[1024];
sprintf(path, "/proc/%d/exe", getpid());
readlink(path, buf, size);
char* pos = strrchr(buf, '/');
if(pos)
pos[1]=0;
}
struct timeval startTime;
@ -200,6 +204,7 @@ struct timeval costStart; /*only used for recording the cost*/
double totalCost = 0;
void cost_start()
{
totalCost = 0;

271
deps/SZ/sz/src/sz_double.c vendored
View File

@ -19,12 +19,9 @@
#include "DynamicIntArray.h"
#include "TightDataPointStorageD.h"
#include "sz_double.h"
#include "sz_double_pwr.h"
#include "szd_double.h"
#include "szd_double_pwr.h"
#include "zlib.h"
#include "rw.h"
#include "sz_double_ts.h"
#include "utility.h"
#include "CacheTable.h"
#include "MultiLevelCacheTableWideInterval.h"
@ -51,11 +48,6 @@ inline void computeReqLength_double(double realPrecision, short radExpo, int* re
}
}
inline short computeReqLength_double_MSST19(double realPrecision)
{
short reqExpo = getPrecisionReqLength_double(realPrecision);
return 12-reqExpo;
}
unsigned int optimize_intervals_double_1D(double *oriData, size_t dataLength, double realPrecision)
{
@ -117,9 +109,8 @@ size_t dataLength, double realPrecision, double valueRangeSize, double medianVal
short radExpo = getExponent_double(valueRangeSize/2);
computeReqLength_double(realPrecision, radExpo, &reqLength, &medianValue);
int* type = (int*) malloc(dataLength*sizeof(int));
int* type = (int*) malloc(dataLength*sizeof(int));
double* spaceFillingValue = oriData; //
DynamicIntArray *exactLeadNumArray;
@ -141,8 +132,8 @@ size_t dataLength, double realPrecision, double valueRangeSize, double medianVal
DoubleValueCompressElement *vce = (DoubleValueCompressElement*)malloc(sizeof(DoubleValueCompressElement));
LossyCompressionElement *lce = (LossyCompressionElement*)malloc(sizeof(LossyCompressionElement));
//add the first data
type[0] = 0;
//add the first data
type[0] = 0;
compressSingleDoubleValue(vce, spaceFillingValue[0], realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,8);
@ -166,7 +157,7 @@ size_t dataLength, double realPrecision, double valueRangeSize, double medianVal
double interval = 2*realPrecision;
double recip_realPrecision = 1/realPrecision;
for(i=2;i<dataLength;i++)
for(i=2; i < dataLength; i++)
{
//printf("%.30G\n",last3CmprsData[0]);
curData = spaceFillingValue[i];
@ -190,8 +181,8 @@ size_t dataLength, double realPrecision, double valueRangeSize, double medianVal
continue;
}
//unpredictable data processing
type[i] = 0;
//unpredictable data processing
type[i] = 0;
compressSingleDoubleValue(vce, curData, realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,8);
@ -199,7 +190,6 @@ size_t dataLength, double realPrecision, double valueRangeSize, double medianVal
//listAdd_double(last3CmprsData, vce->data);
pred = vce->data;
}//end of for
@ -212,7 +202,7 @@ size_t dataLength, double realPrecision, double valueRangeSize, double medianVal
exactLeadNumArray->array,
resiBitArray->array, resiBitArray->size,
resiBitsLength,
realPrecision, medianValue, (char)reqLength, quantization_intervals, NULL, 0, 0);
realPrecision, medianValue, (char)reqLength, quantization_intervals, 0);
// printf("exactDataNum=%d, expSegmentsInBytes_size=%d, exactMidByteArray->size=%d\n",
// exactDataNum, expSegmentsInBytes_size, exactMidByteArray->size);
@ -283,171 +273,16 @@ size_t dataLength, double realPrecision, size_t *outSize, double valueRangeSize,
return true;
}
/*MSST19*/
TightDataPointStorageD* SZ_compress_double_1D_MDQ_MSST19(double *oriData,
size_t dataLength, double realPrecision, double valueRangeSize, double medianValue_f)
{
//struct ClockPoint clockPointBuild;
//TimeDurationStart("build", &clockPointBuild);
unsigned int quantization_intervals;
if(exe_params->optQuantMode==1)
quantization_intervals = optimize_intervals_double_1D_opt_MSST19(oriData, dataLength, realPrecision);
else
quantization_intervals = exe_params->intvCapacity;
//updateQuantizationInfo(quantization_intervals);
int intvRadius = quantization_intervals/2;
double* precisionTable = (double*)malloc(sizeof(double) * quantization_intervals);
double inv = 2.0-pow(2, -(confparams_cpr->plus_bits));
for(int i=0; i<quantization_intervals; i++){
double test = pow((1+realPrecision), inv*(i - intvRadius));
precisionTable[i] = test;
}
struct TopLevelTableWideInterval levelTable;
MultiLevelCacheTableWideIntervalBuild(&levelTable, precisionTable, quantization_intervals, realPrecision, confparams_cpr->plus_bits);
size_t i;
int reqLength;
double medianValue = medianValue_f;
//double medianInverse = 1 / medianValue_f;
//short radExpo = getExponent_double(realPrecision);
reqLength = computeReqLength_double_MSST19(realPrecision);
int* type = (int*) malloc(dataLength*sizeof(int));
double* spaceFillingValue = oriData; //
DynamicIntArray *exactLeadNumArray;
new_DIA(&exactLeadNumArray, dataLength/2/8);
DynamicByteArray *exactMidByteArray;
new_DBA(&exactMidByteArray, dataLength/2);
DynamicIntArray *resiBitArray;
new_DIA(&resiBitArray, DynArrayInitLen);
unsigned char preDataBytes[8];
intToBytes_bigEndian(preDataBytes, 0);
int reqBytesLength = reqLength/8;
int resiBitsLength = reqLength%8;
double last3CmprsData[3] = {0};
//size_t miss=0, hit=0;
DoubleValueCompressElement *vce = (DoubleValueCompressElement*)malloc(sizeof(DoubleValueCompressElement));
LossyCompressionElement *lce = (LossyCompressionElement*)malloc(sizeof(LossyCompressionElement));
//add the first data
type[0] = 0;
compressSingleDoubleValue_MSST19(vce, spaceFillingValue[0], realPrecision, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,8);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
listAdd_double(last3CmprsData, vce->data);
//miss++;
//add the second data
type[1] = 0;
compressSingleDoubleValue_MSST19(vce, spaceFillingValue[1], realPrecision, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,8);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
listAdd_double(last3CmprsData, vce->data);
//miss++;
int state;
//double checkRadius;
double curData;
double pred = vce->data;
double predRelErrRatio;
const uint64_t top = levelTable.topIndex, base = levelTable.baseIndex;
const uint64_t range = top - base;
const int bits = levelTable.bits;
uint64_t* const buffer = (uint64_t*)&predRelErrRatio;
const int shift = 52-bits;
uint64_t expoIndex, mantiIndex;
uint16_t* tables[range+1];
for(int i=0; i<=range; i++){
tables[i] = levelTable.subTables[i].table;
}
for(i=2;i<dataLength;i++)
{
curData = spaceFillingValue[i];
predRelErrRatio = curData / pred;
expoIndex = ((*buffer & 0x7fffffffffffffff) >> 52) - base;
if(expoIndex <= range){
mantiIndex = (*buffer & 0x000fffffffffffff) >> shift;
state = tables[expoIndex][mantiIndex];
}else{
state = 0;
}
if(state)
{
type[i] = state;
pred *= precisionTable[state];
//hit++;
continue;
}
//unpredictable data processing
type[i] = 0;
compressSingleDoubleValue_MSST19(vce, curData, realPrecision, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,8);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
pred = vce->data;
//miss++;
}//end of for
// printf("miss:%d, hit:%d\n", miss, hit);
size_t exactDataNum = exactLeadNumArray->size;
TightDataPointStorageD* tdps;
new_TightDataPointStorageD(&tdps, dataLength, exactDataNum,
type, exactMidByteArray->array, exactMidByteArray->size,
exactLeadNumArray->array,
resiBitArray->array, resiBitArray->size,
resiBitsLength,
realPrecision, medianValue, (char)reqLength, quantization_intervals, NULL, 0, 0);
tdps->plus_bits = confparams_cpr->plus_bits;
//free memory
free_DIA(exactLeadNumArray);
free_DIA(resiBitArray);
free(type);
free(vce);
free(lce);
free(exactMidByteArray); //exactMidByteArray->array has been released in free_TightDataPointStorageF(tdps);
free(precisionTable);
freeTopLevelTableWideInterval(&levelTable);
return tdps;
}
void SZ_compress_args_double_withinRange(unsigned char* newByteData, double *oriData, size_t dataLength, size_t *outSize)
{
TightDataPointStorageD* tdps = (TightDataPointStorageD*) malloc(sizeof(TightDataPointStorageD));
tdps->rtypeArray = NULL;
tdps->typeArray = NULL;
tdps->leadNumArray = NULL;
tdps->residualMidBits = NULL;
tdps->allSameData = 1;
tdps->dataSeriesLength = dataLength;
tdps->exactMidBytes = (unsigned char*)malloc(sizeof(unsigned char)*8);
tdps->pwrErrBoundBytes = NULL;
tdps->isLossless = 0;
double value = oriData[0];
doubleToBytes(tdps->exactMidBytes, value);
@ -472,12 +307,8 @@ int SZ_compress_args_double(double *oriData, size_t r1, unsigned char* newByteDa
double valueRangeSize = 0, medianValue = 0;
unsigned char * signs = NULL;
bool positive = true;
double nearZero = 0.0;
double min = 0;
if(params->pw_relBoundRatio < 0.000009999)
params->accelerate_pw_rel_compression = 0;
// check at least elements count
if(dataLength <= MIN_NUM_OF_ELEMENTS)
@ -486,14 +317,8 @@ int SZ_compress_args_double(double *oriData, size_t r1, unsigned char* newByteDa
return SZ_LITTER_ELEMENT;
}
if(params->errorBoundMode == PW_REL && params->accelerate_pw_rel_compression == 1)
{
signs = (unsigned char *) malloc(dataLength);
memset(signs, 0, dataLength);
min = computeRangeSize_double_MSST19(oriData, dataLength, &valueRangeSize, &medianValue, signs, &positive, &nearZero);
}
else
min = computeRangeSize_double(oriData, dataLength, &valueRangeSize, &medianValue);
double min = computeRangeSize_double(oriData, dataLength, &valueRangeSize, &medianValue);
double max = min+valueRangeSize;
params->dmin = min;
params->dmax = max;
@ -517,9 +342,7 @@ int SZ_compress_args_double(double *oriData, size_t r1, unsigned char* newByteDa
params->absErrBound = realPrecision;
}
if(valueRangeSize <= realPrecision)
{
if(params->errorBoundMode>=PW_REL && params->accelerate_pw_rel_compression == 1)
free(signs);
{
SZ_compress_args_double_withinRange(newByteData, oriData, dataLength, outSize);
}
else
@ -532,25 +355,15 @@ int SZ_compress_args_double(double *oriData, size_t r1, unsigned char* newByteDa
tmpByteData = (unsigned char*)malloc(r1*sizeof(double)*1.2);
}
if(params->errorBoundMode>=PW_REL)
if(!SZ_compress_args_double_NoCkRngeNoGzip_1D(tmpByteData, oriData, r1, realPrecision, &tmpOutSize, valueRangeSize, medianValue))
{
if(params->accelerate_pw_rel_compression && params->maxRangeRadius <= 32768)
SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log_MSST19(tmpByteData, oriData, params->pw_relBoundRatio, r1, &tmpOutSize, valueRangeSize, medianValue, signs, &positive, min, max, nearZero);
else
SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log(tmpByteData, oriData, params->pw_relBoundRatio, r1, &tmpOutSize, min, max);
}
else
{
if(!SZ_compress_args_double_NoCkRngeNoGzip_1D(tmpByteData, oriData, r1, realPrecision, &tmpOutSize, valueRangeSize, medianValue))
{
if(twoStage)
free(tmpByteData);
if(twoStage)
free(tmpByteData);
return SZ_ALGORITHM_ERR;
}
//if(tmpOutSize>=dataLength*sizeof(double) + 3 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + 1)
// SZ_compress_args_double_StoreOriData(oriData, dataLength, tmpByteData, &tmpOutSize);
return SZ_ALGORITHM_ERR;
}
//if(tmpOutSize>=dataLength*sizeof(double) + 3 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + 1)
// SZ_compress_args_double_StoreOriData(oriData, dataLength, tmpByteData, &tmpOutSize);
//
//Call Gzip to do the further compression.
@ -569,58 +382,6 @@ int SZ_compress_args_double(double *oriData, size_t r1, unsigned char* newByteDa
return status;
}
unsigned int optimize_intervals_double_1D_opt_MSST19(double *oriData, size_t dataLength, double realPrecision)
{
size_t i = 0, radiusIndex;
double pred_value = 0;
double pred_err;
size_t *intervals = (size_t*)malloc(confparams_cpr->maxRangeRadius*sizeof(size_t));
memset(intervals, 0, confparams_cpr->maxRangeRadius*sizeof(size_t));
size_t totalSampleSize = 0;//dataLength/confparams_cpr->sampleDistance;
double * data_pos = oriData + 2;
double divider = log2(1+realPrecision)*2;
int tempIndex = 0;
while(data_pos - oriData < dataLength){
if(*data_pos == 0){
data_pos += confparams_cpr->sampleDistance;
continue;
}
tempIndex++;
totalSampleSize++;
pred_value = data_pos[-1];
pred_err = fabs((double)*data_pos / pred_value);
radiusIndex = (unsigned long)fabs(log2(pred_err)/divider+0.5);
if(radiusIndex>=confparams_cpr->maxRangeRadius)
radiusIndex = confparams_cpr->maxRangeRadius - 1;
intervals[radiusIndex]++;
data_pos += confparams_cpr->sampleDistance;
}
//compute the appropriate number
size_t targetCount = totalSampleSize*confparams_cpr->predThreshold;
size_t sum = 0;
for(i=0;i<confparams_cpr->maxRangeRadius;i++)
{
sum += intervals[i];
if(sum>targetCount)
break;
}
if(i>=confparams_cpr->maxRangeRadius)
i = confparams_cpr->maxRangeRadius-1;
unsigned int accIntervals = 2*(i+1);
unsigned int powerOf2 = roundUpToPowerOf2(accIntervals);
if(powerOf2<64)
powerOf2 = 64;
free(intervals);
return powerOf2;
}
unsigned int optimize_intervals_double_1D_opt(double *oriData, size_t dataLength, double realPrecision)
{
size_t i = 0, radiusIndex;

752
deps/SZ/sz/src/sz_double_pwr.c vendored
View File

@ -1,752 +0,0 @@
/**
* @file sz_double_pwr.c
* @author Sheng Di, Dingwen Tao, Xin Liang, Xiangyu Zou, Tao Lu, Wen Xia, Xuan Wang, Weizhe Zhang
* @date Aug, 2016
* @brief SZ_Init, Compression and Decompression functions
* This file contains the compression/decompression functions related to point-wise relative errors
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <math.h>
#include "sz.h"
#include "CompressElement.h"
#include "DynamicByteArray.h"
#include "TightDataPointStorageD.h"
#include "sz_double.h"
#include "sz_double_pwr.h"
#include "zlib.h"
#include "rw.h"
#include "utility.h"
void compute_segment_precisions_double_1D(double *oriData, size_t dataLength, double* pwrErrBound, unsigned char* pwrErrBoundBytes, double globalPrecision)
{
size_t i = 0, j = 0, k = 0;
double realPrecision = oriData[0]!=0?fabs(confparams_cpr->pw_relBoundRatio*oriData[0]):confparams_cpr->pw_relBoundRatio;
double approxPrecision;
unsigned char realPrecBytes[8];
double curPrecision;
double curValue;
double sum = 0;
for(i=0;i<dataLength;i++)
{
curValue = oriData[i];
if(i%confparams_cpr->segment_size==0&&i>0)
{
//get two first bytes of the realPrecision
if(confparams_cpr->pwr_type==SZ_PWR_AVG_TYPE)
{
realPrecision = sum/confparams_cpr->segment_size;
sum = 0;
}
realPrecision *= confparams_cpr->pw_relBoundRatio;
if(confparams_cpr->errorBoundMode==ABS_AND_PW_REL||confparams_cpr->errorBoundMode==REL_AND_PW_REL)
realPrecision = realPrecision<globalPrecision?realPrecision:globalPrecision;
else if(confparams_cpr->errorBoundMode==ABS_OR_PW_REL||confparams_cpr->errorBoundMode==REL_OR_PW_REL)
realPrecision = realPrecision<globalPrecision?globalPrecision:realPrecision;
doubleToBytes(realPrecBytes, realPrecision);
memset(&realPrecBytes[2], 0, 6);
approxPrecision = bytesToDouble(realPrecBytes);
//put the realPrecision in double* pwrErBound
pwrErrBound[j++] = approxPrecision;
//put the two bytes in pwrErrBoundBytes
pwrErrBoundBytes[k++] = realPrecBytes[0];
pwrErrBoundBytes[k++] = realPrecBytes[1];
realPrecision = fabs(curValue);
}
if(curValue!=0)
{
curPrecision = fabs(curValue);
switch(confparams_cpr->pwr_type)
{
case SZ_PWR_MIN_TYPE:
if(realPrecision>curPrecision)
realPrecision = curPrecision;
break;
case SZ_PWR_AVG_TYPE:
sum += curPrecision;
break;
case SZ_PWR_MAX_TYPE:
if(realPrecision<curPrecision)
realPrecision = curPrecision;
break;
}
}
}
if(confparams_cpr->pwr_type==SZ_PWR_AVG_TYPE)
{
int size = dataLength%confparams_cpr->segment_size==0?confparams_cpr->segment_size:dataLength%confparams_cpr->segment_size;
realPrecision = sum/size;
}
if(confparams_cpr->errorBoundMode==ABS_AND_PW_REL||confparams_cpr->errorBoundMode==REL_AND_PW_REL)
realPrecision = realPrecision<globalPrecision?realPrecision:globalPrecision;
else if(confparams_cpr->errorBoundMode==ABS_OR_PW_REL||confparams_cpr->errorBoundMode==REL_OR_PW_REL)
realPrecision = realPrecision<globalPrecision?globalPrecision:realPrecision;
doubleToBytes(realPrecBytes, realPrecision);
memset(&realPrecBytes[2], 0, 6);
approxPrecision = bytesToDouble(realPrecBytes);
//put the realPrecision in double* pwrErBound
pwrErrBound[j++] = approxPrecision;
//put the two bytes in pwrErrBoundBytes
pwrErrBoundBytes[k++] = realPrecBytes[0];
pwrErrBoundBytes[k++] = realPrecBytes[1];
}
unsigned int optimize_intervals_double_1D_pwr(double *oriData, size_t dataLength, double* pwrErrBound)
{
size_t i = 0, j = 0;
double realPrecision = pwrErrBound[j++];
unsigned long radiusIndex;
double pred_value = 0, pred_err;
int *intervals = (int*)malloc(confparams_cpr->maxRangeRadius*sizeof(int));
memset(intervals, 0, confparams_cpr->maxRangeRadius*sizeof(int));
int totalSampleSize = dataLength/confparams_cpr->sampleDistance;
for(i=2;i<dataLength;i++)
{
if(i%confparams_cpr->segment_size==0)
realPrecision = pwrErrBound[j++];
if(i%confparams_cpr->sampleDistance==0)
{
//pred_value = 2*oriData[i-1] - oriData[i-2];
pred_value = oriData[i-1];
pred_err = fabs(pred_value - oriData[i]);
radiusIndex = (unsigned long)((pred_err/realPrecision+1)/2);
if(radiusIndex>=confparams_cpr->maxRangeRadius)
radiusIndex = confparams_cpr->maxRangeRadius - 1;
intervals[radiusIndex]++;
}
}
//compute the appropriate number
size_t targetCount = totalSampleSize*confparams_cpr->predThreshold;
size_t sum = 0;
for(i=0;i<confparams_cpr->maxRangeRadius;i++)
{
sum += intervals[i];
if(sum>targetCount)
break;
}
if(i>=confparams_cpr->maxRangeRadius)
i = confparams_cpr->maxRangeRadius-1;
unsigned int accIntervals = 2*(i+1);
unsigned int powerOf2 = roundUpToPowerOf2(accIntervals);
if(powerOf2<32)
powerOf2 = 32;
free(intervals);
//printf("accIntervals=%d, powerOf2=%d\n", accIntervals, powerOf2);
return powerOf2;
}
void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr(unsigned char* newByteData, double *oriData, double globalPrecision,
size_t dataLength, size_t *outSize, double min, double max)
{
size_t pwrLength = dataLength%confparams_cpr->segment_size==0?dataLength/confparams_cpr->segment_size:dataLength/confparams_cpr->segment_size+1;
double* pwrErrBound = (double*)malloc(sizeof(double)*pwrLength);
size_t pwrErrBoundBytes_size = sizeof(unsigned char)*pwrLength*2;
unsigned char* pwrErrBoundBytes = (unsigned char*)malloc(pwrErrBoundBytes_size);
compute_segment_precisions_double_1D(oriData, dataLength, pwrErrBound, pwrErrBoundBytes, globalPrecision);
unsigned int quantization_intervals;
if(exe_params->optQuantMode==1)
{
quantization_intervals = optimize_intervals_double_1D_pwr(oriData, dataLength, pwrErrBound);
updateQuantizationInfo(quantization_intervals);
}
else
quantization_intervals = exe_params->intvCapacity;
size_t i = 0, j = 0;
int reqLength;
double realPrecision = pwrErrBound[j++];
double medianValue = 0;
double radius = fabs(max)<fabs(min)?fabs(min):fabs(max);
short radExpo = getExponent_double(radius);
computeReqLength_double(realPrecision, radExpo, &reqLength, &medianValue);
int* type = (int*) malloc(dataLength*sizeof(int));
//type[dataLength]=0;
double* spaceFillingValue = oriData; //
DynamicByteArray *resiBitLengthArray;
new_DBA(&resiBitLengthArray, DynArrayInitLen);
DynamicIntArray *exactLeadNumArray;
new_DIA(&exactLeadNumArray, DynArrayInitLen);
DynamicByteArray *exactMidByteArray;
new_DBA(&exactMidByteArray, DynArrayInitLen);
DynamicIntArray *resiBitArray;
new_DIA(&resiBitArray, DynArrayInitLen);
type[0] = 0;
unsigned char preDataBytes[8] = {0};
intToBytes_bigEndian(preDataBytes, 0);
int reqBytesLength = reqLength/8;
int resiBitsLength = reqLength%8;
double last3CmprsData[3] = {0};
DoubleValueCompressElement *vce = (DoubleValueCompressElement*)malloc(sizeof(DoubleValueCompressElement));
LossyCompressionElement *lce = (LossyCompressionElement*)malloc(sizeof(LossyCompressionElement));
//add the first data
addDBA_Data(resiBitLengthArray, (unsigned char)resiBitsLength);
compressSingleDoubleValue(vce, spaceFillingValue[0], realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,8);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
listAdd_double(last3CmprsData, vce->data);
//printf("%.30G\n",last3CmprsData[0]);
//add the second data
type[1] = 0;
addDBA_Data(resiBitLengthArray, (unsigned char)resiBitsLength);
compressSingleDoubleValue(vce, spaceFillingValue[1], realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,8);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
listAdd_double(last3CmprsData, vce->data);
//printf("%.30G\n",last3CmprsData[0]);
int state;
double checkRadius;
double curData;
double pred;
double predAbsErr;
checkRadius = (exe_params->intvCapacity-1)*realPrecision;
double interval = 2*realPrecision;
int updateReqLength = 0; //a marker: 1 means already updated
for(i=2;i<dataLength;i++)
{
curData = spaceFillingValue[i];
if(i%confparams_cpr->segment_size==0)
{
realPrecision = pwrErrBound[j++];
checkRadius = (exe_params->intvCapacity-1)*realPrecision;
interval = 2*realPrecision;
updateReqLength = 0;
}
//pred = 2*last3CmprsData[0] - last3CmprsData[1];
pred = last3CmprsData[0];
predAbsErr = fabs(curData - pred);
if(predAbsErr<checkRadius)
{
state = (predAbsErr/realPrecision+1)/2;
if(curData>=pred)
{
type[i] = exe_params->intvRadius+state;
pred = pred + state*interval;
}
else //curData<pred
{
type[i] = exe_params->intvRadius-state;
pred = pred - state*interval;
}
listAdd_double(last3CmprsData, pred);
continue;
}
//unpredictable data processing
if(updateReqLength==0)
{
computeReqLength_double(realPrecision, radExpo, &reqLength, &medianValue);
reqBytesLength = reqLength/8;
resiBitsLength = reqLength%8;
updateReqLength = 1;
}
type[i] = 0;
addDBA_Data(resiBitLengthArray, (unsigned char)resiBitsLength);
compressSingleDoubleValue(vce, curData, realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,8);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
listAdd_double(last3CmprsData, vce->data);
}//end of for
// char* expSegmentsInBytes;
// int expSegmentsInBytes_size = convertESCToBytes(esc, &expSegmentsInBytes);
int exactDataNum = exactLeadNumArray->size;
TightDataPointStorageD* tdps;
new_TightDataPointStorageD2(&tdps, dataLength, exactDataNum,
type, exactMidByteArray->array, exactMidByteArray->size,
exactLeadNumArray->array,
resiBitArray->array, resiBitArray->size,
resiBitLengthArray->array, resiBitLengthArray->size,
realPrecision, medianValue, (char)reqLength, quantization_intervals, pwrErrBoundBytes, pwrErrBoundBytes_size, radExpo);
//sdi:Debug
/* int sum =0;
for(i=0;i<dataLength;i++)
if(type[i]==0) sum++;
printf("opt_quantizations=%d, exactDataNum=%d, sum=%d\n",quantization_intervals, exactDataNum, sum);
*/
// writeUShortData(type, dataLength, "compressStateBytes.sb");
// unsigned short type_[dataLength];
// SZ_Reset();
// decode_withTree(tdps->typeArray, tdps->typeArray_size, type_);
// printf("tdps->typeArray_size=%d\n", tdps->typeArray_size);
//free memory
free_DBA(resiBitLengthArray);
free_DIA(exactLeadNumArray);
free_DIA(resiBitArray);
free(type);
convertTDPStoFlatBytes_double(tdps, newByteData, outSize);
int doubleSize=sizeof(double);
if(*outSize>dataLength*doubleSize)
{
size_t k = 0, i;
tdps->isLossless = 1;
size_t totalByteLength = 1 + exe_params->SZ_SIZE_TYPE + 1 + doubleSize*dataLength;
//*newByteData = (unsigned char*)malloc(totalByteLength); comment by tickduan
unsigned char dsLengthBytes[exe_params->SZ_SIZE_TYPE];
intToBytes_bigEndian(dsLengthBytes, dataLength);//4
newByteData[k++] = versionNumber;
if(exe_params->SZ_SIZE_TYPE==4)
{
newByteData[k++] = 16; //=00010000
}
else
{
newByteData[k++] = 80;
}
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)//4 or 8
newByteData[k++] = dsLengthBytes[i];
if(sysEndianType==BIG_ENDIAN_SYSTEM)
memcpy(newByteData+4+exe_params->SZ_SIZE_TYPE, oriData, dataLength*doubleSize);
else
{
unsigned char* p = newByteData+4+exe_params->SZ_SIZE_TYPE;
for(i=0;i<dataLength;i++,p+=doubleSize)
doubleToBytes(p, oriData[i]);
}
*outSize = totalByteLength;
}
free(pwrErrBound);
free(vce);
free(lce);
free_TightDataPointStorageD(tdps);
free(exactMidByteArray);
}
void createRangeGroups_double(double** posGroups, double** negGroups, int** posFlags, int** negFlags)
{
size_t size = GROUP_COUNT*sizeof(double);
size_t size2 = GROUP_COUNT*sizeof(int);
*posGroups = (double*)malloc(size);
*negGroups = (double*)malloc(size);
*posFlags = (int*)malloc(size2);
*negFlags = (int*)malloc(size2);
memset(*posGroups, 0, size);
memset(*negGroups, 0, size);
memset(*posFlags, 0, size2);
memset(*negFlags, 0, size2);
}
void compressGroupIDArray_double(char* groupID, TightDataPointStorageD* tdps)
{
size_t dataLength = tdps->dataSeriesLength;
int* standGroupID = (int*)malloc(dataLength*sizeof(int));
size_t i;
standGroupID[0] = groupID[0]+GROUP_COUNT; //plus an offset such that it would not be a negative number.
char lastGroupIDValue = groupID[0], curGroupIDValue;
int offset = 2*(GROUP_COUNT + 2);
for(i=1; i<dataLength;i++)
{
curGroupIDValue = groupID[i];
standGroupID[i] = (curGroupIDValue - lastGroupIDValue) + offset;
lastGroupIDValue = curGroupIDValue;
}
unsigned char* out = NULL;
size_t outSize;
HuffmanTree* huffmanTree = SZ_Reset();
encode_withTree(huffmanTree, standGroupID, dataLength, &out, &outSize);
SZ_ReleaseHuffman(huffmanTree);
tdps->pwrErrBoundBytes = out; //groupIDArray
tdps->pwrErrBoundBytes_size = outSize;
free(standGroupID);
}
TightDataPointStorageD* SZ_compress_double_1D_MDQ_pwrGroup(double* oriData, size_t dataLength, int errBoundMode,
double absErrBound, double relBoundRatio, double pwrErrRatio, double valueRangeSize, double medianValue_f)
{
size_t i;
double *posGroups, *negGroups, *groups;
double pos_01_group = 0, neg_01_group = 0; //[0,1] and [-1,0]
int *posFlags, *negFlags, *flags;
int pos_01_flag = 0, neg_01_flag = 0;
createRangeGroups_double(&posGroups, &negGroups, &posFlags, &negFlags);
size_t nbBins = (size_t)(1/pwrErrRatio);
if(nbBins%2==1)
nbBins++;
exe_params->intvRadius = nbBins;
int reqLength, status;
double medianValue = medianValue_f;
double realPrecision = (double)getRealPrecision_double(valueRangeSize, errBoundMode, absErrBound, relBoundRatio, &status);
if(realPrecision<0)
realPrecision = pwrErrRatio;
double realGroupPrecision; //precision (error) based on group ID
getPrecisionReqLength_double(realPrecision);
short radExpo = getExponent_double(valueRangeSize/2);
short lastGroupNum = 0, groupNum, grpNum = 0;
double* groupErrorBounds = generateGroupErrBounds(errBoundMode, realPrecision, pwrErrRatio);
exe_params->intvRadius = generateGroupMaxIntervalCount(groupErrorBounds);
computeReqLength_double(realPrecision, radExpo, &reqLength, &medianValue);
int* type = (int*) malloc(dataLength*sizeof(int));
char *groupID = (char*) malloc(dataLength*sizeof(char));
char *gp = groupID;
double* spaceFillingValue = oriData;
DynamicIntArray *exactLeadNumArray;
new_DIA(&exactLeadNumArray, DynArrayInitLen);
DynamicByteArray *exactMidByteArray;
new_DBA(&exactMidByteArray, DynArrayInitLen);
DynamicIntArray *resiBitArray;
new_DIA(&resiBitArray, DynArrayInitLen);
unsigned char preDataBytes[8];
intToBytes_bigEndian(preDataBytes, 0);
int reqBytesLength = reqLength/8;
int resiBitsLength = reqLength%8;
DoubleValueCompressElement *vce = (DoubleValueCompressElement*)malloc(sizeof(DoubleValueCompressElement));
LossyCompressionElement *lce = (LossyCompressionElement*)malloc(sizeof(LossyCompressionElement));
int state;
double curData, decValue;
double pred;
double predAbsErr;
double interval = 0;
//add the first data
type[0] = 0;
compressSingleDoubleValue(vce, spaceFillingValue[0], realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,8);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
curData = spaceFillingValue[0];
groupNum = computeGroupNum_double(vce->data);
if(curData > 0 && groupNum >= 0)
{
groups = posGroups;
flags = posFlags;
grpNum = groupNum;
}
else if(curData < 0 && groupNum >= 0)
{
groups = negGroups;
flags = negFlags;
grpNum = groupNum;
}
else if(curData >= 0 && groupNum == -1)
{
groups = &pos_01_group;
flags = &pos_01_flag;
grpNum = 0;
}
else //curData < 0 && groupNum == -1
{
groups = &neg_01_group;
flags = &neg_01_flag;
grpNum = 0;
}
listAdd_double_group(groups, flags, groupNum, spaceFillingValue[0], vce->data, gp);
gp++;
for(i=1;i<dataLength;i++)
{
curData = oriData[i];
//printf("i=%d, posGroups[3]=%f, negGroups[3]=%f\n", i, posGroups[3], negGroups[3]);
groupNum = computeGroupNum_double(curData);
if(curData > 0 && groupNum >= 0)
{
groups = posGroups;
flags = posFlags;
grpNum = groupNum;
}
else if(curData < 0 && groupNum >= 0)
{
groups = negGroups;
flags = negFlags;
grpNum = groupNum;
}
else if(curData >= 0 && groupNum == -1)
{
groups = &pos_01_group;
flags = &pos_01_flag;
grpNum = 0;
}
else //curData < 0 && groupNum == -1
{
groups = &neg_01_group;
flags = &neg_01_flag;
grpNum = 0;
}
if(groupNum>=GROUP_COUNT)
{
type[i] = 0;
compressSingleDoubleValue(vce, curData, realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,8);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
listAdd_double_group(groups, flags, lastGroupNum, curData, vce->data, gp); //set the group number to be last one in order to get the groupID array as smooth as possible.
}
else if(flags[grpNum]==0) //the dec value may not be in the same group
{
type[i] = 0;
compressSingleDoubleValue(vce, curData, realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,8);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
//decGroupNum = computeGroupNum_double(vce->data);
//if(decGroupNum < groupNum)
// decValue = curData>0?pow(2, groupNum):-pow(2, groupNum);
//else if(decGroupNum > groupNum)
// decValue = curData>0?pow(2, groupNum+1):-pow(2, groupNum+1);
//else
// decValue = vce->data;
decValue = vce->data;
listAdd_double_group(groups, flags, groupNum, curData, decValue, gp);
lastGroupNum = curData>0?groupNum + 2: -(groupNum+2);
}
else //if flags[groupNum]==1, the dec value must be in the same group
{
pred = groups[grpNum];
predAbsErr = fabs(curData - pred);
realGroupPrecision = groupErrorBounds[grpNum]; //compute real error bound
interval = realGroupPrecision*2;
state = (predAbsErr/realGroupPrecision+1)/2;
if(curData>=pred)
{
type[i] = exe_params->intvRadius+state;
decValue = pred + state*interval;
}
else //curData<pred
{
type[i] = exe_params->intvRadius-state;
decValue = pred - state*interval;
}
//decGroupNum = computeGroupNum_double(pred);
if((decValue>0&&curData<0)||(decValue<0&&curData>=0))
decValue = 0;
//else
//{
// if(decGroupNum < groupNum)
// decValue = curData>0?pow(2, groupNum):-pow(2, groupNum);
// else if(decGroupNum > groupNum)
// decValue = curData>0?pow(2, groupNum+1):-pow(2, groupNum+1);
// else
// decValue = pred;
//}
if(fabs(curData-decValue)>realGroupPrecision)
{
type[i] = 0;
compressSingleDoubleValue(vce, curData, realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,8);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
decValue = vce->data;
}
listAdd_double_group(groups, flags, groupNum, curData, decValue, gp);
lastGroupNum = curData>=0?groupNum + 2: -(groupNum+2);
}
gp++;
}
int exactDataNum = exactLeadNumArray->size;
TightDataPointStorageD* tdps;
//combineTypeAndGroupIDArray(nbBins, dataLength, &type, groupID);
new_TightDataPointStorageD(&tdps, dataLength, exactDataNum,
type, exactMidByteArray->array, exactMidByteArray->size,
exactLeadNumArray->array,
resiBitArray->array, resiBitArray->size,
resiBitsLength,
realPrecision, medianValue, (char)reqLength, nbBins, NULL, 0, radExpo);
compressGroupIDArray_double(groupID, tdps);
free(posGroups);
free(negGroups);
free(posFlags);
free(negFlags);
free(groupID);
free(groupErrorBounds);
free_DIA(exactLeadNumArray);
free_DIA(resiBitArray);
free(type);
free(vce);
free(lce);
free(exactMidByteArray); //exactMidByteArray->array has been released in free_TightDataPointStorageD(tdps);
return tdps;
}
void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwrgroup(unsigned char* newByteData, double *oriData,
size_t dataLength, double absErrBound, double relBoundRatio, double pwrErrRatio, double valueRangeSize, double medianValue_f, size_t *outSize)
{
TightDataPointStorageD* tdps = SZ_compress_double_1D_MDQ_pwrGroup(oriData, dataLength, confparams_cpr->errorBoundMode,
absErrBound, relBoundRatio, pwrErrRatio,
valueRangeSize, medianValue_f);
convertTDPStoFlatBytes_double(tdps, newByteData, outSize);
if(*outSize > 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(double)*dataLength)
SZ_compress_args_double_StoreOriData(oriData, dataLength, newByteData, outSize);
free_TightDataPointStorageD(tdps);
}
#include <stdbool.h>
void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log(unsigned char* newByteData, double *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, double min, double max){
double * log_data = (double *) malloc(dataLength * sizeof(double));
unsigned char * signs = (unsigned char *) malloc(dataLength);
memset(signs, 0, dataLength);
// preprocess
double max_abs_log_data;
if(min == 0) max_abs_log_data = fabs(log2(fabs(max)));
else if(max == 0) max_abs_log_data = fabs(log2(fabs(min)));
else max_abs_log_data = fabs(log2(fabs(min))) > fabs(log2(fabs(max))) ? fabs(log2(fabs(min))) : fabs(log2(fabs(max)));
double min_log_data = max_abs_log_data;
bool positive = true;
for(size_t i=0; i<dataLength; i++){
if(oriData[i] < 0){
signs[i] = 1;
log_data[i] = -oriData[i];
positive = false;
}
else
log_data[i] = oriData[i];
if(log_data[i] > 0){
log_data[i] = log2(log_data[i]);
if(log_data[i] > max_abs_log_data) max_abs_log_data = log_data[i];
if(log_data[i] < min_log_data) min_log_data = log_data[i];
}
}
double valueRangeSize, medianValue_f;
computeRangeSize_double(log_data, dataLength, &valueRangeSize, &medianValue_f);
if(fabs(min_log_data) > max_abs_log_data) max_abs_log_data = fabs(min_log_data);
double realPrecision = log2(1.0 + pwrErrRatio) - max_abs_log_data * 2.23e-16;
for(size_t i=0; i<dataLength; i++){
if(oriData[i] == 0){
log_data[i] = min_log_data - 2.0001*realPrecision;
}
}
TightDataPointStorageD* tdps = SZ_compress_double_1D_MDQ(log_data, dataLength, realPrecision, valueRangeSize, medianValue_f);
tdps->minLogValue = min_log_data - 1.0001*realPrecision;
free(log_data);
if(!positive){
unsigned char * comp_signs;
// compress signs
unsigned long signSize = sz_lossless_compress(ZSTD_COMPRESSOR, 3, signs, dataLength, &comp_signs);
tdps->pwrErrBoundBytes = comp_signs;
tdps->pwrErrBoundBytes_size = signSize;
}
else{
tdps->pwrErrBoundBytes = NULL;
tdps->pwrErrBoundBytes_size = 0;
}
free(signs);
convertTDPStoFlatBytes_double(tdps, newByteData, outSize);
if(*outSize>1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(double)*dataLength)
SZ_compress_args_double_StoreOriData(oriData, dataLength, newByteData, outSize);
free_TightDataPointStorageD(tdps);
}
void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log_MSST19(unsigned char* newByteData, double *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, double valueRangeSize, double medianValue_f,
unsigned char* signs, bool* positive, double min, double max, double nearZero){
double multiplier = pow((1+pwrErrRatio), -3.0001);
for(int i=0; i<dataLength; i++){
if(oriData[i] == 0){
oriData[i] = nearZero * multiplier;
}
}
double median_log = sqrt(fabs(nearZero * max));
TightDataPointStorageD* tdps = SZ_compress_double_1D_MDQ_MSST19(oriData, dataLength, pwrErrRatio, valueRangeSize, median_log);
tdps->minLogValue = nearZero / ((1+pwrErrRatio)*(1+pwrErrRatio));
if(!(*positive)){
unsigned char * comp_signs;
// compress signs
unsigned long signSize = sz_lossless_compress(ZSTD_COMPRESSOR, 3, signs, dataLength, &comp_signs);
tdps->pwrErrBoundBytes = comp_signs;
tdps->pwrErrBoundBytes_size = signSize;
}
else{
tdps->pwrErrBoundBytes = NULL;
tdps->pwrErrBoundBytes_size = 0;
}
free(signs);
convertTDPStoFlatBytes_double(tdps, newByteData, outSize);
if(*outSize > 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(double)*dataLength)
SZ_compress_args_double_StoreOriData(oriData, dataLength, newByteData, outSize);
free_TightDataPointStorageD(tdps);
}

191
deps/SZ/sz/src/sz_double_ts.c vendored
View File

@ -1,191 +0,0 @@
/**
* @file sz_double_ts.c
* @author Sheng Di and Dingwen Tao
* @date Aug, 2016
* @brief
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <math.h>
#include "sz.h"
#include "CompressElement.h"
#include "DynamicByteArray.h"
#include "DynamicIntArray.h"
#include "TightDataPointStorageD.h"
#include "zlib.h"
#include "rw.h"
#include "sz_double_ts.h"
unsigned int optimize_intervals_double_1D_ts(double *oriData, size_t dataLength, double* preData, double realPrecision)
{
size_t i = 0, radiusIndex;
double pred_value = 0, pred_err;
size_t *intervals = (size_t*)malloc(confparams_cpr->maxRangeRadius*sizeof(size_t));
memset(intervals, 0, confparams_cpr->maxRangeRadius*sizeof(size_t));
size_t totalSampleSize = dataLength/confparams_cpr->sampleDistance;
for(i=2;i<dataLength;i++)
{
if(i%confparams_cpr->sampleDistance==0)
{
pred_value = preData[i];
pred_err = fabs(pred_value - oriData[i]);
radiusIndex = (unsigned long)((pred_err/realPrecision+1)/2);
if(radiusIndex>=confparams_cpr->maxRangeRadius)
radiusIndex = confparams_cpr->maxRangeRadius - 1;
intervals[radiusIndex]++;
}
}
//compute the appropriate number
size_t targetCount = totalSampleSize*confparams_cpr->predThreshold;
size_t sum = 0;
for(i=0;i<confparams_cpr->maxRangeRadius;i++)
{
sum += intervals[i];
if(sum>targetCount)
break;
}
if(i>=confparams_cpr->maxRangeRadius)
i = confparams_cpr->maxRangeRadius-1;
unsigned int accIntervals = 2*(i+1);
unsigned int powerOf2 = roundUpToPowerOf2(accIntervals);
if(powerOf2<32)
powerOf2 = 32;
free(intervals);
return powerOf2;
}
TightDataPointStorageD* SZ_compress_double_1D_MDQ_ts(double *oriData, size_t dataLength, sz_multisteps* multisteps,
double realPrecision, double valueRangeSize, double medianValue_d)
{
double* preStepData = (double*)(multisteps->hist_data);
//store the decompressed data
//double* decData = (double*)malloc(sizeof(double)*dataLength);
//memset(decData, 0, sizeof(double)*dataLength);
double* decData = preStepData;
unsigned int quantization_intervals;
if(exe_params->optQuantMode==1)
quantization_intervals = optimize_intervals_double_1D_ts(oriData, dataLength, preStepData, realPrecision);
else
quantization_intervals = exe_params->intvCapacity;
updateQuantizationInfo(quantization_intervals);
size_t i;
int reqLength;
double medianValue = medianValue_d;
short radExpo = getExponent_double(valueRangeSize/2);
computeReqLength_double(realPrecision, radExpo, &reqLength, &medianValue);
int* type = (int*) malloc(dataLength*sizeof(int));
double* spaceFillingValue = oriData; //
DynamicIntArray *exactLeadNumArray;
new_DIA(&exactLeadNumArray, DynArrayInitLen);
DynamicByteArray *exactMidByteArray;
new_DBA(&exactMidByteArray, DynArrayInitLen);
DynamicIntArray *resiBitArray;
new_DIA(&resiBitArray, DynArrayInitLen);
unsigned char preDataBytes[8];
longToBytes_bigEndian(preDataBytes, 0);
int reqBytesLength = reqLength/8;
int resiBitsLength = reqLength%8;
DoubleValueCompressElement *vce = (DoubleValueCompressElement*)malloc(sizeof(DoubleValueCompressElement));
LossyCompressionElement *lce = (LossyCompressionElement*)malloc(sizeof(LossyCompressionElement));
//add the first data
type[0] = 0;
compressSingleDoubleValue(vce, spaceFillingValue[0], realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,8);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
decData[0] = vce->data;
//add the second data
type[1] = 0;
compressSingleDoubleValue(vce, spaceFillingValue[1], realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,8);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
decData[1] = vce->data;
int state = 0;
double checkRadius = 0;
double curData = 0;
double pred = 0;
double predAbsErr = 0;
checkRadius = (exe_params->intvCapacity-1)*realPrecision;
double interval = 2*realPrecision;
for(i=2;i<dataLength;i++)
{
curData = spaceFillingValue[i];
pred = preStepData[i];
predAbsErr = fabs(curData - pred);
if(predAbsErr<=checkRadius)
{
state = (predAbsErr/realPrecision+1)/2;
if(curData>=pred)
{
type[i] = exe_params->intvRadius+state;
pred = pred + state*interval;
}
else //curData<pred
{
type[i] = exe_params->intvRadius-state;
pred = pred - state*interval;
}
decData[i] = pred;
continue;
}
//unpredictable data processing
type[i] = 0;
compressSingleDoubleValue(vce, curData, realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,8);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
decData[i] = vce->data;
}//end of for
size_t exactDataNum = exactLeadNumArray->size;
TightDataPointStorageD* tdps;
new_TightDataPointStorageD(&tdps, dataLength, exactDataNum,
type, exactMidByteArray->array, exactMidByteArray->size,
exactLeadNumArray->array,
resiBitArray->array, resiBitArray->size,
resiBitsLength,
realPrecision, medianValue, (char)reqLength, quantization_intervals, NULL, 0, 0);
//free memory
free_DIA(exactLeadNumArray);
free_DIA(resiBitArray);
free(type);
free(vce);
free(lce);
free(exactMidByteArray); //exactMidByteArray->array has been released in free_TightDataPointStorageF(tdps);
//memcpy(preStepData, decData, dataLength*sizeof(double)); //update the data
//free(decData);
return tdps;
}

258
deps/SZ/sz/src/sz_float.c vendored
View File

@ -19,9 +19,7 @@
#include "DynamicByteArray.h"
#include "TightDataPointStorageF.h"
#include "sz_float.h"
#include "sz_float_pwr.h"
#include "szd_float.h"
#include "szd_float_pwr.h"
#include "zlib.h"
#include "rw.h"
#include "utility.h"
@ -53,11 +51,6 @@ void computeReqLength_float(double realPrecision, short rangeExpo, int* reqLengt
}
}
inline short computeReqLength_float_MSST19(double realPrecision)
{
short reqExpo = getPrecisionReqLength_float(realPrecision);
return 9-reqExpo;
}
unsigned int optimize_intervals_float_1D(float *oriData, size_t dataLength, double realPrecision)
{
@ -240,7 +233,7 @@ TightDataPointStorageF* SZ_compress_float_1D_MDQ(float *oriData,
exactLeadNumArray->array,
resiBitArray->array, resiBitArray->size,
resiBitsLength,
realPrecision, medianValue, (char)reqLength, quantization_intervals, NULL, 0, 0);
realPrecision, medianValue, (char)reqLength, quantization_intervals, 0);
//sdi:Debug
/* int sum =0;
@ -286,176 +279,16 @@ bool SZ_compress_args_float_NoCkRngeNoGzip_1D( unsigned char* newByteData, float
return true;
}
/*MSST19*/
TightDataPointStorageF* SZ_compress_float_1D_MDQ_MSST19(float *oriData,
size_t dataLength, double realPrecision, float valueRangeSize, float medianValue_f)
{
//struct ClockPoint clockPointBuild;
//TimeDurationStart("build", &clockPointBuild);
unsigned int quantization_intervals;
if(exe_params->optQuantMode==1)
quantization_intervals = optimize_intervals_float_1D_opt_MSST19(oriData, dataLength, realPrecision);
else
quantization_intervals = exe_params->intvCapacity;
//updateQuantizationInfo(quantization_intervals);
int intvRadius = quantization_intervals/2;
double* precisionTable = (double*)malloc(sizeof(double) * quantization_intervals);
double inv = 2.0-pow(2, -(confparams_cpr->plus_bits));
for(int i=0; i<quantization_intervals; i++){
double test = pow((1+realPrecision), inv*(i - intvRadius));
precisionTable[i] = test;
// if(i>30000 && i<40000)
// printf("%d %.30G\n", i, test);
}
//float smallest_precision = precisionTable[0], largest_precision = precisionTable[quantization_intervals-1];
struct TopLevelTableWideInterval levelTable;
MultiLevelCacheTableWideIntervalBuild(&levelTable, precisionTable, quantization_intervals, realPrecision, confparams_cpr->plus_bits);
size_t i;
int reqLength;
float medianValue = medianValue_f;
//float medianInverse = 1 / medianValue_f;
//short radExpo = getExponent_float(valueRangeSize/2);
reqLength = computeReqLength_float_MSST19(realPrecision);
int* type = (int*) malloc(dataLength*sizeof(int));
float* spaceFillingValue = oriData; //
DynamicIntArray *exactLeadNumArray;
new_DIA(&exactLeadNumArray, dataLength/2/8);
DynamicByteArray *exactMidByteArray;
new_DBA(&exactMidByteArray, dataLength/2);
DynamicIntArray *resiBitArray;
new_DIA(&resiBitArray, DynArrayInitLen);
unsigned char preDataBytes[4];
intToBytes_bigEndian(preDataBytes, 0);
int reqBytesLength = reqLength/8;
int resiBitsLength = reqLength%8;
float last3CmprsData[3] = {0};
//size_t miss=0, hit=0;
FloatValueCompressElement *vce = (FloatValueCompressElement*)malloc(sizeof(FloatValueCompressElement));
LossyCompressionElement *lce = (LossyCompressionElement*)malloc(sizeof(LossyCompressionElement));
//add the first data
type[0] = 0;
compressSingleFloatValue_MSST19(vce, spaceFillingValue[0], realPrecision, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Float(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,4);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
listAdd_float(last3CmprsData, vce->data);
//miss++;
//add the second data
type[1] = 0;
compressSingleFloatValue_MSST19(vce, spaceFillingValue[1], realPrecision, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Float(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,4);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
listAdd_float(last3CmprsData, vce->data);
//miss++;
int state;
//double checkRadius;
float curData;
float pred = vce->data;
double predRelErrRatio;
const uint64_t top = levelTable.topIndex, base = levelTable.baseIndex;
const uint64_t range = top - base;
const int bits = levelTable.bits;
uint64_t* const buffer = (uint64_t*)&predRelErrRatio;
const int shift = 52-bits;
uint64_t expoIndex, mantiIndex;
uint16_t* tables[range+1];
for(int i=0; i<=range; i++){
tables[i] = levelTable.subTables[i].table;
}
for(i=2;i<dataLength;i++)
{
curData = spaceFillingValue[i];
predRelErrRatio = curData / pred;
expoIndex = ((*buffer & 0x7fffffffffffffff) >> 52) - base;
if(expoIndex <= range){
mantiIndex = (*buffer & 0x000fffffffffffff) >> shift;
state = tables[expoIndex][mantiIndex];
}else{
state = 0;
}
if(state)
{
type[i] = state;
pred *= precisionTable[state];
//hit++;
continue;
}
//unpredictable data processing
type[i] = 0;
compressSingleFloatValue_MSST19(vce, curData, realPrecision, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Float(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,4);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
pred = vce->data;
//miss++;
#ifdef HAVE_TIMECMPR
if(confparams_cpr->szMode == SZ_TEMPORAL_COMPRESSION)
decData[i] = vce->data;
#endif
}//end of for
// printf("miss:%d, hit:%d\n", miss, hit);
size_t exactDataNum = exactLeadNumArray->size;
TightDataPointStorageF* tdps;
new_TightDataPointStorageF(&tdps, dataLength, exactDataNum,
type, exactMidByteArray->array, exactMidByteArray->size,
exactLeadNumArray->array,
resiBitArray->array, resiBitArray->size,
resiBitsLength,
realPrecision, medianValue, (char)reqLength, quantization_intervals, NULL, 0, 0);
tdps->plus_bits = confparams_cpr->plus_bits;
//free memory
free_DIA(exactLeadNumArray);
free_DIA(resiBitArray);
free(type);
free(vce);
free(lce);
free(exactMidByteArray); //exactMidByteArray->array has been released in free_TightDataPointStorageF(tdps);
free(precisionTable);
freeTopLevelTableWideInterval(&levelTable);
return tdps;
}
void SZ_compress_args_float_withinRange(unsigned char* newByteData, float *oriData, size_t dataLength, size_t *outSize)
{
TightDataPointStorageF* tdps = (TightDataPointStorageF*) malloc(sizeof(TightDataPointStorageF));
tdps->rtypeArray = NULL;
tdps->typeArray = NULL;
tdps->leadNumArray = NULL;
tdps->residualMidBits = NULL;
tdps->allSameData = 1;
tdps->dataSeriesLength = dataLength;
tdps->exactMidBytes = (unsigned char*)malloc(sizeof(unsigned char)*4);
tdps->pwrErrBoundBytes = NULL;
tdps->isLossless = 0;
float value = oriData[0];
floatToBytes(tdps->exactMidBytes, value);
@ -487,9 +320,6 @@ int SZ_compress_args_float(float *oriData, size_t r1, unsigned char* newByteData
printf("error, input elements count=%d less than %d, so need not do compress.\n", dataLength, MIN_NUM_OF_ELEMENTS);
return SZ_LITTER_ELEMENT;
}
if(params->pw_relBoundRatio < 0.000009999)
params->accelerate_pw_rel_compression = 0;
float valueRangeSize = 0, medianValue = 0;
unsigned char * signs = NULL;
@ -497,17 +327,7 @@ int SZ_compress_args_float(float *oriData, size_t r1, unsigned char* newByteData
float nearZero = 0.0;
float min = 0;
// calc min
if(params->errorBoundMode == PW_REL && params->accelerate_pw_rel_compression)
{
signs = (unsigned char *) malloc(dataLength);
memset(signs, 0, dataLength);
min = computeRangeSize_float_MSST19(oriData, dataLength, &valueRangeSize, &medianValue, signs, &positive, &nearZero);
free(signs); // tickduan move to here
signs = NULL;
}
else
min = computeRangeSize_float(oriData, dataLength, &valueRangeSize, &medianValue);
min = computeRangeSize_float(oriData, dataLength, &valueRangeSize, &medianValue);
// calc max
float max = min+valueRangeSize;
@ -557,23 +377,13 @@ int SZ_compress_args_float(float *oriData, size_t r1, unsigned char* newByteData
tmpByteData = (unsigned char*)malloc(r1*sizeof(float) + 1024);
}
if(params->errorBoundMode>=PW_REL)
// compress core algorithm
if(!SZ_compress_args_float_NoCkRngeNoGzip_1D(tmpByteData, oriData, r1, realPrecision, &tmpOutSize, valueRangeSize, medianValue))
{
if(params->accelerate_pw_rel_compression && params->maxRangeRadius <= 32768)
SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_pre_log_MSST19(tmpByteData, oriData, params->pw_relBoundRatio, r1, &tmpOutSize, valueRangeSize, medianValue, signs, &positive, min, max, nearZero);
else
SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_pre_log(tmpByteData, oriData, params->pw_relBoundRatio, r1, &tmpOutSize, min, max);
}
else
{
// compress core algorithm
if(!SZ_compress_args_float_NoCkRngeNoGzip_1D(tmpByteData, oriData, r1, realPrecision, &tmpOutSize, valueRangeSize, medianValue))
{
*outSize = 0;
if(twoStage)
free(tmpByteData);
return SZ_ALGORITHM_ERR;
}
*outSize = 0;
if(twoStage)
free(tmpByteData);
return SZ_ALGORITHM_ERR;
}
//cost_end(" sz_first_compress");
@ -597,58 +407,6 @@ int SZ_compress_args_float(float *oriData, size_t r1, unsigned char* newByteData
return SZ_SUCCESS;
}
unsigned int optimize_intervals_float_1D_opt_MSST19(float *oriData, size_t dataLength, double realPrecision)
{
size_t i = 0, radiusIndex;
float pred_value = 0;
double pred_err;
size_t *intervals = (size_t*)malloc(confparams_cpr->maxRangeRadius*sizeof(size_t));
memset(intervals, 0, confparams_cpr->maxRangeRadius*sizeof(size_t));
size_t totalSampleSize = 0;//dataLength/confparams_cpr->sampleDistance;
float * data_pos = oriData + 2;
float divider = log2(1+realPrecision)*2;
int tempIndex = 0;
while(data_pos - oriData < dataLength){
if(*data_pos == 0){
data_pos += confparams_cpr->sampleDistance;
continue;
}
tempIndex++;
totalSampleSize++;
pred_value = data_pos[-1];
pred_err = fabs((double)*data_pos / pred_value);
radiusIndex = (unsigned long)fabs(log2(pred_err)/divider+0.5);
if(radiusIndex>=confparams_cpr->maxRangeRadius)
radiusIndex = confparams_cpr->maxRangeRadius - 1;
intervals[radiusIndex]++;
data_pos += confparams_cpr->sampleDistance;
}
//compute the appropriate number
size_t targetCount = totalSampleSize*confparams_cpr->predThreshold;
size_t sum = 0;
for(i=0;i<confparams_cpr->maxRangeRadius;i++)
{
sum += intervals[i];
if(sum>targetCount)
break;
}
if(i>=confparams_cpr->maxRangeRadius)
i = confparams_cpr->maxRangeRadius-1;
unsigned int accIntervals = 2*(i+1);
unsigned int powerOf2 = roundUpToPowerOf2(accIntervals);
if(powerOf2<32)
powerOf2 = 32;
free(intervals);
return powerOf2;
}
unsigned int optimize_intervals_float_1D_opt(float *oriData, size_t dataLength, double realPrecision)
{
size_t i = 0, radiusIndex;

736
deps/SZ/sz/src/sz_float_pwr.c vendored
View File

@ -1,736 +0,0 @@
/**
* @file sz_float_pwr.c
* @author Sheng Di, Dingwen Tao, Xin Liang, Xiangyu Zou, Tao Lu, Wen Xia, Xuan Wang, Weizhe Zhang
* @date Aug, 2016
* @brief SZ_Init, Compression and Decompression functions
* This file contains the compression/decompression functions related to point-wise relative errors
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <math.h>
#include "sz.h"
#include "CompressElement.h"
#include "DynamicByteArray.h"
#include "DynamicIntArray.h"
#include "TightDataPointStorageF.h"
#include "sz_float.h"
#include "sz_float_pwr.h"
#include "zlib.h"
#include "rw.h"
#include "utility.h"
void compute_segment_precisions_float_1D(float *oriData, size_t dataLength, float* pwrErrBound, unsigned char* pwrErrBoundBytes, double globalPrecision)
{
size_t i = 0, j = 0, k = 0;
float realPrecision = oriData[0]!=0?fabs(confparams_cpr->pw_relBoundRatio*oriData[0]):confparams_cpr->pw_relBoundRatio;
float approxPrecision;
unsigned char realPrecBytes[4];
float curPrecision;
float curValue;
float sum = 0;
for(i=0;i<dataLength;i++)
{
curValue = oriData[i];
if(i%confparams_cpr->segment_size==0&&i>0)
{
//get two first bytes of the realPrecision
if(confparams_cpr->pwr_type==SZ_PWR_AVG_TYPE)
{
realPrecision = sum/confparams_cpr->segment_size;
sum = 0;
}
realPrecision *= confparams_cpr->pw_relBoundRatio;
if(confparams_cpr->errorBoundMode==ABS_AND_PW_REL||confparams_cpr->errorBoundMode==REL_AND_PW_REL)
realPrecision = realPrecision<globalPrecision?realPrecision:globalPrecision;
else if(confparams_cpr->errorBoundMode==ABS_OR_PW_REL||confparams_cpr->errorBoundMode==REL_OR_PW_REL)
realPrecision = realPrecision<globalPrecision?globalPrecision:realPrecision;
floatToBytes(realPrecBytes, realPrecision);
realPrecBytes[2] = realPrecBytes[3] = 0;
approxPrecision = bytesToFloat(realPrecBytes);
//put the realPrecision in float* pwrErBound
pwrErrBound[j++] = approxPrecision;
//put the two bytes in pwrErrBoundBytes
pwrErrBoundBytes[k++] = realPrecBytes[0];
pwrErrBoundBytes[k++] = realPrecBytes[1];
realPrecision = fabs(curValue);
}
if(curValue!=0)
{
curPrecision = fabs(curValue);
switch(confparams_cpr->pwr_type)
{
case SZ_PWR_MIN_TYPE:
if(realPrecision>curPrecision)
realPrecision = curPrecision;
break;
case SZ_PWR_AVG_TYPE:
sum += curPrecision;
break;
case SZ_PWR_MAX_TYPE:
if(realPrecision<curPrecision)
realPrecision = curPrecision;
break;
}
}
}
if(confparams_cpr->pwr_type==SZ_PWR_AVG_TYPE)
{
int size = dataLength%confparams_cpr->segment_size==0?confparams_cpr->segment_size:dataLength%confparams_cpr->segment_size;
realPrecision = sum/size;
}
if(confparams_cpr->errorBoundMode==ABS_AND_PW_REL||confparams_cpr->errorBoundMode==REL_AND_PW_REL)
realPrecision = realPrecision<globalPrecision?realPrecision:globalPrecision;
else if(confparams_cpr->errorBoundMode==ABS_OR_PW_REL||confparams_cpr->errorBoundMode==REL_OR_PW_REL)
realPrecision = realPrecision<globalPrecision?globalPrecision:realPrecision;
floatToBytes(realPrecBytes, realPrecision);
realPrecBytes[2] = realPrecBytes[3] = 0;
approxPrecision = bytesToFloat(realPrecBytes);
//put the realPrecision in float* pwrErBound
pwrErrBound[j++] = approxPrecision;
//put the two bytes in pwrErrBoundBytes
pwrErrBoundBytes[k++] = realPrecBytes[0];
pwrErrBoundBytes[k++] = realPrecBytes[1];
}
unsigned int optimize_intervals_float_1D_pwr(float *oriData, size_t dataLength, float* pwrErrBound)
{
size_t i = 0, j = 0;
float realPrecision = pwrErrBound[j++];
unsigned long radiusIndex;
float pred_value = 0, pred_err;
int *intervals = (int*)malloc(confparams_cpr->maxRangeRadius*sizeof(int));
memset(intervals, 0, confparams_cpr->maxRangeRadius*sizeof(int));
int totalSampleSize = dataLength/confparams_cpr->sampleDistance;
for(i=2;i<dataLength;i++)
{
if(i%confparams_cpr->segment_size==0)
realPrecision = pwrErrBound[j++];
if(i%confparams_cpr->sampleDistance==0)
{
//pred_value = 2*oriData[i-1] - oriData[i-2];
pred_value = oriData[i-1];
pred_err = fabs(pred_value - oriData[i]);
radiusIndex = (unsigned long)((pred_err/realPrecision+1)/2);
if(radiusIndex>=confparams_cpr->maxRangeRadius)
radiusIndex = confparams_cpr->maxRangeRadius - 1;
intervals[radiusIndex]++;
}
}
//compute the appropriate number
size_t targetCount = totalSampleSize*confparams_cpr->predThreshold;
size_t sum = 0;
for(i=0;i<confparams_cpr->maxRangeRadius;i++)
{
sum += intervals[i];
if(sum>targetCount)
break;
}
if(i>=confparams_cpr->maxRangeRadius)
i = confparams_cpr->maxRangeRadius-1;
unsigned int accIntervals = 2*(i+1);
unsigned int powerOf2 = roundUpToPowerOf2(accIntervals);
if(powerOf2<32)
powerOf2 = 32;
free(intervals);
//printf("accIntervals=%d, powerOf2=%d\n", accIntervals, powerOf2);
return powerOf2;
}
void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr(unsigned char* newByteData, float *oriData, double globalPrecision,
size_t dataLength, size_t *outSize, float min, float max)
{
size_t pwrLength = dataLength%confparams_cpr->segment_size==0?dataLength/confparams_cpr->segment_size:dataLength/confparams_cpr->segment_size+1;
float* pwrErrBound = (float*)malloc(sizeof(float)*pwrLength);
size_t pwrErrBoundBytes_size = sizeof(unsigned char)*pwrLength*2;
unsigned char* pwrErrBoundBytes = (unsigned char*)malloc(pwrErrBoundBytes_size);
compute_segment_precisions_float_1D(oriData, dataLength, pwrErrBound, pwrErrBoundBytes, globalPrecision);
unsigned int quantization_intervals;
if(exe_params->optQuantMode==1)
{
quantization_intervals = optimize_intervals_float_1D_pwr(oriData, dataLength, pwrErrBound);
updateQuantizationInfo(quantization_intervals);
}
else
quantization_intervals = exe_params->intvCapacity;
size_t i = 0, j = 0;
int reqLength;
float realPrecision = pwrErrBound[j++];
float medianValue = 0;
float radius = fabs(max)<fabs(min)?fabs(min):fabs(max);
short radExpo = getExponent_float(radius);
computeReqLength_float(realPrecision, radExpo, &reqLength, &medianValue);
int* type = (int*) malloc(dataLength*sizeof(int));
//type[dataLength]=0;
float* spaceFillingValue = oriData; //
DynamicByteArray *resiBitLengthArray;
new_DBA(&resiBitLengthArray, DynArrayInitLen);
DynamicIntArray *exactLeadNumArray;
new_DIA(&exactLeadNumArray, DynArrayInitLen);
DynamicByteArray *exactMidByteArray;
new_DBA(&exactMidByteArray, DynArrayInitLen);
DynamicIntArray *resiBitArray;
new_DIA(&resiBitArray, DynArrayInitLen);
type[0] = 0;
unsigned char preDataBytes[4] = {0};
intToBytes_bigEndian(preDataBytes, 0);
int reqBytesLength = reqLength/8;
int resiBitsLength = reqLength%8;
float last3CmprsData[3] = {0};
FloatValueCompressElement *vce = (FloatValueCompressElement*)malloc(sizeof(FloatValueCompressElement));
LossyCompressionElement *lce = (LossyCompressionElement*)malloc(sizeof(LossyCompressionElement));
//add the first data
addDBA_Data(resiBitLengthArray, (unsigned char)resiBitsLength);
compressSingleFloatValue(vce, spaceFillingValue[0], realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Float(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,4);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
listAdd_float(last3CmprsData, vce->data);
//printf("%.30G\n",last3CmprsData[0]);
//add the second data
type[1] = 0;
addDBA_Data(resiBitLengthArray, (unsigned char)resiBitsLength);
compressSingleFloatValue(vce, spaceFillingValue[1], realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Float(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,4);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
listAdd_float(last3CmprsData, vce->data);
//printf("%.30G\n",last3CmprsData[0]);
int state;
double checkRadius;
float curData;
float pred;
double predAbsErr;
checkRadius = (exe_params->intvCapacity-1)*realPrecision;
double interval = 2*realPrecision;
int updateReqLength = 0; //a marker: 1 means already updated
for(i=2;i<dataLength;i++)
{
curData = spaceFillingValue[i];
if(i%confparams_cpr->segment_size==0)
{
realPrecision = pwrErrBound[j++];
checkRadius = (exe_params->intvCapacity-1)*realPrecision;
interval = 2*realPrecision;
updateReqLength = 0;
}
//pred = 2*last3CmprsData[0] - last3CmprsData[1];
pred = last3CmprsData[0];
predAbsErr = fabs(curData - pred);
if(predAbsErr<checkRadius)
{
state = (predAbsErr/realPrecision+1)/2;
if(curData>=pred)
{
type[i] = exe_params->intvRadius+state;
pred = pred + state*interval;
}
else //curData<pred
{
type[i] = exe_params->intvRadius-state;
pred = pred - state*interval;
}
listAdd_float(last3CmprsData, pred);
continue;
}
//unpredictable data processing
if(updateReqLength==0)
{
computeReqLength_float(realPrecision, radExpo, &reqLength, &medianValue);
reqBytesLength = reqLength/8;
resiBitsLength = reqLength%8;
updateReqLength = 1;
}
type[i] = 0;
addDBA_Data(resiBitLengthArray, (unsigned char)resiBitsLength);
compressSingleFloatValue(vce, curData, realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Float(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,4);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
listAdd_float(last3CmprsData, vce->data);
}//end of for
int exactDataNum = exactLeadNumArray->size;
TightDataPointStorageF* tdps;
new_TightDataPointStorageF2(&tdps, dataLength, exactDataNum,
type, exactMidByteArray->array, exactMidByteArray->size,
exactLeadNumArray->array,
resiBitArray->array, resiBitArray->size,
resiBitLengthArray->array, resiBitLengthArray->size,
realPrecision, medianValue, (char)reqLength, quantization_intervals, pwrErrBoundBytes, pwrErrBoundBytes_size, radExpo);
//free memory
free_DBA(resiBitLengthArray);
free_DIA(exactLeadNumArray);
free_DIA(resiBitArray);
free(type);
convertTDPStoFlatBytes_float(tdps, newByteData, outSize);
int floatSize=sizeof(float);
if(*outSize>dataLength*floatSize)
{
size_t k = 0, i;
tdps->isLossless = 1;
size_t totalByteLength = 1 + exe_params->SZ_SIZE_TYPE + 1 + floatSize*dataLength;
//*newByteData = (unsigned char*)malloc(totalByteLength); comment by tickduan
unsigned char dsLengthBytes[exe_params->SZ_SIZE_TYPE];
intToBytes_bigEndian(dsLengthBytes, dataLength);//4
newByteData[k++] = versionNumber;
if(exe_params->SZ_SIZE_TYPE==4)
{
newByteData[k++] = 16; //=00010000
}
else
{
newByteData[k++] = 80;
}
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)//4 or 8
newByteData[k++] = dsLengthBytes[i];
if(sysEndianType==BIG_ENDIAN_SYSTEM)
memcpy(newByteData+4+exe_params->SZ_SIZE_TYPE, oriData, dataLength*floatSize);
else
{
unsigned char* p = newByteData+4+exe_params->SZ_SIZE_TYPE;
for(i=0;i<dataLength;i++,p+=floatSize)
floatToBytes(p, oriData[i]);
}
*outSize = totalByteLength;
}
free(pwrErrBound);
free(vce);
free(lce);
free_TightDataPointStorageF(tdps);
free(exactMidByteArray);
}
void createRangeGroups_float(float** posGroups, float** negGroups, int** posFlags, int** negFlags)
{
size_t size = GROUP_COUNT*sizeof(float);
size_t size2 = GROUP_COUNT*sizeof(int);
*posGroups = (float*)malloc(size);
*negGroups = (float*)malloc(size);
*posFlags = (int*)malloc(size2);
*negFlags = (int*)malloc(size2);
memset(*posGroups, 0, size);
memset(*negGroups, 0, size);
memset(*posFlags, 0, size2);
memset(*negFlags, 0, size2);
}
void compressGroupIDArray_float(char* groupID, TightDataPointStorageF* tdps)
{
size_t dataLength = tdps->dataSeriesLength;
int* standGroupID = (int*)malloc(dataLength*sizeof(int));
size_t i;
standGroupID[0] = groupID[0]+GROUP_COUNT; //plus an offset such that it would not be a negative number.
char lastGroupIDValue = groupID[0], curGroupIDValue;
int offset = 2*(GROUP_COUNT + 2);
for(i=1; i<dataLength;i++)
{
curGroupIDValue = groupID[i];
standGroupID[i] = (curGroupIDValue - lastGroupIDValue) + offset;
lastGroupIDValue = curGroupIDValue;
}
unsigned char* out = NULL;
size_t outSize;
HuffmanTree* huffmanTree = SZ_Reset();
encode_withTree(huffmanTree, standGroupID, dataLength, &out, &outSize);
SZ_ReleaseHuffman(huffmanTree);
tdps->pwrErrBoundBytes = out; //groupIDArray
tdps->pwrErrBoundBytes_size = outSize;
free(standGroupID);
}
TightDataPointStorageF* SZ_compress_float_1D_MDQ_pwrGroup(float* oriData, size_t dataLength, int errBoundMode,
double absErrBound, double relBoundRatio, double pwrErrRatio, float valueRangeSize, float medianValue_f)
{
size_t i;
float *posGroups, *negGroups, *groups;
float pos_01_group = 0, neg_01_group = 0; //[0,1] and [-1,0]
int *posFlags, *negFlags, *flags;
int pos_01_flag = 0, neg_01_flag = 0;
createRangeGroups_float(&posGroups, &negGroups, &posFlags, &negFlags);
size_t nbBins = (size_t)(1/pwrErrRatio);
if(nbBins%2==1)
nbBins++;
exe_params->intvRadius = nbBins;
int reqLength, status;
float medianValue = medianValue_f;
float realPrecision = (float)getRealPrecision_float(valueRangeSize, errBoundMode, absErrBound, relBoundRatio, &status);
if(realPrecision<0)
realPrecision = pwrErrRatio;
float realGroupPrecision; //precision (error) based on group ID
getPrecisionReqLength_float(realPrecision);
short radExpo = getExponent_float(valueRangeSize/2);
short lastGroupNum = 0, groupNum, grpNum = 0;
double* groupErrorBounds = generateGroupErrBounds(errBoundMode, realPrecision, pwrErrRatio);
exe_params->intvRadius = generateGroupMaxIntervalCount(groupErrorBounds);
computeReqLength_float(realPrecision, radExpo, &reqLength, &medianValue);
int* type = (int*) malloc(dataLength*sizeof(int));
char *groupID = (char*) malloc(dataLength*sizeof(char));
char *gp = groupID;
float* spaceFillingValue = oriData;
DynamicIntArray *exactLeadNumArray;
new_DIA(&exactLeadNumArray, DynArrayInitLen);
DynamicByteArray *exactMidByteArray;
new_DBA(&exactMidByteArray, DynArrayInitLen);
DynamicIntArray *resiBitArray;
new_DIA(&resiBitArray, DynArrayInitLen);
unsigned char preDataBytes[4];
intToBytes_bigEndian(preDataBytes, 0);
int reqBytesLength = reqLength/8;
int resiBitsLength = reqLength%8;
FloatValueCompressElement *vce = (FloatValueCompressElement*)malloc(sizeof(FloatValueCompressElement));
LossyCompressionElement *lce = (LossyCompressionElement*)malloc(sizeof(LossyCompressionElement));
int state;
float curData, decValue;
float pred;
float predAbsErr;
double interval = 0;
//add the first data
type[0] = 0;
compressSingleFloatValue(vce, spaceFillingValue[0], realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Float(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,4);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
curData = spaceFillingValue[0];
groupNum = computeGroupNum_float(vce->data);
if(curData > 0 && groupNum >= 0)
{
groups = posGroups;
flags = posFlags;
grpNum = groupNum;
}
else if(curData < 0 && groupNum >= 0)
{
groups = negGroups;
flags = negFlags;
grpNum = groupNum;
}
else if(curData >= 0 && groupNum == -1)
{
groups = &pos_01_group;
flags = &pos_01_flag;
grpNum = 0;
}
else //curData < 0 && groupNum == -1
{
groups = &neg_01_group;
flags = &neg_01_flag;
grpNum = 0;
}
listAdd_float_group(groups, flags, groupNum, spaceFillingValue[0], vce->data, gp);
gp++;
for(i=1;i<dataLength;i++)
{
curData = oriData[i];
//printf("i=%d, posGroups[3]=%f, negGroups[3]=%f\n", i, posGroups[3], negGroups[3]);
groupNum = computeGroupNum_float(curData);
if(curData > 0 && groupNum >= 0)
{
groups = posGroups;
flags = posFlags;
grpNum = groupNum;
}
else if(curData < 0 && groupNum >= 0)
{
groups = negGroups;
flags = negFlags;
grpNum = groupNum;
}
else if(curData >= 0 && groupNum == -1)
{
groups = &pos_01_group;
flags = &pos_01_flag;
grpNum = 0;
}
else //curData < 0 && groupNum == -1
{
groups = &neg_01_group;
flags = &neg_01_flag;
grpNum = 0;
}
if(groupNum>=GROUP_COUNT)
{
type[i] = 0;
compressSingleFloatValue(vce, curData, realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Float(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,4);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
listAdd_float_group(groups, flags, lastGroupNum, curData, vce->data, gp); //set the group number to be last one in order to get the groupID array as smooth as possible.
}
else if(flags[grpNum]==0) //the dec value may not be in the same group
{
type[i] = 0;
compressSingleFloatValue(vce, curData, realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Float(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,4);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
//decGroupNum = computeGroupNum_float(vce->data);
//if(decGroupNum < groupNum)
// decValue = curData>0?pow(2, groupNum):-pow(2, groupNum);
//else if(decGroupNum > groupNum)
// decValue = curData>0?pow(2, groupNum+1):-pow(2, groupNum+1);
//else
// decValue = vce->data;
decValue = vce->data;
listAdd_float_group(groups, flags, groupNum, curData, decValue, gp);
lastGroupNum = curData>0?groupNum + 2: -(groupNum+2);
}
else //if flags[groupNum]==1, the dec value must be in the same group
{
pred = groups[grpNum];
predAbsErr = fabs(curData - pred);
realGroupPrecision = groupErrorBounds[grpNum]; //compute real error bound
interval = realGroupPrecision*2;
state = (predAbsErr/realGroupPrecision+1)/2;
if(curData>=pred)
{
type[i] = exe_params->intvRadius+state;
decValue = pred + state*interval;
}
else //curData<pred
{
type[i] = exe_params->intvRadius-state;
decValue = pred - state*interval;
}
//decGroupNum = computeGroupNum_float(pred);
if((decValue>0&&curData<0)||(decValue<0&&curData>=0))
decValue = 0;
//else
//{
// if(decGroupNum < groupNum)
// decValue = curData>0?pow(2, groupNum):-pow(2, groupNum);
// else if(decGroupNum > groupNum)
// decValue = curData>0?pow(2, groupNum+1):-pow(2, groupNum+1);
// else
// decValue = pred;
//}
if(fabs(curData-decValue)>realGroupPrecision)
{
type[i] = 0;
compressSingleFloatValue(vce, curData, realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Float(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,4);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
decValue = vce->data;
}
listAdd_float_group(groups, flags, groupNum, curData, decValue, gp);
lastGroupNum = curData>=0?groupNum + 2: -(groupNum+2);
}
gp++;
}
int exactDataNum = exactLeadNumArray->size;
TightDataPointStorageF* tdps;
//combineTypeAndGroupIDArray(nbBins, dataLength, &type, groupID);
new_TightDataPointStorageF(&tdps, dataLength, exactDataNum,
type, exactMidByteArray->array, exactMidByteArray->size,
exactLeadNumArray->array,
resiBitArray->array, resiBitArray->size,
resiBitsLength,
realPrecision, medianValue, (char)reqLength, nbBins, NULL, 0, radExpo);
compressGroupIDArray_float(groupID, tdps);
free(posGroups);
free(negGroups);
free(posFlags);
free(negFlags);
free(groupID);
free(groupErrorBounds);
free_DIA(exactLeadNumArray);
free_DIA(resiBitArray);
free(type);
free(vce);
free(lce);
free(exactMidByteArray); //exactMidByteArray->array has been released in free_TightDataPointStorageF(tdps);
return tdps;
}
void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwrgroup(unsigned char* newByteData, float *oriData,
size_t dataLength, double absErrBound, double relBoundRatio, double pwrErrRatio, float valueRangeSize, float medianValue_f, size_t *outSize)
{
TightDataPointStorageF* tdps = SZ_compress_float_1D_MDQ_pwrGroup(oriData, dataLength, confparams_cpr->errorBoundMode,
absErrBound, relBoundRatio, pwrErrRatio,
valueRangeSize, medianValue_f);
convertTDPStoFlatBytes_float(tdps, newByteData, outSize);
free_TightDataPointStorageF(tdps);
}
#include <stdbool.h>
void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_pre_log(unsigned char* newByteData, float *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, float min, float max){
float * log_data = (float *) malloc(dataLength * sizeof(float));
unsigned char * signs = (unsigned char *) malloc(dataLength);
memset(signs, 0, dataLength);
// preprocess
float max_abs_log_data;
if(min == 0) max_abs_log_data = fabs(log2(fabs(max)));
else if(max == 0) max_abs_log_data = fabs(log2(fabs(min)));
else max_abs_log_data = fabs(log2(fabs(min))) > fabs(log2(fabs(max))) ? fabs(log2(fabs(min))) : fabs(log2(fabs(max)));
float min_log_data = max_abs_log_data;
bool positive = true;
for(size_t i=0; i<dataLength; i++){
if(oriData[i] < 0){
signs[i] = 1;
log_data[i] = -oriData[i];
positive = false;
}
else
log_data[i] = oriData[i];
if(log_data[i] > 0){
log_data[i] = log2(log_data[i]);
if(log_data[i] > max_abs_log_data) max_abs_log_data = log_data[i];
if(log_data[i] < min_log_data) min_log_data = log_data[i];
}
}
float valueRangeSize, medianValue_f;
computeRangeSize_float(log_data, dataLength, &valueRangeSize, &medianValue_f);
if(fabs(min_log_data) > max_abs_log_data) max_abs_log_data = fabs(min_log_data);
double realPrecision = log2(1.0 + pwrErrRatio) - max_abs_log_data * 1.2e-7;
for(size_t i=0; i<dataLength; i++){
if(oriData[i] == 0){
log_data[i] = min_log_data - 2.0001*realPrecision;
}
}
TightDataPointStorageF* tdps = SZ_compress_float_1D_MDQ(log_data, dataLength, realPrecision, valueRangeSize, medianValue_f);
tdps->minLogValue = min_log_data - 1.0001*realPrecision;
free(log_data);
if(!positive){
unsigned char * comp_signs;
// compress signs
unsigned long signSize = sz_lossless_compress(ZSTD_COMPRESSOR, 3, signs, dataLength, &comp_signs);
tdps->pwrErrBoundBytes = comp_signs;
tdps->pwrErrBoundBytes_size = signSize;
}
else{
tdps->pwrErrBoundBytes = NULL;
tdps->pwrErrBoundBytes_size = 0;
}
free(signs);
convertTDPStoFlatBytes_float(tdps, newByteData, outSize);
free_TightDataPointStorageF(tdps);
}
void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_pre_log_MSST19(unsigned char* newByteData, float *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, float valueRangeSize, float medianValue_f,
unsigned char* signs, bool* positive, float min, float max, float nearZero){
float multiplier = pow((1+pwrErrRatio), -3.0001);
for(int i=0; i<dataLength; i++){
if(oriData[i] == 0){
oriData[i] = nearZero * multiplier;
}
}
float median_log = sqrt(fabs(nearZero * max));
TightDataPointStorageF* tdps = SZ_compress_float_1D_MDQ_MSST19(oriData, dataLength, pwrErrRatio, valueRangeSize, median_log);
tdps->minLogValue = nearZero / ((1+pwrErrRatio)*(1+pwrErrRatio));
if(!(*positive)){
unsigned char * comp_signs;
// compress signs
unsigned long signSize = sz_lossless_compress(ZSTD_COMPRESSOR, 3, signs, dataLength, &comp_signs);
tdps->pwrErrBoundBytes = comp_signs;
tdps->pwrErrBoundBytes_size = signSize;
}
else{
tdps->pwrErrBoundBytes = NULL;
tdps->pwrErrBoundBytes_size = 0;
}
free(signs);
convertTDPStoFlatBytes_float(tdps, newByteData, outSize);
free_TightDataPointStorageF(tdps);
}

139
deps/SZ/sz/src/szd_double.c vendored
View File

@ -13,10 +13,8 @@
#include "szd_double.h"
#include "TightDataPointStorageD.h"
#include "sz.h"
#include "Huffman.h"
#include "szd_double_pwr.h"
#include "szd_double_ts.h"
#include "utility.h"
#include "Huffman.h"
int SZ_decompress_args_double(double* newData, size_t r1, unsigned char* cmpBytes,
size_t cmpSize, int compressionType, double* hist_data, sz_exedata* pde_exe, sz_params* pde_params)
@ -68,8 +66,12 @@ int SZ_decompress_args_double(double* newData, size_t r1, unsigned char* cmpByte
// calc postion
//pde_params->sol_ID = szTmpBytes[1+3-2+14-4]; //szTmpBytes: version(3bytes), samebyte(1byte), [14]:sol_ID=SZ or SZ_Transpose
//TODO: convert szTmpBytes to double array.
TightDataPointStorageD* tdps;
TightDataPointStorageD* tdps = NULL;
int errBoundMode = new_TightDataPointStorageD_fromFlatBytes(&tdps, szTmpBytes, tmpSize, pde_exe, pde_params);
if(tdps == NULL)
{
return SZ_FORMAT_ERR;
}
int dim = r1;
int doubleSize = sizeof(double);
@ -124,7 +126,6 @@ void decompressDataSeries_double_1D(double* data, size_t dataSeriesLength, doubl
//*data = (double*)malloc(sizeof(double)*dataSeriesLength); comment by tickduan
int* type = (int*)malloc(dataSeriesLength*sizeof(int));
HuffmanTree* huffmanTree = createHuffmanTree(tdps->stateNum);
decode_withTree(huffmanTree, tdps->typeArray, dataSeriesLength, type);
SZ_ReleaseHuffman(huffmanTree);
@ -143,8 +144,10 @@ void decompressDataSeries_double_1D(double* data, size_t dataSeriesLength, doubl
resiBitsLength = tdps->reqLength%8;
medianValue = tdps->medianValue;
int type_;
for (i = 0; i < dataSeriesLength; i++) {
for (i = 0; i < dataSeriesLength; i++)
{
type_ = type[i];
switch (type_) {
case 0:
@ -203,108 +206,6 @@ void decompressDataSeries_double_1D(double* data, size_t dataSeriesLength, doubl
return;
}
/*MSST19*/
void decompressDataSeries_double_1D_MSST19(double* data, size_t dataSeriesLength, TightDataPointStorageD* tdps)
{
//updateQuantizationInfo(tdps->intervals);
int intvRadius = tdps->intervals/2;
int intvCapacity = tdps->intervals;
size_t i, j, k = 0, p = 0, l = 0; // k is to track the location of residual_bit
// in resiMidBits, p is to track the
// byte_index of resiMidBits, l is for
// leadNum
unsigned char* leadNum;
//double interval = tdps->realPrecision*2;
convertByteArray2IntArray_fast_2b(tdps->exactDataNum, tdps->leadNumArray, tdps->leadNumArray_size, &leadNum);
//*data = (double*)malloc(sizeof(double)*dataSeriesLength); comment by tickduan
int* type = (int*)malloc(dataSeriesLength*sizeof(int));
HuffmanTree* huffmanTree = createHuffmanTree(tdps->stateNum);
decode_withTree_MSST19(huffmanTree, tdps->typeArray, dataSeriesLength, type, tdps->max_bits);
//decode_withTree(huffmanTree, tdps->typeArray, dataSeriesLength, type);
SZ_ReleaseHuffman(huffmanTree);
unsigned char preBytes[8];
unsigned char curBytes[8];
memset(preBytes, 0, 8);
size_t curByteIndex = 0;
int reqBytesLength, resiBitsLength, resiBits;
unsigned char leadingNum;
double exactData, predValue = 0;
reqBytesLength = tdps->reqLength/8;
resiBitsLength = tdps->reqLength%8;
//float threshold = tdps->minLogValue;
double* precisionTable = (double*)malloc(sizeof(double) * intvCapacity);
double inv = 2.0-pow(2, -(tdps->plus_bits));
for(int i=0; i<intvCapacity; i++){
double test = pow((1+tdps->realPrecision), inv*(i - intvRadius));
precisionTable[i] = test;
}
int type_;
for (i = 0; i < dataSeriesLength; i++) {
type_ = type[i];
switch (type_) {
case 0:
// compute resiBits
resiBits = 0;
if (resiBitsLength != 0) {
int kMod8 = k % 8;
int rightMovSteps = getRightMovingSteps(kMod8, resiBitsLength);
if (rightMovSteps > 0) {
int code = getRightMovingCode(kMod8, resiBitsLength);
resiBits = (tdps->residualMidBits[p] & code) >> rightMovSteps;
} else if (rightMovSteps < 0) {
int code1 = getLeftMovingCode(kMod8);
int code2 = getRightMovingCode(kMod8, resiBitsLength);
int leftMovSteps = -rightMovSteps;
rightMovSteps = 8 - leftMovSteps;
resiBits = (tdps->residualMidBits[p] & code1) << leftMovSteps;
p++;
resiBits = resiBits
| ((tdps->residualMidBits[p] & code2) >> rightMovSteps);
} else // rightMovSteps == 0
{
int code = getRightMovingCode(kMod8, resiBitsLength);
resiBits = (tdps->residualMidBits[p] & code);
p++;
}
k += resiBitsLength;
}
// recover the exact data
memset(curBytes, 0, 8);
leadingNum = leadNum[l++];
memcpy(curBytes, preBytes, leadingNum);
for (j = leadingNum; j < reqBytesLength; j++)
curBytes[j] = tdps->exactMidBytes[curByteIndex++];
if (resiBitsLength != 0) {
unsigned char resiByte = (unsigned char) (resiBits << (8 - resiBitsLength));
curBytes[reqBytesLength] = resiByte;
}
exactData = bytesToDouble(curBytes);
data[i] = exactData;
memcpy(preBytes,curBytes,8);
predValue = data[i];
break;
default:
//predValue = 2 * data[i-1] - data[i-2];
//predValue = data[i-1];
predValue = fabs(predValue) * precisionTable[type_];
data[i] = predValue;
break;
}
}
free(precisionTable);
free(leadNum);
free(type);
return;
}
void getSnapshotData_double_1D(double* data, size_t dataSeriesLength, TightDataPointStorageD* tdps, int errBoundMode, int compressionType, double* hist_data, sz_params* pde_params)
{
@ -315,24 +216,10 @@ void getSnapshotData_double_1D(double* data, size_t dataSeriesLength, TightDataP
//*data = (double*)malloc(sizeof(double)*dataSeriesLength); comment by tickduan
for (i = 0; i < dataSeriesLength; i++)
data[i] = value;
} else {
if (tdps->rtypeArray == NULL) {
if(errBoundMode < PW_REL)
{
decompressDataSeries_double_1D(data, dataSeriesLength, hist_data, tdps);
}
else
{
if(confparams_dec->accelerate_pw_rel_compression)
decompressDataSeries_double_1D_pwr_pre_log_MSST19(data, dataSeriesLength, tdps);
else
decompressDataSeries_double_1D_pwr_pre_log(data, dataSeriesLength, tdps);
//decompressDataSeries_double_1D_pwrgroup(data, dataSeriesLength, tdps);
}
return;
} else {
//TODO
}
}
else
{
decompressDataSeries_double_1D(data, dataSeriesLength, hist_data, tdps);
}
}

371
deps/SZ/sz/src/szd_double_pwr.c vendored
View File

@ -1,371 +0,0 @@
/**
* @file szd_double_pwr.c
* @author Sheng Di, Dingwen Tao, Xin Liang, Xiangyu Zou, Tao Lu, Wen Xia, Xuan Wang, Weizhe Zhang
* @date Feb, 2019
* @brief
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <math.h>
#include "TightDataPointStorageD.h"
#include "CompressElement.h"
#include "sz.h"
#include "Huffman.h"
#include "sz_double_pwr.h"
#include "utility.h"
//#include "rw.h"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wchar-subscripts"
void decompressDataSeries_double_1D_pwr(double* data, size_t dataSeriesLength, TightDataPointStorageD* tdps)
{
updateQuantizationInfo(tdps->intervals);
unsigned char tmpPrecBytes[8] = {0}; //used when needing to convert bytes to double values
unsigned char* bp = tdps->pwrErrBoundBytes;
size_t i, j, k = 0, p = 0, l = 0; // k is to track the location of residual_bit
// in resiMidBits, p is to track the
// byte_index of resiMidBits, l is for
// leadNum
unsigned char* leadNum;
double interval = 0;// = (double)tdps->realPrecision*2;
convertByteArray2IntArray_fast_2b(tdps->exactDataNum, tdps->leadNumArray, tdps->leadNumArray_size, &leadNum);
//*data = (double*)malloc(sizeof(double)*dataSeriesLength); comment by tickduan
int* type = (int*)malloc(dataSeriesLength*sizeof(int));
HuffmanTree* huffmanTree = createHuffmanTree(tdps->stateNum);
decode_withTree(huffmanTree, tdps->typeArray, dataSeriesLength, type);
SZ_ReleaseHuffman(huffmanTree);
unsigned char preBytes[8];
unsigned char curBytes[8];
memset(preBytes, 0, 8);
size_t curByteIndex = 0;
int reqLength = 0, reqBytesLength = 0, resiBitsLength = 0, resiBits = 0;
unsigned char leadingNum;
double medianValue, exactData, predValue = 0, realPrecision = 0;
medianValue = tdps->medianValue;
int type_, updateReqLength = 0;
for (i = 0; i < dataSeriesLength; i++)
{
if(i%tdps->segment_size==0)
{
tmpPrecBytes[0] = *(bp++);
tmpPrecBytes[1] = *(bp++);
memset(&tmpPrecBytes[2], 0, 6*sizeof(unsigned char));
realPrecision = bytesToDouble(tmpPrecBytes);
interval = realPrecision*2;
updateReqLength = 0;
}
type_ = type[i];
switch (type_) {
case 0:
// compute resiBits
if(updateReqLength==0)
{
computeReqLength_double(realPrecision, tdps->radExpo, &reqLength, &medianValue);
reqBytesLength = reqLength/8;
resiBitsLength = reqLength%8;
updateReqLength = 1;
}
resiBits = 0;
if (resiBitsLength != 0) {
int kMod8 = k % 8;
int rightMovSteps = getRightMovingSteps(kMod8, resiBitsLength);
if (rightMovSteps > 0) {
int code = getRightMovingCode(kMod8, resiBitsLength);
resiBits = (tdps->residualMidBits[p] & code) >> rightMovSteps;
} else if (rightMovSteps < 0) {
int code1 = getLeftMovingCode(kMod8);
int code2 = getRightMovingCode(kMod8, resiBitsLength);
int leftMovSteps = -rightMovSteps;
rightMovSteps = 8 - leftMovSteps;
resiBits = (tdps->residualMidBits[p] & code1) << leftMovSteps;
p++;
resiBits = resiBits
| ((tdps->residualMidBits[p] & code2) >> rightMovSteps);
} else // rightMovSteps == 0
{
int code = getRightMovingCode(kMod8, resiBitsLength);
resiBits = (tdps->residualMidBits[p] & code);
p++;
}
k += resiBitsLength;
}
// recover the exact data
memset(curBytes, 0, 8);
leadingNum = leadNum[l++];
memcpy(curBytes, preBytes, leadingNum);
for (j = leadingNum; j < reqBytesLength; j++)
curBytes[j] = tdps->exactMidBytes[curByteIndex++];
if (resiBitsLength != 0) {
unsigned char resiByte = (unsigned char) (resiBits << (8 - resiBitsLength));
curBytes[reqBytesLength] = resiByte;
}
exactData = bytesToDouble(curBytes);
data[i] = exactData + medianValue;
memcpy(preBytes,curBytes,8);
break;
default:
//predValue = 2 * data[i-1] - data[i-2];
predValue = data[i-1];
data[i] = predValue + (type_-exe_params->intvRadius)*interval;
break;
}
//printf("%.30G\n",data[i]);
}
free(leadNum);
free(type);
return;
}
void decompressDataSeries_double_1D_pwrgroup(double* data, size_t dataSeriesLength, TightDataPointStorageD* tdps)
{
double *posGroups, *negGroups, *groups;
double pos_01_group, neg_01_group;
int *posFlags, *negFlags;
updateQuantizationInfo(tdps->intervals);
unsigned char* leadNum;
double interval;// = (double)tdps->realPrecision*2;
convertByteArray2IntArray_fast_2b(tdps->exactDataNum, tdps->leadNumArray, tdps->leadNumArray_size, &leadNum);
//*data = (double*)malloc(sizeof(double)*dataSeriesLength); comment by tickduan
int* type = (int*)malloc(dataSeriesLength*sizeof(int));
HuffmanTree* huffmanTree = createHuffmanTree(tdps->stateNum);
decode_withTree(huffmanTree, tdps->typeArray, dataSeriesLength, type);
SZ_ReleaseHuffman(huffmanTree);
createRangeGroups_double(&posGroups, &negGroups, &posFlags, &negFlags);
double realGroupPrecision;
double realPrecision = tdps->realPrecision;
char* groupID = decompressGroupIDArray(tdps->pwrErrBoundBytes, tdps->dataSeriesLength);
//note that the groupID values here are [1,2,3,....,18] or [-1,-2,...,-18]
double* groupErrorBounds = generateGroupErrBounds(confparams_dec->errorBoundMode, realPrecision, confparams_dec->pw_relBoundRatio);
exe_params->intvRadius = generateGroupMaxIntervalCount(groupErrorBounds);
size_t nbBins = (size_t)(1/confparams_dec->pw_relBoundRatio + 0.5);
if(nbBins%2==1)
nbBins++;
exe_params->intvRadius = nbBins;
unsigned char preBytes[8];
unsigned char curBytes[8];
memset(preBytes, 0, 8);
size_t curByteIndex = 0;
int reqLength, reqBytesLength = 0, resiBitsLength = 0, resiBits;
unsigned char leadingNum;
double medianValue, exactData, curValue, predValue;
medianValue = tdps->medianValue;
size_t i, j, k = 0, p = 0, l = 0; // k is to track the location of residual_bit
// in resiMidBits, p is to track the
// byte_index of resiMidBits, l is for
// leadNum
int type_, updateReqLength = 0;
char rawGrpID = 0, indexGrpID = 0;
for (i = 0; i < dataSeriesLength; i++)
{
rawGrpID = groupID[i];
if(rawGrpID >= 2)
{
groups = posGroups;
indexGrpID = rawGrpID - 2;
}
else if(rawGrpID <= -2)
{
groups = negGroups;
indexGrpID = -rawGrpID - 2; }
else if(rawGrpID == 1)
{
groups = &pos_01_group;
indexGrpID = 0;
}
else //rawGrpID == -1
{
groups = &neg_01_group;
indexGrpID = 0;
}
type_ = type[i];
switch (type_) {
case 0:
// compute resiBits
if(updateReqLength==0)
{
computeReqLength_double(realPrecision, tdps->radExpo, &reqLength, &medianValue);
reqBytesLength = reqLength/8;
resiBitsLength = reqLength%8;
updateReqLength = 1;
}
resiBits = 0;
if (resiBitsLength != 0) {
int kMod8 = k % 8;
int rightMovSteps = getRightMovingSteps(kMod8, resiBitsLength);
if (rightMovSteps > 0) {
int code = getRightMovingCode(kMod8, resiBitsLength);
resiBits = (tdps->residualMidBits[p] & code) >> rightMovSteps;
} else if (rightMovSteps < 0) {
int code1 = getLeftMovingCode(kMod8);
int code2 = getRightMovingCode(kMod8, resiBitsLength);
int leftMovSteps = -rightMovSteps;
rightMovSteps = 8 - leftMovSteps;
resiBits = (tdps->residualMidBits[p] & code1) << leftMovSteps;
p++;
resiBits = resiBits
| ((tdps->residualMidBits[p] & code2) >> rightMovSteps);
} else // rightMovSteps == 0
{
int code = getRightMovingCode(kMod8, resiBitsLength);
resiBits = (tdps->residualMidBits[p] & code);
p++;
}
k += resiBitsLength;
}
// recover the exact data
memset(curBytes, 0, 8);
leadingNum = leadNum[l++];
memcpy(curBytes, preBytes, leadingNum);
for (j = leadingNum; j < reqBytesLength; j++)
curBytes[j] = tdps->exactMidBytes[curByteIndex++];
if (resiBitsLength != 0) {
unsigned char resiByte = (unsigned char) (resiBits << (8 - resiBitsLength));
curBytes[reqBytesLength] = resiByte;
}
exactData = bytesToDouble(curBytes);
exactData = exactData + medianValue;
data[i] = exactData;
memcpy(preBytes,curBytes,8);
groups[indexGrpID] = exactData;
break;
default:
predValue = groups[indexGrpID]; //Here, groups[indexGrpID] is the previous value.
realGroupPrecision = groupErrorBounds[indexGrpID];
interval = realGroupPrecision*2;
curValue = predValue + (type_-exe_params->intvRadius)*interval;
//groupNum = computeGroupNum_double(curValue);
if((curValue>0&&rawGrpID<0)||(curValue<0&&rawGrpID>0))
curValue = 0;
//else
//{
// realGrpID = fabs(rawGrpID)-2;
// if(groupNum<realGrpID)
// curValue = rawGrpID>0?pow(2,realGrpID):-pow(2,realGrpID);
// else if(groupNum>realGrpID)
// curValue = rawGrpID>0?pow(2,groupNum):-pow(2,groupNum);
//}
data[i] = curValue;
groups[indexGrpID] = curValue;
break;
}
}
free(leadNum);
free(type);
free(posGroups);
free(negGroups);
free(posFlags);
free(negFlags);
free(groupErrorBounds);
free(groupID);
}
void decompressDataSeries_double_1D_pwr_pre_log(double* data, size_t dataSeriesLength, TightDataPointStorageD* tdps) {
decompressDataSeries_double_1D(data, dataSeriesLength, NULL, tdps);
double threshold = tdps->minLogValue;
if(tdps->pwrErrBoundBytes_size > 0){
unsigned char * signs;
sz_lossless_decompress(ZSTD_COMPRESSOR, tdps->pwrErrBoundBytes, tdps->pwrErrBoundBytes_size, &signs, dataSeriesLength);
for(size_t i=0; i<dataSeriesLength; i++){
if(data[i] < threshold) data[i] = 0;
else data[i] = exp2(data[i]);
if(signs[i]) data[i] = -(data[i]);
}
free(signs);
}
else{
for(size_t i=0; i<dataSeriesLength; i++){
if(data[i] < threshold) data[i] = 0;
else data[i] = exp2(data[i]);
}
}
}
void decompressDataSeries_double_1D_pwr_pre_log_MSST19(double* data, size_t dataSeriesLength, TightDataPointStorageD* tdps)
{
decompressDataSeries_double_1D_MSST19(data, dataSeriesLength, tdps);
double threshold = tdps->minLogValue;
uint64_t* ptr;
if(tdps->pwrErrBoundBytes_size > 0){
unsigned char * signs = NULL;
if(tdps->pwrErrBoundBytes_size==0)
{
signs = (unsigned char*)malloc(dataSeriesLength);
memset(signs, 0, dataSeriesLength);
}
else
sz_lossless_decompress(ZSTD_COMPRESSOR, tdps->pwrErrBoundBytes, tdps->pwrErrBoundBytes_size, &signs, dataSeriesLength);
for(size_t i=0; i<dataSeriesLength; i++){
if(data[i] < threshold && data[i] >= 0){
data[i] = 0;
continue;
}
if(signs[i]){
ptr = (uint64_t*)data + i;
*ptr |= 0x8000000000000000;
}
}
free(signs);
}
else{
for(size_t i=0; i<dataSeriesLength; i++){
if(data[i] < threshold) data[i] = 0;
}
}
}
#pragma GCC diagnostic pop

185
deps/SZ/sz/src/szd_float.c vendored
View File

@ -13,30 +13,8 @@
#include "szd_float.h"
#include "TightDataPointStorageF.h"
#include "sz.h"
#include "Huffman.h"
#include "szd_float_pwr.h"
#include "utility.h"
//struct timeval startTime_;
//struct timeval endTime_; /* Start and end times */
//struct timeval costStart_; /*only used for recording the cost*/
//double totalCost_ = 0;
/*void cost_start_()
{
totalCost_ = 0;
gettimeofday(&costStart_, NULL);
}
void cost_end_()
{
double elapsed;
struct timeval costEnd;
gettimeofday(&costEnd, NULL);
elapsed = ((costEnd.tv_sec*1000000+costEnd.tv_usec)-(costStart_.tv_sec*1000000+costStart_.tv_usec))/1000000.0;
totalCost_ += elapsed;
}*/
#include "Huffman.h"
/**
@ -95,10 +73,13 @@ int SZ_decompress_args_float(float* newData, size_t r1, unsigned char* cmpBytes,
//pde_params->sol_ID = szTmpBytes[1+3-2+14-4]; //szTmpBytes: version(1bytes), samebyte(1byte), [14-4]:sol_ID=SZ or SZ_Transpose
//TODO: convert szTmpBytes to data array.
TightDataPointStorageF* tdps;
TightDataPointStorageF* tdps = NULL;
int errBoundMode = new_TightDataPointStorageF_fromFlatBytes(&tdps, szTmpBytes, tmpSize, pde_exe, pde_params);
if(tdps == NULL)
{
return SZ_FORMAT_ERR;
}
//writeByteData(tdps->typeArray, tdps->typeArray_size, "decompress-typebytes.tbt");
int floatSize = sizeof(float);
if(tdps->isLossless)
{
@ -145,14 +126,14 @@ void decompressDataSeries_float_1D(float* data, size_t dataSeriesLength, float*
// leadNum
unsigned char* leadNum;
float interval = tdps->realPrecision*2;
convertByteArray2IntArray_fast_2b(tdps->exactDataNum, tdps->leadNumArray, tdps->leadNumArray_size, &leadNum);
//data = (float*)malloc(sizeof(float)*dataSeriesLength); // comment by tickduan
int* type = (int*)malloc(dataSeriesLength*sizeof(int));
// type tree
int* types = (int*)malloc(dataSeriesLength*sizeof(int));
HuffmanTree* huffmanTree = createHuffmanTree(tdps->stateNum);
decode_withTree(huffmanTree, tdps->typeArray, dataSeriesLength, type);
decode_withTree(huffmanTree, tdps->typeArray, dataSeriesLength, types);
SZ_ReleaseHuffman(huffmanTree);
unsigned char preBytes[4];
@ -167,12 +148,15 @@ void decompressDataSeries_float_1D(float* data, size_t dataSeriesLength, float*
reqBytesLength = tdps->reqLength/8;
resiBitsLength = tdps->reqLength%8;
medianValue = tdps->medianValue;
medianValue = tdps->medianValue;
int type_;
for (i = 0; i < dataSeriesLength; i++) {
type_ = type[i];
switch (type_) {
// decompress core
int type;
for (i = 0; i < dataSeriesLength; i++)
{
type = types[i];
switch (type)
{
case 0:
// compute resiBits
resiBits = 0;
@ -218,147 +202,30 @@ void decompressDataSeries_float_1D(float* data, size_t dataSeriesLength, float*
default:
//predValue = 2 * data[i-1] - data[i-2];
predValue = data[i-1];
data[i] = predValue + (float)(type_-intvRadius)*interval;
data[i] = predValue + (float)(type - intvRadius) * interval;
break;
}
//printf("%.30G\n",data[i]);
}
free(leadNum);
free(type);
free(types);
return;
}
/*MSST19*/
void decompressDataSeries_float_1D_MSST19(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps)
{
//updateQuantizationInfo(tdps->intervals);
int intvRadius = tdps->intervals/2;
int intvCapacity = tdps->intervals;
size_t i, j, k = 0, p = 0, l = 0; // k is to track the location of residual_bit
// in resiMidBits, p is to track the
// byte_index of resiMidBits, l is for
// leadNum
unsigned char* leadNum;
//double interval = tdps->realPrecision*2;
convertByteArray2IntArray_fast_2b(tdps->exactDataNum, tdps->leadNumArray, tdps->leadNumArray_size, &leadNum);
// *data = (float*)malloc(sizeof(float)*dataSeriesLength); comment by tickduan
int* type = (int*)malloc(dataSeriesLength*sizeof(int));
HuffmanTree* huffmanTree = createHuffmanTree(tdps->stateNum);
decode_withTree_MSST19(huffmanTree, tdps->typeArray, dataSeriesLength, type, tdps->max_bits);
SZ_ReleaseHuffman(huffmanTree);
unsigned char preBytes[4];
unsigned char curBytes[4];
memset(preBytes, 0, 4);
size_t curByteIndex = 0;
int reqBytesLength, resiBitsLength, resiBits;
unsigned char leadingNum;
float exactData, predValue = 0;
reqBytesLength = tdps->reqLength/8;
resiBitsLength = tdps->reqLength%8;
//float threshold = tdps->minLogValue;
double* precisionTable = (double*)malloc(sizeof(double) * intvCapacity);
double inv = 2.0-pow(2, -(tdps->plus_bits));
for(int i=0; i<intvCapacity; i++){
double test = pow((1+tdps->realPrecision), inv*(i - intvRadius));
precisionTable[i] = test;
}
int type_;
for (i = 0; i < dataSeriesLength; i++) {
type_ = type[i];
switch (type_) {
case 0:
// compute resiBits
resiBits = 0;
if (resiBitsLength != 0) {
int kMod8 = k % 8;
int rightMovSteps = getRightMovingSteps(kMod8, resiBitsLength);
if (rightMovSteps > 0) {
int code = getRightMovingCode(kMod8, resiBitsLength);
resiBits = (tdps->residualMidBits[p] & code) >> rightMovSteps;
} else if (rightMovSteps < 0) {
int code1 = getLeftMovingCode(kMod8);
int code2 = getRightMovingCode(kMod8, resiBitsLength);
int leftMovSteps = -rightMovSteps;
rightMovSteps = 8 - leftMovSteps;
resiBits = (tdps->residualMidBits[p] & code1) << leftMovSteps;
p++;
resiBits = resiBits
| ((tdps->residualMidBits[p] & code2) >> rightMovSteps);
} else // rightMovSteps == 0
{
int code = getRightMovingCode(kMod8, resiBitsLength);
resiBits = (tdps->residualMidBits[p] & code);
p++;
}
k += resiBitsLength;
}
// recover the exact data
memset(curBytes, 0, 4);
leadingNum = leadNum[l++];
memcpy(curBytes, preBytes, leadingNum);
for (j = leadingNum; j < reqBytesLength; j++)
curBytes[j] = tdps->exactMidBytes[curByteIndex++];
if (resiBitsLength != 0) {
unsigned char resiByte = (unsigned char) (resiBits << (8 - resiBitsLength));
curBytes[reqBytesLength] = resiByte;
}
exactData = bytesToFloat(curBytes);
data[i] = exactData;
memcpy(preBytes,curBytes,4);
predValue = data[i];
break;
default:
//predValue = 2 * data[i-1] - data[i-2];
//predValue = data[i-1];
predValue = fabs(predValue) * precisionTable[type_];
data[i] = predValue;
break;
}
//printf("%.30G\n",data[i]);
}
free(precisionTable);
free(leadNum);
free(type);
return;
}
void getSnapshotData_float_1D(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps, int errBoundMode, int compressionType, float* hist_data, sz_params* pde_params)
{
size_t i;
if (tdps->allSameData) {
if (tdps->allSameData)
{
float value = bytesToFloat(tdps->exactMidBytes);
//*data = (float*)malloc(sizeof(float)*dataSeriesLength); commnet by tickduan
for (i = 0; i < dataSeriesLength; i++)
data[i] = value;
} else {
if (tdps->rtypeArray == NULL) {
if(errBoundMode < PW_REL)
{
decompressDataSeries_float_1D(data, dataSeriesLength, hist_data, tdps);
}
else
{
if(pde_params->accelerate_pw_rel_compression)
decompressDataSeries_float_1D_pwr_pre_log_MSST19(data, dataSeriesLength, tdps);
else
decompressDataSeries_float_1D_pwr_pre_log(data, dataSeriesLength, tdps);
//decompressDataSeries_float_1D_pwrgroup(data, dataSeriesLength, tdps);
}
return;
} else { //the special version supporting one value to reserve
//TODO
}
}
else
{
decompressDataSeries_float_1D(data, dataSeriesLength, hist_data, tdps);
}
}

372
deps/SZ/sz/src/szd_float_pwr.c vendored
View File

@ -1,372 +0,0 @@
/**
* @file szd_float_pwr.c
* @author Sheng Di, Dingwen Tao, Xin Liang, Xiangyu Zou, Tao Lu, Wen Xia, Xuan Wang, Weizhe Zhang
* @date Feb., 2019
* @brief
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <math.h>
#include "TightDataPointStorageF.h"
#include "CompressElement.h"
#include "sz.h"
#include "Huffman.h"
#include "sz_float_pwr.h"
#include "utility.h"
//#include "rw.h"
//
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wchar-subscripts"
void decompressDataSeries_float_1D_pwr(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps)
{
updateQuantizationInfo(tdps->intervals);
unsigned char tmpPrecBytes[4] = {0}; //used when needing to convert bytes to float values
unsigned char* bp = tdps->pwrErrBoundBytes;
size_t i, j, k = 0, p = 0, l = 0; // k is to track the location of residual_bit
// in resiMidBits, p is to track the
// byte_index of resiMidBits, l is for
// leadNum
unsigned char* leadNum;
float interval = 0;// = (float)tdps->realPrecision*2;
convertByteArray2IntArray_fast_2b(tdps->exactDataNum, tdps->leadNumArray, tdps->leadNumArray_size, &leadNum);
//*data = (float*)malloc(sizeof(float)*dataSeriesLength); // comment by tickduan
int* type = (int*)malloc(dataSeriesLength*sizeof(int));
HuffmanTree* huffmanTree = createHuffmanTree(tdps->stateNum);
decode_withTree(huffmanTree, tdps->typeArray, dataSeriesLength, type);
SZ_ReleaseHuffman(huffmanTree);
//sdi:Debug
//writeUShortData(type, dataSeriesLength, "decompressStateBytes.sb");
unsigned char preBytes[4];
unsigned char curBytes[4];
memset(preBytes, 0, 4);
size_t curByteIndex = 0;
int reqLength = 0, reqBytesLength = 0, resiBitsLength = 0, resiBits = 0;
unsigned char leadingNum;
float medianValue, exactData, predValue = 0, realPrecision = 0;
medianValue = tdps->medianValue;
int type_, updateReqLength = 0;
for (i = 0; i < dataSeriesLength; i++)
{
if(i%tdps->segment_size==0)
{
tmpPrecBytes[0] = *(bp++);
tmpPrecBytes[1] = *(bp++);
tmpPrecBytes[2] = 0;
tmpPrecBytes[3] = 0;
realPrecision = bytesToFloat(tmpPrecBytes);
interval = realPrecision*2;
updateReqLength = 0;
}
type_ = type[i];
switch (type_) {
case 0:
// compute resiBits
if(updateReqLength==0)
{
computeReqLength_float(realPrecision, tdps->radExpo, &reqLength, &medianValue);
reqBytesLength = reqLength/8;
resiBitsLength = reqLength%8;
updateReqLength = 1;
}
resiBits = 0;
if (resiBitsLength != 0) {
int kMod8 = k % 8;
int rightMovSteps = getRightMovingSteps(kMod8, resiBitsLength);
if (rightMovSteps > 0) {
int code = getRightMovingCode(kMod8, resiBitsLength);
resiBits = (tdps->residualMidBits[p] & code) >> rightMovSteps;
} else if (rightMovSteps < 0) {
int code1 = getLeftMovingCode(kMod8);
int code2 = getRightMovingCode(kMod8, resiBitsLength);
int leftMovSteps = -rightMovSteps;
rightMovSteps = 8 - leftMovSteps;
resiBits = (tdps->residualMidBits[p] & code1) << leftMovSteps;
p++;
resiBits = resiBits
| ((tdps->residualMidBits[p] & code2) >> rightMovSteps);
} else // rightMovSteps == 0
{
int code = getRightMovingCode(kMod8, resiBitsLength);
resiBits = (tdps->residualMidBits[p] & code);
p++;
}
k += resiBitsLength;
}
// recover the exact data
memset(curBytes, 0, 4);
leadingNum = leadNum[l++];
memcpy(curBytes, preBytes, leadingNum);
for (j = leadingNum; j < reqBytesLength; j++)
curBytes[j] = tdps->exactMidBytes[curByteIndex++];
if (resiBitsLength != 0) {
unsigned char resiByte = (unsigned char) (resiBits << (8 - resiBitsLength));
curBytes[reqBytesLength] = resiByte;
}
exactData = bytesToFloat(curBytes);
data[i] = exactData + medianValue;
memcpy(preBytes,curBytes,4);
break;
default:
//predValue = 2 * data[i-1] - data[i-2];
predValue = data[i-1];
data[i] = predValue + (type_-exe_params->intvRadius)*interval;
break;
}
//printf("%.30G\n",data[i]);
}
free(leadNum);
free(type);
return;
}
void decompressDataSeries_float_1D_pwrgroup(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps)
{
float *posGroups, *negGroups, *groups;
float pos_01_group, neg_01_group;
int *posFlags, *negFlags;
updateQuantizationInfo(tdps->intervals);
unsigned char* leadNum;
double interval;// = (float)tdps->realPrecision*2;
convertByteArray2IntArray_fast_2b(tdps->exactDataNum, tdps->leadNumArray, tdps->leadNumArray_size, &leadNum);
//*data = (float*)malloc(sizeof(float)*dataSeriesLength); // comment by tickduan
int* type = (int*)malloc(dataSeriesLength*sizeof(int));
HuffmanTree* huffmanTree = createHuffmanTree(tdps->stateNum);
decode_withTree(huffmanTree, tdps->typeArray, dataSeriesLength, type);
SZ_ReleaseHuffman(huffmanTree);
createRangeGroups_float(&posGroups, &negGroups, &posFlags, &negFlags);
float realGroupPrecision;
float realPrecision = tdps->realPrecision;
char* groupID = decompressGroupIDArray(tdps->pwrErrBoundBytes, tdps->dataSeriesLength);
//note that the groupID values here are [1,2,3,....,18] or [-1,-2,...,-18]
double* groupErrorBounds = generateGroupErrBounds(confparams_dec->errorBoundMode, realPrecision, confparams_dec->pw_relBoundRatio);
exe_params->intvRadius = generateGroupMaxIntervalCount(groupErrorBounds);
size_t nbBins = (size_t)(1/confparams_dec->pw_relBoundRatio + 0.5);
if(nbBins%2==1)
nbBins++;
exe_params->intvRadius = nbBins;
unsigned char preBytes[4];
unsigned char curBytes[4];
memset(preBytes, 0, 4);
size_t curByteIndex = 0;
int reqLength, reqBytesLength = 0, resiBitsLength = 0, resiBits;
unsigned char leadingNum;
float medianValue, exactData, curValue, predValue;
medianValue = tdps->medianValue;
size_t i, j, k = 0, p = 0, l = 0; // k is to track the location of residual_bit
// in resiMidBits, p is to track the
// byte_index of resiMidBits, l is for
// leadNum
int type_, updateReqLength = 0;
char rawGrpID = 0, indexGrpID = 0;
for (i = 0; i < dataSeriesLength; i++)
{
rawGrpID = groupID[i];
if(rawGrpID >= 2)
{
groups = posGroups;
indexGrpID = rawGrpID - 2;
}
else if(rawGrpID <= -2)
{
groups = negGroups;
indexGrpID = -rawGrpID - 2; }
else if(rawGrpID == 1)
{
groups = &pos_01_group;
indexGrpID = 0;
}
else //rawGrpID == -1
{
groups = &neg_01_group;
indexGrpID = 0;
}
type_ = type[i];
switch (type_) {
case 0:
// compute resiBits
if(updateReqLength==0)
{
computeReqLength_float(realPrecision, tdps->radExpo, &reqLength, &medianValue);
reqBytesLength = reqLength/8;
resiBitsLength = reqLength%8;
updateReqLength = 1;
}
resiBits = 0;
if (resiBitsLength != 0) {
int kMod8 = k % 8;
int rightMovSteps = getRightMovingSteps(kMod8, resiBitsLength);
if (rightMovSteps > 0) {
int code = getRightMovingCode(kMod8, resiBitsLength);
resiBits = (tdps->residualMidBits[p] & code) >> rightMovSteps;
} else if (rightMovSteps < 0) {
int code1 = getLeftMovingCode(kMod8);
int code2 = getRightMovingCode(kMod8, resiBitsLength);
int leftMovSteps = -rightMovSteps;
rightMovSteps = 8 - leftMovSteps;
resiBits = (tdps->residualMidBits[p] & code1) << leftMovSteps;
p++;
resiBits = resiBits
| ((tdps->residualMidBits[p] & code2) >> rightMovSteps);
} else // rightMovSteps == 0
{
int code = getRightMovingCode(kMod8, resiBitsLength);
resiBits = (tdps->residualMidBits[p] & code);
p++;
}
k += resiBitsLength;
}
// recover the exact data
memset(curBytes, 0, 4);
leadingNum = leadNum[l++];
memcpy(curBytes, preBytes, leadingNum);
for (j = leadingNum; j < reqBytesLength; j++)
curBytes[j] = tdps->exactMidBytes[curByteIndex++];
if (resiBitsLength != 0) {
unsigned char resiByte = (unsigned char) (resiBits << (8 - resiBitsLength));
curBytes[reqBytesLength] = resiByte;
}
exactData = bytesToFloat(curBytes);
exactData = exactData + medianValue;
data[i] = exactData;
memcpy(preBytes,curBytes,4);
groups[indexGrpID] = exactData;
break;
default:
predValue = groups[indexGrpID]; //Here, groups[indexGrpID] is the previous value.
realGroupPrecision = groupErrorBounds[indexGrpID];
interval = realGroupPrecision*2;
curValue = predValue + (type_-exe_params->intvRadius)*interval;
//groupNum = computeGroupNum_float(curValue);
if((curValue>0&&rawGrpID<0)||(curValue<0&&rawGrpID>0))
curValue = 0;
//else
//{
// realGrpID = fabs(rawGrpID)-2;
// if(groupNum<realGrpID)
// curValue = rawGrpID>0?pow(2,realGrpID):-pow(2,realGrpID);
// else if(groupNum>realGrpID)
// curValue = rawGrpID>0?pow(2,groupNum):-pow(2,groupNum);
//}
data[i] = curValue;
groups[indexGrpID] = curValue;
break;
}
}
free(leadNum);
free(type);
free(posGroups);
free(negGroups);
free(posFlags);
free(negFlags);
free(groupErrorBounds);
free(groupID);
}
void decompressDataSeries_float_1D_pwr_pre_log(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps) {
decompressDataSeries_float_1D(data, dataSeriesLength, NULL, tdps);
float threshold = tdps->minLogValue;
if(tdps->pwrErrBoundBytes_size > 0){
unsigned char * signs;
sz_lossless_decompress(ZSTD_COMPRESSOR, tdps->pwrErrBoundBytes, tdps->pwrErrBoundBytes_size, &signs, dataSeriesLength);
for(size_t i=0; i<dataSeriesLength; i++){
if(data[i] < threshold) data[i] = 0;
else data[i] = exp2(data[i]);
if(signs[i]) data[i] = -(data[i]);
}
free(signs);
}
else{
for(size_t i=0; i<dataSeriesLength; i++){
if(data[i] < threshold) data[i] = 0;
else data[i] = exp2(data[i]);
}
}
}
void decompressDataSeries_float_1D_pwr_pre_log_MSST19(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps)
{
decompressDataSeries_float_1D_MSST19(data, dataSeriesLength, tdps);
float threshold = tdps->minLogValue;
uint32_t* ptr;
if(tdps->pwrErrBoundBytes_size > 0){
unsigned char * signs = NULL;
if(tdps->pwrErrBoundBytes_size==0)
{
signs = (unsigned char*)malloc(dataSeriesLength);
memset(signs, 0, dataSeriesLength);
}
else
sz_lossless_decompress(ZSTD_COMPRESSOR, tdps->pwrErrBoundBytes, tdps->pwrErrBoundBytes_size, &signs, dataSeriesLength);
for(size_t i=0; i<dataSeriesLength; i++){
if(data[i] < threshold && data[i] >= 0){
data[i] = 0;
continue;
}
if(signs[i]){
ptr = (uint32_t*)data + i;
*ptr |= 0x80000000;
}
}
free(signs);
}
else{
for(size_t i=0; i<dataSeriesLength; i++){
if(data[i] < threshold) data[i] = 0;
}
}
}
#pragma GCC diagnostic pop

26
src/kit/taospack/taospack.c
View File

@ -451,7 +451,7 @@ int memTest() {
void* memTestThread(void* lparam) {
//memTest();
printf(" enter thread ....\n");
for(int i=0; i< 5; i++)
for(int i=0; i<1; i++)
{
memTest();
printf(" start i=%d .... \n", i);
@ -484,7 +484,7 @@ void test_threadsafe(int thread_count){
void* memTestThreadDouble(void* lparam) {
//memTest();
printf(" enter thread ....\n");
for(int i=0; i< 10; i++)
for(int i=0; i< 1; i++)
{
memTest();
printf(" double start i=%d .... \n", i);
@ -577,19 +577,32 @@ void unitTestFloat() {
}
void modulePath(char *buf, int size)
{
char path[1024];
sprintf(path, "/proc/%d/exe", getpid());
readlink(path, buf, size);
char* pos = strrchr(buf, '/');
if(pos)
pos[1]=0;
}
//
// ----------------- main ----------------------
//
int main(int argc, char *argv[]) {
printf("welcome to use taospack tools v1.2 SZ_SIZE_TYPE=%ld\n", sizeof(size_t));
printf("welcome to use taospack tools v1.3\n");
char szbuf[512];
modulePath(szbuf, 512);
printf(szbuf);
gOpenLossy = false;
tsLossyInit();
//
//tsCompressExit();
//return 1;
//printf(" SZ_SIZE_TYPE=%d", )
if(argc == 3){
char algo = 0;
@ -598,7 +611,7 @@ int main(int argc, char *argv[]) {
algo = ONE_STAGE_COMP;
gOpenLossy = true;
}
if(strcmp(argv[1], "-ttwo") == 0) {
if(strcmp(argv[1], "-tw") == 0) {
algo = TWO_STAGE_COMP;
gOpenLossy = false;
}
@ -612,8 +625,9 @@ int main(int argc, char *argv[]) {
test_threadsafe_double(atoi(argv[2]));
return 0;
}
if(algo == 0){
printf(" no param -tone -ttwo \n");
printf(" no param -tone -tw \n");
return 0;
}