xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

reduce head meta from 36 to 2 bytes

This commit is contained in:
tickduan 2021-07-05 10:12:51 +08:00
parent 6438e19373
commit 3e8cfacdae
17 changed files with 117 additions and 264 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
deps/SZ/sz/include/conf.h vendored
View File

@ -20,7 +20,7 @@ extern "C" {
void updateQuantizationInfo(int quant_intervals); void updateQuantizationInfo(int quant_intervals);
int SZ_ReadConf(const char* sz_cfgFile); int SZ_ReadConf(const char* sz_cfgFile);
int SZ_LoadConf(const char* sz_cfgFile); int SZ_LoadConf(const char* sz_cfgFile);
int checkVersion(char* version); int checkVersion(unsigned char version);
int computeVersion(int major, int minor, int revision); int computeVersion(int major, int minor, int revision);
int checkVersion2(char* version); int checkVersion2(char* version);

1
deps/SZ/sz/include/dataCompression.h vendored
View File

@ -82,7 +82,6 @@ int getPredictionCoefficients(int layers, int dimension, int **coeff_array, int
int computeBlockEdgeSize_3D(int segmentSize); int computeBlockEdgeSize_3D(int segmentSize);
int computeBlockEdgeSize_2D(int segmentSize); int computeBlockEdgeSize_2D(int segmentSize);
int initRandomAccessBytes(unsigned char* raBytes);
int generateLossyCoefficients_float(float* oriData, double precision, size_t nbEle, int* reqBytesLength, int* resiBitsLength, float* medianValue, float* decData); int generateLossyCoefficients_float(float* oriData, double precision, size_t nbEle, int* reqBytesLength, int* resiBitsLength, float* medianValue, float* decData);
int compressExactDataArray_float(float* oriData, double precision, size_t nbEle, unsigned char** leadArray, unsigned char** midArray, unsigned char** resiArray, int compressExactDataArray_float(float* oriData, double precision, size_t nbEle, unsigned char** leadArray, unsigned char** midArray, unsigned char** resiArray,

13
deps/SZ/sz/include/defines.h vendored
View File

@ -11,10 +11,8 @@
#define _SZ_DEFINES_H #define _SZ_DEFINES_H
#define SZ_VERNUM 0x0200 #define SZ_VERNUM 0x0200
#define SZ_VER_MAJOR 2 #define SZ_VER_MAJOR 1 // curretn version
#define SZ_VER_MINOR 1
#define SZ_VER_BUILD 12
#define SZ_VER_REVISION 0
#define PASTRI 103 #define PASTRI 103
#define HZ 102 //deprecated #define HZ 102 //deprecated
@ -95,9 +93,10 @@
#define SZ_DESTROY_WHOLE_VARSET 1 #define SZ_DESTROY_WHOLE_VARSET 1
#define GROUP_COUNT 16 //2^{16}=65536 #define GROUP_COUNT 16 //2^{16}=65536
#define MetaDataByteLength 28 // metaData remove some by tickduan
#define MetaDataByteLength_double 36 //meta data length for double type #define MetaDataByteLength 2 // original is 28 bytes
#define MetaDataByteLength_double 2 // original is 36 bytes
#define numOfBufferedSteps 1 //the number of time steps in the buffer #define numOfBufferedSteps 1 //the number of time steps in the buffer

4
deps/SZ/sz/include/sz.h vendored
View File

@ -142,7 +142,7 @@ typedef union lfloat
typedef struct sz_metadata typedef struct sz_metadata
{ {
int versionNumber[3]; //only used for checking the version by calling SZ_GetMetaData() unsigned char ver; //only used for checking the version by calling SZ_GetMetaData()
int isConstant; //only used for checking if the data are constant values by calling SZ_GetMetaData() int isConstant; //only used for checking if the data are constant values by calling SZ_GetMetaData()
int isLossless; //only used for checking if the data compression was lossless, used only by calling SZ_GetMetaData() int isLossless; //only used for checking if the data compression was lossless, used only by calling SZ_GetMetaData()
int sizeType; //only used for checking whether the size type is "int" or "long" in the compression, used only by calling SZ_GetMetaData() int sizeType; //only used for checking whether the size type is "int" or "long" in the compression, used only by calling SZ_GetMetaData()
@ -165,7 +165,7 @@ typedef struct sz_tsc_metainfo
} sz_tsc_metadata; } sz_tsc_metadata;
extern int versionNumber[4]; extern unsigned char versionNumber;
//-------------------key global variables-------------- //-------------------key global variables--------------
extern int dataEndianType; //*endian type of the data read from disk extern int dataEndianType; //*endian type of the data read from disk

2
deps/SZ/sz/include/sz_float.h vendored
View File

@ -29,8 +29,6 @@ unsigned int optimize_intervals_float_1D_opt_MSST19(float *oriData, size_t dataL
TightDataPointStorageF* SZ_compress_float_1D_MDQ(float *oriData, TightDataPointStorageF* SZ_compress_float_1D_MDQ(float *oriData,
size_t dataLength, float realPrecision, float valueRangeSize, float medianValue_f); size_t dataLength, float realPrecision, float valueRangeSize, float medianValue_f);
void SZ_compress_args_float_StoreOriData(float* oriData, size_t dataLength, unsigned char* newByteData, size_t *outSize);
bool SZ_compress_args_float_NoCkRngeNoGzip_1D( unsigned char* newByteData, float *oriData, bool SZ_compress_args_float_NoCkRngeNoGzip_1D( unsigned char* newByteData, float *oriData,
size_t dataLength, double realPrecision, size_t *outSize, float valueRangeSize, float medianValue_f); size_t dataLength, double realPrecision, size_t *outSize, float valueRangeSize, float medianValue_f);

90
deps/SZ/sz/src/ByteToolkit.c vendored
View File

@ -898,65 +898,66 @@ void convertSZParamsToBytes(sz_params* params, unsigned char* result)
//buf = (buf << 2) | params->pwr_type; //deprecated //buf = (buf << 2) | params->pwr_type; //deprecated
result[0] = buf; result[0] = buf;
//sampleDistance; //2 bytes //sampleDistance; //2 bytes
int16ToBytes_bigEndian(&result[1], params->sampleDistance); //int16ToBytes_bigEndian(&result[1], params->sampleDistance);
//conf_params->predThreshold; // 2 bytes //conf_params->predThreshold; // 2 bytes
short tmp2 = params->predThreshold * 10000; //short tmp2 = params->predThreshold * 10000;
int16ToBytes_bigEndian(&result[3], tmp2); //int16ToBytes_bigEndian(&result[3], tmp2);
//errorBoundMode; //4bits(0.5 byte) //errorBoundMode; //4bits(0.5 byte)
result[5] = params->errorBoundMode; result[1] = params->errorBoundMode;
//data type (float, double, int8, int16, ....) //10 choices, so 4 bits //data type (float, double, int8, int16, ....) //10 choices, so 4 bits
result[5] = (result[5] << 4) | (params->dataType & 0x17); result[1] = (result[1] << 4) | (params->dataType & 0x17);
//result[5]: abs_err_bound or psnr //4 bytes //result[5]: abs_err_bound or psnr //4 bytes
//result[9]: rel_bound_ratio or pwr_err_bound//4 bytes //result[9]: rel_bound_ratio or pwr_err_bound//4 bytes
/*
switch(params->errorBoundMode) switch(params->errorBoundMode)
{ {
case SZ_ABS: case SZ_ABS:
floatToBytes(&result[6], (float)(params->absErrBound)); //big_endian floatToBytes(&result[6-4], (float)(params->absErrBound)); //big_endian
memset(&result[10], 0, 4); memset(&result[10-4], 0, 4);
break; break;
case REL: case REL:
memset(&result[6], 0, 4); memset(&result[6-4], 0, 4);
floatToBytes(&result[10], (float)(params->relBoundRatio)); //big_endian floatToBytes(&result[10-4], (float)(params->relBoundRatio)); //big_endian
break; break;
case ABS_AND_REL: case ABS_AND_REL:
case ABS_OR_REL: case ABS_OR_REL:
floatToBytes(&result[6], (float)(params->absErrBound)); floatToBytes(&result[6-4], (float)(params->absErrBound));
floatToBytes(&result[10], (float)(params->relBoundRatio)); //big_endian floatToBytes(&result[10-4], (float)(params->relBoundRatio)); //big_endian
break; break;
case PSNR: case PSNR:
floatToBytes(&result[6], (float)(params->psnr)); floatToBytes(&result[6-4], (float)(params->psnr));
memset(&result[9], 0, 4); memset(&result[9-4], 0, 4);
break; break;
case ABS_AND_PW_REL: case ABS_AND_PW_REL:
case ABS_OR_PW_REL: case ABS_OR_PW_REL:
floatToBytes(&result[6], (float)(params->absErrBound)); floatToBytes(&result[6-4], (float)(params->absErrBound));
floatToBytes(&result[10], (float)(params->pw_relBoundRatio)); //big_endian floatToBytes(&result[10-4], (float)(params->pw_relBoundRatio)); //big_endian
break; break;
case REL_AND_PW_REL: case REL_AND_PW_REL:
case REL_OR_PW_REL: case REL_OR_PW_REL:
floatToBytes(&result[6], (float)(params->relBoundRatio)); floatToBytes(&result[6-4], (float)(params->relBoundRatio));
floatToBytes(&result[10], (float)(params->pw_relBoundRatio)); //big_endian floatToBytes(&result[10-4], (float)(params->pw_relBoundRatio)); //big_endian
break; break;
case PW_REL: case PW_REL:
memset(&result[6], 0, 4); memset(&result[6-4], 0, 4);
floatToBytes(&result[10], (float)(params->pw_relBoundRatio)); //big_endian floatToBytes(&result[10-4], (float)(params->pw_relBoundRatio)); //big_endian
break; break;
} }
*/
//compressor //compressor
result[14] = (unsigned char)params->sol_ID; //result[14-4] = (unsigned char)params->sol_ID;
//int16ToBytes_bigEndian(&result[14], (short)(params->segment_size)); /* remove fmin fmax intervals
if(exe_params->optQuantMode==1) if(exe_params->optQuantMode==1)
int32ToBytes_bigEndian(&result[16], params->max_quant_intervals); int32ToBytes_bigEndian(&result[16], params->max_quant_intervals);
else else
int32ToBytes_bigEndian(&result[16], params->quantization_intervals); int32ToBytes_bigEndian(&result[16], params->quantization_intervals);
if(params->dataType==SZ_FLOAT) if(params->dataType==SZ_FLOAT)
{ {
@ -968,6 +969,7 @@ void convertSZParamsToBytes(sz_params* params, unsigned char* result)
doubleToBytes(&result[20], params->dmin); doubleToBytes(&result[20], params->dmin);
doubleToBytes(&result[28], params->dmax); doubleToBytes(&result[28], params->dmax);
} }
*/
} }
@ -996,48 +998,45 @@ void convertBytesToSZParams(unsigned char* bytes, sz_params* params, sz_exedata*
//params->pwr_type = (flag1 & 0x03) >> 0; //params->pwr_type = (flag1 & 0x03) >> 0;
params->sampleDistance = bytesToInt16_bigEndian(&bytes[1]); //params->sampleDistance = bytesToInt16_bigEndian(&bytes[1]);
//params->predThreshold = 1.0*bytesToInt16_bigEndian(&bytes[3])/10000.0;
params->predThreshold = 1.0*bytesToInt16_bigEndian(&bytes[3])/10000.0;
params->dataType = bytes[5] & 0x07; params->dataType = bytes[1] & 0x07;
params->errorBoundMode = (bytes[1] & 0xf0) >> 4;
params->errorBoundMode = (bytes[5] & 0xf0) >> 4;
/*
switch(params->errorBoundMode) switch(params->errorBoundMode)
{ {
case SZ_ABS: case SZ_ABS:
params->absErrBound = bytesToFloat(&bytes[6]); params->absErrBound = bytesToFloat(&bytes[6-4]);
break; break;
case REL: case REL:
params->relBoundRatio = bytesToFloat(&bytes[10]); params->relBoundRatio = bytesToFloat(&bytes[10-4]);
break; break;
case ABS_AND_REL: case ABS_AND_REL:
case ABS_OR_REL: case ABS_OR_REL:
params->absErrBound = bytesToFloat(&bytes[6]); params->absErrBound = bytesToFloat(&bytes[6-4]);
params->relBoundRatio = bytesToFloat(&bytes[10]); params->relBoundRatio = bytesToFloat(&bytes[10-4]);
break; break;
case PSNR: case PSNR:
params->psnr = bytesToFloat(&bytes[6]); params->psnr = bytesToFloat(&bytes[6-4]);
break; break;
case ABS_AND_PW_REL: case ABS_AND_PW_REL:
case ABS_OR_PW_REL: case ABS_OR_PW_REL:
params->absErrBound = bytesToFloat(&bytes[6]); params->absErrBound = bytesToFloat(&bytes[6-4]);
params->pw_relBoundRatio = bytesToFloat(&bytes[10]); params->pw_relBoundRatio = bytesToFloat(&bytes[10-4]);
break; break;
case REL_AND_PW_REL: case REL_AND_PW_REL:
case REL_OR_PW_REL: case REL_OR_PW_REL:
params->relBoundRatio = bytesToFloat(&bytes[6]); params->relBoundRatio = bytesToFloat(&bytes[6-4]);
params->pw_relBoundRatio = bytesToFloat(&bytes[10]); params->pw_relBoundRatio = bytesToFloat(&bytes[10-4]);
break; break;
case PW_REL: case PW_REL:
params->pw_relBoundRatio = bytesToFloat(&bytes[10]); params->pw_relBoundRatio = bytesToFloat(&bytes[10-4]);
} }
//segment_size // 2 bytes params->sol_ID = (int)(bytes[14-4]);
//params->segment_size = bytesToInt16_bigEndian(&bytes[14]);
params->sol_ID = (int)(bytes[14]);
if(pde_exe->optQuantMode==1) if(pde_exe->optQuantMode==1)
{ {
params->max_quant_intervals = bytesToInt32_bigEndian(&bytes[16]); params->max_quant_intervals = bytesToInt32_bigEndian(&bytes[16]);
@ -1059,5 +1058,6 @@ void convertBytesToSZParams(unsigned char* bytes, sz_params* params, sz_exedata*
params->dmin = bytesToDouble(&bytes[20]); params->dmin = bytesToDouble(&bytes[20]);
params->dmax = bytesToDouble(&bytes[28]); params->dmax = bytesToDouble(&bytes[28]);
} }
*/
} }

39
deps/SZ/sz/src/TightDataPointStorageD.c vendored
View File

@ -57,24 +57,20 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi
new_TightDataPointStorageD_Empty(this); new_TightDataPointStorageD_Empty(this);
size_t i, index = 0; size_t i, index = 0;
size_t pwrErrBoundBytes_size = 0, segmentL = 0, radExpoL = 0, pwrErrBoundBytesL = 0; size_t pwrErrBoundBytes_size = 0, segmentL = 0, radExpoL = 0, pwrErrBoundBytesL = 0;
char version[3]; unsigned char version = flatBytes[index++]; //3
for (i = 0; i < 3; i++)
version[i] = flatBytes[index++]; //3
unsigned char sameRByte = flatBytes[index++]; //1 unsigned char sameRByte = flatBytes[index++]; //1
if(checkVersion2(version)!=1) if(checkVersion(version)!=1)
{ {
//wrong version //wrong version
printf("Wrong version: \nCompressed-data version (%d.%d.%d)\n",version[0], version[1], version[2]); printf("Wrong version: \nCompressed-data %s . program version=%d\n", version, versionNumber);
printf("Current sz version: (%d.%d.%d)\n", versionNumber[0], versionNumber[1], versionNumber[2]); return 0;
printf("Please double-check if the compressed data (or file) is correct.\n");
exit(0);
} }
int same = sameRByte & 0x01; int same = sameRByte & 0x01;
(*this)->isLossless = (sameRByte & 0x10)>>4; (*this)->isLossless = (sameRByte & 0x10)>>4;
int isPW_REL = (sameRByte & 0x20)>>5; int isPW_REL = (sameRByte & 0x20)>>5;
exe_params->SZ_SIZE_TYPE = ((sameRByte & 0x40)>>6)==1?8:4; exe_params->SZ_SIZE_TYPE = ((sameRByte & 0x40)>>6)==1?8:4;
pde_params->protectValueRange = (sameRByte & 0x04)>>2; //pde_params->protectValueRange = (sameRByte & 0x04)>>2;
pde_params->accelerate_pw_rel_compression = (sameRByte & 0x08) >> 3; pde_params->accelerate_pw_rel_compression = (sameRByte & 0x08) >> 3;
int errorBoundMode = SZ_ABS; int errorBoundMode = SZ_ABS;
if(isPW_REL) if(isPW_REL)
@ -213,7 +209,8 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi
if ((*this)->rtypeArray != NULL) if ((*this)->rtypeArray != NULL)
{ {
(*this)->residualMidBits_size = flatBytesLength - 3 - 1 - MetaDataByteLength_double - exe_params->SZ_SIZE_TYPE - 4 - radExpoL - segmentL - pwrErrBoundBytesL - 4 - 8 - 1 - 8 // -3 -2 modify to -1 -1
(*this)->residualMidBits_size = flatBytesLength - 1 - 1 - MetaDataByteLength_double - exe_params->SZ_SIZE_TYPE - 4 - radExpoL - segmentL - pwrErrBoundBytesL - 4 - 8 - 1 - 8
- exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - minLogValueSize - exe_params->SZ_SIZE_TYPE - 8 - (*this)->rtypeArray_size - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - minLogValueSize - exe_params->SZ_SIZE_TYPE - 8 - (*this)->rtypeArray_size
- minLogValueSize - (*this)->typeArray_size - (*this)->leadNumArray_size - minLogValueSize - (*this)->typeArray_size - (*this)->leadNumArray_size
- (*this)->exactMidBytes_size - pwrErrBoundBytes_size - 1 - 1; - (*this)->exactMidBytes_size - pwrErrBoundBytes_size - 1 - 1;
@ -222,7 +219,7 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi
} }
else else
{ {
(*this)->residualMidBits_size = flatBytesLength - 3 - 1 - MetaDataByteLength_double - exe_params->SZ_SIZE_TYPE - 4 - radExpoL - segmentL - pwrErrBoundBytesL - 4 - 8 - 1 - 8 (*this)->residualMidBits_size = flatBytesLength - 1 - 1 - MetaDataByteLength_double - exe_params->SZ_SIZE_TYPE - 4 - radExpoL - segmentL - pwrErrBoundBytesL - 4 - 8 - 1 - 8
- exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - minLogValueSize - (*this)->typeArray_size - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - minLogValueSize - (*this)->typeArray_size
- (*this)->leadNumArray_size - (*this)->exactMidBytes_size - pwrErrBoundBytes_size - 1 - 1; - (*this)->leadNumArray_size - (*this)->exactMidBytes_size - pwrErrBoundBytes_size - 1 - 1;
} }
@ -381,8 +378,7 @@ void convertTDPStoBytes_double(TightDataPointStorageD* tdps, unsigned char* byte
unsigned char pwrErrBoundBytes_sizeBytes[4]; unsigned char pwrErrBoundBytes_sizeBytes[4];
unsigned char max_quant_intervals_Bytes[4]; unsigned char max_quant_intervals_Bytes[4];
for(i = 0;i<3;i++)//3 bytes bytes[k++] = versionNumber;
bytes[k++] = versionNumber[i];
bytes[k++] = sameByte; //1 byte bytes[k++] = sameByte; //1 byte
convertSZParamsToBytes(confparams_cpr, &(bytes[k])); convertSZParamsToBytes(confparams_cpr, &(bytes[k]));
@ -483,8 +479,7 @@ void convertTDPStoBytes_double_reserve(TightDataPointStorageD* tdps, unsigned ch
unsigned char pwrErrBoundBytes_sizeBytes[4]; unsigned char pwrErrBoundBytes_sizeBytes[4];
unsigned char max_quant_intervals_Bytes[4]; unsigned char max_quant_intervals_Bytes[4];
for(i = 0;i<3;i++)//3 bytes[k++] = versionNumber;
bytes[k++] = versionNumber[i];
bytes[k++] = sameByte; //1 bytes[k++] = sameByte; //1
convertSZParamsToBytes(confparams_cpr, &(bytes[k])); convertSZParamsToBytes(confparams_cpr, &(bytes[k]));
@ -595,15 +590,14 @@ bool convertTDPStoFlatBytes_double(TightDataPointStorageD *tdps, unsigned char*
if(tdps->allSameData==1) if(tdps->allSameData==1)
{ {
size_t totalByteLength = 3 + 1 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + tdps->exactMidBytes_size; size_t totalByteLength = 1 + 1 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + tdps->exactMidBytes_size;
//bytes = (unsigned char *)malloc(sizeof(unsigned char)*totalByteLength); // comment by tickduan //bytes = (unsigned char *)malloc(sizeof(unsigned char)*totalByteLength); // comment by tickduan
if(totalByteLength >= tdps->dataSeriesLength * sizeof(double)) if(totalByteLength >= tdps->dataSeriesLength * sizeof(double))
{ {
return false; return false;
} }
for (i = 0; i < 3; i++)//3 bytes[k++] = versionNumber;
bytes[k++] = versionNumber[i];
bytes[k++] = sameByte; bytes[k++] = sameByte;
convertSZParamsToBytes(confparams_cpr, &(bytes[k])); convertSZParamsToBytes(confparams_cpr, &(bytes[k]));
@ -677,14 +671,13 @@ void convertTDPStoFlatBytes_double_args(TightDataPointStorageD *tdps, unsigned c
sameByte = (unsigned char) (sameByte | 0x40); //0100,0000, the 6th bit sameByte = (unsigned char) (sameByte | 0x40); //0100,0000, the 6th bit
if(confparams_cpr->errorBoundMode == PW_REL && confparams_cpr->accelerate_pw_rel_compression) if(confparams_cpr->errorBoundMode == PW_REL && confparams_cpr->accelerate_pw_rel_compression)
sameByte = (unsigned char) (sameByte | 0x08); //0000,1000, the 7th bit sameByte = (unsigned char) (sameByte | 0x08); //0000,1000, the 7th bit
if(confparams_cpr->protectValueRange) //if(confparams_cpr->protectValueRange)
sameByte = (unsigned char) (sameByte | 0x04); //0000,0100 // sameByte = (unsigned char) (sameByte | 0x04); //0000,0100
if(tdps->allSameData==1) if(tdps->allSameData==1)
{ {
size_t totalByteLength = 3 + 1 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + tdps->exactMidBytes_size; size_t totalByteLength = 1 + 1 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + tdps->exactMidBytes_size;
for (i = 0; i < 3; i++)//3 bytes[k++] = versionNumber;
bytes[k++] = versionNumber[i];
bytes[k++] = sameByte; bytes[k++] = sameByte;
convertSZParamsToBytes(confparams_cpr, &(bytes[k])); convertSZParamsToBytes(confparams_cpr, &(bytes[k]));

49
deps/SZ/sz/src/TightDataPointStorageF.c vendored
View File

@ -56,17 +56,13 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi
new_TightDataPointStorageF_Empty(this); new_TightDataPointStorageF_Empty(this);
size_t i, index = 0; size_t i, index = 0;
size_t pwrErrBoundBytes_size = 0, segmentL = 0, radExpoL = 0, pwrErrBoundBytesL = 0; size_t pwrErrBoundBytes_size = 0, segmentL = 0, radExpoL = 0, pwrErrBoundBytesL = 0;
char version[3]; unsigned char version = flatBytes[index++]; //1
for (i = 0; i < 3; i++)
version[i] = flatBytes[index++]; //3
unsigned char sameRByte = flatBytes[index++]; //1 unsigned char sameRByte = flatBytes[index++]; //1
if(checkVersion2(version)!=1) if(checkVersion(version) != 1)
{ {
//wrong version //wrong version
printf("Wrong version: \nCompressed-data version (%d.%d.%d)\n",version[0], version[1], version[2]); printf("Wrong version: \nCompressed-data version =%d , program version=%d \n", version, versionNumber);
printf("Current sz version: (%d.%d.%d)\n", versionNumber[0], versionNumber[1], versionNumber[2]); return 0;
printf("Please double-check if the compressed data (or file) is correct.\n");
exit(0);
} }
//note that 1000,0000 is reserved for regression tag. //note that 1000,0000 is reserved for regression tag.
int same = sameRByte & 0x01; //0000,0001 int same = sameRByte & 0x01; //0000,0001
@ -76,7 +72,7 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi
//pde_params->randomAccess = (sameRByte & 0x02) >> 1; //pde_params->randomAccess = (sameRByte & 0x02) >> 1;
//pde_params->szMode = (sameRByte & 0x06) >> 1; //0000,0110 (in fact, this szMode could be removed because convertSZParamsToBytes will overwrite it) //pde_params->szMode = (sameRByte & 0x06) >> 1; //0000,0110 (in fact, this szMode could be removed because convertSZParamsToBytes will overwrite it)
pde_params->protectValueRange = (sameRByte & 0x04)>>2; //pde_params->protectValueRange = (sameRByte & 0x04)>>2;
pde_params->accelerate_pw_rel_compression = (sameRByte & 0x08) >> 3;//0000,1000 pde_params->accelerate_pw_rel_compression = (sameRByte & 0x08) >> 3;//0000,1000
@ -108,7 +104,7 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi
else if(same==1) else if(same==1)
{ {
(*this)->allSameData = 1; (*this)->allSameData = 1;
//size_t exactMidBytesLength = sizeof(double);//flatBytesLength - 3 - 1 - MetaDataByteLength -exe_params->SZ_SIZE_TYPE; //size_t exactMidBytesLength = sizeof(double);//flatBytesLength - 1 - 1 - MetaDataByteLength -exe_params->SZ_SIZE_TYPE;
(*this)->exactMidBytes = &(flatBytes[index]); (*this)->exactMidBytes = &(flatBytes[index]);
return errorBoundMode; return errorBoundMode;
} }
@ -116,7 +112,7 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi
(*this)->allSameData = 0; (*this)->allSameData = 0;
if(isRegression == 1) if(isRegression == 1)
{ {
(*this)->raBytes_size = flatBytesLength - 3 - 1 - MetaDataByteLength - exe_params->SZ_SIZE_TYPE; (*this)->raBytes_size = flatBytesLength - 1 - 1 - MetaDataByteLength - exe_params->SZ_SIZE_TYPE;
(*this)->raBytes = &(flatBytes[index]); (*this)->raBytes = &(flatBytes[index]);
return errorBoundMode; return errorBoundMode;
} }
@ -214,7 +210,8 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi
if ((*this)->rtypeArray != NULL) if ((*this)->rtypeArray != NULL)
{ {
(*this)->residualMidBits_size = flatBytesLength - 3 - 1 - MetaDataByteLength - exe_params->SZ_SIZE_TYPE - 4 - radExpoL - segmentL - pwrErrBoundBytesL - 4 - 4 - 1 - 8 // modify from -3 -1 to -1 -1 tickduan
(*this)->residualMidBits_size = flatBytesLength - 1 - 1 - MetaDataByteLength - exe_params->SZ_SIZE_TYPE - 4 - radExpoL - segmentL - pwrErrBoundBytesL - 4 - 4 - 1 - 8
- exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - minLogValueSize - exe_params->SZ_SIZE_TYPE - 4 - (*this)->rtypeArray_size - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - minLogValueSize - exe_params->SZ_SIZE_TYPE - 4 - (*this)->rtypeArray_size
- minLogValueSize - (*this)->typeArray_size - (*this)->leadNumArray_size - minLogValueSize - (*this)->typeArray_size - (*this)->leadNumArray_size
- (*this)->exactMidBytes_size - pwrErrBoundBytes_size - 1 - 1; - (*this)->exactMidBytes_size - pwrErrBoundBytes_size - 1 - 1;
@ -223,7 +220,7 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi
} }
else else
{ {
(*this)->residualMidBits_size = flatBytesLength - 3 - 1 - MetaDataByteLength - exe_params->SZ_SIZE_TYPE - 4 - radExpoL - segmentL - pwrErrBoundBytesL - 4 - 4 - 1 - 8 (*this)->residualMidBits_size = flatBytesLength - 1 - 1 - MetaDataByteLength - exe_params->SZ_SIZE_TYPE - 4 - radExpoL - segmentL - pwrErrBoundBytesL - 4 - 4 - 1 - 8
- exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - minLogValueSize - (*this)->typeArray_size - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - minLogValueSize - (*this)->typeArray_size
- (*this)->leadNumArray_size - (*this)->exactMidBytes_size - pwrErrBoundBytes_size - 1 - 1; - (*this)->leadNumArray_size - (*this)->exactMidBytes_size - pwrErrBoundBytes_size - 1 - 1;
} }
@ -384,15 +381,14 @@ void convertTDPStoBytes_float(TightDataPointStorageF* tdps, unsigned char* bytes
unsigned char max_quant_intervals_Bytes[4]; unsigned char max_quant_intervals_Bytes[4];
// 1 version // 1 version
for(i = 0;i<3;i++)//3 bytes bytes[k++] = versionNumber;
bytes[k++] = versionNumber[i];
// 2 same // 2 same
bytes[k++] = sameByte; //1 byte bytes[k++] = sameByte; //1 byte
// 3 meta // 3 meta
convertSZParamsToBytes(confparams_cpr, &(bytes[k])); convertSZParamsToBytes(confparams_cpr, &(bytes[k]));
k = k + MetaDataByteLength; k = k + MetaDataByteLength;
// 4 element count // 4 element count
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST: 4 or 8 bytes for(i = 0; i < exe_params->SZ_SIZE_TYPE; i++)//ST: 4 or 8 bytes
bytes[k++] = dsLengthBytes[i]; bytes[k++] = dsLengthBytes[i];
intToBytes_bigEndian(max_quant_intervals_Bytes, confparams_cpr->max_quant_intervals); intToBytes_bigEndian(max_quant_intervals_Bytes, confparams_cpr->max_quant_intervals);
// 5 max_quant_intervals length // 5 max_quant_intervals length
@ -495,15 +491,15 @@ bool convertTDPStoFlatBytes_float(TightDataPointStorageF *tdps, unsigned char* b
sameByte = (unsigned char) (sameByte | 0x40); // 0100,0000, the 6th bit sameByte = (unsigned char) (sameByte | 0x40); // 0100,0000, the 6th bit
if(confparams_cpr->errorBoundMode == PW_REL && confparams_cpr->accelerate_pw_rel_compression) if(confparams_cpr->errorBoundMode == PW_REL && confparams_cpr->accelerate_pw_rel_compression)
sameByte = (unsigned char) (sameByte | 0x08); //0000,1000 sameByte = (unsigned char) (sameByte | 0x08); //0000,1000
if(confparams_cpr->protectValueRange) //if(confparams_cpr->protectValueRange)
sameByte = (unsigned char) (sameByte | 0x04); //0000,0100 // sameByte = (unsigned char) (sameByte | 0x04); //0000,0100
if(tdps->allSameData == 1 ) if(tdps->allSameData == 1 )
{ {
// //
// same format // same format
// //
size_t totalByteLength = 3 + 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + tdps->exactMidBytes_size; size_t totalByteLength = 1 + 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + tdps->exactMidBytes_size;
//*bytes = (unsigned char *)malloc(sizeof(unsigned char)*totalByteLength); // not need malloc comment by tickduan //*bytes = (unsigned char *)malloc(sizeof(unsigned char)*totalByteLength); // not need malloc comment by tickduan
// check output buffer enough // check output buffer enough
if(totalByteLength >= tdps->dataSeriesLength * sizeof(float) ) if(totalByteLength >= tdps->dataSeriesLength * sizeof(float) )
@ -512,9 +508,8 @@ bool convertTDPStoFlatBytes_float(TightDataPointStorageF *tdps, unsigned char* b
return false; return false;
} }
// 1 version 3 bytes // 1 version 1 byte
for (i = 0; i < 3; i++)//3 bytes[k++] = versionNumber;
bytes[k++] = versionNumber[i];
// 2 same flag 1 bytes // 2 same flag 1 bytes
bytes[k++] = sameByte; bytes[k++] = sameByte;
// 3 metaData 26 bytes // 3 metaData 26 bytes
@ -545,7 +540,8 @@ bool convertTDPStoFlatBytes_float(TightDataPointStorageF *tdps, unsigned char* b
minLogValueSize = 4; minLogValueSize = 4;
} }
size_t totalByteLength = 3 + 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 4 + radExpoL + segmentL + pwrBoundArrayL + 4 + 4 + 1 + 8 // version(1) + samebyte(1)
size_t totalByteLength = 1 + 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 4 + radExpoL + segmentL + pwrBoundArrayL + 4 + 4 + 1 + 8
+ exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE + minLogValueSize + exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE + minLogValueSize
+ tdps->typeArray_size + tdps->leadNumArray_size + tdps->typeArray_size + tdps->leadNumArray_size
+ tdps->exactMidBytes_size + residualMidBitsLength + tdps->pwrErrBoundBytes_size; + tdps->exactMidBytes_size + residualMidBitsLength + tdps->pwrErrBoundBytes_size;
@ -596,11 +592,10 @@ void convertTDPStoFlatBytes_float_args(TightDataPointStorageF *tdps, unsigned ch
if(tdps->allSameData==1) if(tdps->allSameData==1)
{ {
size_t totalByteLength = 3 + 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + tdps->exactMidBytes_size; size_t totalByteLength = 1 + 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + tdps->exactMidBytes_size;
//*bytes = (unsigned char *)malloc(sizeof(unsigned char)*totalByteLength); //*bytes = (unsigned char *)malloc(sizeof(unsigned char)*totalByteLength);
for (i = 0; i < 3; i++)//3 bytes[k++] = versionNumber;
bytes[k++] = versionNumber[i];
bytes[k++] = sameByte; bytes[k++] = sameByte;
convertSZParamsToBytes(confparams_cpr, &(bytes[k])); convertSZParamsToBytes(confparams_cpr, &(bytes[k]));
@ -624,7 +619,7 @@ void convertTDPStoFlatBytes_float_args(TightDataPointStorageF *tdps, unsigned ch
pwrBoundArrayL = 4; pwrBoundArrayL = 4;
} }
size_t totalByteLength = 3 + 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 4 + radExpoL + segmentL + pwrBoundArrayL + 4 + 4 + 1 + 8 size_t totalByteLength = 1 + 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 4 + radExpoL + segmentL + pwrBoundArrayL + 4 + 4 + 1 + 8
+ exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE
+ tdps->typeArray_size + tdps->leadNumArray_size + tdps->typeArray_size + tdps->leadNumArray_size
+ tdps->exactMidBytes_size + residualMidBitsLength + tdps->pwrErrBoundBytes_size; + tdps->exactMidBytes_size + residualMidBitsLength + tdps->pwrErrBoundBytes_size;

28
deps/SZ/sz/src/conf.c vendored
View File

@ -120,7 +120,7 @@ int SZ_ReadConf(const char* sz_cfgFile) {
confparams_cpr->errorBoundMode = SZ_ABS; confparams_cpr->errorBoundMode = SZ_ABS;
confparams_cpr->psnr = 90; confparams_cpr->psnr = 90;
confparams_cpr->absErrBound = 1E-20; confparams_cpr->absErrBound = 1E-8;
confparams_cpr->relBoundRatio = 1E-8; confparams_cpr->relBoundRatio = 1E-8;
confparams_cpr->accelerate_pw_rel_compression = 1; confparams_cpr->accelerate_pw_rel_compression = 1;
@ -410,33 +410,11 @@ int SZ_LoadConf(const char* sz_cfgFile) {
return SZ_SUCCESS; return SZ_SUCCESS;
} }
int checkVersion(char* version) int checkVersion(unsigned char version)
{ {
int i = 0; return version <= versionNumber;
for(;i<3;i++)
if(version[i]!=versionNumber[i])
return 0;
return 1;
} }
inline int computeVersion(int major, int minor, int revision)
{
return major*10000+minor*100+revision;
}
int checkVersion2(char* version)
{
int major = version[0];
int minor = version[1];
int revision = version[2];
int preVersion = 20108;
int givenVersion = computeVersion(major, minor, revision);
//int currentVersion = computeVersion(SZ_VER_MAJOR, SZ_VER_MINOR, SZ_VER_REVISION);
if(givenVersion < preVersion) //only for old version (older than 2.1.8), we will check whether version is consistent exactly.
return checkVersion(version);
return 1;
}
void initSZ_TSC() void initSZ_TSC()
{ {

26
deps/SZ/sz/src/dataCompression.c vendored
View File

@ -678,32 +678,6 @@ int computeBlockEdgeSize_3D(int segmentSize)
//return (int)(pow(segmentSize, 1.0/3)+1); //return (int)(pow(segmentSize, 1.0/3)+1);
} }
//convert random-access version based bytes to output bytes
int initRandomAccessBytes(unsigned char* raBytes)
{
int k = 0, i = 0;
for (i = 0; i < 3; i++)//3
raBytes[k++] = versionNumber[i];
int sameByte = 0x80; //indicating this is regression-based compression mode
if(exe_params->SZ_SIZE_TYPE==8)
sameByte = (unsigned char) (sameByte | 0x40); // 01000000, the 6th bit
if(confparams_cpr->randomAccess)
sameByte = (unsigned char) (sameByte | 0x02); // 00000010, random access
//sameByte = sameByte | (confparams_cpr->szMode << 1);
if(confparams_cpr->protectValueRange)
sameByte = (unsigned char) (sameByte | 0x04); //00000100, protect value range
raBytes[k++] = sameByte;
convertSZParamsToBytes(confparams_cpr, &(raBytes[k]));
if(confparams_cpr->dataType==SZ_FLOAT)
k = k + MetaDataByteLength;
else if(confparams_cpr->dataType==SZ_DOUBLE)
k = k + MetaDataByteLength_double;
return k;
}
//The following functions are float-precision version of dealing with the unpredictable data points //The following functions are float-precision version of dealing with the unpredictable data points
int generateLossyCoefficients_float(float* oriData, double precision, size_t nbEle, int* reqBytesLength, int* resiBitsLength, float* medianValue, float* decData) int generateLossyCoefficients_float(float* oriData, double precision, size_t nbEle, int* reqBytesLength, int* resiBitsLength, float* medianValue, float* decData)
{ {

2
deps/SZ/sz/src/sz.c vendored
View File

@ -25,7 +25,7 @@
//#include "CurveFillingCompressStorage.h" //#include "CurveFillingCompressStorage.h"
int versionNumber[4] = {SZ_VER_MAJOR,SZ_VER_MINOR,SZ_VER_BUILD,SZ_VER_REVISION}; unsigned char versionNumber = SZ_VER_MAJOR;
int SZ_SIZE_TYPE_DEFUALT = 4; int SZ_SIZE_TYPE_DEFUALT = 4;
int dataEndianType = LITTLE_ENDIAN_DATA; //*endian type of the data read from disk int dataEndianType = LITTLE_ENDIAN_DATA; //*endian type of the data read from disk

5
deps/SZ/sz/src/sz_double.c vendored
View File

@ -232,13 +232,12 @@ void SZ_compress_args_double_StoreOriData(double* oriData, size_t dataLength, un
{ {
int doubleSize = sizeof(double); int doubleSize = sizeof(double);
size_t k = 0, i; size_t k = 0, i;
size_t totalByteLength = 3 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + 1 + doubleSize*dataLength; size_t totalByteLength = 1 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + 1 + doubleSize*dataLength;
/*No need to malloc because newByteData should always already be allocated with no less totalByteLength.*/ /*No need to malloc because newByteData should always already be allocated with no less totalByteLength.*/
//*newByteData = (unsigned char*)malloc(totalByteLength); //*newByteData = (unsigned char*)malloc(totalByteLength);
unsigned char dsLengthBytes[8]; unsigned char dsLengthBytes[8];
for (i = 0; i < 3; i++)//3 newByteData[k++] = versionNumber;
newByteData[k++] = versionNumber[i];
if(exe_params->SZ_SIZE_TYPE==4)//1 if(exe_params->SZ_SIZE_TYPE==4)//1
newByteData[k++] = 16; //00010000 newByteData[k++] = 16; //00010000

13
deps/SZ/sz/src/sz_double_pwr.c vendored
View File

@ -320,13 +320,12 @@ size_t dataLength, size_t *outSize, double min, double max)
{ {
size_t k = 0, i; size_t k = 0, i;
tdps->isLossless = 1; tdps->isLossless = 1;
size_t totalByteLength = 3 + exe_params->SZ_SIZE_TYPE + 1 + doubleSize*dataLength; size_t totalByteLength = 1 + exe_params->SZ_SIZE_TYPE + 1 + doubleSize*dataLength;
*newByteData = (unsigned char*)malloc(totalByteLength); //*newByteData = (unsigned char*)malloc(totalByteLength); comment by tickduan
unsigned char dsLengthBytes[exe_params->SZ_SIZE_TYPE]; unsigned char dsLengthBytes[exe_params->SZ_SIZE_TYPE];
intToBytes_bigEndian(dsLengthBytes, dataLength);//4 intToBytes_bigEndian(dsLengthBytes, dataLength);//4
for (i = 0; i < 3; i++)//3 newByteData[k++] = versionNumber;
newByteData[k++] = versionNumber[i];
if(exe_params->SZ_SIZE_TYPE==4) if(exe_params->SZ_SIZE_TYPE==4)
{ {
@ -650,7 +649,7 @@ size_t dataLength, double absErrBound, double relBoundRatio, double pwrErrRatio,
convertTDPStoFlatBytes_double(tdps, newByteData, outSize); convertTDPStoFlatBytes_double(tdps, newByteData, outSize);
if(*outSize>3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(double)*dataLength) if(*outSize > 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(double)*dataLength)
SZ_compress_args_double_StoreOriData(oriData, dataLength, newByteData, outSize); SZ_compress_args_double_StoreOriData(oriData, dataLength, newByteData, outSize);
free_TightDataPointStorageD(tdps); free_TightDataPointStorageD(tdps);
@ -712,7 +711,7 @@ void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log(unsigned char* newByt
free(signs); free(signs);
convertTDPStoFlatBytes_double(tdps, newByteData, outSize); convertTDPStoFlatBytes_double(tdps, newByteData, outSize);
if(*outSize>3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(double)*dataLength) if(*outSize>1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(double)*dataLength)
SZ_compress_args_double_StoreOriData(oriData, dataLength, newByteData, outSize); SZ_compress_args_double_StoreOriData(oriData, dataLength, newByteData, outSize);
free_TightDataPointStorageD(tdps); free_TightDataPointStorageD(tdps);
@ -746,7 +745,7 @@ void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log_MSST19(unsigned char*
free(signs); free(signs);
convertTDPStoFlatBytes_double(tdps, newByteData, outSize); convertTDPStoFlatBytes_double(tdps, newByteData, outSize);
if(*outSize>3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(double)*dataLength) if(*outSize > 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(double)*dataLength)
SZ_compress_args_double_StoreOriData(oriData, dataLength, newByteData, outSize); SZ_compress_args_double_StoreOriData(oriData, dataLength, newByteData, outSize);
free_TightDataPointStorageD(tdps); free_TightDataPointStorageD(tdps);

42
deps/SZ/sz/src/sz_float.c vendored
View File

@ -103,7 +103,7 @@ unsigned int optimize_intervals_float_1D(float *oriData, size_t dataLength, doub
} }
TightDataPointStorageF* SZ_compress_float_1D_MDQ(float *oriData, TightDataPointStorageF* SZ_compress_float_1D_MDQ(float *oriData,
size_t dataLength, float realPrecision, float valueRangeSize, float medianValue_f) size_t dataLength, float realPrecision, float valueRangeSize, float medianValue_f)
{ {
unsigned int quantization_intervals; unsigned int quantization_intervals;
if(exe_params->optQuantMode==1) if(exe_params->optQuantMode==1)
@ -259,41 +259,6 @@ size_t dataLength, float realPrecision, float valueRangeSize, float medianValue_
return tdps; return tdps;
} }
void SZ_compress_args_float_StoreOriData(float* oriData, size_t dataLength, unsigned char* newByteData, size_t *outSize)
{
int floatSize=sizeof(float);
size_t k = 0, i;
size_t totalByteLength = 3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + floatSize*dataLength;
/*No need to malloc because newByteData should always already be allocated with no less totalByteLength.*/
//*newByteData = (unsigned char*)malloc(totalByteLength);
unsigned char dsLengthBytes[8];
for (i = 0; i < 3; i++)//3
newByteData[k++] = versionNumber[i];
if(exe_params->SZ_SIZE_TYPE==4)//1
newByteData[k++] = 16; //00010000
else
newByteData[k++] = 80; //01010000: 01000000 indicates the SZ_SIZE_TYPE=8
convertSZParamsToBytes(confparams_cpr, &(newByteData[k]));
k = k + MetaDataByteLength;
sizeToBytes(dsLengthBytes,dataLength); //SZ_SIZE_TYPE: 4 or 8
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
newByteData[k++] = dsLengthBytes[i];
if(sysEndianType==BIG_ENDIAN_SYSTEM)
memcpy(newByteData+4+MetaDataByteLength+exe_params->SZ_SIZE_TYPE, oriData, dataLength*floatSize);
else
{
unsigned char* p = newByteData+4+MetaDataByteLength+exe_params->SZ_SIZE_TYPE;
for(i=0;i<dataLength;i++,p+=floatSize)
floatToBytes(p, oriData[i]);
}
*outSize = totalByteLength;
}
// compress core algorithm if success return true else return false // compress core algorithm if success return true else return false
bool SZ_compress_args_float_NoCkRngeNoGzip_1D( unsigned char* newByteData, float *oriData, bool SZ_compress_args_float_NoCkRngeNoGzip_1D( unsigned char* newByteData, float *oriData,
size_t dataLength, double realPrecision, size_t *outSize, float valueRangeSize, float medianValue_f) size_t dataLength, double realPrecision, size_t *outSize, float valueRangeSize, float medianValue_f)
@ -312,9 +277,8 @@ bool SZ_compress_args_float_NoCkRngeNoGzip_1D( unsigned char* newByteData, float
} }
// check compressed size large than original // check compressed size large than original
if(*outSize > 3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(float)*dataLength) if(*outSize > 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(float)*dataLength)
{ {
//SZ_compress_args_float_StoreOriData(oriData, dataLength, newByteData, outSize);
return false; return false;
} }
@ -610,8 +574,6 @@ int SZ_compress_args_float(float *oriData, size_t r1, unsigned char* newByteData
free(tmpByteData); free(tmpByteData);
return SZ_ALGORITHM_ERR; return SZ_ALGORITHM_ERR;
} }
//if(tmpOutSize >= dataLength*sizeof(float) + 3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1)
// SZ_compress_args_float_StoreOriData(oriData, dataLength, tmpByteData, &tmpOutSize);
} }
//cost_end(" sz_first_compress"); //cost_end(" sz_first_compress");

20
deps/SZ/sz/src/sz_float_pwr.c vendored
View File

@ -151,7 +151,7 @@ unsigned int optimize_intervals_float_1D_pwr(float *oriData, size_t dataLength,
} }
void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr(unsigned char* newByteData, float *oriData, double globalPrecision, void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr(unsigned char* newByteData, float *oriData, double globalPrecision,
size_t dataLength, size_t *outSize, float min, float max) size_t dataLength, size_t *outSize, float min, float max)
{ {
size_t pwrLength = dataLength%confparams_cpr->segment_size==0?dataLength/confparams_cpr->segment_size:dataLength/confparams_cpr->segment_size+1; size_t pwrLength = dataLength%confparams_cpr->segment_size==0?dataLength/confparams_cpr->segment_size:dataLength/confparams_cpr->segment_size+1;
float* pwrErrBound = (float*)malloc(sizeof(float)*pwrLength); float* pwrErrBound = (float*)malloc(sizeof(float)*pwrLength);
@ -309,13 +309,13 @@ size_t dataLength, size_t *outSize, float min, float max)
{ {
size_t k = 0, i; size_t k = 0, i;
tdps->isLossless = 1; tdps->isLossless = 1;
size_t totalByteLength = 3 + exe_params->SZ_SIZE_TYPE + 1 + floatSize*dataLength; size_t totalByteLength = 1 + exe_params->SZ_SIZE_TYPE + 1 + floatSize*dataLength;
*newByteData = (unsigned char*)malloc(totalByteLength); //*newByteData = (unsigned char*)malloc(totalByteLength); comment by tickduan
unsigned char dsLengthBytes[exe_params->SZ_SIZE_TYPE]; unsigned char dsLengthBytes[exe_params->SZ_SIZE_TYPE];
intToBytes_bigEndian(dsLengthBytes, dataLength);//4 intToBytes_bigEndian(dsLengthBytes, dataLength);//4
for (i = 0; i < 3; i++)//3
newByteData[k++] = versionNumber[i]; newByteData[k++] = versionNumber;
if(exe_params->SZ_SIZE_TYPE==4) if(exe_params->SZ_SIZE_TYPE==4)
{ {
@ -328,7 +328,6 @@ size_t dataLength, size_t *outSize, float min, float max)
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)//4 or 8 for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)//4 or 8
newByteData[k++] = dsLengthBytes[i]; newByteData[k++] = dsLengthBytes[i];
if(sysEndianType==BIG_ENDIAN_SYSTEM) if(sysEndianType==BIG_ENDIAN_SYSTEM)
memcpy(newByteData+4+exe_params->SZ_SIZE_TYPE, oriData, dataLength*floatSize); memcpy(newByteData+4+exe_params->SZ_SIZE_TYPE, oriData, dataLength*floatSize);
else else
@ -639,9 +638,6 @@ size_t dataLength, double absErrBound, double relBoundRatio, double pwrErrRatio,
convertTDPStoFlatBytes_float(tdps, newByteData, outSize); convertTDPStoFlatBytes_float(tdps, newByteData, outSize);
if(*outSize>3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(float)*dataLength)
SZ_compress_args_float_StoreOriData(oriData, dataLength, newByteData, outSize);
free_TightDataPointStorageF(tdps); free_TightDataPointStorageF(tdps);
} }
@ -702,9 +698,7 @@ void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_pre_log(unsigned char* newByte
free(signs); free(signs);
convertTDPStoFlatBytes_float(tdps, newByteData, outSize); convertTDPStoFlatBytes_float(tdps, newByteData, outSize);
if(*outSize>3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(float)*dataLength)
SZ_compress_args_float_StoreOriData(oriData, dataLength, newByteData, outSize);
free_TightDataPointStorageF(tdps); free_TightDataPointStorageF(tdps);
} }
@ -737,8 +731,6 @@ void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_pre_log_MSST19(unsigned char*
free(signs); free(signs);
convertTDPStoFlatBytes_float(tdps, newByteData, outSize); convertTDPStoFlatBytes_float(tdps, newByteData, outSize);
if(*outSize>3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(float)*dataLength)
SZ_compress_args_float_StoreOriData(oriData, dataLength, newByteData, outSize);
free_TightDataPointStorageF(tdps); free_TightDataPointStorageF(tdps);
} }

22
deps/SZ/sz/src/szd_double.c vendored
View File

@ -64,8 +64,9 @@ int SZ_decompress_args_double(double* newData, size_t r1, unsigned char* cmpByte
} }
else else
szTmpBytes = cmpBytes; szTmpBytes = cmpBytes;
pde_params->sol_ID = szTmpBytes[4+14]; //szTmpBytes: version(3bytes), samebyte(1byte), [14]:sol_ID=SZ or SZ_Transpose // calc postion
//pde_params->sol_ID = szTmpBytes[1+3-2+14-4]; //szTmpBytes: version(3bytes), samebyte(1byte), [14]:sol_ID=SZ or SZ_Transpose
//TODO: convert szTmpBytes to double array. //TODO: convert szTmpBytes to double array.
TightDataPointStorageD* tdps; TightDataPointStorageD* tdps;
int errBoundMode = new_TightDataPointStorageD_fromFlatBytes(&tdps, szTmpBytes, tmpSize, pde_exe, pde_params); int errBoundMode = new_TightDataPointStorageD_fromFlatBytes(&tdps, szTmpBytes, tmpSize, pde_exe, pde_params);
@ -102,23 +103,6 @@ int SZ_decompress_args_double(double* newData, size_t r1, unsigned char* cmpByte
} }
} }
if(pde_params->protectValueRange)
{
double* nd = newData;
double min = pde_params->dmin;
double max = pde_params->dmax;
for(i=0;i<dataLength;i++)
{
double v = nd[i];
if(v <= max && v >= min)
continue;
if(v < min)
nd[i] = min;
else if(v > max)
nd[i] = max;
}
}
free_TightDataPointStorageD2(tdps); free_TightDataPointStorageD2(tdps);
if(pde_params->szMode!=SZ_BEST_SPEED && cmpSize!=12+MetaDataByteLength_double+exe_params->SZ_SIZE_TYPE) if(pde_params->szMode!=SZ_BEST_SPEED && cmpSize!=12+MetaDataByteLength_double+exe_params->SZ_SIZE_TYPE)
free(szTmpBytes); free(szTmpBytes);

23
deps/SZ/sz/src/szd_float.c vendored
View File

@ -91,7 +91,8 @@ int SZ_decompress_args_float(float* newData, size_t r1, unsigned char* cmpBytes,
else else
szTmpBytes = cmpBytes; szTmpBytes = cmpBytes;
pde_params->sol_ID = szTmpBytes[4+14]; //szTmpBytes: version(3bytes), samebyte(1byte), [14]:sol_ID=SZ or SZ_Transpose // calc sol_ID
//pde_params->sol_ID = szTmpBytes[1+3-2+14-4]; //szTmpBytes: version(1bytes), samebyte(1byte), [14-4]:sol_ID=SZ or SZ_Transpose
//TODO: convert szTmpBytes to data array. //TODO: convert szTmpBytes to data array.
TightDataPointStorageF* tdps; TightDataPointStorageF* tdps;
@ -113,10 +114,6 @@ int SZ_decompress_args_float(float* newData, size_t r1, unsigned char* cmpBytes,
newData[i] = bytesToFloat(p); newData[i] = bytesToFloat(p);
} }
} }
else if(pde_params->sol_ID==SZ_Transpose)
{
getSnapshotData_float_1D(newData,dataLength,tdps, errBoundMode, 0, hist_data, pde_params);
}
else //pde_params->sol_ID==SZ else //pde_params->sol_ID==SZ
{ {
if(tdps->raBytes_size > 0) //v2.0 if(tdps->raBytes_size > 0) //v2.0
@ -130,22 +127,6 @@ int SZ_decompress_args_float(float* newData, size_t r1, unsigned char* cmpBytes,
} }
//cost_start_(); //cost_start_();
if(pde_params->protectValueRange)
{
float* nd = newData;
float min = pde_params->fmin;
float max = pde_params->fmax;
for(i=0;i<dataLength;i++)
{
float v = nd[i];
if(v <= max && v >= min)
continue;
if(v < min)
nd[i] = min;
else if(v > max)
nd[i] = max;
}
}
//cost_end_(); //cost_end_();
//printf("totalCost_=%f\n", totalCost_); //printf("totalCost_=%f\n", totalCost_);
free_TightDataPointStorageF2(tdps); free_TightDataPointStorageF2(tdps);