xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fixed open lossy then close, read old data crash bug

This commit is contained in:
tickduan 2021-07-16 17:00:48 +08:00
parent efdb71325b
commit 04aa94dd0e
8 changed files with 52 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
deps/SZ/sz/include/ByteToolkit.h vendored
View File

@ -36,8 +36,8 @@ int getLeftMovingCode(int kMod8);
int getRightMovingSteps(int kMod8, int resiBitLength);
int getRightMovingCode(int kMod8, int resiBitLength);
size_t bytesToSize(unsigned char* bytes);
void sizeToBytes(unsigned char* outBytes, size_t size);
size_t bytesToSize(unsigned char* bytes, int size_type);
void sizeToBytes(unsigned char* outBytes, size_t size, int size_type);
#ifdef __cplusplus
}

2
deps/SZ/sz/include/sz.h vendored
View File

@ -186,7 +186,7 @@ size_t SZ_compress_args(int dataType, void *data, size_t r1, unsigned char* outD
size_t SZ_decompress(int dataType, unsigned char *bytes, size_t byteLength, size_t r1, unsigned char* outData);
void convertSZParamsToBytes(sz_params* params, unsigned char* result);
void convertSZParamsToBytes(sz_params* params, unsigned char* result, char optQuantMode);
void convertBytesToSZParams(unsigned char* bytes, sz_params* params, sz_exedata* pde_exe);
#ifdef __cplusplus

13
deps/SZ/sz/src/ByteToolkit.c vendored
View File

@ -260,30 +260,29 @@ INLINE int getRightMovingCode(int kMod8, int resiBitLength)
}
}
INLINE size_t bytesToSize(unsigned char* bytes)
INLINE size_t bytesToSize(unsigned char* bytes, int size_type)
{
size_t result = 0;
if(exe_params->SZ_SIZE_TYPE==4)
if(size_type == 4)
result = bytesToInt_bigEndian(bytes);//4
else
result = bytesToLong_bigEndian(bytes);//8
return result;
}
INLINE void sizeToBytes(unsigned char* outBytes, size_t size)
INLINE void sizeToBytes(unsigned char* outBytes, size_t size, int size_type)
{
if(exe_params->SZ_SIZE_TYPE==4)
if(size_type == 4)
intToBytes_bigEndian(outBytes, (unsigned int)size);//4
else
longToBytes_bigEndian(outBytes, (unsigned long)size);//8
}
void convertSZParamsToBytes(sz_params* params, unsigned char* result)
void convertSZParamsToBytes(sz_params* params, unsigned char* result, char optQuantMode)
{
//unsigned char* result = (unsigned char*)malloc(16);
unsigned char buf = 0;
//flag1: exe_params->optQuantMode(1bit), dataEndianType(1bit), sysEndianType(1bit), conf_params->szMode (1bit), conf_params->gzipMode (2bits), pwrType (2bits)
buf = exe_params->optQuantMode;
buf = optQuantMode;
buf = (buf << 1) | dataEndianType;
buf = (buf << 1) | sysEndianType;
buf = (buf << 2) | params->szMode;

35
deps/SZ/sz/src/TightDataPointStorageD.c vendored
View File

@ -41,7 +41,7 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi
int same = sameRByte & 0x01;
(*this)->isLossless = (sameRByte & 0x10)>>4;
exe_params->SZ_SIZE_TYPE = ((sameRByte & 0x40)>>6)==1?8:4;
pde_exe->SZ_SIZE_TYPE = ((sameRByte & 0x40)>>6)==1?8:4;
//pde_params->protectValueRange = (sameRByte & 0x04)>>2;
pde_params->accelerate_pw_rel_compression = (sameRByte & 0x08) >> 3;
int errorBoundMode = SZ_ABS;
@ -53,9 +53,9 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi
int isRegression = (sameRByte >> 7) & 0x01;
unsigned char dsLengthBytes[8];
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++)
dsLengthBytes[i] = flatBytes[index++];
(*this)->dataSeriesLength = bytesToSize(dsLengthBytes);
(*this)->dataSeriesLength = bytesToSize(dsLengthBytes, pde_exe->SZ_SIZE_TYPE);
if((*this)->isLossless==1)
{
@ -65,7 +65,6 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi
else if(same==1)
{
(*this)->allSameData = 1;
//size_t exactMidBytesLength = sizeof(double);//flatBytesLength - 3 - 1 - MetaDataByteLength_double -exe_params->SZ_SIZE_TYPE;
(*this)->exactMidBytes = &(flatBytes[index]);
return errorBoundMode;
}
@ -74,7 +73,7 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi
if(isRegression == 1)
{
(*this)->raBytes_size = flatBytesLength - 3 - 1 - MetaDataByteLength_double - exe_params->SZ_SIZE_TYPE;
(*this)->raBytes_size = flatBytesLength - 3 - 1 - MetaDataByteLength_double - pde_exe->SZ_SIZE_TYPE;
(*this)->raBytes = &(flatBytes[index]);
return errorBoundMode;
}
@ -101,17 +100,17 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi
byteBuf[i] = flatBytes[index++];
(*this)->realPrecision = bytesToDouble(byteBuf);//8
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++)
byteBuf[i] = flatBytes[index++];
(*this)->typeArray_size = bytesToSize(byteBuf);// exe_params->SZ_SIZE_TYPE
(*this)->typeArray_size = bytesToSize(byteBuf, pde_exe->SZ_SIZE_TYPE);
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++)
byteBuf[i] = flatBytes[index++];
(*this)->exactDataNum = bytesToSize(byteBuf);// ST
(*this)->exactDataNum = bytesToSize(byteBuf, pde_exe->SZ_SIZE_TYPE);// ST
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++)
byteBuf[i] = flatBytes[index++];
(*this)->exactMidBytes_size = bytesToSize(byteBuf);// ST
(*this)->exactMidBytes_size = bytesToSize(byteBuf, pde_exe->SZ_SIZE_TYPE);// ST
size_t logicLeadNumBitsNum = (*this)->exactDataNum * 2;
if (logicLeadNumBitsNum % 8 == 0)
@ -132,8 +131,8 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi
// todo need check length
(*this)->residualMidBits_size = flatBytesLength - 1 - 1 - MetaDataByteLength - exe_params->SZ_SIZE_TYPE - 4 - 4 - 4 - 1 - 8
- exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE
(*this)->residualMidBits_size = flatBytesLength - 1 - 1 - MetaDataByteLength - pde_exe->SZ_SIZE_TYPE - 4 - 4 - 4 - 1 - 8
- pde_exe->SZ_SIZE_TYPE - pde_exe->SZ_SIZE_TYPE - pde_exe->SZ_SIZE_TYPE
- (*this)->leadNumArray_size - (*this)->exactMidBytes_size - (*this)->typeArray_size;
(*this)->leadNumArray = &flatBytes[index];
@ -211,7 +210,7 @@ void convertTDPStoBytes_double(TightDataPointStorageD* tdps, unsigned char* byte
bytes[k++] = versionNumber;
bytes[k++] = sameByte; //1 byte
convertSZParamsToBytes(confparams_cpr, &(bytes[k]));
convertSZParamsToBytes(confparams_cpr, &(bytes[k]), exe_params->optQuantMode);
k = k + MetaDataByteLength_double;
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST: 4 or 8 bytes
@ -234,15 +233,15 @@ void convertTDPStoBytes_double(TightDataPointStorageD* tdps, unsigned char* byte
for (i = 0; i < 8; i++)// 8
bytes[k++] = realPrecisionBytes[i];
sizeToBytes(typeArrayLengthBytes, tdps->typeArray_size);
sizeToBytes(typeArrayLengthBytes, tdps->typeArray_size, exe_params->SZ_SIZE_TYPE);
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
bytes[k++] = typeArrayLengthBytes[i];
sizeToBytes(exactLengthBytes, tdps->exactDataNum);
sizeToBytes(exactLengthBytes, tdps->exactDataNum, exe_params->SZ_SIZE_TYPE);
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
bytes[k++] = exactLengthBytes[i];
sizeToBytes(exactMidBytesLength, tdps->exactMidBytes_size);
sizeToBytes(exactMidBytesLength, tdps->exactMidBytes_size, exe_params->SZ_SIZE_TYPE);
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
bytes[k++] = exactMidBytesLength[i];
@ -302,7 +301,7 @@ bool convertTDPStoFlatBytes_double(TightDataPointStorageD *tdps, unsigned char*
bytes[k++] = versionNumber;
bytes[k++] = sameByte;
convertSZParamsToBytes(confparams_cpr, &(bytes[k]));
convertSZParamsToBytes(confparams_cpr, &(bytes[k]), exe_params->optQuantMode);
k = k + MetaDataByteLength_double;
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)

35
deps/SZ/sz/src/TightDataPointStorageF.c vendored
View File

@ -48,16 +48,16 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi
// 2 same(1) //note that 1000,0000 is reserved for regression tag.
int same = sameRByte & 0x01; //0000,0001
(*this)->isLossless = (sameRByte & 0x10)>>4; //0001,0000 //0010,0000
exe_params->SZ_SIZE_TYPE = ((sameRByte & 0x40)>>6)==1?8:4; //0100,0000
pde_exe->SZ_SIZE_TYPE = ((sameRByte & 0x40)>>6)==1?8:4; //0100,0000
int errorBoundMode = SZ_ABS;
// 3 meta(2)
convertBytesToSZParams(&(flatBytes[index]), pde_params, pde_exe);
index += MetaDataByteLength;
// 4 element count(4)
unsigned char dsLengthBytes[8];
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++)
dsLengthBytes[i] = flatBytes[index++];
(*this)->dataSeriesLength = bytesToSize(dsLengthBytes);// 4 or 8
(*this)->dataSeriesLength = bytesToSize(dsLengthBytes, pde_exe->SZ_SIZE_TYPE);// 4 or 8
if((*this)->isLossless==1)
{
//(*this)->exactMidBytes = flatBytes+8;
@ -66,7 +66,6 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi
else if(same==1)
{
(*this)->allSameData = 1;
//size_t exactMidBytesLength = sizeof(double);//flatBytesLength - 1 - 1 - MetaDataByteLength -exe_params->SZ_SIZE_TYPE;
(*this)->exactMidBytes = &(flatBytes[index]);
return errorBoundMode;
}
@ -76,7 +75,7 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi
int isRegression = (sameRByte >> 7) & 0x01;
if(isRegression == 1)
{
(*this)->raBytes_size = flatBytesLength - 1 - 1 - MetaDataByteLength - exe_params->SZ_SIZE_TYPE;
(*this)->raBytes_size = flatBytesLength - 1 - 1 - MetaDataByteLength - pde_exe->SZ_SIZE_TYPE;
(*this)->raBytes = &(flatBytes[index]);
return errorBoundMode;
}
@ -101,17 +100,17 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi
byteBuf[i] = flatBytes[index++];
(*this)->realPrecision = bytesToDouble(byteBuf);//8
// 10 typeArray_size
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++)
byteBuf[i] = flatBytes[index++];
(*this)->typeArray_size = bytesToSize(byteBuf);// 4
(*this)->typeArray_size = bytesToSize(byteBuf, pde_exe->SZ_SIZE_TYPE);// 4
// 11 exactNum
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++)
byteBuf[i] = flatBytes[index++];
(*this)->exactDataNum = bytesToSize(byteBuf);// ST
(*this)->exactDataNum = bytesToSize(byteBuf, pde_exe->SZ_SIZE_TYPE);// ST
// 12 mid size
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++)
byteBuf[i] = flatBytes[index++];
(*this)->exactMidBytes_size = bytesToSize(byteBuf);// ST
(*this)->exactMidBytes_size = bytesToSize(byteBuf, pde_exe->SZ_SIZE_TYPE);// STqq
// calc leadNumArray_size
size_t logicLeadNumBitsNum = (*this)->exactDataNum * 2;
@ -141,8 +140,8 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi
(*this)->residualMidBits = &flatBytes[index];
// calc residualMidBits_size
(*this)->residualMidBits_size = flatBytesLength - 1 - 1 - MetaDataByteLength - exe_params->SZ_SIZE_TYPE - 4 - 4 - 4 - 1 - 8
- exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE
(*this)->residualMidBits_size = flatBytesLength - 1 - 1 - MetaDataByteLength - pde_exe->SZ_SIZE_TYPE - 4 - 4 - 4 - 1 - 8
- pde_exe->SZ_SIZE_TYPE - pde_exe->SZ_SIZE_TYPE - pde_exe->SZ_SIZE_TYPE
- (*this)->leadNumArray_size - (*this)->exactMidBytes_size - (*this)->typeArray_size;
@ -211,7 +210,7 @@ void convertTDPStoBytes_float(TightDataPointStorageF* tdps, unsigned char* bytes
// 2 same
bytes[k++] = sameByte; //1 byte
// 3 meta
convertSZParamsToBytes(confparams_cpr, &(bytes[k]));
convertSZParamsToBytes(confparams_cpr, &(bytes[k]), exe_params->optQuantMode);
k = k + MetaDataByteLength;
// 4 element count
for(i = 0; i < exe_params->SZ_SIZE_TYPE; i++)//ST: 4 or 8 bytes
@ -235,15 +234,15 @@ void convertTDPStoBytes_float(TightDataPointStorageF* tdps, unsigned char* bytes
for (i = 0; i < 8; i++)// 8
bytes[k++] = realPrecisionBytes[i];
// 10 typeArray size
sizeToBytes(typeArrayLengthBytes, tdps->typeArray_size);
sizeToBytes(typeArrayLengthBytes, tdps->typeArray_size, exe_params->SZ_SIZE_TYPE);
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
bytes[k++] = typeArrayLengthBytes[i];
// 11 exactDataNum leadNum calc by this , so not save leadNum
sizeToBytes(exactLengthBytes, tdps->exactDataNum);
sizeToBytes(exactLengthBytes, tdps->exactDataNum, exe_params->SZ_SIZE_TYPE);
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
bytes[k++] = exactLengthBytes[i];
// 12 Mid size
sizeToBytes(exactMidBytesLength, tdps->exactMidBytes_size);
sizeToBytes(exactMidBytesLength, tdps->exactMidBytes_size, exe_params->SZ_SIZE_TYPE);
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
bytes[k++] = exactMidBytesLength[i];
// 13 typeArray
@ -306,7 +305,7 @@ bool convertTDPStoFlatBytes_float(TightDataPointStorageF *tdps, unsigned char* b
// 2 same flag 1 bytes
bytes[k++] = sameByte;
// 3 metaData 26 bytes
convertSZParamsToBytes(confparams_cpr, &(bytes[k]));
convertSZParamsToBytes(confparams_cpr, &(bytes[k]), exe_params->optQuantMode);
k = k + MetaDataByteLength;
// 4 data Length 4 or 8 bytes
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)

4
deps/SZ/sz/src/sz_double.c vendored
View File

@ -229,10 +229,10 @@ void SZ_compress_args_double_StoreOriData(double* oriData, size_t dataLength, un
else
newByteData[k++] = 80; //01010000: 01000000 indicates the SZ_SIZE_TYPE=8
convertSZParamsToBytes(confparams_cpr, &(newByteData[k]));
convertSZParamsToBytes(confparams_cpr, &(newByteData[k]), exe_params->optQuantMode);
k = k + MetaDataByteLength_double;
sizeToBytes(dsLengthBytes,dataLength);
sizeToBytes(dsLengthBytes,dataLength, exe_params->SZ_SIZE_TYPE);
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)//ST: 4 or 8
newByteData[k++] = dsLengthBytes[i];

8
deps/SZ/sz/src/szd_double.c vendored
View File

@ -25,7 +25,7 @@ int SZ_decompress_args_double(double* newData, size_t r1, unsigned char* cmpByte
//unsigned char* tmpBytes;
size_t targetUncompressSize = dataLength <<3; //i.e., *8
//tmpSize must be "much" smaller than dataLength
size_t i, tmpSize = 12+MetaDataByteLength_double+exe_params->SZ_SIZE_TYPE;
size_t i, tmpSize = 12+MetaDataByteLength_double+8;
unsigned char* szTmpBytes = NULL;
bool needFree = false;
@ -50,7 +50,7 @@ int SZ_decompress_args_double(double* newData, size_t r1, unsigned char* cmpByte
{
if(targetUncompressSize<MIN_ZLIB_DEC_ALLOMEM_BYTES) //Considering the minimum size
targetUncompressSize = MIN_ZLIB_DEC_ALLOMEM_BYTES;
tmpSize = sz_lossless_decompress(pde_params->losslessCompressor, cmpBytes, (unsigned long)cmpSize, &szTmpBytes, (unsigned long)targetUncompressSize+4+MetaDataByteLength_double+exe_params->SZ_SIZE_TYPE);
tmpSize = sz_lossless_decompress(pde_params->losslessCompressor, cmpBytes, (unsigned long)cmpSize, &szTmpBytes, (unsigned long)targetUncompressSize+4+MetaDataByteLength_double+8);
needFree = true;
}
}
@ -73,11 +73,11 @@ int SZ_decompress_args_double(double* newData, size_t r1, unsigned char* cmpByte
// *newData = (double*)malloc(doubleSize*dataLength); comment by tickduan
if(sysEndianType==BIG_ENDIAN_SYSTEM)
{
memcpy(newData, szTmpBytes+4+MetaDataByteLength_double+exe_params->SZ_SIZE_TYPE, dataLength*doubleSize);
memcpy(newData, szTmpBytes+4+MetaDataByteLength_double+pde_exe->SZ_SIZE_TYPE, dataLength*doubleSize);
}
else
{
unsigned char* p = szTmpBytes+4+MetaDataByteLength_double+exe_params->SZ_SIZE_TYPE;
unsigned char* p = szTmpBytes+4+MetaDataByteLength_double+pde_exe->SZ_SIZE_TYPE;
for(i=0;i<dataLength;i++,p+=doubleSize)
newData[i] = bytesToDouble(p);
}

6
deps/SZ/sz/src/szd_float.c vendored
View File

@ -57,7 +57,7 @@ int SZ_decompress_args_float(float* newData, size_t r1, unsigned char* cmpBytes,
{
if(targetUncompressSize<MIN_ZLIB_DEC_ALLOMEM_BYTES) //Considering the minimum size
targetUncompressSize = MIN_ZLIB_DEC_ALLOMEM_BYTES;
tmpSize = sz_lossless_decompress(pde_params->losslessCompressor, cmpBytes, (unsigned long)cmpSize, &szTmpBytes, (unsigned long)targetUncompressSize+4+MetaDataByteLength+exe_params->SZ_SIZE_TYPE);// (unsigned long)targetUncompressSize+8: consider the total length under lossless compression mode is actually 3+4+1+targetUncompressSize
tmpSize = sz_lossless_decompress(pde_params->losslessCompressor, cmpBytes, (unsigned long)cmpSize, &szTmpBytes, (unsigned long)targetUncompressSize+4+MetaDataByteLength+8);
needFree = true;
}
}
@ -81,11 +81,11 @@ int SZ_decompress_args_float(float* newData, size_t r1, unsigned char* cmpBytes,
//*newData = (float*)malloc(floatSize*dataLength); comment by tickduan
if(sysEndianType==BIG_ENDIAN_SYSTEM)
{
memcpy(newData, szTmpBytes+4+MetaDataByteLength+exe_params->SZ_SIZE_TYPE, dataLength*floatSize);
memcpy(newData, szTmpBytes+4+MetaDataByteLength+pde_exe->SZ_SIZE_TYPE, dataLength*floatSize);
}
else
{
unsigned char* p = szTmpBytes+4+MetaDataByteLength+exe_params->SZ_SIZE_TYPE;
unsigned char* p = szTmpBytes+4+MetaDataByteLength+pde_exe->SZ_SIZE_TYPE;
for(i=0;i<dataLength;i++,p+=floatSize)
newData[i] = bytesToFloat(p);
}