add TSZ and ADT-FSE algorithm
This commit is contained in:
lx1zhong
parent
5a556ef33a
commit
6da4088012
|
|
@ -83,6 +83,16 @@ ELSE ()
|
|||
SET(TAOS_LIB taos)
|
||||
ENDIF ()
|
||||
|
||||
# build TSZ by default
|
||||
IF ("${TSZ_ENABLED}" MATCHES "false")
|
||||
set(VAR_TSZ "" CACHE INTERNAL "global variant empty" )
|
||||
ELSE()
|
||||
# define add
|
||||
MESSAGE(STATUS "build with TSZ enabled")
|
||||
ADD_DEFINITIONS(-DTD_TSZ)
|
||||
set(VAR_TSZ "TSZ" CACHE INTERNAL "global variant tsz" )
|
||||
ENDIF()
|
||||
|
||||
IF (TD_WINDOWS)
|
||||
MESSAGE("${Yellow} set compiler flag for Windows! ${ColourReset}")
|
||||
SET(COMMON_FLAGS "/w /D_WIN32 /DWIN32 /Zi /MTd")
|
||||
|
|
|
|||
|
|
@ -163,6 +163,7 @@ extern double tsFPrecision;
|
|||
extern double tsDPrecision;
|
||||
extern uint32_t tsMaxRange;
|
||||
extern uint32_t tsCurRange;
|
||||
extern bool tsIfAdtFse;
|
||||
extern char tsCompressor[];
|
||||
|
||||
// tfs
|
||||
|
|
|
|||
|
|
@ -0,0 +1,51 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
#ifndef _TD_SZ_H
|
||||
#define _TD_SZ_H
|
||||
#include "defines.h"
|
||||
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
void cost_start();
|
||||
double cost_end(const char* tag);
|
||||
|
||||
//
|
||||
// Init success return 1 else 0
|
||||
//
|
||||
void tdszInit(double fPrecision, double dPrecision, unsigned int maxIntervals, unsigned int intervals, int ifAdtFse, const char* compressor);
|
||||
|
||||
//
|
||||
// compress interface to tdengine return value is count of output with bytes
|
||||
//
|
||||
int tdszCompress(int type, const char * input, const int nelements, const char * output);
|
||||
|
||||
//
|
||||
// decompress interface to tdengine return value is count of output with bytes
|
||||
//
|
||||
int tdszDecompress(int type, const char * input, int compressedSize, const int nelements, const char * output);
|
||||
|
||||
//
|
||||
// Exit call
|
||||
//
|
||||
void tdszExit();
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ----- #ifndef _SZ_H ----- */
|
||||
|
|
@ -57,6 +57,11 @@ extern bool lossyDouble;
|
|||
int32_t tsCompressInit();
|
||||
void tsCompressExit();
|
||||
|
||||
int32_t tsCompressFloatLossyImp(const char *const input, const int32_t nelements, char *const output);
|
||||
int32_t tsDecompressFloatLossyImp(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output);
|
||||
int32_t tsCompressDoubleLossyImp(const char *const input, const int32_t nelements, char *const output);
|
||||
int32_t tsDecompressDoubleLossyImp(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output);
|
||||
|
||||
static FORCE_INLINE int32_t tsCompressFloatLossy(const char *const input, int32_t inputSize, const int32_t nelements,
|
||||
char *const output, int32_t outputSize, char algorithm,
|
||||
char *const buffer, int32_t bufferSize) {
|
||||
|
|
|
|||
|
|
@ -43,6 +43,7 @@ typedef enum {
|
|||
CFG_DTYPE_INT32,
|
||||
CFG_DTYPE_INT64,
|
||||
CFG_DTYPE_FLOAT,
|
||||
CFG_DTYPE_DOUBLE,
|
||||
CFG_DTYPE_STRING,
|
||||
CFG_DTYPE_DIR,
|
||||
CFG_DTYPE_LOCALE,
|
||||
|
|
@ -64,6 +65,7 @@ typedef struct SConfigItem {
|
|||
union {
|
||||
bool bval;
|
||||
float fval;
|
||||
double dval;
|
||||
int32_t i32;
|
||||
int64_t i64;
|
||||
char *str;
|
||||
|
|
@ -102,6 +104,7 @@ int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal, int8_t scop
|
|||
int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, int8_t scope);
|
||||
int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval, int8_t scope);
|
||||
int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, double minval, double maxval, int8_t scope);
|
||||
int32_t cfgAddDouble(SConfig *pCfg, const char *name, double defaultVal, double minval, double maxval, int8_t scope);
|
||||
int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope);
|
||||
int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope);
|
||||
int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope);
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@
|
|||
#include "tgrant.h"
|
||||
#include "tlog.h"
|
||||
#include "tmisce.h"
|
||||
#include "defines.h"
|
||||
|
||||
#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL)
|
||||
#include "cus_name.h"
|
||||
|
|
@ -211,14 +212,15 @@ int64_t tsTickPerMin[] = {60000L, 60000000L, 60000000000L};
|
|||
*/
|
||||
int64_t tsTickPerHour[] = {3600000L, 3600000000L, 3600000000000L};
|
||||
|
||||
// lossy compress 6
|
||||
// lossy compress 7
|
||||
char tsLossyColumns[32] = ""; // "float|double" means all float and double columns can be lossy compressed. set empty
|
||||
// can close lossy compress.
|
||||
// below option can take effect when tsLossyColumns not empty
|
||||
double tsFPrecision = 1E-8; // float column precision
|
||||
double tsDPrecision = 1E-16; // double column precision
|
||||
uint32_t tsMaxRange = 500; // max range
|
||||
uint32_t tsCurRange = 100; // range
|
||||
uint32_t tsMaxRange = 500; // max quantization intervals
|
||||
uint32_t tsCurRange = 100; // current quantization intervals
|
||||
bool tsIfAdtFse = true; // ADT-FSE algorithom or original huffman algorithom
|
||||
char tsCompressor[32] = "ZSTD_COMPRESSOR"; // ZSTD_COMPRESSOR or GZIP_COMPRESSOR
|
||||
|
||||
// udf
|
||||
|
|
@ -646,6 +648,14 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
|
|||
if (cfgAddInt32(pCfg, "cacheLazyLoadThreshold", tsCacheLazyLoadThreshold, 0, 100000, CFG_SCOPE_SERVER) != 0)
|
||||
return -1;
|
||||
|
||||
if (cfgAddString(pCfg, "LossyColumns", tsLossyColumns, CFG_SCOPE_SERVER) != 0) return -1;
|
||||
if (cfgAddDouble(pCfg, "FPrecision", tsFPrecision, 0.0f, 1000000.0f, CFG_SCOPE_SERVER) != 0) return -1;
|
||||
if (cfgAddDouble(pCfg, "DPrecision", tsDPrecision, 0.0f, 1000000.0f, CFG_SCOPE_SERVER) != 0) return -1;
|
||||
if (cfgAddInt32(pCfg, "MaxRange", tsMaxRange, 0, 65536, CFG_SCOPE_SERVER) != 0) return -1;
|
||||
if (cfgAddInt32(pCfg, "CurRange", tsCurRange, 0, 65536, CFG_SCOPE_SERVER) != 0) return -1;
|
||||
if (cfgAddBool(pCfg, "IfAdtFse", tsIfAdtFse, CFG_SCOPE_SERVER) != 0) return -1;
|
||||
if (cfgAddString(pCfg, "Compressor", tsCompressor, CFG_SCOPE_SERVER) != 0) return -1;
|
||||
|
||||
if (cfgAddBool(pCfg, "filterScalarMode", tsFilterScalarMode, CFG_SCOPE_SERVER) != 0) return -1;
|
||||
if (cfgAddInt32(pCfg, "keepTimeOffset", tsKeepTimeOffset, 0, 23, CFG_SCOPE_SERVER) != 0) return -1;
|
||||
if (cfgAddInt32(pCfg, "maxStreamBackendCache", tsMaxStreamBackendCache, 16, 1024, CFG_SCOPE_SERVER) != 0) return -1;
|
||||
|
|
@ -1060,6 +1070,15 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
|
|||
|
||||
tsCacheLazyLoadThreshold = cfgGetItem(pCfg, "cacheLazyLoadThreshold")->i32;
|
||||
|
||||
tstrncpy(tsLossyColumns, cfgGetItem(pCfg, "LossyColumns")->str, sizeof(tsLossyColumns));
|
||||
tsFPrecision = cfgGetItem(pCfg, "FPrecision")->fval;
|
||||
tsDPrecision = cfgGetItem(pCfg, "DPrecision")->dval;
|
||||
tsMaxRange = cfgGetItem(pCfg, "MaxRange")->i32;
|
||||
tsCurRange = cfgGetItem(pCfg, "CurRange")->i32;
|
||||
tsIfAdtFse = cfgGetItem(pCfg, "IfAdtFse")->bval;
|
||||
tstrncpy(tsCompressor, cfgGetItem(pCfg, "Compressor")->str, sizeof(tsCompressor));
|
||||
|
||||
|
||||
tsDisableStream = cfgGetItem(pCfg, "disableStream")->bval;
|
||||
tsStreamBufferSize = cfgGetItem(pCfg, "streamBufferSize")->i64;
|
||||
|
||||
|
|
|
|||
|
|
@ -111,6 +111,11 @@ int32_t dmInitDnode(SDnode *pDnode) {
|
|||
goto _OVER;
|
||||
}
|
||||
|
||||
#ifdef TD_TSZ
|
||||
// compress module init
|
||||
tsCompressInit();
|
||||
#endif
|
||||
|
||||
pDnode->wrappers[DNODE].func = dmGetMgmtFunc();
|
||||
pDnode->wrappers[MNODE].func = mmGetMgmtFunc();
|
||||
pDnode->wrappers[VNODE].func = vmGetMgmtFunc();
|
||||
|
|
@ -180,6 +185,12 @@ void dmCleanupDnode(SDnode *pDnode) {
|
|||
streamMetaCleanup();
|
||||
indexCleanup();
|
||||
taosConvDestroy();
|
||||
|
||||
#ifdef TD_TSZ
|
||||
// compress destroy
|
||||
tsCompressExit();
|
||||
#endif
|
||||
|
||||
dDebug("dnode is closed, ptr:%p", pDnode);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -18,12 +18,13 @@ target_include_directories(
|
|||
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
|
||||
PRIVATE "${TD_SOURCE_DIR}/include/common"
|
||||
PRIVATE "${GRANT_CFG_INCLUDE_DIR}"
|
||||
PRIVATE "${TD_SOURCE_DIR}/utils/TSZ/sz/inc"
|
||||
)
|
||||
target_link_libraries(
|
||||
util
|
||||
PUBLIC os
|
||||
PUBLIC lz4_static
|
||||
PUBLIC api cjson geos_c
|
||||
PUBLIC api cjson geos_c TSZ
|
||||
)
|
||||
|
||||
if(${BUILD_TEST})
|
||||
|
|
|
|||
|
|
@ -52,6 +52,7 @@
|
|||
#include "lz4.h"
|
||||
#include "tRealloc.h"
|
||||
#include "tlog.h"
|
||||
#include "tglobal.h"
|
||||
|
||||
#ifdef TD_TSZ
|
||||
#include "td_sz.h"
|
||||
|
|
@ -72,18 +73,18 @@ bool lossyDouble = false;
|
|||
// init call
|
||||
int32_t tsCompressInit() {
|
||||
// config
|
||||
if (lossyColumns[0] == 0) {
|
||||
if (tsLossyColumns[0] == 0) {
|
||||
lossyFloat = false;
|
||||
lossyDouble = false;
|
||||
return 0;
|
||||
}
|
||||
|
||||
lossyFloat = strstr(lossyColumns, "float") != NULL;
|
||||
lossyDouble = strstr(lossyColumns, "double") != NULL;
|
||||
lossyFloat = strstr(tsLossyColumns, "float") != NULL;
|
||||
lossyDouble = strstr(tsLossyColumns, "double") != NULL;
|
||||
|
||||
if (lossyFloat == false && lossyDouble == false) return 0;
|
||||
|
||||
tdszInit(fPrecision, dPrecision, maxRange, curRange, Compressor);
|
||||
tdszInit(tsFPrecision, tsDPrecision, tsMaxRange, tsCurRange, (int)tsIfAdtFse, tsCompressor);
|
||||
if (lossyFloat) uTrace("lossy compression float is opened. ");
|
||||
if (lossyDouble) uTrace("lossy compression double is opened. ");
|
||||
return 1;
|
||||
|
|
@ -2377,7 +2378,7 @@ int32_t tsCompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_
|
|||
int32_t tsDecompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
|
||||
int32_t nBuf) {
|
||||
#ifdef TD_TSZ
|
||||
if (HEAD_ALGO(pIn[0]) == ALGO_SZ_LOSSY) {
|
||||
if (HEAD_ALGO(((uint8_t *)pIn)[0]) == ALGO_SZ_LOSSY) {
|
||||
// decompress lossy
|
||||
return tsDecompressFloatLossyImp(pIn, nIn, nEle, pOut);
|
||||
} else {
|
||||
|
|
@ -2424,7 +2425,7 @@ int32_t tsCompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32
|
|||
int32_t tsDecompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
|
||||
int32_t nBuf) {
|
||||
#ifdef TD_TSZ
|
||||
if (HEAD_ALGO(input[0]) == ALGO_SZ_LOSSY) {
|
||||
if (HEAD_ALGO(((uint8_t *)pIn)[0]) == ALGO_SZ_LOSSY) {
|
||||
// decompress lossy
|
||||
return tsDecompressDoubleLossyImp(pIn, nIn, nEle, pOut);
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -415,6 +415,16 @@ int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, double mi
|
|||
return cfgAddItem(pCfg, &item, name);
|
||||
}
|
||||
|
||||
int32_t cfgAddDouble(SConfig *pCfg, const char *name, double defaultVal, double minval, double maxval, int8_t scope) {
|
||||
if (defaultVal < minval || defaultVal > maxval) {
|
||||
terrno = TSDB_CODE_OUT_OF_RANGE;
|
||||
return -1;
|
||||
}
|
||||
|
||||
SConfigItem item = {.dtype = CFG_DTYPE_DOUBLE, .fval = defaultVal, .fmin = minval, .fmax = maxval, .scope = scope};
|
||||
return cfgAddItem(pCfg, &item, name);
|
||||
}
|
||||
|
||||
int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope) {
|
||||
SConfigItem item = {.dtype = CFG_DTYPE_STRING, .scope = scope};
|
||||
item.str = taosStrdup(defaultVal);
|
||||
|
|
|
|||
|
|
@ -2,3 +2,7 @@
|
|||
ADD_SUBDIRECTORY(tsim)
|
||||
ADD_SUBDIRECTORY(test/c)
|
||||
#ADD_SUBDIRECTORY(comparisonTest/tdengine)
|
||||
|
||||
IF (NOT "${TSZ_ENABLED}" MATCHES "false")
|
||||
ADD_SUBDIRECTORY(TSZ)
|
||||
ENDIF()
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
CMAKE_MINIMUM_REQUIRED(VERSION 3.0)
|
||||
PROJECT(TDengine)
|
||||
|
||||
# include
|
||||
INCLUDE_DIRECTORIES(sz/inc)
|
||||
INCLUDE_DIRECTORIES(zstd/)
|
||||
INCLUDE_DIRECTORIES(zstd/common/)
|
||||
|
||||
|
||||
# source
|
||||
AUX_SOURCE_DIRECTORY(sz/src SRC1)
|
||||
AUX_SOURCE_DIRECTORY(zstd/dictBuilder SRC2)
|
||||
AUX_SOURCE_DIRECTORY(zstd/common SRC3)
|
||||
AUX_SOURCE_DIRECTORY(zstd/compress SRC4)
|
||||
AUX_SOURCE_DIRECTORY(zstd/decompress SRC5)
|
||||
AUX_SOURCE_DIRECTORY(zstd/deprecated SRC6)
|
||||
AUX_SOURCE_DIRECTORY(zstd/legacy SRC7)
|
||||
|
||||
|
||||
# archive
|
||||
ADD_LIBRARY(TSZ STATIC ${SRC1} ${SRC2} ${SRC3} ${SRC4} ${SRC5} ${SRC6} ${SRC7})
|
||||
TARGET_INCLUDE_DIRECTORIES(TSZ PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/sz/inc ${TD_SOURCE_DIR}/include)
|
||||
|
||||
# windows ignore warning
|
||||
IF (TD_WINDOWS)
|
||||
SET_TARGET_PROPERTIES(TSZ PROPERTIES COMPILE_FLAGS -w)
|
||||
ENDIF ()
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
BSD 3-Clause License
|
||||
|
||||
Copyright (c) 2021, taosdata
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
1. Redistributions of source code must retain the above copyright notice, this
|
||||
list of conditions and the following disclaimer.
|
||||
|
||||
2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
|
||||
3. Neither the name of the copyright holder nor the names of its
|
||||
contributors may be used to endorse or promote products derived from
|
||||
this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
# TSZ
|
||||
Error-bounded Lossy Data Compressor For Float/Double
|
||||
|
||||
TSZ algorithm is come from SZ algorithm, Github url is https://github.com/szcompressor.
|
||||
|
||||
Bellow is aspect of improvement :
|
||||
1) Better speed and size
|
||||
|
||||
SZ head size about 24 bytes, we are reduced to 2 bytes.
|
||||
we delete some no use code and some unnecessary function could be droped.
|
||||
2) Support multi-threads, interface is thread-safety.
|
||||
3) Remove 2D 3D 4D 5D function, only 1D be remained.
|
||||
4) Remove int8 int16 int32 and other datatype, only float double be remained.
|
||||
5) Optimize code speed
|
||||
6) Other optimize...
|
||||
|
||||
After modify, TSZ become faster、smaller and independent. TSZ more suitable for small block data compression.
|
||||
|
||||
## ADT-FSE
|
||||
ADT-FSE is an optimization algorithm of TSZ based on timeseries database scenario, which can provide higher compression rate and faster compression and decompression speed for small files.
|
||||
|
||||
Usage:
|
||||
By default, ADT-FSE algorithm is enabled. Control it by `ifAdtFse` configuration.
|
||||
|
||||
Citation:
|
||||
```
|
||||
@inproceedings{
|
||||
adtfse_sc2023,
|
||||
author = {Tao Lu, Yu Zhong, Zibin Sun, Xiang Chen, You Zhou, Fei Wu, Ying Yang, Yunxin Huang, and Yafei Yang},
|
||||
title = {ADT-FSE: A New Encoder for SZ},
|
||||
year = {2023},
|
||||
publisher = {IEEE Press},
|
||||
booktitle = {Proceedings of the International Conference on High Performance Computing, Networking, Storage and Analysis},
|
||||
location = {Denver, Colorado},
|
||||
series = {SC'23}
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
/**
|
||||
* @file ByteToolkit.h
|
||||
* @author Sheng Di
|
||||
* @date July, 2017
|
||||
* @brief Header file for the ByteToolkit.c.
|
||||
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
|
||||
* See COPYRIGHT in top-level directory.
|
||||
*/
|
||||
|
||||
#ifndef _ByteToolkit_H
|
||||
#define _ByteToolkit_H
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include <stdio.h>
|
||||
|
||||
int bytesToInt_bigEndian(unsigned char* bytes);
|
||||
void intToBytes_bigEndian(unsigned char *b, unsigned int num);
|
||||
|
||||
long bytesToLong_bigEndian(unsigned char* b);
|
||||
void longToBytes_bigEndian(unsigned char *b, long num);
|
||||
|
||||
short getExponent_float(float value);
|
||||
short getPrecisionReqLength_float(float precision);
|
||||
short getExponent_double(double value);
|
||||
short getPrecisionReqLength_double(double precision);
|
||||
|
||||
float bytesToFloat(unsigned char* bytes);
|
||||
void floatToBytes(unsigned char *b, float num);
|
||||
double bytesToDouble(unsigned char* bytes);
|
||||
void doubleToBytes(unsigned char *b, double num);
|
||||
int getMaskRightCode(int m);
|
||||
int getLeftMovingCode(int kMod8);
|
||||
int getRightMovingSteps(int kMod8, int resiBitLength);
|
||||
int getRightMovingCode(int kMod8, int resiBitLength);
|
||||
|
||||
size_t bytesToSize(unsigned char* bytes, int size_type);
|
||||
void sizeToBytes(unsigned char* outBytes, size_t size, int size_type);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ----- #ifndef _ByteToolkit_H ----- */
|
||||
|
||||
|
|
@ -0,0 +1,75 @@
|
|||
/**
|
||||
* @file CompressElement.h
|
||||
* @author Sheng Di
|
||||
* @date April, 2016
|
||||
* @brief Header file for Compress Elements such as DoubleCompressELement.
|
||||
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
|
||||
* See COPYRIGHT in top-level directory.
|
||||
*/
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#ifndef _CompressElement_H
|
||||
#define _CompressElement_H
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
typedef struct DoubleValueCompressElement
|
||||
{
|
||||
double data;
|
||||
long curValue;
|
||||
unsigned char curBytes[8]; //big_endian
|
||||
int reqBytesLength;
|
||||
int resiBitsLength;
|
||||
} DoubleValueCompressElement;
|
||||
|
||||
typedef struct FloatValueCompressElement
|
||||
{
|
||||
float data; // diffValue + medianValue
|
||||
int curValue; // diff int value
|
||||
unsigned char curBytes[4]; // dif bytes value diffValue->iValue big_endian
|
||||
int reqBytesLength;
|
||||
int resiBitsLength;
|
||||
} FloatValueCompressElement;
|
||||
|
||||
typedef struct LossyCompressionElement
|
||||
{
|
||||
int leadingZeroBytes; //0,1,2,or 3
|
||||
unsigned char integerMidBytes[8];
|
||||
int integerMidBytes_Length; //they are mid_bits actually
|
||||
//char curBytes[8];
|
||||
//int curBytes_Length; //4 for single_precision or 8 for double_precision
|
||||
int resMidBitsLength;
|
||||
int residualMidBits;
|
||||
} LossyCompressionElement;
|
||||
|
||||
|
||||
short computeGroupNum_float(float value);
|
||||
short computeGroupNum_double(double value);
|
||||
|
||||
void listAdd_double(double last3CmprsData[3], double value);
|
||||
void listAdd_float(float last3CmprsData[3], float value);
|
||||
void listAdd_int(int64_t last3CmprsData[3], int64_t value);
|
||||
void listAdd_int32(int32_t last3CmprsData[3], int32_t value);
|
||||
void listAdd_float_group(float *groups, int *flags, char groupNum, float oriValue, float decValue, char* curGroupID);
|
||||
void listAdd_double_group(double *groups, int *flags, char groupNum, double oriValue, double decValue, char* curGroupID);
|
||||
|
||||
int validPrediction_double(double minErr, double precision);
|
||||
int validPrediction_float(float minErr, float precision);
|
||||
double* generateGroupErrBounds(int errorBoundMode, double realPrecision, double pwrErrBound);
|
||||
int generateGroupMaxIntervalCount(double* groupErrBounds);
|
||||
|
||||
void new_LossyCompressionElement(LossyCompressionElement *lce, int leadingNum, unsigned char* intMidBytes,
|
||||
int intMidBytes_Length, int resiMidBitsLength, int resiBits);
|
||||
void updateLossyCompElement_Double(unsigned char* curBytes, unsigned char* preBytes,
|
||||
int reqBytesLength, int resiBitsLength, LossyCompressionElement *lce);
|
||||
void updateLossyCompElement_Float(unsigned char* curBytes, unsigned char* preBytes,
|
||||
int reqBytesLength, int resiBitsLength, LossyCompressionElement *lce);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ----- #ifndef _CompressElement_H ----- */
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
/**
|
||||
* @file DynamicByteArray.h
|
||||
* @author Sheng Di
|
||||
* @date April, 2016
|
||||
* @brief Header file for Dynamic Byte Array.
|
||||
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
|
||||
* See COPYRIGHT in top-level directory.
|
||||
*/
|
||||
|
||||
#ifndef _DynamicByteArray_H
|
||||
#define _DynamicByteArray_H
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include <stdio.h>
|
||||
typedef struct DynamicByteArray
|
||||
{
|
||||
unsigned char* array;
|
||||
size_t size;
|
||||
size_t capacity;
|
||||
} DynamicByteArray;
|
||||
|
||||
void new_DBA(DynamicByteArray **dba, size_t cap);
|
||||
void convertDBAtoBytes(DynamicByteArray *dba, unsigned char** bytes);
|
||||
void free_DBA(DynamicByteArray *dba);
|
||||
unsigned char getDBA_Data(DynamicByteArray *dba, size_t pos);
|
||||
void addDBA_Data(DynamicByteArray *dba, unsigned char value);
|
||||
void memcpyDBA_Data(DynamicByteArray *dba, unsigned char* data, size_t length);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ----- #ifndef _DynamicByteArray_H ----- */
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
/**
|
||||
* @file DynamicIntArray.h
|
||||
* @author Sheng Di
|
||||
* @date April, 2016
|
||||
* @brief Header file for Dynamic Int Array.
|
||||
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
|
||||
* See COPYRIGHT in top-level directory.
|
||||
*/
|
||||
|
||||
#ifndef _DynamicIntArray_H
|
||||
#define _DynamicIntArray_H
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include <stdio.h>
|
||||
typedef struct DynamicIntArray
|
||||
{
|
||||
unsigned char* array; //char* (one byte) is enough, don't have to be int*
|
||||
size_t size;
|
||||
size_t capacity;
|
||||
} DynamicIntArray;
|
||||
|
||||
void new_DIA(DynamicIntArray **dia, size_t cap);
|
||||
void convertDIAtoInts(DynamicIntArray *dia, unsigned char **data);
|
||||
void free_DIA(DynamicIntArray *dia);
|
||||
int getDIA_Data(DynamicIntArray *dia, size_t pos);
|
||||
void addDIA_Data(DynamicIntArray *dia, int value);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ----- #ifndef _DynamicIntArray_H ----- */
|
||||
|
|
@ -0,0 +1,71 @@
|
|||
/**
|
||||
* @file Huffman.h
|
||||
* @author Sheng Di
|
||||
* @date Aug., 2016
|
||||
* @brief Header file for the exponential segment constructor.
|
||||
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
|
||||
* See COPYRIGHT in top-level directory.
|
||||
*/
|
||||
|
||||
#ifndef _Huffman_H
|
||||
#define _Huffman_H
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
//Note: when changing the following settings, intvCapacity in sz.h should be changed as well.
|
||||
//#define allNodes 131072
|
||||
//#define stateNum 65536
|
||||
|
||||
typedef struct node_t {
|
||||
struct node_t *left, *right;
|
||||
size_t freq;
|
||||
char t; //in_node:0; otherwise:1
|
||||
unsigned int c;
|
||||
} *node;
|
||||
|
||||
typedef struct HuffmanTree {
|
||||
unsigned int stateNum;
|
||||
unsigned int allNodes;
|
||||
struct node_t* pool;
|
||||
node *qqq, *qq; //the root node of the HuffmanTree is qq[1]
|
||||
int n_nodes; //n_nodes is for compression
|
||||
int qend;
|
||||
unsigned long **code;
|
||||
unsigned char *cout;
|
||||
int n_inode; //n_inode is for decompression
|
||||
int maxBitCount;
|
||||
} HuffmanTree;
|
||||
|
||||
HuffmanTree* createHuffmanTree(int stateNum);
|
||||
HuffmanTree* createDefaultHuffmanTree();
|
||||
|
||||
node new_node(HuffmanTree *huffmanTree, size_t freq, unsigned int c, node a, node b);
|
||||
node new_node2(HuffmanTree *huffmanTree, unsigned int c, unsigned char t);
|
||||
void qinsert(HuffmanTree *huffmanTree, node n);
|
||||
node qremove(HuffmanTree *huffmanTree);
|
||||
void build_code(HuffmanTree *huffmanTree, node n, int len, unsigned long out1, unsigned long out2);
|
||||
void init(HuffmanTree *huffmanTree, int *s, size_t length);
|
||||
void init_static(HuffmanTree *huffmanTree, int *s, size_t length);
|
||||
void encode(HuffmanTree *huffmanTree, int *s, size_t length, unsigned char *out, size_t *outSize);
|
||||
|
||||
void decode(unsigned char *s, size_t targetLength, node t, int *out);
|
||||
|
||||
void pad_tree_uchar(HuffmanTree* huffmanTree, unsigned char* L, unsigned char* R, unsigned int* C, unsigned char* t, unsigned int i, node root);
|
||||
void pad_tree_ushort(HuffmanTree* huffmanTree, unsigned short* L, unsigned short* R, unsigned int* C, unsigned char* t, unsigned int i, node root);
|
||||
void pad_tree_uint(HuffmanTree* huffmanTree, unsigned int* L, unsigned int* R, unsigned int* C, unsigned char* t, unsigned int i, node root);
|
||||
unsigned int convert_HuffTree_to_bytes_anyStates(HuffmanTree* huffmanTree, int nodeCount, unsigned char** out);
|
||||
void unpad_tree_uchar(HuffmanTree* huffmanTree, unsigned char* L, unsigned char* R, unsigned int* C, unsigned char *t, unsigned int i, node root);
|
||||
void unpad_tree_ushort(HuffmanTree* huffmanTree, unsigned short* L, unsigned short* R, unsigned int* C, unsigned char* t, unsigned int i, node root);
|
||||
void unpad_tree_uint(HuffmanTree* huffmanTree, unsigned int* L, unsigned int* R, unsigned int* C, unsigned char* t, unsigned int i, node root);
|
||||
node reconstruct_HuffTree_from_bytes_anyStates(HuffmanTree *huffmanTree, unsigned char* bytes, int nodeCount);
|
||||
void encode_withTree(HuffmanTree* huffmanTree, int *s, size_t length, unsigned char **out, size_t *outSize);
|
||||
void decode_withTree(HuffmanTree* huffmanTree, unsigned char *s, size_t targetLength, int *out);
|
||||
void SZ_ReleaseHuffman(HuffmanTree* huffmanTree);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
|
@ -0,0 +1,92 @@
|
|||
/**
|
||||
* @file TightDataPointStorageD.h
|
||||
* @author Sheng Di
|
||||
* @date April, 2016
|
||||
* @brief Header file for the tight data point storage (TDPS).
|
||||
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
|
||||
* See COPYRIGHT in top-level directory.
|
||||
*/
|
||||
|
||||
#ifndef _TightDataPointStorageD_H
|
||||
#define _TightDataPointStorageD_H
|
||||
|
||||
#include <stdbool.h>
|
||||
#include "pub.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
typedef struct TightDataPointStorageD
|
||||
{
|
||||
int ifAdtFse;
|
||||
size_t dataSeriesLength;
|
||||
int allSameData;
|
||||
double realPrecision;
|
||||
double medianValue;
|
||||
char reqLength;
|
||||
char radExpo; //used to compute reqLength based on segmented precisions in "pw_rel_compression"
|
||||
|
||||
double minLogValue;
|
||||
|
||||
int stateNum;
|
||||
int allNodes;
|
||||
|
||||
size_t exactDataNum;
|
||||
double reservedValue;
|
||||
|
||||
unsigned char* FseCode; // fse code of tp_code
|
||||
size_t FseCode_size;
|
||||
|
||||
unsigned char* transCodeBits; // extra bitstream of transcoding
|
||||
size_t transCodeBits_size;
|
||||
|
||||
unsigned char* typeArray; //its size is dataSeriesLength/4 (or xxx/4+1)
|
||||
size_t typeArray_size;
|
||||
|
||||
unsigned char* leadNumArray; //its size is exactDataNum/4 (or exactDataNum/4+1)
|
||||
size_t leadNumArray_size;
|
||||
|
||||
unsigned char* exactMidBytes;
|
||||
size_t exactMidBytes_size;
|
||||
|
||||
unsigned char* residualMidBits;
|
||||
size_t residualMidBits_size;
|
||||
|
||||
unsigned int intervals;
|
||||
|
||||
unsigned char isLossless; //a mark to denote whether it's lossless compression (1 is yes, 0 is no)
|
||||
|
||||
size_t segment_size;
|
||||
|
||||
unsigned char* raBytes;
|
||||
size_t raBytes_size;
|
||||
|
||||
unsigned char plus_bits;
|
||||
unsigned char max_bits;
|
||||
|
||||
} TightDataPointStorageD;
|
||||
|
||||
void new_TightDataPointStorageD_Empty(TightDataPointStorageD **self);
|
||||
int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **self, unsigned char* flatBytes, size_t flatBytesLength, sz_exedata* pde_exe, sz_params* pde_params);
|
||||
|
||||
void new_TightDataPointStorageD(TightDataPointStorageD **self,
|
||||
size_t dataSeriesLength, size_t exactDataNum,
|
||||
int* type, unsigned char* exactMidBytes, size_t exactMidBytes_size,
|
||||
unsigned char* leadNumIntArray, //leadNumIntArray contains readable numbers....
|
||||
unsigned char* resiMidBits, size_t resiMidBits_size,
|
||||
unsigned char resiBitLength,
|
||||
double realPrecision, double medianValue, char reqLength,
|
||||
unsigned int intervals, unsigned char radExpo);
|
||||
|
||||
void convertTDPStoBytes_double(TightDataPointStorageD* tdps, unsigned char* bytes, unsigned char* dsLengthBytes, unsigned char sameByte);
|
||||
bool convertTDPStoFlatBytes_double(TightDataPointStorageD *tdps, unsigned char* bytes, size_t *size);
|
||||
|
||||
void free_TightDataPointStorageD(TightDataPointStorageD *tdps);
|
||||
void free_TightDataPointStorageD2(TightDataPointStorageD *tdps);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ----- #ifndef _TightDataPointStorageD_H ----- */
|
||||
|
|
@ -0,0 +1,97 @@
|
|||
/**
|
||||
* @file TightDataPointStorageF.h
|
||||
* @author Sheng Di and Dingwen Tao
|
||||
* @date Aug, 2016
|
||||
* @brief Header file for the tight data point storage (TDPS).
|
||||
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
|
||||
* See COPYRIGHT in top-level directory.
|
||||
*/
|
||||
|
||||
#ifndef _TightDataPointStorageF_H
|
||||
#define _TightDataPointStorageF_H
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdbool.h>
|
||||
#include "pub.h"
|
||||
#include "bitstream.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
typedef struct TightDataPointStorageF
|
||||
{
|
||||
int ifAdtFse;
|
||||
size_t dataSeriesLength;
|
||||
int allSameData;
|
||||
double realPrecision; //it's used as the pwrErrBoundRatio when errBoundMode==PW_REL
|
||||
float medianValue;
|
||||
char reqLength;
|
||||
char radExpo; //used to compute reqLength based on segmented precisions in "pw_rel_compression"
|
||||
|
||||
int stateNum;
|
||||
int allNodes;
|
||||
|
||||
size_t exactDataNum;
|
||||
float reservedValue;
|
||||
|
||||
float minLogValue;
|
||||
|
||||
unsigned char* FseCode; // fse code of tp_code
|
||||
size_t FseCode_size;
|
||||
|
||||
unsigned char* transCodeBits; // extra bitstream of transcoding
|
||||
size_t transCodeBits_size;
|
||||
|
||||
unsigned char* typeArray; //its size is dataSeriesLength/4 (or xxx/4+1)
|
||||
size_t typeArray_size;
|
||||
|
||||
unsigned char* leadNumArray; //its size is exactDataNum/4 (or exactDataNum/4+1)
|
||||
size_t leadNumArray_size;
|
||||
|
||||
unsigned char* exactMidBytes;
|
||||
size_t exactMidBytes_size;
|
||||
|
||||
unsigned char* residualMidBits;
|
||||
size_t residualMidBits_size;
|
||||
|
||||
unsigned int intervals; //quantization_intervals
|
||||
|
||||
unsigned char isLossless; //a mark to denote whether it's lossless compression (1 is yes, 0 is no)
|
||||
|
||||
size_t segment_size;
|
||||
|
||||
|
||||
unsigned char* raBytes;
|
||||
size_t raBytes_size;
|
||||
|
||||
unsigned char plus_bits;
|
||||
unsigned char max_bits;
|
||||
|
||||
} TightDataPointStorageF;
|
||||
|
||||
void new_TightDataPointStorageF_Empty(TightDataPointStorageF **self);
|
||||
int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **self, unsigned char* flatBytes, size_t flatBytesLength, sz_exedata* pde_exe, sz_params* pde_params);
|
||||
|
||||
void new_TightDataPointStorageF(TightDataPointStorageF **self,
|
||||
size_t dataSeriesLength, size_t exactDataNum,
|
||||
int* type, unsigned char* exactMidBytes, size_t exactMidBytes_size,
|
||||
unsigned char* leadNumIntArray, //leadNumIntArray contains readable numbers....
|
||||
unsigned char* resiMidBits, size_t resiMidBits_size,
|
||||
unsigned char resiBitLength,
|
||||
double realPrecision, float medianValue, char reqLength, unsigned int intervals,
|
||||
unsigned char radExpo);
|
||||
|
||||
void convertTDPStoBytes_float(TightDataPointStorageF* tdps, unsigned char* bytes, unsigned char* dsLengthBytes, unsigned char sameByte);
|
||||
bool convertTDPStoFlatBytes_float(TightDataPointStorageF *tdps, unsigned char* bytes, size_t *size);
|
||||
|
||||
void free_TightDataPointStorageF(TightDataPointStorageF *tdps);
|
||||
void free_TightDataPointStorageF2(TightDataPointStorageF *tdps);
|
||||
|
||||
void printTDPS(TightDataPointStorageF *tdps);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ----- #ifndef _TightDataPointStorageF_H ----- */
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
/**
|
||||
* @file TypeManager.h
|
||||
* @author Sheng Di
|
||||
* @date July, 2017
|
||||
* @brief Header file for the TypeManager.c.
|
||||
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
|
||||
* See COPYRIGHT in top-level directory.
|
||||
*/
|
||||
|
||||
#ifndef _TypeManager_H
|
||||
#define _TypeManager_H
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdint.h>
|
||||
|
||||
//TypeManager.c
|
||||
void convertByteArray2IntArray_fast_2b(size_t stepLength, unsigned char* byteArray, size_t byteArrayLength, unsigned char **intArray);
|
||||
size_t convertIntArray2ByteArray_fast_2b(unsigned char* timeStepType, size_t timeStepTypeLength, unsigned char **result);
|
||||
|
||||
|
||||
int getLeftMovingSteps(size_t k, unsigned char resiBitLength);
|
||||
size_t convertIntArray2ByteArray_fast_dynamic(unsigned char* timeStepType, unsigned char resiBitLength, size_t nbEle, unsigned char **bytes);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ----- #ifndef _TypeManager_H ----- */
|
||||
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
/**
|
||||
* @file conf.h
|
||||
* @author Sheng Di
|
||||
* @date July, 2017
|
||||
* @brief Header file for the conf.c.
|
||||
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
|
||||
* See COPYRIGHT in top-level directory.
|
||||
*/
|
||||
|
||||
#ifndef _Conf_H
|
||||
#define _Conf_H
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include <stdio.h>
|
||||
|
||||
//
|
||||
// set default value
|
||||
//
|
||||
void setDefaulParams(sz_exedata* exedata, sz_params* params);
|
||||
|
||||
//conf.c
|
||||
void updateQuantizationInfo(int quant_intervals);
|
||||
int SZ_ReadConf(const char* sz_cfgFile);
|
||||
int SZ_LoadConf(const char* sz_cfgFile);
|
||||
|
||||
|
||||
unsigned int roundUpToPowerOf2(unsigned int base);
|
||||
double computeABSErrBoundFromPSNR(double psnr, double threshold, double value_range);
|
||||
double computeABSErrBoundFromNORM_ERR(double normErr, size_t nbEle);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ----- #ifndef _Conf_H ----- */
|
||||
|
||||
|
|
@ -0,0 +1,79 @@
|
|||
/**
|
||||
* @file dataCompression.h
|
||||
* @author Sheng Di
|
||||
* @date July, 2017
|
||||
* @brief Header file for the dataCompression.c.
|
||||
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
|
||||
* See COPYRIGHT in top-level directory.
|
||||
*/
|
||||
|
||||
#ifndef _DataCompression_H
|
||||
#define _DataCompression_H
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include "sz.h"
|
||||
#include <stdio.h>
|
||||
#include <stdbool.h>
|
||||
|
||||
#define computeMinMax(data) \
|
||||
for(i=1;i<size;i++)\
|
||||
{\
|
||||
data_ = data[i];\
|
||||
if(min>data_)\
|
||||
min = data_;\
|
||||
else if(max<data_)\
|
||||
max = data_;\
|
||||
}\
|
||||
|
||||
|
||||
//dataCompression.c
|
||||
double computeRangeSize_double(double* oriData, size_t size, double* valueRangeSize, double* medianValue);
|
||||
float computeRangeSize_float(float* oriData, size_t size, float* valueRangeSize, float* medianValue);
|
||||
|
||||
double min_d(double a, double b);
|
||||
double max_d(double a, double b);
|
||||
float min_f(float a, float b);
|
||||
float max_f(float a, float b);
|
||||
double getRealPrecision_double(double valueRangeSize, int errBoundMode, double absErrBound, double relBoundRatio, int *status);
|
||||
double getRealPrecision_float(float valueRangeSize, int errBoundMode, double absErrBound, double relBoundRatio, int *status);
|
||||
double getRealPrecision_int(long valueRangeSize, int errBoundMode, double absErrBound, double relBoundRatio, int *status);
|
||||
void symTransform_8bytes(unsigned char data[8]);
|
||||
void symTransform_2bytes(unsigned char data[2]);
|
||||
void symTransform_4bytes(unsigned char data[4]);
|
||||
|
||||
void compressSingleFloatValue(FloatValueCompressElement *vce, float tgtValue, float precision, float medianValue,
|
||||
int reqLength, int reqBytesLength, int resiBitsLength);
|
||||
void compressSingleDoubleValue(DoubleValueCompressElement *vce, double tgtValue, double precision, double medianValue,
|
||||
int reqLength, int reqBytesLength, int resiBitsLength);
|
||||
|
||||
int compIdenticalLeadingBytesCount_double(unsigned char* preBytes, unsigned char* curBytes);
|
||||
int compIdenticalLeadingBytesCount_float(unsigned char* preBytes, unsigned char* curBytes);
|
||||
void addExactData(DynamicByteArray *exactMidByteArray, DynamicIntArray *exactLeadNumArray,
|
||||
DynamicIntArray *resiBitArray, LossyCompressionElement *lce);
|
||||
|
||||
int getPredictionCoefficients(int layers, int dimension, int **coeff_array, int *status);
|
||||
|
||||
int computeBlockEdgeSize_3D(int segmentSize);
|
||||
int computeBlockEdgeSize_2D(int segmentSize);
|
||||
|
||||
int generateLossyCoefficients_float(float* oriData, double precision, size_t nbEle, int* reqBytesLength, int* resiBitsLength, float* medianValue, float* decData);
|
||||
int compressExactDataArray_float(float* oriData, double precision, size_t nbEle, unsigned char** leadArray, unsigned char** midArray, unsigned char** resiArray,
|
||||
int reqLength, int reqBytesLength, int resiBitsLength, float medianValue);
|
||||
|
||||
void decompressExactDataArray_float(unsigned char* leadNum, unsigned char* exactMidBytes, unsigned char* residualMidBits, size_t nbEle, int reqLength, float medianValue, float** decData);
|
||||
|
||||
int generateLossyCoefficients_double(double* oriData, double precision, size_t nbEle, int* reqBytesLength, int* resiBitsLength, double* medianValue, double* decData);
|
||||
int compressExactDataArray_double(double* oriData, double precision, size_t nbEle, unsigned char** leadArray, unsigned char** midArray, unsigned char** resiArray,
|
||||
int reqLength, int reqBytesLength, int resiBitsLength, double medianValue);
|
||||
|
||||
void decompressExactDataArray_double(unsigned char* leadNum, unsigned char* exactMidBytes, unsigned char* residualMidBits, size_t nbEle, int reqLength, double medianValue, double** decData);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ----- #ifndef _DataCompression_H ----- */
|
||||
|
||||
|
|
@ -0,0 +1,108 @@
|
|||
/**
|
||||
* @file defines.h
|
||||
* @author Sheng Di
|
||||
* @date July, 2019
|
||||
* @brief Header file for the dataCompression.c.
|
||||
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
|
||||
* See COPYRIGHT in top-level directory.
|
||||
*/
|
||||
|
||||
#ifndef _SZ_DEFINES_H
|
||||
#define _SZ_DEFINES_H
|
||||
|
||||
// define data format version
|
||||
#define DATA_FROMAT_VER1 1 // curretn version
|
||||
|
||||
|
||||
#define PASTRI 103
|
||||
// #define HZ 102 //deprecated
|
||||
#define SZ 101
|
||||
#define SZ_Transpose 104
|
||||
|
||||
//prediction mode of temporal dimension based compression
|
||||
#define SZ_PREVIOUS_VALUE_ESTIMATE 0
|
||||
|
||||
#define MIN_NUM_OF_ELEMENTS 10 //if the # elements <= 20, skip the compression
|
||||
|
||||
#define SZ_ABS 0
|
||||
#define REL 1
|
||||
#define VR_REL 1 //alternative name to REL
|
||||
#define ABS_AND_REL 2
|
||||
#define ABS_OR_REL 3
|
||||
#define PSNR 4
|
||||
#define NORM 5
|
||||
|
||||
#define PW_REL 10
|
||||
#define ABS_AND_PW_REL 11
|
||||
#define ABS_OR_PW_REL 12
|
||||
#define REL_AND_PW_REL 13
|
||||
#define REL_OR_PW_REL 14
|
||||
|
||||
#define SZ_FLOAT 0
|
||||
#define SZ_DOUBLE 1
|
||||
#define SZ_UINT8 2
|
||||
#define SZ_INT8 3
|
||||
#define SZ_UINT16 4
|
||||
#define SZ_INT16 5
|
||||
#define SZ_UINT32 6
|
||||
#define SZ_INT32 7
|
||||
#define SZ_UINT64 8
|
||||
#define SZ_INT64 9
|
||||
|
||||
#define LITTLE_ENDIAN_DATA 0 //refers to the endian type of the data read from the disk
|
||||
#define BIG_ENDIAN_DATA 1 //big_endian (ppc, max, etc.) ; little_endian (x86, x64, etc.)
|
||||
|
||||
#define LITTLE_ENDIAN_SYSTEM 0 //refers to the endian type of the system
|
||||
#define BIG_ENDIAN_SYSTEM 1
|
||||
|
||||
#define DynArrayInitLen 1024
|
||||
|
||||
#define MIN_ZLIB_DEC_ALLOMEM_BYTES 1000000
|
||||
|
||||
//#define maxRangeRadius 32768
|
||||
//#define maxRangeRadius 1048576//131072
|
||||
|
||||
#define SZ_BEST_SPEED 0
|
||||
#define SZ_BEST_COMPRESSION 1
|
||||
#define SZ_DEFAULT_COMPRESSION 2
|
||||
#define SZ_TEMPORAL_COMPRESSION 3
|
||||
|
||||
#define SZ_NO_REGRESSION 0
|
||||
#define SZ_WITH_LINEAR_REGRESSION 1
|
||||
|
||||
#define SZ_PWR_MIN_TYPE 0
|
||||
#define SZ_PWR_AVG_TYPE 1
|
||||
#define SZ_PWR_MAX_TYPE 2
|
||||
|
||||
#define SZ_FORCE_SNAPSHOT_COMPRESSION 0
|
||||
#define SZ_FORCE_TEMPORAL_COMPRESSION 1
|
||||
#define SZ_PERIO_TEMPORAL_COMPRESSION 2
|
||||
|
||||
//SUCCESS returning status
|
||||
#define SZ_SUCCESS 0 //successful
|
||||
#define SZ_FAILED -1 //Not successful
|
||||
#define SZ_FERR -2 //Failed to open input file
|
||||
#define SZ_TERR -3 //wrong data type (should be only float or double)
|
||||
#define SZ_DERR -4 //dimension error
|
||||
#define SZ_MERR -5 //sz_mode error
|
||||
#define SZ_BERR -6 //bound-mode error (should be only SZ_ABS, REL, ABS_AND_REL, ABS_OR_REL, or PW_REL)
|
||||
#define SZ_LITTER_ELEMENT -7
|
||||
#define SZ_ALGORITHM_ERR -8
|
||||
#define SZ_FORMAT_ERR -9
|
||||
|
||||
#define SZ_MAINTAIN_VAR_DATA 0
|
||||
#define SZ_DESTROY_WHOLE_VARSET 1
|
||||
|
||||
#define GROUP_COUNT 16 //2^{16}=65536
|
||||
|
||||
// metaData remove some by tickduan
|
||||
#define MetaDataByteLength 2 // original is 28 bytes
|
||||
#define MetaDataByteLength_double 2 // original is 36 bytes
|
||||
|
||||
#define numOfBufferedSteps 1 //the number of time steps in the buffer
|
||||
|
||||
|
||||
#define GZIP_COMPRESSOR 0 //i.e., ZLIB_COMPRSSOR
|
||||
#define ZSTD_COMPRESSOR 1
|
||||
|
||||
#endif /* _SZ_DEFINES_H */
|
||||
|
|
@ -0,0 +1,171 @@
|
|||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@file dictionary.h
|
||||
@author N. Devillard
|
||||
@brief Implements a dictionary for string variables.
|
||||
|
||||
This module implements a simple dictionary object, i.e. a list
|
||||
of string/string associations. This object is useful to store e.g.
|
||||
informations retrieved from a configuration file (ini files).
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
|
||||
#ifndef _DICTIONARY_H_
|
||||
#define _DICTIONARY_H_
|
||||
|
||||
/*---------------------------------------------------------------------------
|
||||
Includes
|
||||
---------------------------------------------------------------------------*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
/*---------------------------------------------------------------------------
|
||||
New types
|
||||
---------------------------------------------------------------------------*/
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Dictionary object
|
||||
|
||||
This object contains a list of string/string associations. Each
|
||||
association is identified by a unique string key. Looking up values
|
||||
in the dictionary is speeded up by the use of a (hopefully collision-free)
|
||||
hash function.
|
||||
*/
|
||||
/*-------------------------------------------------------------------------*/
|
||||
typedef struct _dictionary_ {
|
||||
int n ; /** Number of entries in dictionary */
|
||||
int size ; /** Storage size */
|
||||
char ** val ; /** List of string values */
|
||||
char ** key ; /** List of string keys */
|
||||
unsigned * hash ; /** List of hash values for keys */
|
||||
} dictionary ;
|
||||
|
||||
|
||||
/*---------------------------------------------------------------------------
|
||||
Function prototypes
|
||||
---------------------------------------------------------------------------*/
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Compute the hash key for a string.
|
||||
@param key Character string to use for key.
|
||||
@return 1 unsigned int on at least 32 bits.
|
||||
|
||||
This hash function has been taken from an Article in Dr Dobbs Journal.
|
||||
This is normally a collision-free function, distributing keys evenly.
|
||||
The key is stored anyway in the struct so that collision can be avoided
|
||||
by comparing the key itself in last resort.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
unsigned dictionary_hash(const char * key);
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Create a new dictionary object.
|
||||
@param size Optional initial size of the dictionary.
|
||||
@return 1 newly allocated dictionary objet.
|
||||
|
||||
This function allocates a new dictionary object of given size and returns
|
||||
it. If you do not know in advance (roughly) the number of entries in the
|
||||
dictionary, give size=0.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
dictionary * dictionary_new(int size);
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Delete a dictionary object
|
||||
@param d dictionary object to deallocate.
|
||||
@return void
|
||||
|
||||
Deallocate a dictionary object and all memory associated to it.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
void dictionary_del(dictionary * vd);
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Get a value from a dictionary.
|
||||
@param d dictionary object to search.
|
||||
@param key Key to look for in the dictionary.
|
||||
@param def Default value to return if key not found.
|
||||
@return 1 pointer to internally allocated character string.
|
||||
|
||||
This function locates a key in a dictionary and returns a pointer to its
|
||||
value, or the passed 'def' pointer if no such key can be found in
|
||||
dictionary. The returned character pointer points to data internal to the
|
||||
dictionary object, you should not try to free it or modify it.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
char * dictionary_get(dictionary * d, const char * key, char * def);
|
||||
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Set a value in a dictionary.
|
||||
@param d dictionary object to modify.
|
||||
@param key Key to modify or add.
|
||||
@param val Value to add.
|
||||
@return int 0 if Ok, anything else otherwise
|
||||
|
||||
If the given key is found in the dictionary, the associated value is
|
||||
replaced by the provided one. If the key cannot be found in the
|
||||
dictionary, it is added to it.
|
||||
|
||||
It is Ok to provide a NULL value for val, but NULL values for the dictionary
|
||||
or the key are considered as errors: the function will return immediately
|
||||
in such a case.
|
||||
|
||||
Notice that if you dictionary_set a variable to NULL, a call to
|
||||
dictionary_get will return a NULL value: the variable will be found, and
|
||||
its value (NULL) is returned. In other words, setting the variable
|
||||
content to NULL is equivalent to deleting the variable from the
|
||||
dictionary. It is not possible (in this implementation) to have a key in
|
||||
the dictionary without value.
|
||||
|
||||
This function returns non-zero in case of failure.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
int dictionary_set(dictionary * vd, const char * key, const char * val);
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Delete a key in a dictionary
|
||||
@param d dictionary object to modify.
|
||||
@param key Key to remove.
|
||||
@return void
|
||||
|
||||
This function deletes a key in a dictionary. Nothing is done if the
|
||||
key cannot be found.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
void dictionary_unset(dictionary * d, const char * key);
|
||||
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Dump a dictionary to an opened file pointer.
|
||||
@param d Dictionary to dump
|
||||
@param f Opened file pointer.
|
||||
@return void
|
||||
|
||||
Dumps a dictionary onto an opened file pointer. Key pairs are printed out
|
||||
as @c [Key]=[Value], one per line. It is Ok to provide stdout or stderr as
|
||||
output file pointers.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
void dictionary_dump(dictionary * d, FILE * out);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
|
@ -0,0 +1,321 @@
|
|||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@file iniparser.h
|
||||
@author N. Devillard
|
||||
@brief Parser for ini files.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
|
||||
#ifndef _INIPARSER_H_
|
||||
#define _INIPARSER_H_
|
||||
|
||||
/*---------------------------------------------------------------------------
|
||||
Includes
|
||||
---------------------------------------------------------------------------*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
/*
|
||||
* The following #include is necessary on many Unixes but not Linux.
|
||||
* It is not needed for Windows platforms.
|
||||
* Uncomment it if needed.
|
||||
*/
|
||||
/* #include <unistd.h> */
|
||||
|
||||
#include "dictionary.h"
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Get number of sections in a dictionary
|
||||
@param d Dictionary to examine
|
||||
@return int Number of sections found in dictionary
|
||||
|
||||
This function returns the number of sections found in a dictionary.
|
||||
The test to recognize sections is done on the string stored in the
|
||||
dictionary: a section name is given as "section" whereas a key is
|
||||
stored as "section:key", thus the test looks for entries that do not
|
||||
contain a colon.
|
||||
|
||||
This clearly fails in the case a section name contains a colon, but
|
||||
this should simply be avoided.
|
||||
|
||||
This function returns -1 in case of error.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
|
||||
int iniparser_getnsec(dictionary * d);
|
||||
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Get name for section n in a dictionary.
|
||||
@param d Dictionary to examine
|
||||
@param n Section number (from 0 to nsec-1).
|
||||
@return Pointer to char string
|
||||
|
||||
This function locates the n-th section in a dictionary and returns
|
||||
its name as a pointer to a string statically allocated inside the
|
||||
dictionary. Do not free or modify the returned string!
|
||||
|
||||
This function returns NULL in case of error.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
|
||||
char * iniparser_getsecname(dictionary * d, int n);
|
||||
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Save a dictionary to a loadable ini file
|
||||
@param d Dictionary to dump
|
||||
@param f Opened file pointer to dump to
|
||||
@return void
|
||||
|
||||
This function dumps a given dictionary into a loadable ini file.
|
||||
It is Ok to specify @c stderr or @c stdout as output files.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
|
||||
void iniparser_dump_ini(dictionary * d, FILE * f);
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Save a dictionary section to a loadable ini file
|
||||
@param d Dictionary to dump
|
||||
@param s Section name of dictionary to dump
|
||||
@param f Opened file pointer to dump to
|
||||
@return void
|
||||
|
||||
This function dumps a given section of a given dictionary into a loadable ini
|
||||
file. It is Ok to specify @c stderr or @c stdout as output files.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
|
||||
void iniparser_dumpsection_ini(dictionary * d, char * s, FILE * f);
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Dump a dictionary to an opened file pointer.
|
||||
@param d Dictionary to dump.
|
||||
@param f Opened file pointer to dump to.
|
||||
@return void
|
||||
|
||||
This function prints out the contents of a dictionary, one element by
|
||||
line, onto the provided file pointer. It is OK to specify @c stderr
|
||||
or @c stdout as output files. This function is meant for debugging
|
||||
purposes mostly.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
void iniparser_dump(dictionary * d, FILE * f);
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Get the number of keys in a section of a dictionary.
|
||||
@param d Dictionary to examine
|
||||
@param s Section name of dictionary to examine
|
||||
@return Number of keys in section
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
int iniparser_getsecnkeys(dictionary * d, char * s);
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Get the number of keys in a section of a dictionary.
|
||||
@param d Dictionary to examine
|
||||
@param s Section name of dictionary to examine
|
||||
@return pointer to statically allocated character strings
|
||||
|
||||
This function queries a dictionary and finds all keys in a given section.
|
||||
Each pointer in the returned char pointer-to-pointer is pointing to
|
||||
a string allocated in the dictionary; do not free or modify them.
|
||||
|
||||
This function returns NULL in case of error.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
char ** iniparser_getseckeys(dictionary * d, char * s);
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Get the string associated to a key
|
||||
@param d Dictionary to search
|
||||
@param key Key string to look for
|
||||
@param def Default value to return if key not found.
|
||||
@return pointer to statically allocated character string
|
||||
|
||||
This function queries a dictionary for a key. A key as read from an
|
||||
ini file is given as "section:key". If the key cannot be found,
|
||||
the pointer passed as 'def' is returned.
|
||||
The returned char pointer is pointing to a string allocated in
|
||||
the dictionary, do not free or modify it.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
char * iniparser_getstring(dictionary * d, const char * key, char * def);
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Get the string associated to a key, convert to an int
|
||||
@param d Dictionary to search
|
||||
@param key Key string to look for
|
||||
@param notfound Value to return in case of error
|
||||
@return integer
|
||||
|
||||
This function queries a dictionary for a key. A key as read from an
|
||||
ini file is given as "section:key". If the key cannot be found,
|
||||
the notfound value is returned.
|
||||
|
||||
Supported values for integers include the usual C notation
|
||||
so decimal, octal (starting with 0) and hexadecimal (starting with 0x)
|
||||
are supported. Examples:
|
||||
|
||||
- "42" -> 42
|
||||
- "042" -> 34 (octal -> decimal)
|
||||
- "0x42" -> 66 (hexa -> decimal)
|
||||
|
||||
Warning: the conversion may overflow in various ways. Conversion is
|
||||
totally outsourced to strtol(), see the associated man page for overflow
|
||||
handling.
|
||||
|
||||
Credits: Thanks to A. Becker for suggesting strtol()
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
int iniparser_getint(dictionary * d, const char * key, int notfound);
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Get the string associated to a key, convert to a long
|
||||
@param d Dictionary to search
|
||||
@param key Key string to look for
|
||||
@param notfound Value to return in case of error
|
||||
@return long
|
||||
|
||||
Credits: This function bases completely on int iniparser_getint and was
|
||||
slightly modified to return long instead of int.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
long iniparser_getlint(dictionary * d, const char * key, int notfound);
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Get the string associated to a key, convert to a double
|
||||
@param d Dictionary to search
|
||||
@param key Key string to look for
|
||||
@param notfound Value to return in case of error
|
||||
@return double
|
||||
|
||||
This function queries a dictionary for a key. A key as read from an
|
||||
ini file is given as "section:key". If the key cannot be found,
|
||||
the notfound value is returned.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
double iniparser_getdouble(dictionary * d, const char * key, double notfound);
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Get the string associated to a key, convert to a boolean
|
||||
@param d Dictionary to search
|
||||
@param key Key string to look for
|
||||
@param notfound Value to return in case of error
|
||||
@return integer
|
||||
|
||||
This function queries a dictionary for a key. A key as read from an
|
||||
ini file is given as "section:key". If the key cannot be found,
|
||||
the notfound value is returned.
|
||||
|
||||
A true boolean is found if one of the following is matched:
|
||||
|
||||
- A string starting with 'y'
|
||||
- A string starting with 'Y'
|
||||
- A string starting with 't'
|
||||
- A string starting with 'T'
|
||||
- A string starting with '1'
|
||||
|
||||
A false boolean is found if one of the following is matched:
|
||||
|
||||
- A string starting with 'n'
|
||||
- A string starting with 'N'
|
||||
- A string starting with 'f'
|
||||
- A string starting with 'F'
|
||||
- A string starting with '0'
|
||||
|
||||
The notfound value returned if no boolean is identified, does not
|
||||
necessarily have to be 0 or 1.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
int iniparser_getboolean(dictionary * d, const char * key, int notfound);
|
||||
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Set an entry in a dictionary.
|
||||
@param ini Dictionary to modify.
|
||||
@param entry Entry to modify (entry name)
|
||||
@param val New value to associate to the entry.
|
||||
@return int 0 if Ok, -1 otherwise.
|
||||
|
||||
If the given entry can be found in the dictionary, it is modified to
|
||||
contain the provided value. If it cannot be found, -1 is returned.
|
||||
It is Ok to set val to NULL.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
int iniparser_set(dictionary * ini, const char * entry, const char * val);
|
||||
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Delete an entry in a dictionary
|
||||
@param ini Dictionary to modify
|
||||
@param entry Entry to delete (entry name)
|
||||
@return void
|
||||
|
||||
If the given entry can be found, it is deleted from the dictionary.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
void iniparser_unset(dictionary * ini, const char * entry);
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Finds out if a given entry exists in a dictionary
|
||||
@param ini Dictionary to search
|
||||
@param entry Name of the entry to look for
|
||||
@return integer 1 if entry exists, 0 otherwise
|
||||
|
||||
Finds out if a given entry exists in the dictionary. Since sections
|
||||
are stored as keys with NULL associated values, this is the only way
|
||||
of querying for the presence of sections in a dictionary.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
int iniparser_find_entry(dictionary * ini, const char * entry) ;
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Parse an ini file and return an allocated dictionary object
|
||||
@param ininame Name of the ini file to read.
|
||||
@return Pointer to newly allocated dictionary
|
||||
|
||||
This is the parser for ini files. This function is called, providing
|
||||
the name of the file to be read. It returns a dictionary object that
|
||||
should not be accessed directly, but through accessor functions
|
||||
instead.
|
||||
|
||||
The returned dictionary must be freed using iniparser_freedict().
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
dictionary * iniparser_load(const char * ininame);
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Free all memory associated to an ini dictionary
|
||||
@param d Dictionary to free
|
||||
@return void
|
||||
|
||||
Free all memory associated to an ini dictionary.
|
||||
It is mandatory to call this function before the dictionary object
|
||||
gets out of the current context.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
void iniparser_freedict(dictionary * d);
|
||||
|
||||
#endif
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
/**
|
||||
* @file sz.h
|
||||
* @author Sheng Di
|
||||
* @date April, 2015
|
||||
* @brief Header file for the whole compressor.
|
||||
* (C) 2015 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
|
||||
* See COPYRIGHT in top-level directory.
|
||||
*/
|
||||
|
||||
#ifndef _PUB_H
|
||||
#define _PUB_H
|
||||
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
/* array meta data and compression parameters for SZ_Init_Params() */
|
||||
typedef struct sz_params
|
||||
{
|
||||
int dataType;
|
||||
int ifAdtFse; // 0 (ADT-FSE algorithm) or 1 (original Huffman encoding)
|
||||
unsigned int max_quant_intervals; //max number of quantization intervals for quantization
|
||||
unsigned int quantization_intervals;
|
||||
unsigned int maxRangeRadius;
|
||||
int sol_ID;// it's SZ or SZ_Transpose, unless the setting is PASTRI compression mode (./configure --enable-pastri)
|
||||
int losslessCompressor;
|
||||
int sampleDistance; //2 bytes
|
||||
float predThreshold; // 2 bytes
|
||||
int szMode; //* 0 (best speed) or 1 (better compression with Zstd/Gzip) or 3 temporal-dimension based compression
|
||||
int errorBoundMode; //4bits (0.5byte), //SZ_ABS, REL, ABS_AND_REL, or ABS_OR_REL, PSNR, or PW_REL, PSNR
|
||||
double absErrBound; //absolute error bound for float
|
||||
double absErrBoundDouble; // for double
|
||||
double relBoundRatio; //value range based relative error bound ratio
|
||||
double psnr; //PSNR
|
||||
double normErr;
|
||||
double pw_relBoundRatio; //point-wise relative error bound
|
||||
int segment_size; //only used for 2D/3D data compression with pw_relBoundRatio (deprecated)
|
||||
int pwr_type; //only used for 2D/3D data compression with pw_relBoundRatio
|
||||
|
||||
int protectValueRange; //0 or 1
|
||||
float fmin, fmax;
|
||||
double dmin, dmax;
|
||||
|
||||
int snapshotCmprStep; //perform single-snapshot-based compression if time_step == snapshotCmprStep
|
||||
int predictionMode;
|
||||
|
||||
int accelerate_pw_rel_compression;
|
||||
int plus_bits;
|
||||
|
||||
int randomAccess;
|
||||
int withRegression;
|
||||
|
||||
} sz_params;
|
||||
|
||||
typedef struct sz_exedata
|
||||
{
|
||||
char optQuantMode; //opt Quantization (0: fixed ; 1: optimized)
|
||||
int intvCapacity; // the number of intervals for the linear-scaling quantization
|
||||
int intvRadius; // the number of intervals for the radius of the quantization range (intvRadius=intvCapacity/2)
|
||||
unsigned int SZ_SIZE_TYPE; //the length (# bytes) of the size_t in the system at runtime //4 or 8: sizeof(size_t)
|
||||
} sz_exedata;
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ----- #ifndef _PUB_H ----- */
|
||||
|
|
@ -0,0 +1,196 @@
|
|||
/**
|
||||
* @file sz.h
|
||||
* @author Sheng Di
|
||||
* @date April, 2015
|
||||
* @brief Header file for the whole compressor.
|
||||
* (C) 2015 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
|
||||
* See COPYRIGHT in top-level directory.
|
||||
*/
|
||||
|
||||
#ifndef _SZ_H
|
||||
#define _SZ_H
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdint.h>
|
||||
#include <stdbool.h>
|
||||
#include <time.h> /* For time(), in seconds */
|
||||
#include "pub.h"
|
||||
#include "CompressElement.h"
|
||||
#include "DynamicByteArray.h"
|
||||
#include "DynamicIntArray.h"
|
||||
#include "TightDataPointStorageD.h"
|
||||
#include "TightDataPointStorageF.h"
|
||||
#include "conf.h"
|
||||
#include "dataCompression.h"
|
||||
#include "ByteToolkit.h"
|
||||
#include "TypeManager.h"
|
||||
#include "sz_float.h"
|
||||
#include "sz_double.h"
|
||||
#include "szd_float.h"
|
||||
#include "szd_double.h"
|
||||
#include "utility.h"
|
||||
|
||||
#ifdef _WIN32
|
||||
#define PATH_SEPARATOR ';'
|
||||
#define INLINE
|
||||
#else
|
||||
#define PATH_SEPARATOR ':'
|
||||
#define INLINE inline
|
||||
#endif
|
||||
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
void cost_start();
|
||||
double cost_end(const char* tag);
|
||||
void show_rate( int in_len, int out_len);
|
||||
|
||||
//typedef char int8_t;
|
||||
//typedef unsigned char uint8_t;
|
||||
//typedef short int16_t;
|
||||
//typedef unsigned short uint16_t;
|
||||
//typedef int int32_t;
|
||||
//typedef unsigned int uint32_t;
|
||||
//typedef long int64_t;
|
||||
//typedef unsigned long uint64_t;
|
||||
|
||||
#include "defines.h"
|
||||
|
||||
//Note: the following setting should be consistent with stateNum in Huffman.h
|
||||
//#define intvCapacity 65536
|
||||
//#define intvRadius 32768
|
||||
//#define intvCapacity 131072
|
||||
//#define intvRadius 65536
|
||||
|
||||
#define SZ_COMPUTE_1D_NUMBER_OF_BLOCKS( COUNT, NUM_BLOCKS, BLOCK_SIZE ) \
|
||||
if (COUNT <= BLOCK_SIZE){ \
|
||||
NUM_BLOCKS = 1; \
|
||||
} \
|
||||
else{ \
|
||||
NUM_BLOCKS = COUNT / BLOCK_SIZE; \
|
||||
} \
|
||||
|
||||
#define SZ_COMPUTE_2D_NUMBER_OF_BLOCKS( COUNT, NUM_BLOCKS, BLOCK_SIZE ) \
|
||||
if (COUNT <= BLOCK_SIZE){ \
|
||||
NUM_BLOCKS = 1; \
|
||||
} \
|
||||
else{ \
|
||||
NUM_BLOCKS = COUNT / BLOCK_SIZE; \
|
||||
} \
|
||||
|
||||
#define SZ_COMPUTE_3D_NUMBER_OF_BLOCKS( COUNT, NUM_BLOCKS, BLOCK_SIZE ) \
|
||||
if (COUNT <= BLOCK_SIZE){ \
|
||||
NUM_BLOCKS = 1; \
|
||||
} \
|
||||
else{ \
|
||||
NUM_BLOCKS = COUNT / BLOCK_SIZE; \
|
||||
} \
|
||||
|
||||
#define SZ_COMPUTE_BLOCKCOUNT( COUNT, NUM_BLOCKS, SPLIT_INDEX, \
|
||||
EARLY_BLOCK_COUNT, LATE_BLOCK_COUNT ) \
|
||||
EARLY_BLOCK_COUNT = LATE_BLOCK_COUNT = COUNT / NUM_BLOCKS; \
|
||||
SPLIT_INDEX = COUNT % NUM_BLOCKS; \
|
||||
if (0 != SPLIT_INDEX) { \
|
||||
EARLY_BLOCK_COUNT = EARLY_BLOCK_COUNT + 1; \
|
||||
} \
|
||||
|
||||
//typedef unsigned long unsigned long;
|
||||
//typedef unsigned int uint;
|
||||
|
||||
typedef union lint16
|
||||
{
|
||||
unsigned short usvalue;
|
||||
short svalue;
|
||||
unsigned char byte[2];
|
||||
} lint16;
|
||||
|
||||
typedef union lint32
|
||||
{
|
||||
int ivalue;
|
||||
unsigned int uivalue;
|
||||
unsigned char byte[4];
|
||||
} lint32;
|
||||
|
||||
typedef union lint64
|
||||
{
|
||||
long lvalue;
|
||||
unsigned long ulvalue;
|
||||
unsigned char byte[8];
|
||||
} lint64;
|
||||
|
||||
typedef union ldouble
|
||||
{
|
||||
double value;
|
||||
unsigned long lvalue;
|
||||
unsigned char byte[8];
|
||||
} ldouble;
|
||||
|
||||
typedef union lfloat
|
||||
{
|
||||
float value;
|
||||
unsigned int ivalue;
|
||||
unsigned char byte[4];
|
||||
} lfloat;
|
||||
|
||||
|
||||
typedef struct sz_metadata
|
||||
{
|
||||
unsigned char ver; //only used for checking the version by calling SZ_GetMetaData()
|
||||
int isConstant; //only used for checking if the data are constant values by calling SZ_GetMetaData()
|
||||
int isLossless; //only used for checking if the data compression was lossless, used only by calling SZ_GetMetaData()
|
||||
int sizeType; //only used for checking whether the size type is "int" or "long" in the compression, used only by calling SZ_GetMetaData()
|
||||
size_t dataSeriesLength; //# number of data points in the dataset
|
||||
int defactoNBBins; //real number of quantization bins
|
||||
struct sz_params* conf_params; //configuration parameters
|
||||
} sz_metadata;
|
||||
|
||||
|
||||
/*We use a linked list to maintain time-step meta info for time-step based compression*/
|
||||
typedef struct sz_tsc_metainfo
|
||||
{
|
||||
int totalNumOfSteps;
|
||||
int currentStep;
|
||||
char metadata_filename[256];
|
||||
FILE *metadata_file;
|
||||
unsigned char* bit_array; //sihuan added
|
||||
size_t intersect_size; //sihuan added
|
||||
int64_t* hist_index; //sihuan added: prestep index
|
||||
|
||||
} sz_tsc_metadata;
|
||||
|
||||
extern unsigned char versionNumber;
|
||||
|
||||
//-------------------key global variables--------------
|
||||
extern int dataEndianType; //*endian type of the data read from disk
|
||||
extern int sysEndianType; //*sysEndianType is actually set automatically.
|
||||
|
||||
extern sz_params *confparams_cpr;
|
||||
extern sz_exedata *exe_params;
|
||||
|
||||
|
||||
void SZ_Finalize();
|
||||
int SZ_Init(const char *configFilePath);
|
||||
int SZ_Init_Params(sz_params *params);
|
||||
|
||||
//
|
||||
// compress output data to outData and return outSize
|
||||
//
|
||||
size_t SZ_compress_args(int dataType, void *data, size_t r1, unsigned char* outData, sz_params* params);
|
||||
|
||||
|
||||
//
|
||||
// decompress output data to outData and return outSize
|
||||
//
|
||||
size_t SZ_decompress(int dataType, unsigned char *bytes, size_t byteLength, size_t r1, unsigned char* outData);
|
||||
|
||||
|
||||
void convertSZParamsToBytes(sz_params* params, unsigned char* result, char optQuantMode);
|
||||
void convertBytesToSZParams(unsigned char* bytes, sz_params* params, sz_exedata* pde_exe);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ----- #ifndef _SZ_H ----- */
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
/**
|
||||
* @file sz_double.h
|
||||
* @author Sheng Di
|
||||
* @date July, 2017
|
||||
* @brief Header file for the sz_double.c.
|
||||
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
|
||||
* See COPYRIGHT in top-level directory.
|
||||
*/
|
||||
|
||||
#ifndef _SZ_Double_H
|
||||
#define _SZ_Double_H
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdbool.h>
|
||||
|
||||
unsigned char* SZ_skip_compress_double(double* data, size_t dataLength, size_t* outSize);
|
||||
|
||||
void computeReqLength_double(double realPrecision, short radExpo, int* reqLength, double* medianValue);
|
||||
|
||||
unsigned int optimize_intervals_double_1D(double *oriData, size_t dataLength, double realPrecision);
|
||||
|
||||
unsigned int optimize_intervals_double_1D_opt(double *oriData, size_t dataLength, double realPrecision);
|
||||
|
||||
|
||||
TightDataPointStorageD* SZ_compress_double_1D_MDQ(double *oriData,
|
||||
size_t dataLength, double realPrecision, double valueRangeSize, double medianValue_d);
|
||||
void SZ_compress_args_double_StoreOriData(double* oriData, size_t dataLength, unsigned char* newByteData, size_t *outSize);
|
||||
|
||||
bool SZ_compress_args_double_NoCkRngeNoGzip_1D( unsigned char* newByteData, double *oriData, size_t dataLength, double realPrecision, size_t *outSize, double valueRangeSize, double medianValue_d);
|
||||
|
||||
void SZ_compress_args_double_withinRange(unsigned char* newByteData, double *oriData, size_t dataLength, size_t *outSize);
|
||||
|
||||
|
||||
int SZ_compress_args_double(double *oriData, size_t r1, unsigned char* newByteData, size_t *outSize, sz_params* params);
|
||||
|
||||
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ----- #ifndef _SZ_Double_H ----- */
|
||||
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
/**
|
||||
* @file sz_float.h
|
||||
* @author Sheng Di
|
||||
* @date July, 2017
|
||||
* @brief Header file for the sz_float.c.
|
||||
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
|
||||
* See COPYRIGHT in top-level directory.
|
||||
*/
|
||||
|
||||
#ifndef _SZ_Float_H
|
||||
#define _SZ_Float_H
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
void computeReqLength_float(double realPrecision, short radExpo, int* reqLength, float* medianValue);
|
||||
|
||||
unsigned int optimize_intervals_float_1D(float *oriData, size_t dataLength, double realPrecision);
|
||||
|
||||
unsigned int optimize_intervals_float_1D_opt(float *oriData, size_t dataLength, double realPrecision);
|
||||
|
||||
TightDataPointStorageF* SZ_compress_float_1D_MDQ(float *oriData,
|
||||
size_t dataLength, float realPrecision, float valueRangeSize, float medianValue_f);
|
||||
|
||||
bool SZ_compress_args_float_NoCkRngeNoGzip_1D( unsigned char* newByteData, float *oriData,
|
||||
size_t dataLength, double realPrecision, size_t *outSize, float valueRangeSize, float medianValue_f);
|
||||
|
||||
|
||||
void SZ_compress_args_float_withinRange(unsigned char* newByteData, float *oriData, size_t dataLength, size_t *outSize);
|
||||
|
||||
|
||||
int SZ_compress_args_float(float *oriData, size_t r1, unsigned char* newByteData, size_t *outSize, sz_params* params);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ----- #ifndef _SZ_Float_H ----- */
|
||||
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
/**
|
||||
* @file szd_double.h
|
||||
* @author Sheng Di
|
||||
* @date July, 2017
|
||||
* @brief Header file for the szd_double.c.
|
||||
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
|
||||
* See COPYRIGHT in top-level directory.
|
||||
*/
|
||||
|
||||
#ifndef _SZD_Double_H
|
||||
#define _SZD_Double_H
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include "TightDataPointStorageD.h"
|
||||
|
||||
void decompressDataSeries_double_1D(double* data, size_t dataSeriesLength, double* hist_data, TightDataPointStorageD* tdps);
|
||||
|
||||
void getSnapshotData_double_1D(double* data, size_t dataSeriesLength, TightDataPointStorageD* tdps, int errBoundMode, int compressionType, double* hist_data, sz_params* pde_params);
|
||||
|
||||
int SZ_decompress_args_double(double* newData, size_t r1, unsigned char* cmpBytes, size_t cmpSize, int compressionType, double* hist_data, sz_exedata* pde_exe, sz_params* pde_params);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ----- #ifndef _SZD_Double_H ----- */
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
/**
|
||||
* @file szd_float.h
|
||||
* @author Sheng Di
|
||||
* @date July, 2017
|
||||
* @brief Header file for the szd_float.c.
|
||||
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
|
||||
* See COPYRIGHT in top-level directory.
|
||||
*/
|
||||
|
||||
#ifndef _SZD_Float_H
|
||||
#define _SZD_Float_H
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include "TightDataPointStorageF.h"
|
||||
|
||||
void decompressDataSeries_float_1D(float* data, size_t dataSeriesLength, float* hist_data, TightDataPointStorageF* tdps);
|
||||
|
||||
void getSnapshotData_float_1D(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps, int errBoundMode, int compressionType, float* hist_data, sz_params* pde_params);
|
||||
|
||||
int SZ_decompress_args_float(float* newData, size_t r1, unsigned char* cmpBytes, size_t cmpSize, int compressionType, float* hist_data, sz_exedata* pde_exe, sz_params* pde_params);
|
||||
|
||||
|
||||
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ----- #ifndef _SZD_Float_H ----- */
|
||||
|
|
@ -0,0 +1,117 @@
|
|||
/**
|
||||
* @file transcode.h
|
||||
* @author Yu Zhong (lx1zhong@qq.com)
|
||||
* @brief Header file for ADT-FSE algorithm
|
||||
* @date 2022-08-25
|
||||
*
|
||||
* @copyright Copyright (c) 2022
|
||||
*
|
||||
*/
|
||||
|
||||
#ifndef _Transcode_H
|
||||
#define _Transcode_H
|
||||
#include "sz.h"
|
||||
#include "mem.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/**
|
||||
* dict = {
|
||||
0: [0,0], 1:[1,0], 2:[2,0], 3:[3,0],
|
||||
4: [4,0], 5:[5,0], 6:[6,0], 7:[7,0],
|
||||
8: [8,0], 9:[9,0], 10:[10,0], 11:[11,0],
|
||||
12: [12,0], 13:[13,0], 14:[14,0], 15:[15,0],
|
||||
16: [16,1], 18:[17,1], 20:[18,1], 22:[19,1],
|
||||
24: [20,2], 28:[21,2], 32:[22,3], 40:[23,3],
|
||||
48: [24,4], 64:[25,6], 128:[26,7], 256:[27,8],
|
||||
512: [28,9], 1024:[29,10], 2048:[30,11], 4096:[31,12],
|
||||
8192: [32,13], 16384:[33,14],
|
||||
-8192: [35,13], -16384:[34,14],
|
||||
-512: [39,9], -1024:[38,10], -2048:[37,11], -4096:[36,12],
|
||||
-48: [43,4], -64:[42,6], -128:[41,7], -256:[40,8],
|
||||
-24: [47,2], -28:[46,2], -32:[45,3], -40:[44,3],
|
||||
-16: [51,1], -18:[50,1], -20:[49,1], -22:[48,1],
|
||||
-12: [55,0], -13:[54,0], -14:[53,0], -15:[52,0],
|
||||
-8: [59,0], -9:[58,0], -10:[57,0], -11:[56,0],
|
||||
-4: [63,0], -5:[62,0], -6:[61,0], -7:[60,0],
|
||||
-1:[66,0], -2:[65,0], -3:[64,0],
|
||||
#-16384:[67,0]
|
||||
}
|
||||
*
|
||||
*/
|
||||
|
||||
MEM_STATIC int Int2code(int factor)
|
||||
{
|
||||
static const uint8_t Ft_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7,
|
||||
8, 9, 10, 11, 12, 13, 14, 15,
|
||||
16, 16, 17, 17, 18, 18, 19, 19,
|
||||
20, 20, 20, 20, 21, 21, 21, 21,
|
||||
22, 22, 22, 22, 22, 22, 22, 22,
|
||||
23, 23, 23, 23, 23, 23, 23, 23,
|
||||
24, 24, 24, 24, 24, 24, 24, 24,
|
||||
24, 24, 24, 24, 24, 24, 24, 24 };
|
||||
if (factor >= 0)
|
||||
return (factor > 63) ? 50 - __builtin_clz(factor) : Ft_Code[factor];
|
||||
else {
|
||||
factor = -factor;
|
||||
return (factor > 63) ? 17 + __builtin_clz(factor) : 67 - Ft_Code[factor];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* reverse_dict = {
|
||||
0: [0,0], 1:[1,0], 2:[2,0], 3:[3,0],
|
||||
4: [4,0], 5:[5,0], 6:[6,0], 7:[7,0],
|
||||
8: [8,0], 9:[9,0], 10:[10,0], 11:[11,0],
|
||||
12: [12,0], 13:[13,0], 14:[14,0], 15:[15,0],
|
||||
16: [16,1], 17:[18,1], 18:[20,1], 19:[22,1],
|
||||
20: [24,2], 21:[28,2], 22:[32,3], 23:[40,3],
|
||||
24: [48,4], 25:[64,6], 26:[128,7], 27:[256,8],
|
||||
28: [512,9], 29:[1024,10], 30:[2048,11], 31:[4096,12],
|
||||
32: [8192,13], 33:[16384,14],
|
||||
35:[-8192,13], 34:[-16384,14],
|
||||
39:[-512,9], 38:[-1024,10], 37:[-2048,11], 36:[-4096,12],
|
||||
43:[-48,4], 42:[-64,6], 41:[-128,7], 40:[-256,8],
|
||||
47:[-24,2], 46:[-28,2], 45:[-32,3], 44:[-40,3],
|
||||
51:[-16,1], 50:[-18,1], 49:[-20,1], 48:[-22,1],
|
||||
55:[-12,0], 54:[-13,0], 53:[-14,0], 52:[-15,0],
|
||||
59:[-8,0], 58:[-9,0], 57:[-10,0], 56:[-11,0],
|
||||
63:[-4,0], 62:[-5,0], 61:[-6,0], 60:[-7,0],
|
||||
66:[-1,0], 65:[-2,0], 64:[-3,0],
|
||||
#67:[-16384,0]
|
||||
}
|
||||
*
|
||||
*/
|
||||
static const int code2int[67][2] = { {0,0},{1,0},{2,0},{3,0},
|
||||
{4,0},{5,0},{6,0},{7,0},
|
||||
{8,0},{9,0},{10,0},{11,0},
|
||||
{12,0},{13,0},{14,0},{15,0},
|
||||
{16,1},{18,1},{20,1},{22,1},
|
||||
{24,2},{28,2},{32,3},{40,3},
|
||||
{48,4},{64,6},{128,7},{256,8},
|
||||
{512,9},{1024,10},{2048,11},{4096,12},
|
||||
{8192,13},{16384,14},
|
||||
{-16384,14}, {-8192,13},
|
||||
{-4096,12},{-2048,11},{-1024,10},{-512,9},
|
||||
{-256,8},{-128,7},{-64,6},{-48,4},
|
||||
{-40,3},{-32,3},{-28,2},{-24,2},
|
||||
{-22,1},{-20,1},{-18,1},{-16,1},
|
||||
{-15,0},{-14,0},{-13,0},{-12,0},
|
||||
{-11,0},{-10,0},{-9,0},{-8,0},
|
||||
{-7,0},{-6,0},{-5,0},{-4,0},
|
||||
{-3,0},{-2,0},{-1,0} };
|
||||
|
||||
void encode_with_fse(int *type, size_t dataSeriesLength, unsigned int intervals,
|
||||
unsigned char **FseCode, size_t *FseCode_size,
|
||||
unsigned char **transCodeBits, size_t *transCodeBits_size);
|
||||
void decode_with_fse(int *type, size_t dataSeriesLength, unsigned int intervals,
|
||||
unsigned char *FseCode, size_t FseCode_size,
|
||||
unsigned char *transCodeBits, size_t transCodeBits_size);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
/**
|
||||
* @file utility.h
|
||||
* @author Sheng Di, Sihuan Li
|
||||
* @date July, 2018
|
||||
* @brief Header file for the utility.c.
|
||||
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
|
||||
* See COPYRIGHT in top-level directory.
|
||||
*/
|
||||
|
||||
#ifndef _UTILITY_H
|
||||
#define _UTILITY_H
|
||||
|
||||
#include "sz.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
int is_lossless_compressed_data(unsigned char* compressedBytes, size_t cmpSize);
|
||||
unsigned long sz_lossless_compress(int losslessCompressor, unsigned char* data, unsigned long dataLength, unsigned char* compressBytes);
|
||||
unsigned long sz_lossless_decompress(int losslessCompressor, unsigned char* compressBytes, unsigned long cmpSize, unsigned char** oriData, unsigned long targetOriSize);
|
||||
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ----- #ifndef _UTILITY_H ----- */
|
||||
|
|
@ -0,0 +1,298 @@
|
|||
/**
|
||||
* @file ByteToolkit.c
|
||||
* @author Sheng Di
|
||||
* @date April, 2016
|
||||
* @brief Byte Toolkit
|
||||
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
|
||||
* See COPYRIGHT in top-level directory.
|
||||
*/
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include "sz.h"
|
||||
|
||||
INLINE int bytesToInt_bigEndian(unsigned char* bytes)
|
||||
{
|
||||
int res;
|
||||
unsigned char* des = (unsigned char*)&res;
|
||||
des[0] = bytes[3];
|
||||
des[1] = bytes[2];
|
||||
des[2] = bytes[1];
|
||||
des[3] = bytes[0];
|
||||
return res;
|
||||
}
|
||||
|
||||
/**
|
||||
* @unsigned char *b the variable to store the converted bytes (length=4)
|
||||
* @unsigned int num
|
||||
* */
|
||||
INLINE void intToBytes_bigEndian(unsigned char *b, unsigned int num)
|
||||
{
|
||||
unsigned char* sou =(unsigned char*)#
|
||||
b[0] = sou[3];
|
||||
b[1] = sou[2];
|
||||
b[2] = sou[1];
|
||||
b[3] = sou[0];
|
||||
}
|
||||
|
||||
/**
|
||||
* @endianType: refers to the endian_type of unsigned char* b.
|
||||
* */
|
||||
INLINE long bytesToLong_bigEndian(unsigned char* b) {
|
||||
|
||||
long temp = 0;
|
||||
long res = 0;
|
||||
|
||||
res <<= 8;
|
||||
temp = b[0] & 0xff;
|
||||
res |= temp;
|
||||
|
||||
res <<= 8;
|
||||
temp = b[1] & 0xff;
|
||||
res |= temp;
|
||||
|
||||
res <<= 8;
|
||||
temp = b[2] & 0xff;
|
||||
res |= temp;
|
||||
|
||||
res <<= 8;
|
||||
temp = b[3] & 0xff;
|
||||
res |= temp;
|
||||
|
||||
res <<= 8;
|
||||
temp = b[4] & 0xff;
|
||||
res |= temp;
|
||||
|
||||
res <<= 8;
|
||||
temp = b[5] & 0xff;
|
||||
res |= temp;
|
||||
|
||||
res <<= 8;
|
||||
temp = b[6] & 0xff;
|
||||
res |= temp;
|
||||
|
||||
res <<= 8;
|
||||
temp = b[7] & 0xff;
|
||||
res |= temp;
|
||||
|
||||
return res;
|
||||
|
||||
}
|
||||
|
||||
INLINE void longToBytes_bigEndian(unsigned char *b, long num)
|
||||
{
|
||||
unsigned char* sou = (unsigned char*)#
|
||||
#if defined(_TD_LINUX_64) || defined(_TD_ARM_64) || defined(_TD_DARWIN_64)
|
||||
// 8 bytes
|
||||
b[7] = sou[0];
|
||||
b[6] = sou[1];
|
||||
b[5] = sou[2];
|
||||
b[4] = sou[3];
|
||||
b[3] = sou[4];
|
||||
b[2] = sou[5];
|
||||
b[1] = sou[6];
|
||||
b[0] = sou[7];
|
||||
#else
|
||||
memset(b, 0, 4);
|
||||
b[7] = sou[0];
|
||||
b[6] = sou[1];
|
||||
b[5] = sou[2];
|
||||
b[4] = sou[3];
|
||||
#endif
|
||||
}
|
||||
|
||||
//TODO: debug: lfBuf.lvalue could be actually little_endian....
|
||||
INLINE short getExponent_float(float value)
|
||||
{
|
||||
//int ivalue = floatToBigEndianInt(value);
|
||||
|
||||
lfloat lbuf;
|
||||
lbuf.value = value;
|
||||
int ivalue = lbuf.ivalue;
|
||||
|
||||
int expValue = (ivalue & 0x7F800000) >> 23;
|
||||
expValue -= 127;
|
||||
return (short)expValue;
|
||||
}
|
||||
|
||||
INLINE short getPrecisionReqLength_float(float precision)
|
||||
{
|
||||
lfloat lbuf;
|
||||
lbuf.value = precision;
|
||||
int ivalue = lbuf.ivalue;
|
||||
|
||||
int expValue = (ivalue & 0x7F800000) >> 23;
|
||||
expValue -= 127;
|
||||
return (short)expValue;
|
||||
}
|
||||
|
||||
INLINE short getExponent_double(double value)
|
||||
{
|
||||
//long lvalue = doubleToBigEndianLong(value);
|
||||
|
||||
ldouble lbuf;
|
||||
lbuf.value = value;
|
||||
long lvalue = lbuf.lvalue;
|
||||
|
||||
int expValue = (int)((lvalue & 0x7FF0000000000000) >> 52);
|
||||
expValue -= 1023;
|
||||
return (short)expValue;
|
||||
}
|
||||
|
||||
INLINE short getPrecisionReqLength_double(double precision)
|
||||
{
|
||||
ldouble lbuf;
|
||||
lbuf.value = precision;
|
||||
long lvalue = lbuf.lvalue;
|
||||
|
||||
int expValue = (int)((lvalue & 0x7FF0000000000000) >> 52);
|
||||
expValue -= 1023;
|
||||
// unsigned char the1stManBit = (unsigned char)((lvalue & 0x0008000000000000) >> 51);
|
||||
// if(the1stManBit==1)
|
||||
// expValue--;
|
||||
return (short)expValue;
|
||||
}
|
||||
|
||||
|
||||
//the byte to input is in the big-endian format
|
||||
INLINE float bytesToFloat(unsigned char* bytes)
|
||||
{
|
||||
lfloat buf;
|
||||
memcpy(buf.byte, bytes, 4);
|
||||
if(sysEndianType==LITTLE_ENDIAN_SYSTEM)
|
||||
symTransform_4bytes(buf.byte);
|
||||
return buf.value;
|
||||
}
|
||||
|
||||
INLINE void floatToBytes(unsigned char *b, float num)
|
||||
{
|
||||
lfloat buf;
|
||||
buf.value = num;
|
||||
memcpy(b, buf.byte, 4);
|
||||
if(sysEndianType==LITTLE_ENDIAN_SYSTEM)
|
||||
symTransform_4bytes(b);
|
||||
}
|
||||
|
||||
//the byte to input is in the big-endian format
|
||||
INLINE double bytesToDouble(unsigned char* bytes)
|
||||
{
|
||||
ldouble buf;
|
||||
memcpy(buf.byte, bytes, 8);
|
||||
if(sysEndianType==LITTLE_ENDIAN_SYSTEM)
|
||||
symTransform_8bytes(buf.byte);
|
||||
return buf.value;
|
||||
}
|
||||
|
||||
INLINE void doubleToBytes(unsigned char *b, double num)
|
||||
{
|
||||
ldouble buf;
|
||||
buf.value = num;
|
||||
memcpy(b, buf.byte, 8);
|
||||
if(sysEndianType==LITTLE_ENDIAN_SYSTEM)
|
||||
symTransform_8bytes(b);
|
||||
}
|
||||
|
||||
|
||||
INLINE int getMaskRightCode(int m) {
|
||||
switch (m) {
|
||||
case 1:
|
||||
return 0x01;
|
||||
case 2:
|
||||
return 0x03;
|
||||
case 3:
|
||||
return 0x07;
|
||||
case 4:
|
||||
return 0x0F;
|
||||
case 5:
|
||||
return 0x1F;
|
||||
case 6:
|
||||
return 0x3F;
|
||||
case 7:
|
||||
return 0X7F;
|
||||
case 8:
|
||||
return 0XFF;
|
||||
default:
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
INLINE int getLeftMovingCode(int kMod8)
|
||||
{
|
||||
return getMaskRightCode(8 - kMod8);
|
||||
}
|
||||
|
||||
INLINE int getRightMovingSteps(int kMod8, int resiBitLength) {
|
||||
return 8 - kMod8 - resiBitLength;
|
||||
}
|
||||
|
||||
INLINE int getRightMovingCode(int kMod8, int resiBitLength)
|
||||
{
|
||||
int rightMovingSteps = 8 - kMod8 - resiBitLength;
|
||||
if(rightMovingSteps < 0)
|
||||
{
|
||||
switch(-rightMovingSteps)
|
||||
{
|
||||
case 1:
|
||||
return 0x80;
|
||||
case 2:
|
||||
return 0xC0;
|
||||
case 3:
|
||||
return 0xE0;
|
||||
case 4:
|
||||
return 0xF0;
|
||||
case 5:
|
||||
return 0xF8;
|
||||
case 6:
|
||||
return 0xFC;
|
||||
case 7:
|
||||
return 0XFE;
|
||||
default:
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
else //if(rightMovingSteps >= 0)
|
||||
{
|
||||
int a = getMaskRightCode(8 - kMod8);
|
||||
int b = getMaskRightCode(8 - kMod8 - resiBitLength);
|
||||
int c = a - b;
|
||||
return c;
|
||||
}
|
||||
}
|
||||
|
||||
INLINE size_t bytesToSize(unsigned char* bytes, int size_type)
|
||||
{
|
||||
size_t result = 0;
|
||||
if(size_type == 4)
|
||||
result = bytesToInt_bigEndian(bytes);//4
|
||||
else
|
||||
result = bytesToLong_bigEndian(bytes);//8
|
||||
return result;
|
||||
}
|
||||
|
||||
INLINE void sizeToBytes(unsigned char* outBytes, size_t size, int size_type)
|
||||
{
|
||||
if(size_type == 4)
|
||||
intToBytes_bigEndian(outBytes, (unsigned int)size);//4
|
||||
else
|
||||
longToBytes_bigEndian(outBytes, (unsigned long)size);//8
|
||||
}
|
||||
|
||||
void convertSZParamsToBytes(sz_params* params, unsigned char* result, char optQuantMode)
|
||||
{
|
||||
//unsigned char* result = (unsigned char*)malloc(16);
|
||||
unsigned char buf = 0;
|
||||
buf = optQuantMode;
|
||||
buf = (buf << 1) | dataEndianType;
|
||||
buf = (buf << 1) | sysEndianType;
|
||||
buf = (buf << 2) | params->szMode;
|
||||
|
||||
result[0] = buf;
|
||||
}
|
||||
|
||||
void convertBytesToSZParams(unsigned char* bytes, sz_params* params, sz_exedata* pde_exe)
|
||||
{
|
||||
unsigned char flag1 = bytes[0];
|
||||
pde_exe->optQuantMode = (flag1 & 0x40) >> 6;
|
||||
dataEndianType = (flag1 & 0x20) >> 5;
|
||||
params->szMode = (flag1 & 0x0c) >> 2;
|
||||
}
|
||||
|
|
@ -0,0 +1,237 @@
|
|||
/**
|
||||
* @file CompressElement.c
|
||||
* @author Sheng Di
|
||||
* @date May, 2016
|
||||
* @brief Functions of CompressElement
|
||||
* (C) 2015 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
|
||||
* See COPYRIGHT in top-level directory.
|
||||
*/
|
||||
#ifndef WINDOWS
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wchar-subscripts"
|
||||
#endif
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <stdio.h>
|
||||
#include <math.h>
|
||||
#include <string.h>
|
||||
#include "sz.h"
|
||||
#include "CompressElement.h"
|
||||
|
||||
|
||||
INLINE short computeGroupNum_float(float value)
|
||||
{
|
||||
short expo = getExponent_float(value);
|
||||
if(expo < 0)
|
||||
expo = -1;
|
||||
return expo;
|
||||
}
|
||||
|
||||
INLINE short computeGroupNum_double(double value)
|
||||
{
|
||||
short expo = getExponent_double(value);
|
||||
if(expo < 0)
|
||||
expo = -1;
|
||||
return expo;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add preceding neighbor values to a buffer.
|
||||
* @param last3CmprsData buffer
|
||||
* @param value the value to be added to the buffer
|
||||
* */
|
||||
INLINE void listAdd_double(double last3CmprsData[3], double value)
|
||||
{
|
||||
last3CmprsData[2] = last3CmprsData[1];
|
||||
last3CmprsData[1] = last3CmprsData[0];
|
||||
last3CmprsData[0] = value;
|
||||
}
|
||||
|
||||
INLINE void listAdd_float(float last3CmprsData[3], float value)
|
||||
{
|
||||
last3CmprsData[2] = last3CmprsData[1];
|
||||
last3CmprsData[1] = last3CmprsData[0];
|
||||
last3CmprsData[0] = value;
|
||||
}
|
||||
|
||||
INLINE void listAdd_int(int64_t last3CmprsData[3], int64_t value)
|
||||
{
|
||||
last3CmprsData[2] = last3CmprsData[1];
|
||||
last3CmprsData[1] = last3CmprsData[0];
|
||||
last3CmprsData[0] = value;
|
||||
}
|
||||
|
||||
INLINE void listAdd_int32(int32_t last3CmprsData[3], int32_t value)
|
||||
{
|
||||
last3CmprsData[2] = last3CmprsData[1];
|
||||
last3CmprsData[1] = last3CmprsData[0];
|
||||
last3CmprsData[0] = value;
|
||||
}
|
||||
|
||||
INLINE void listAdd_float_group(float *groups, int *flags, char groupNum, float oriValue, float decValue, char* curGroupID)
|
||||
{
|
||||
if(groupNum>=0)
|
||||
{
|
||||
if(flags[groupNum]==0)
|
||||
flags[groupNum] = 1;
|
||||
groups[groupNum] = decValue;
|
||||
}
|
||||
else
|
||||
{
|
||||
groups[0] = decValue;
|
||||
flags[0] = 1;
|
||||
}
|
||||
|
||||
if(oriValue>=0)
|
||||
*curGroupID = groupNum+2; //+[-1,0,1,2,3,....,16] is mapped to [1,2,....,18]
|
||||
else
|
||||
*curGroupID = -(groupNum+2); //-[-1,0,1,2,3,....,16] is mapped to [-1,-2,....,-18]
|
||||
}
|
||||
|
||||
INLINE void listAdd_double_group(double *groups, int *flags, char groupNum, double oriValue, double decValue, char* curGroupID)
|
||||
{
|
||||
if(groupNum>=0)
|
||||
{
|
||||
if(flags[groupNum]==0)
|
||||
flags[groupNum] = 1;
|
||||
groups[groupNum] = decValue;
|
||||
}
|
||||
else
|
||||
{
|
||||
groups[0] = decValue;
|
||||
flags[0] = 1;
|
||||
}
|
||||
|
||||
if(oriValue>=0)
|
||||
*curGroupID = groupNum+2; //+[-1,0,1,2,3,....,16] is mapped to [1,2,....,18]
|
||||
else
|
||||
*curGroupID = -(groupNum+2); //-[-1,0,1,2,3,....,16] is mapped to [-1,-2,....,-18]
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine whether the prediction value minErr is valid.
|
||||
*
|
||||
* */
|
||||
INLINE int validPrediction_double(double minErr, double precision)
|
||||
{
|
||||
if(minErr<=precision)
|
||||
return 1;
|
||||
else
|
||||
return 0;
|
||||
}
|
||||
|
||||
INLINE int validPrediction_float(float minErr, float precision)
|
||||
{
|
||||
if(minErr<=precision)
|
||||
return 1;
|
||||
else
|
||||
return 0;
|
||||
}
|
||||
|
||||
double* generateGroupErrBounds(int errorBoundMode, double realPrecision, double pwrErrBound)
|
||||
{
|
||||
double pwrError;
|
||||
double* result = (double*)malloc(GROUP_COUNT*sizeof(double));
|
||||
int i = 0;
|
||||
for(i=0;i<GROUP_COUNT;i++)
|
||||
{
|
||||
pwrError = ((double)pow(2, i))*pwrErrBound;
|
||||
switch(errorBoundMode)
|
||||
{
|
||||
case ABS_AND_PW_REL:
|
||||
case REL_AND_PW_REL:
|
||||
result[i] = pwrError<realPrecision?pwrError:realPrecision;
|
||||
break;
|
||||
case ABS_OR_PW_REL:
|
||||
case REL_OR_PW_REL:
|
||||
result[i] = pwrError<realPrecision?realPrecision:pwrError;
|
||||
break;
|
||||
case PW_REL:
|
||||
result[i] = pwrError;
|
||||
break;
|
||||
}
|
||||
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
int generateGroupMaxIntervalCount(double* groupErrBounds)
|
||||
{
|
||||
int i = 0;
|
||||
int maxCount = 0, count = 0;
|
||||
for(i=0;i<GROUP_COUNT;i++)
|
||||
{
|
||||
count = (int)(pow(2, i)/groupErrBounds[i] + 0.5);
|
||||
if(maxCount<count)
|
||||
maxCount = count;
|
||||
}
|
||||
|
||||
return maxCount;
|
||||
}
|
||||
|
||||
void new_LossyCompressionElement(LossyCompressionElement *lce, int leadingNum, unsigned char* intMidBytes,
|
||||
int intMidBytes_Length, int resiMidBitsLength, int resiBits)
|
||||
{
|
||||
lce->leadingZeroBytes = leadingNum; //0,1,2,or 3
|
||||
memcpy(lce->integerMidBytes,intMidBytes,intMidBytes_Length);
|
||||
lce->integerMidBytes_Length = intMidBytes_Length; //they are mid_bits actually
|
||||
lce->resMidBitsLength = resiMidBitsLength;
|
||||
lce->residualMidBits = resiBits;
|
||||
}
|
||||
|
||||
void updateLossyCompElement_Double(unsigned char* curBytes, unsigned char* preBytes,
|
||||
int reqBytesLength, int resiBitsLength, LossyCompressionElement *lce)
|
||||
{
|
||||
int resiIndex, intMidBytes_Length = 0;
|
||||
int leadingNum = compIdenticalLeadingBytesCount_double(preBytes, curBytes); //in fact, float is enough for both single-precision and double-precisiond ata.
|
||||
int fromByteIndex = leadingNum;
|
||||
int toByteIndex = reqBytesLength; //later on: should use "< toByteIndex" to tarverse....
|
||||
if(fromByteIndex < toByteIndex)
|
||||
{
|
||||
intMidBytes_Length = reqBytesLength - leadingNum;
|
||||
memcpy(lce->integerMidBytes, &(curBytes[fromByteIndex]), intMidBytes_Length);
|
||||
}
|
||||
int resiBits = 0;
|
||||
if(resiBitsLength!=0)
|
||||
{
|
||||
resiIndex = reqBytesLength;
|
||||
if(resiIndex < 8)
|
||||
resiBits = (curBytes[resiIndex] & 0xFF) >> (8-resiBitsLength);
|
||||
}
|
||||
lce->leadingZeroBytes = leadingNum;
|
||||
lce->integerMidBytes_Length = intMidBytes_Length;
|
||||
lce->resMidBitsLength = resiBitsLength;
|
||||
lce->residualMidBits = resiBits;
|
||||
}
|
||||
|
||||
INLINE void updateLossyCompElement_Float(unsigned char* diffBytes, unsigned char* preDiffBytes,
|
||||
int reqBytesLength, int resiBitsLength, LossyCompressionElement *lce)
|
||||
{
|
||||
int resiIndex, intMidBytes_Length = 0;
|
||||
int leadingNum = compIdenticalLeadingBytesCount_float(preDiffBytes, diffBytes); //in fact, float is enough for both single-precision and double-precisiond ata.
|
||||
int fromByteIndex = leadingNum;
|
||||
int toByteIndex = reqBytesLength; //later on: should use "< toByteIndex" to tarverse....
|
||||
if(fromByteIndex < toByteIndex)
|
||||
{
|
||||
intMidBytes_Length = reqBytesLength - leadingNum;
|
||||
// set lce mid data
|
||||
memcpy(lce->integerMidBytes, &(diffBytes[fromByteIndex]), intMidBytes_Length);
|
||||
}
|
||||
int resiBits = 0;
|
||||
if(resiBitsLength!=0)
|
||||
{
|
||||
resiIndex = reqBytesLength;
|
||||
if(resiIndex < 8)
|
||||
resiBits = (diffBytes[resiIndex] & 0xFF) >> (8-resiBitsLength);
|
||||
}
|
||||
|
||||
// set lce
|
||||
lce->leadingZeroBytes = leadingNum;
|
||||
lce->integerMidBytes_Length = intMidBytes_Length;
|
||||
lce->resMidBitsLength = resiBitsLength;
|
||||
lce->residualMidBits = resiBits;
|
||||
}
|
||||
|
||||
#ifndef WINDOWS
|
||||
#pragma GCC diagnostic pop
|
||||
#endif
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
/**
|
||||
* @file DynamicByteArray.c
|
||||
* @author Sheng Di
|
||||
* @date May, 2016
|
||||
* @brief Dynamic Byte Array
|
||||
* (C) 2015 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
|
||||
* See COPYRIGHT in top-level directory.
|
||||
*/
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include "DynamicByteArray.h"
|
||||
#include "sz.h"
|
||||
|
||||
void new_DBA(DynamicByteArray **dba, size_t cap) {
|
||||
*dba = (DynamicByteArray *)malloc(sizeof(DynamicByteArray));
|
||||
(*dba)->size = 0;
|
||||
(*dba)->capacity = cap;
|
||||
(*dba)->array = (unsigned char*)malloc(sizeof(unsigned char)*cap);
|
||||
}
|
||||
|
||||
void convertDBAtoBytes(DynamicByteArray *dba, unsigned char** bytes)
|
||||
{
|
||||
size_t size = dba->size;
|
||||
if(size>0)
|
||||
*bytes = (unsigned char*)malloc(size * sizeof(unsigned char));
|
||||
else
|
||||
{
|
||||
*bytes = NULL;
|
||||
return ;
|
||||
}
|
||||
memcpy(*bytes, dba->array, size*sizeof(unsigned char));
|
||||
}
|
||||
|
||||
void free_DBA(DynamicByteArray *dba)
|
||||
{
|
||||
free(dba->array);
|
||||
free(dba);
|
||||
}
|
||||
|
||||
INLINE unsigned char getDBA_Data(DynamicByteArray *dba, size_t pos)
|
||||
{
|
||||
if(pos>=dba->size)
|
||||
{
|
||||
printf("Error: wrong position of DBA (impossible case unless bugs elsewhere in the code?).\n");
|
||||
exit(0);
|
||||
}
|
||||
return dba->array[pos];
|
||||
}
|
||||
|
||||
INLINE void addDBA_Data(DynamicByteArray *dba, unsigned char value)
|
||||
{
|
||||
if(dba->size==dba->capacity)
|
||||
{
|
||||
dba->capacity = dba->capacity << 1;
|
||||
dba->array = (unsigned char *)realloc(dba->array, dba->capacity*sizeof(unsigned char));
|
||||
}
|
||||
dba->array[dba->size] = value;
|
||||
dba->size ++;
|
||||
}
|
||||
|
||||
INLINE void memcpyDBA_Data(DynamicByteArray *dba, unsigned char* data, size_t length)
|
||||
{
|
||||
if(dba->size + length > dba->capacity)
|
||||
{
|
||||
dba->capacity = dba->size + length;
|
||||
dba->array = (unsigned char *)realloc(dba->array, dba->capacity*sizeof(unsigned char));
|
||||
}
|
||||
memcpy(&(dba->array[dba->size]), data, length);
|
||||
dba->size += length;
|
||||
}
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
/**
|
||||
* @file DynamicIntArray.c
|
||||
* @author Sheng Di
|
||||
* @date May, 2016
|
||||
* @brief Dynamic Int Array
|
||||
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
|
||||
* See COPYRIGHT in top-level directory.
|
||||
*/
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include "DynamicIntArray.h"
|
||||
#include "sz.h"
|
||||
|
||||
void new_DIA(DynamicIntArray **dia, size_t cap) {
|
||||
*dia = (DynamicIntArray *)malloc(sizeof(DynamicIntArray));
|
||||
(*dia)->size = 0;
|
||||
(*dia)->capacity = cap;
|
||||
(*dia)->array = (unsigned char*)malloc(sizeof(unsigned char)*cap);
|
||||
}
|
||||
|
||||
void convertDIAtoInts(DynamicIntArray *dia, unsigned char **data)
|
||||
{
|
||||
size_t size = dia->size;
|
||||
if(size>0)
|
||||
*data = (unsigned char*)malloc(size * sizeof(char));
|
||||
else
|
||||
*data = NULL;
|
||||
memcpy(*data, dia->array, size*sizeof(unsigned char));
|
||||
}
|
||||
|
||||
void free_DIA(DynamicIntArray *dia)
|
||||
{
|
||||
free(dia->array);
|
||||
free(dia);
|
||||
}
|
||||
|
||||
int getDIA_Data(DynamicIntArray *dia, size_t pos)
|
||||
{
|
||||
if(pos>=dia->size)
|
||||
{
|
||||
printf("Error: wrong position of DIA.\n");
|
||||
exit(0);
|
||||
}
|
||||
return dia->array[pos];
|
||||
}
|
||||
|
||||
INLINE void addDIA_Data(DynamicIntArray *dia, int value)
|
||||
{
|
||||
if(dia->size==dia->capacity)
|
||||
{
|
||||
dia->capacity = dia->capacity << 1;
|
||||
dia->array = (unsigned char *)realloc(dia->array, dia->capacity*sizeof(unsigned char));
|
||||
}
|
||||
dia->array[dia->size] = (unsigned char)value;
|
||||
dia->size ++;
|
||||
}
|
||||
|
|
@ -0,0 +1,765 @@
|
|||
/**
|
||||
* @file Huffman.c
|
||||
* @author Sheng Di
|
||||
* @date Aug., 2016
|
||||
* @brief Customized Huffman Encoding, Compression and Decompression functions
|
||||
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
|
||||
* See COPYRIGHT in top-level directory.
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include "Huffman.h"
|
||||
#include "sz.h"
|
||||
|
||||
|
||||
HuffmanTree* createHuffmanTree(int stateNum)
|
||||
{
|
||||
HuffmanTree *huffmanTree = (HuffmanTree*)malloc(sizeof(HuffmanTree));
|
||||
memset(huffmanTree, 0, sizeof(HuffmanTree));
|
||||
huffmanTree->stateNum = stateNum;
|
||||
huffmanTree->allNodes = 2*stateNum;
|
||||
|
||||
huffmanTree->pool = (struct node_t*)malloc(huffmanTree->allNodes*2*sizeof(struct node_t));
|
||||
huffmanTree->qqq = (node*)malloc(huffmanTree->allNodes*2*sizeof(node));
|
||||
huffmanTree->code = (unsigned long**)malloc(huffmanTree->stateNum*sizeof(unsigned long*));
|
||||
huffmanTree->cout = (unsigned char *)malloc(huffmanTree->stateNum*sizeof(unsigned char));
|
||||
|
||||
memset(huffmanTree->pool, 0, huffmanTree->allNodes*2*sizeof(struct node_t));
|
||||
memset(huffmanTree->qqq, 0, huffmanTree->allNodes*2*sizeof(node));
|
||||
memset(huffmanTree->code, 0, huffmanTree->stateNum*sizeof(unsigned long*));
|
||||
memset(huffmanTree->cout, 0, huffmanTree->stateNum*sizeof(unsigned char));
|
||||
huffmanTree->qq = huffmanTree->qqq - 1;
|
||||
huffmanTree->n_nodes = 0;
|
||||
huffmanTree->n_inode = 0;
|
||||
huffmanTree->qend = 1;
|
||||
|
||||
return huffmanTree;
|
||||
}
|
||||
|
||||
HuffmanTree* createDefaultHuffmanTree()
|
||||
{
|
||||
int maxRangeRadius = 32768;
|
||||
int stateNum = maxRangeRadius << 1; //*2
|
||||
|
||||
return createHuffmanTree(stateNum);
|
||||
}
|
||||
|
||||
node new_node(HuffmanTree* huffmanTree, size_t freq, unsigned int c, node a, node b)
|
||||
{
|
||||
node n = huffmanTree->pool + huffmanTree->n_nodes++;
|
||||
if (freq)
|
||||
{
|
||||
n->c = c;
|
||||
n->freq = freq;
|
||||
n->t = 1;
|
||||
}
|
||||
else {
|
||||
n->left = a;
|
||||
n->right = b;
|
||||
n->freq = a->freq + b->freq;
|
||||
n->t = 0;
|
||||
//n->c = 0;
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
node new_node2(HuffmanTree *huffmanTree, unsigned int c, unsigned char t)
|
||||
{
|
||||
huffmanTree->pool[huffmanTree->n_nodes].c = c;
|
||||
huffmanTree->pool[huffmanTree->n_nodes].t = t;
|
||||
return huffmanTree->pool + huffmanTree->n_nodes++;
|
||||
}
|
||||
|
||||
/* priority queue */
|
||||
void qinsert(HuffmanTree *huffmanTree, node n)
|
||||
{
|
||||
int j, i = huffmanTree->qend++;
|
||||
while ((j = (i>>1))) //j=i/2
|
||||
{
|
||||
if (huffmanTree->qq[j]->freq <= n->freq) break;
|
||||
huffmanTree->qq[i] = huffmanTree->qq[j], i = j;
|
||||
}
|
||||
huffmanTree->qq[i] = n;
|
||||
}
|
||||
|
||||
node qremove(HuffmanTree* huffmanTree)
|
||||
{
|
||||
int i, l;
|
||||
node n = huffmanTree->qq[i = 1];
|
||||
node p;
|
||||
if (huffmanTree->qend < 2) return 0;
|
||||
huffmanTree->qend --;
|
||||
huffmanTree->qq[i] = huffmanTree->qq[huffmanTree->qend];
|
||||
|
||||
while ((l = (i<<1)) < huffmanTree->qend) //l=(i*2)
|
||||
{
|
||||
if (l + 1 < huffmanTree->qend && huffmanTree->qq[l + 1]->freq < huffmanTree->qq[l]->freq) l++;
|
||||
if(huffmanTree->qq[i]->freq > huffmanTree->qq[l]->freq)
|
||||
{
|
||||
p = huffmanTree->qq[i];
|
||||
huffmanTree->qq[i] = huffmanTree->qq[l];
|
||||
huffmanTree->qq[l] = p;
|
||||
i = l;
|
||||
}
|
||||
else
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return n;
|
||||
}
|
||||
|
||||
/* walk the tree and put 0s and 1s */
|
||||
/**
|
||||
* @out1 should be set to 0.
|
||||
* @out2 should be 0 as well.
|
||||
* @index: the index of the byte
|
||||
* */
|
||||
void build_code(HuffmanTree *huffmanTree, node n, int len, unsigned long out1, unsigned long out2)
|
||||
{
|
||||
if (n->t) {
|
||||
huffmanTree->code[n->c] = (unsigned long*)malloc(2*sizeof(unsigned long));
|
||||
if(len<=64)
|
||||
{
|
||||
(huffmanTree->code[n->c])[0] = out1 << (64 - len);
|
||||
(huffmanTree->code[n->c])[1] = out2;
|
||||
}
|
||||
else
|
||||
{
|
||||
(huffmanTree->code[n->c])[0] = out1;
|
||||
(huffmanTree->code[n->c])[1] = out2 << (128 - len);
|
||||
}
|
||||
huffmanTree->cout[n->c] = (unsigned char)len;
|
||||
return;
|
||||
}
|
||||
int index = len >> 6; //=len/64
|
||||
if(index == 0)
|
||||
{
|
||||
out1 = out1 << 1;
|
||||
out1 = out1 | 0;
|
||||
build_code(huffmanTree, n->left, len + 1, out1, 0);
|
||||
out1 = out1 | 1;
|
||||
build_code(huffmanTree, n->right, len + 1, out1, 0);
|
||||
}
|
||||
else
|
||||
{
|
||||
if(len%64!=0)
|
||||
out2 = out2 << 1;
|
||||
out2 = out2 | 0;
|
||||
build_code(huffmanTree, n->left, len + 1, out1, out2);
|
||||
out2 = out2 | 1;
|
||||
build_code(huffmanTree, n->right, len + 1, out1, out2);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the frequency of the data and build the Huffman tree
|
||||
* @param HuffmanTree* huffmanTree (output)
|
||||
* @param int *s (input)
|
||||
* @param size_t length (input)
|
||||
* */
|
||||
void init(HuffmanTree* huffmanTree, int *s, size_t length)
|
||||
{
|
||||
size_t i, index;
|
||||
size_t *freq = (size_t *)malloc(huffmanTree->allNodes*sizeof(size_t));
|
||||
memset(freq, 0, huffmanTree->allNodes*sizeof(size_t));
|
||||
for(i = 0;i < length;i++)
|
||||
{
|
||||
index = s[i];
|
||||
freq[index]++;
|
||||
}
|
||||
|
||||
for (i = 0; i < huffmanTree->allNodes; i++)
|
||||
if (freq[i])
|
||||
qinsert(huffmanTree, new_node(huffmanTree, freq[i], i, 0, 0));
|
||||
|
||||
while (huffmanTree->qend > 2)
|
||||
qinsert(huffmanTree, new_node(huffmanTree, 0, 0, qremove(huffmanTree), qremove(huffmanTree)));
|
||||
|
||||
build_code(huffmanTree, huffmanTree->qq[1], 0, 0, 0);
|
||||
free(freq);
|
||||
}
|
||||
|
||||
void init_static(HuffmanTree* huffmanTree, int *s, size_t length)
|
||||
{
|
||||
size_t i;
|
||||
size_t *freq = (size_t *)malloc(huffmanTree->allNodes*sizeof(size_t));
|
||||
memset(freq, 0, huffmanTree->allNodes*sizeof(size_t));
|
||||
|
||||
|
||||
for (i = 0; i < huffmanTree->allNodes; i++)
|
||||
if (freq[i])
|
||||
qinsert(huffmanTree, new_node(huffmanTree, freq[i], i, 0, 0));
|
||||
|
||||
while (huffmanTree->qend > 2)
|
||||
qinsert(huffmanTree, new_node(huffmanTree, 0, 0, qremove(huffmanTree), qremove(huffmanTree)));
|
||||
|
||||
build_code(huffmanTree, huffmanTree->qq[1], 0, 0, 0);
|
||||
free(freq);
|
||||
}
|
||||
|
||||
void encode(HuffmanTree *huffmanTree, int *s, size_t length, unsigned char *out, size_t *outSize)
|
||||
{
|
||||
size_t i = 0;
|
||||
unsigned char bitSize = 0, byteSize, byteSizep;
|
||||
int state;
|
||||
unsigned char *p = out;
|
||||
int lackBits = 0;
|
||||
//long totalBitSize = 0, maxBitSize = 0, bitSize21 = 0, bitSize32 = 0;
|
||||
for (i = 0;i<length;i++)
|
||||
{
|
||||
state = s[i];
|
||||
bitSize = huffmanTree->cout[state];
|
||||
|
||||
//printf("%d %d : %d %u\n",i, state, bitSize, (code[state])[0] >> (64-cout[state]));
|
||||
//debug: compute the average bitSize and the count that is over 32...
|
||||
/*if(bitSize>=21)
|
||||
bitSize21++;
|
||||
if(bitSize>=32)
|
||||
bitSize32++;
|
||||
if(maxBitSize<bitSize)
|
||||
maxBitSize = bitSize;
|
||||
totalBitSize+=bitSize;*/
|
||||
|
||||
if(lackBits==0)
|
||||
{
|
||||
byteSize = bitSize%8==0 ? bitSize/8 : bitSize/8+1; //it's equal to the number of bytes involved (for *outSize)
|
||||
byteSizep = bitSize/8; //it's used to move the pointer p for next data
|
||||
if(byteSize<=8)
|
||||
{
|
||||
longToBytes_bigEndian(p, (huffmanTree->code[state])[0]);
|
||||
p += byteSizep;
|
||||
}
|
||||
else //byteSize>8
|
||||
{
|
||||
longToBytes_bigEndian(p, (huffmanTree->code[state])[0]);
|
||||
p += 8;
|
||||
longToBytes_bigEndian(p, (huffmanTree->code[state])[1]);
|
||||
p += (byteSizep - 8);
|
||||
}
|
||||
*outSize += byteSize;
|
||||
lackBits = bitSize%8==0 ? 0 : 8 - bitSize%8;
|
||||
}
|
||||
else
|
||||
{
|
||||
*p = (*p) | (unsigned char)((huffmanTree->code[state])[0] >> (64 - lackBits));
|
||||
if(lackBits < bitSize)
|
||||
{
|
||||
p++;
|
||||
//(*outSize)++;
|
||||
long newCode = (huffmanTree->code[state])[0] << lackBits;
|
||||
longToBytes_bigEndian(p, newCode);
|
||||
|
||||
if(bitSize<=64)
|
||||
{
|
||||
bitSize -= lackBits;
|
||||
byteSize = bitSize%8==0 ? bitSize/8 : bitSize/8+1;
|
||||
byteSizep = bitSize/8;
|
||||
p += byteSizep;
|
||||
(*outSize)+=byteSize;
|
||||
lackBits = bitSize%8==0 ? 0 : 8 - bitSize%8;
|
||||
}
|
||||
else //bitSize > 64
|
||||
{
|
||||
byteSizep = 7; //must be 7 bytes, because lackBits!=0
|
||||
p+=byteSizep;
|
||||
(*outSize)+=byteSize;
|
||||
|
||||
bitSize -= 64;
|
||||
if(lackBits < bitSize)
|
||||
{
|
||||
*p = (*p) | (unsigned char)((huffmanTree->code[state])[0] >> (64 - lackBits));
|
||||
p++;
|
||||
//(*outSize)++;
|
||||
newCode = (huffmanTree->code[state])[1] << lackBits;
|
||||
longToBytes_bigEndian(p, newCode);
|
||||
bitSize -= lackBits;
|
||||
byteSize = bitSize%8==0 ? bitSize/8 : bitSize/8+1;
|
||||
byteSizep = bitSize/8;
|
||||
p += byteSizep;
|
||||
(*outSize)+=byteSize;
|
||||
lackBits = bitSize%8==0 ? 0 : 8 - bitSize%8;
|
||||
}
|
||||
else //lackBits >= bitSize
|
||||
{
|
||||
*p = (*p) | (unsigned char)((huffmanTree->code[state])[0] >> (64 - bitSize));
|
||||
lackBits -= bitSize;
|
||||
}
|
||||
}
|
||||
}
|
||||
else //lackBits >= bitSize
|
||||
{
|
||||
lackBits -= bitSize;
|
||||
if(lackBits==0)
|
||||
p++;
|
||||
}
|
||||
}
|
||||
}
|
||||
// for(i=0;i<stateNum;i++)
|
||||
// if(code[i]!=NULL) free(code[i]);
|
||||
/*printf("max bitsize = %d\n", maxBitSize);
|
||||
printf("bitSize21 ratio = %f\n", ((float)bitSize21)/length);
|
||||
printf("bitSize32 ratio = %f\n", ((float)bitSize32)/length);
|
||||
printf("avg bit size = %f\n", ((float)totalBitSize)/length);*/
|
||||
}
|
||||
|
||||
void decode(unsigned char *s, size_t targetLength, node t, int *out)
|
||||
{
|
||||
size_t i = 0, byteIndex = 0, count = 0;
|
||||
int r;
|
||||
node n = t;
|
||||
|
||||
if(n->t) //root->t==1 means that all state values are the same (constant)
|
||||
{
|
||||
for(count=0;count<targetLength;count++)
|
||||
out[count] = n->c;
|
||||
return;
|
||||
}
|
||||
|
||||
for(i=0;count<targetLength;i++)
|
||||
{
|
||||
|
||||
byteIndex = i>>3; //i/8
|
||||
r = i%8;
|
||||
if(((s[byteIndex] >> (7-r)) & 0x01) == 0)
|
||||
n = n->left;
|
||||
else
|
||||
n = n->right;
|
||||
|
||||
if (n->t) {
|
||||
//putchar(n->c);
|
||||
out[count] = n->c;
|
||||
n = t;
|
||||
count++;
|
||||
}
|
||||
}
|
||||
// putchar('\n');
|
||||
if (t != n) printf("garbage input\n");
|
||||
return;
|
||||
}
|
||||
|
||||
void pad_tree_uchar(HuffmanTree* huffmanTree, unsigned char* L, unsigned char* R, unsigned int* C, unsigned char* t, unsigned int i, node root)
|
||||
{
|
||||
C[i] = root->c;
|
||||
t[i] = root->t;
|
||||
node lroot = root->left;
|
||||
if(lroot!=0)
|
||||
{
|
||||
huffmanTree->n_inode++;
|
||||
L[i] = huffmanTree->n_inode;
|
||||
pad_tree_uchar(huffmanTree, L,R,C,t, huffmanTree->n_inode, lroot);
|
||||
}
|
||||
node rroot = root->right;
|
||||
if(rroot!=0)
|
||||
{
|
||||
huffmanTree->n_inode++;
|
||||
R[i] = huffmanTree->n_inode;
|
||||
pad_tree_uchar(huffmanTree, L,R,C,t, huffmanTree->n_inode, rroot);
|
||||
}
|
||||
}
|
||||
|
||||
void pad_tree_ushort(HuffmanTree* huffmanTree, unsigned short* L, unsigned short* R, unsigned int* C, unsigned char* t, unsigned int i, node root)
|
||||
{
|
||||
C[i] = root->c;
|
||||
t[i] = root->t;
|
||||
node lroot = root->left;
|
||||
if(lroot!=0)
|
||||
{
|
||||
huffmanTree->n_inode++;
|
||||
L[i] = huffmanTree->n_inode;
|
||||
pad_tree_ushort(huffmanTree,L,R,C,t,huffmanTree->n_inode, lroot);
|
||||
}
|
||||
node rroot = root->right;
|
||||
if(rroot!=0)
|
||||
{
|
||||
huffmanTree->n_inode++;
|
||||
R[i] = huffmanTree->n_inode;
|
||||
pad_tree_ushort(huffmanTree,L,R,C,t,huffmanTree->n_inode, rroot);
|
||||
}
|
||||
}
|
||||
|
||||
void pad_tree_uint(HuffmanTree* huffmanTree, unsigned int* L, unsigned int* R, unsigned int* C, unsigned char* t, unsigned int i, node root)
|
||||
{
|
||||
C[i] = root->c;
|
||||
t[i] = root->t;
|
||||
node lroot = root->left;
|
||||
if(lroot!=0)
|
||||
{
|
||||
huffmanTree->n_inode++;
|
||||
L[i] = huffmanTree->n_inode;
|
||||
pad_tree_uint(huffmanTree,L,R,C,t,huffmanTree->n_inode, lroot);
|
||||
}
|
||||
node rroot = root->right;
|
||||
if(rroot!=0)
|
||||
{
|
||||
huffmanTree->n_inode++;
|
||||
R[i] = huffmanTree->n_inode;
|
||||
pad_tree_uint(huffmanTree,L,R,C,t,huffmanTree->n_inode, rroot);
|
||||
}
|
||||
}
|
||||
|
||||
unsigned int convert_HuffTree_to_bytes_anyStates(HuffmanTree* huffmanTree, int nodeCount, unsigned char** out)
|
||||
{
|
||||
if(nodeCount<=256)
|
||||
{
|
||||
unsigned char* L = (unsigned char*)malloc(nodeCount*sizeof(unsigned char));
|
||||
memset(L, 0, nodeCount*sizeof(unsigned char));
|
||||
unsigned char* R = (unsigned char*)malloc(nodeCount*sizeof(unsigned char));
|
||||
memset(R, 0, nodeCount*sizeof(unsigned char));
|
||||
unsigned int* C = (unsigned int*)malloc(nodeCount*sizeof(unsigned int));
|
||||
memset(C, 0, nodeCount*sizeof(unsigned int));
|
||||
unsigned char* t = (unsigned char*)malloc(nodeCount*sizeof(unsigned char));
|
||||
memset(t, 0, nodeCount*sizeof(unsigned char));
|
||||
|
||||
pad_tree_uchar(huffmanTree,L,R,C,t,0,huffmanTree->qq[1]);
|
||||
|
||||
unsigned int totalSize = 1+3*nodeCount*sizeof(unsigned char)+nodeCount*sizeof(unsigned int);
|
||||
*out = (unsigned char*)malloc(totalSize*sizeof(unsigned char));
|
||||
(*out)[0] = (unsigned char)sysEndianType;
|
||||
memcpy(*out+1, L, nodeCount*sizeof(unsigned char));
|
||||
memcpy((*out)+1+nodeCount*sizeof(unsigned char),R,nodeCount*sizeof(unsigned char));
|
||||
memcpy((*out)+1+2*nodeCount*sizeof(unsigned char),C,nodeCount*sizeof(unsigned int));
|
||||
memcpy((*out)+1+2*nodeCount*sizeof(unsigned char)+nodeCount*sizeof(unsigned int), t, nodeCount*sizeof(unsigned char));
|
||||
free(L);
|
||||
free(R);
|
||||
free(C);
|
||||
free(t);
|
||||
return totalSize;
|
||||
|
||||
}
|
||||
else if(nodeCount<=65536)
|
||||
{
|
||||
unsigned short* L = (unsigned short*)malloc(nodeCount*sizeof(unsigned short));
|
||||
memset(L, 0, nodeCount*sizeof(unsigned short));
|
||||
unsigned short* R = (unsigned short*)malloc(nodeCount*sizeof(unsigned short));
|
||||
memset(R, 0, nodeCount*sizeof(unsigned short));
|
||||
unsigned int* C = (unsigned int*)malloc(nodeCount*sizeof(unsigned int));
|
||||
memset(C, 0, nodeCount*sizeof(unsigned int));
|
||||
unsigned char* t = (unsigned char*)malloc(nodeCount*sizeof(unsigned char));
|
||||
memset(t, 0, nodeCount*sizeof(unsigned char));
|
||||
pad_tree_ushort(huffmanTree,L,R,C,t,0,huffmanTree->qq[1]);
|
||||
unsigned int totalSize = 1+2*nodeCount*sizeof(unsigned short)+nodeCount*sizeof(unsigned char) + nodeCount*sizeof(unsigned int);
|
||||
*out = (unsigned char*)malloc(totalSize);
|
||||
(*out)[0] = (unsigned char)sysEndianType;
|
||||
memcpy(*out+1, L, nodeCount*sizeof(unsigned short));
|
||||
memcpy((*out)+1+nodeCount*sizeof(unsigned short),R,nodeCount*sizeof(unsigned short));
|
||||
memcpy((*out)+1+2*nodeCount*sizeof(unsigned short),C,nodeCount*sizeof(unsigned int));
|
||||
memcpy((*out)+1+2*nodeCount*sizeof(unsigned short)+nodeCount*sizeof(unsigned int),t,nodeCount*sizeof(unsigned char));
|
||||
free(L);
|
||||
free(R);
|
||||
free(C);
|
||||
free(t);
|
||||
return totalSize;
|
||||
}
|
||||
else //nodeCount>65536
|
||||
{
|
||||
unsigned int* L = (unsigned int*)malloc(nodeCount*sizeof(unsigned int));
|
||||
memset(L, 0, nodeCount*sizeof(unsigned int));
|
||||
unsigned int* R = (unsigned int*)malloc(nodeCount*sizeof(unsigned int));
|
||||
memset(R, 0, nodeCount*sizeof(unsigned int));
|
||||
unsigned int* C = (unsigned int*)malloc(nodeCount*sizeof(unsigned int));
|
||||
memset(C, 0, nodeCount*sizeof(unsigned int));
|
||||
unsigned char* t = (unsigned char*)malloc(nodeCount*sizeof(unsigned char));
|
||||
memset(t, 0, nodeCount*sizeof(unsigned char));
|
||||
pad_tree_uint(huffmanTree, L,R,C,t,0,huffmanTree->qq[1]);
|
||||
|
||||
//debug
|
||||
//node root = new_node2(0,0);
|
||||
//unpad_tree_uint(L,R,C,t,0,root);
|
||||
|
||||
unsigned int totalSize = 1+3*nodeCount*sizeof(unsigned int)+nodeCount*sizeof(unsigned char);
|
||||
*out = (unsigned char*)malloc(totalSize);
|
||||
(*out)[0] = (unsigned char)sysEndianType;
|
||||
memcpy(*out+1, L, nodeCount*sizeof(unsigned int));
|
||||
memcpy((*out)+1+nodeCount*sizeof(unsigned int),R,nodeCount*sizeof(unsigned int));
|
||||
memcpy((*out)+1+2*nodeCount*sizeof(unsigned int),C,nodeCount*sizeof(unsigned int));
|
||||
memcpy((*out)+1+3*nodeCount*sizeof(unsigned int),t,nodeCount*sizeof(unsigned char));
|
||||
free(L);
|
||||
free(R);
|
||||
free(C);
|
||||
free(t);
|
||||
return totalSize;
|
||||
}
|
||||
}
|
||||
|
||||
void unpad_tree_uchar(HuffmanTree* huffmanTree, unsigned char* L, unsigned char* R, unsigned int* C, unsigned char *t, unsigned int i, node root)
|
||||
{
|
||||
//root->c = C[i];
|
||||
if(root->t==0)
|
||||
{
|
||||
unsigned char l, r;
|
||||
l = L[i];
|
||||
if(l!=0)
|
||||
{
|
||||
node lroot = new_node2(huffmanTree,C[l],t[l]);
|
||||
root->left = lroot;
|
||||
unpad_tree_uchar(huffmanTree,L,R,C,t,l,lroot);
|
||||
}
|
||||
r = R[i];
|
||||
if(r!=0)
|
||||
{
|
||||
node rroot = new_node2(huffmanTree,C[r],t[r]);
|
||||
root->right = rroot;
|
||||
unpad_tree_uchar(huffmanTree,L,R,C,t,r,rroot);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void unpad_tree_ushort(HuffmanTree* huffmanTree, unsigned short* L, unsigned short* R, unsigned int* C, unsigned char* t, unsigned int i, node root)
|
||||
{
|
||||
//root->c = C[i];
|
||||
if(root->t==0)
|
||||
{
|
||||
unsigned short l, r;
|
||||
l = L[i];
|
||||
if(l!=0)
|
||||
{
|
||||
node lroot = new_node2(huffmanTree,C[l],t[l]);
|
||||
root->left = lroot;
|
||||
unpad_tree_ushort(huffmanTree,L,R,C,t,l,lroot);
|
||||
}
|
||||
r = R[i];
|
||||
if(r!=0)
|
||||
{
|
||||
node rroot = new_node2(huffmanTree,C[r],t[r]);
|
||||
root->right = rroot;
|
||||
unpad_tree_ushort(huffmanTree,L,R,C,t,r,rroot);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void unpad_tree_uint(HuffmanTree* huffmanTree, unsigned int* L, unsigned int* R, unsigned int* C, unsigned char* t, unsigned int i, node root)
|
||||
{
|
||||
//root->c = C[i];
|
||||
if(root->t==0)
|
||||
{
|
||||
unsigned int l, r;
|
||||
l = L[i];
|
||||
if(l!=0)
|
||||
{
|
||||
node lroot = new_node2(huffmanTree,C[l],t[l]);
|
||||
root->left = lroot;
|
||||
unpad_tree_uint(huffmanTree,L,R,C,t,l,lroot);
|
||||
}
|
||||
r = R[i];
|
||||
if(r!=0)
|
||||
{
|
||||
node rroot = new_node2(huffmanTree,C[r],t[r]);
|
||||
root->right = rroot;
|
||||
unpad_tree_uint(huffmanTree,L,R,C,t,r,rroot);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
node reconstruct_HuffTree_from_bytes_anyStates(HuffmanTree *huffmanTree, unsigned char* bytes, int nodeCount)
|
||||
{
|
||||
if(nodeCount<=256)
|
||||
{
|
||||
unsigned char* L = (unsigned char*)malloc(nodeCount*sizeof(unsigned char));
|
||||
memset(L, 0, nodeCount*sizeof(unsigned char));
|
||||
unsigned char* R = (unsigned char*)malloc(nodeCount*sizeof(unsigned char));
|
||||
memset(R, 0, nodeCount*sizeof(unsigned char));
|
||||
unsigned int* C = (unsigned int*)malloc(nodeCount*sizeof(unsigned int));
|
||||
memset(C, 0, nodeCount*sizeof(unsigned int));
|
||||
unsigned char* t = (unsigned char*)malloc(nodeCount*sizeof(unsigned char));
|
||||
memset(t, 0, nodeCount*sizeof(unsigned char));
|
||||
unsigned char cmpSysEndianType = bytes[0];
|
||||
if(cmpSysEndianType!=(unsigned char)sysEndianType)
|
||||
{
|
||||
unsigned char* p = (unsigned char*)(bytes+1+2*nodeCount*sizeof(unsigned char));
|
||||
size_t i = 0, size = nodeCount*sizeof(unsigned int);
|
||||
while(1)
|
||||
{
|
||||
symTransform_4bytes(p);
|
||||
i+=sizeof(unsigned int);
|
||||
if(i<size)
|
||||
p+=sizeof(unsigned int);
|
||||
else
|
||||
break;
|
||||
}
|
||||
}
|
||||
memcpy(L, bytes+1, nodeCount*sizeof(unsigned char));
|
||||
memcpy(R, bytes+1+nodeCount*sizeof(unsigned char), nodeCount*sizeof(unsigned char));
|
||||
memcpy(C, bytes+1+2*nodeCount*sizeof(unsigned char), nodeCount*sizeof(unsigned int));
|
||||
memcpy(t, bytes+1+2*nodeCount*sizeof(unsigned char)+nodeCount*sizeof(unsigned int), nodeCount*sizeof(unsigned char));
|
||||
node root = new_node2(huffmanTree, C[0],t[0]);
|
||||
unpad_tree_uchar(huffmanTree,L,R,C,t,0,root);
|
||||
free(L);
|
||||
free(R);
|
||||
free(C);
|
||||
free(t);
|
||||
return root;
|
||||
}
|
||||
else if(nodeCount<=65536)
|
||||
{
|
||||
unsigned short* L = (unsigned short*)malloc(nodeCount*sizeof(unsigned short));
|
||||
memset(L, 0, nodeCount*sizeof(unsigned short));
|
||||
unsigned short* R = (unsigned short*)malloc(nodeCount*sizeof(unsigned short));
|
||||
memset(R, 0, nodeCount*sizeof(unsigned short));
|
||||
unsigned int* C = (unsigned int*)malloc(nodeCount*sizeof(unsigned int));
|
||||
memset(C, 0, nodeCount*sizeof(unsigned int));
|
||||
unsigned char* t = (unsigned char*)malloc(nodeCount*sizeof(unsigned char));
|
||||
memset(t, 0, nodeCount*sizeof(unsigned char));
|
||||
|
||||
unsigned char cmpSysEndianType = bytes[0];
|
||||
if(cmpSysEndianType!=(unsigned char)sysEndianType)
|
||||
{
|
||||
unsigned char* p = (unsigned char*)(bytes+1);
|
||||
size_t i = 0, size = 2*nodeCount*sizeof(unsigned short);
|
||||
|
||||
while(1)
|
||||
{
|
||||
symTransform_2bytes(p);
|
||||
i+=sizeof(unsigned short);
|
||||
if(i<size)
|
||||
p+=sizeof(unsigned short);
|
||||
else
|
||||
break;
|
||||
}
|
||||
|
||||
size = nodeCount*sizeof(unsigned int);
|
||||
while(1)
|
||||
{
|
||||
symTransform_4bytes(p);
|
||||
i+=sizeof(unsigned int);
|
||||
if(i<size)
|
||||
p+=sizeof(unsigned int);
|
||||
else
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
memcpy(L, bytes+1, nodeCount*sizeof(unsigned short));
|
||||
memcpy(R, bytes+1+nodeCount*sizeof(unsigned short), nodeCount*sizeof(unsigned short));
|
||||
memcpy(C, bytes+1+2*nodeCount*sizeof(unsigned short), nodeCount*sizeof(unsigned int));
|
||||
|
||||
memcpy(t, bytes+1+2*nodeCount*sizeof(unsigned short)+nodeCount*sizeof(unsigned int), nodeCount*sizeof(unsigned char));
|
||||
|
||||
node root = new_node2(huffmanTree,0,0);
|
||||
unpad_tree_ushort(huffmanTree,L,R,C,t,0,root);
|
||||
free(L);
|
||||
free(R);
|
||||
free(C);
|
||||
free(t);
|
||||
return root;
|
||||
}
|
||||
else //nodeCount>65536
|
||||
{
|
||||
unsigned int* L = (unsigned int*)malloc(nodeCount*sizeof(unsigned int));
|
||||
memset(L, 0, nodeCount*sizeof(unsigned int));
|
||||
unsigned int* R = (unsigned int*)malloc(nodeCount*sizeof(unsigned int));
|
||||
memset(R, 0, nodeCount*sizeof(unsigned int));
|
||||
unsigned int* C = (unsigned int*)malloc(nodeCount*sizeof(unsigned int));
|
||||
memset(C, 0, nodeCount*sizeof(unsigned int));
|
||||
unsigned char* t = (unsigned char*)malloc(nodeCount*sizeof(unsigned char));
|
||||
memset(t, 0, nodeCount*sizeof(unsigned char));
|
||||
unsigned char cmpSysEndianType = bytes[0];
|
||||
if(cmpSysEndianType!=(unsigned char)sysEndianType)
|
||||
{
|
||||
unsigned char* p = (unsigned char*)(bytes+1);
|
||||
size_t i = 0, size = 3*nodeCount*sizeof(unsigned int);
|
||||
while(1)
|
||||
{
|
||||
symTransform_4bytes(p);
|
||||
i+=sizeof(unsigned int);
|
||||
if(i<size)
|
||||
p+=sizeof(unsigned int);
|
||||
else
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
memcpy(L, bytes+1, nodeCount*sizeof(unsigned int));
|
||||
memcpy(R, bytes+1+nodeCount*sizeof(unsigned int), nodeCount*sizeof(unsigned int));
|
||||
memcpy(C, bytes+1+2*nodeCount*sizeof(unsigned int), nodeCount*sizeof(unsigned int));
|
||||
|
||||
memcpy(t, bytes+1+3*nodeCount*sizeof(unsigned int), nodeCount*sizeof(unsigned char));
|
||||
|
||||
node root = new_node2(huffmanTree,0,0);
|
||||
unpad_tree_uint(huffmanTree,L,R,C,t,0,root);
|
||||
free(L);
|
||||
free(R);
|
||||
free(C);
|
||||
free(t);
|
||||
return root;
|
||||
}
|
||||
}
|
||||
|
||||
void encode_withTree(HuffmanTree* huffmanTree, int *s, size_t length, unsigned char **out, size_t *outSize)
|
||||
{
|
||||
size_t i;
|
||||
int nodeCount = 0;
|
||||
unsigned char *treeBytes, buffer[4];
|
||||
|
||||
init(huffmanTree, s, length);
|
||||
for (i = 0; i < huffmanTree->stateNum; i++)
|
||||
if (huffmanTree->code[i]) nodeCount++;
|
||||
nodeCount = nodeCount*2-1;
|
||||
unsigned int treeByteSize = convert_HuffTree_to_bytes_anyStates(huffmanTree,nodeCount, &treeBytes);
|
||||
//printf("treeByteSize = %d\n", treeByteSize);
|
||||
|
||||
*out = (unsigned char*)malloc(length*sizeof(int)+treeByteSize);
|
||||
intToBytes_bigEndian(buffer, nodeCount);
|
||||
memcpy(*out, buffer, 4);
|
||||
intToBytes_bigEndian(buffer, huffmanTree->stateNum/2); //real number of intervals
|
||||
memcpy(*out+4, buffer, 4);
|
||||
memcpy(*out+8, treeBytes, treeByteSize);
|
||||
free(treeBytes);
|
||||
size_t enCodeSize = 0;
|
||||
encode(huffmanTree, s, length, *out+8+treeByteSize, &enCodeSize);
|
||||
*outSize = 8+treeByteSize+enCodeSize;
|
||||
}
|
||||
|
||||
/**
|
||||
* @par *out rememmber to allocate targetLength short_type data for it beforehand.
|
||||
*
|
||||
* */
|
||||
void decode_withTree(HuffmanTree* huffmanTree, unsigned char *s, size_t targetLength, int *out)
|
||||
{
|
||||
size_t encodeStartIndex;
|
||||
size_t nodeCount = bytesToInt_bigEndian(s);
|
||||
node root = reconstruct_HuffTree_from_bytes_anyStates(huffmanTree,s+8, nodeCount);
|
||||
|
||||
//sdi: Debug
|
||||
/* build_code(root, 0, 0, 0);
|
||||
int i;
|
||||
unsigned long code_1, code_2;
|
||||
for (i = 0; i < stateNum; i++)
|
||||
if (code[i])
|
||||
{
|
||||
printf("%d: %lu,%lu ; %u\n", i, (code[i])[0],(code[i])[1], cout[i]);
|
||||
//code_1 = (code[i])[0];
|
||||
}*/
|
||||
|
||||
if(nodeCount<=256)
|
||||
encodeStartIndex = 1+3*nodeCount*sizeof(unsigned char)+nodeCount*sizeof(unsigned int);
|
||||
else if(nodeCount<=65536)
|
||||
encodeStartIndex = 1+2*nodeCount*sizeof(unsigned short)+nodeCount*sizeof(unsigned char)+nodeCount*sizeof(unsigned int);
|
||||
else
|
||||
encodeStartIndex = 1+3*nodeCount*sizeof(unsigned int)+nodeCount*sizeof(unsigned char);
|
||||
decode(s+8+encodeStartIndex, targetLength, root, out);
|
||||
}
|
||||
|
||||
void SZ_ReleaseHuffman(HuffmanTree* huffmanTree)
|
||||
{
|
||||
size_t i;
|
||||
free(huffmanTree->pool);
|
||||
huffmanTree->pool = NULL;
|
||||
free(huffmanTree->qqq);
|
||||
huffmanTree->qqq = NULL;
|
||||
for(i=0;i<huffmanTree->stateNum;i++)
|
||||
{
|
||||
if(huffmanTree->code[i]!=NULL)
|
||||
free(huffmanTree->code[i]);
|
||||
}
|
||||
free(huffmanTree->code);
|
||||
huffmanTree->code = NULL;
|
||||
free(huffmanTree->cout);
|
||||
huffmanTree->cout = NULL;
|
||||
free(huffmanTree);
|
||||
huffmanTree = NULL;
|
||||
}
|
||||
|
|
@ -0,0 +1,421 @@
|
|||
/**
|
||||
* @file TightPointDataStorageD.c
|
||||
* @author Sheng Di and Dingwen Tao
|
||||
* @date Aug, 2016
|
||||
* @brief The functions used to construct the tightPointDataStorage element for storing compressed bytes.
|
||||
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
|
||||
* See COPYRIGHT in top-level directory.
|
||||
*/
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include "TightDataPointStorageD.h"
|
||||
#include "sz.h"
|
||||
#include "Huffman.h"
|
||||
#include "transcode.h"
|
||||
#include "fse.h"
|
||||
#include <sys/time.h>
|
||||
|
||||
void new_TightDataPointStorageD_Empty(TightDataPointStorageD **this)
|
||||
{
|
||||
TightDataPointStorageD* tdps = (TightDataPointStorageD*)malloc(sizeof(TightDataPointStorageD));
|
||||
memset(tdps, 0, sizeof(TightDataPointStorageD));
|
||||
*this = tdps;
|
||||
}
|
||||
|
||||
int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsigned char* flatBytes, size_t flatBytesLength, sz_exedata* pde_exe, sz_params* pde_params)
|
||||
{
|
||||
new_TightDataPointStorageD_Empty(this);
|
||||
size_t i, index = 0;
|
||||
unsigned char version = flatBytes[index++]; //3
|
||||
unsigned char sameRByte = flatBytes[index++]; //1
|
||||
|
||||
// parse data format
|
||||
switch (version)
|
||||
{
|
||||
case DATA_FROMAT_VER1:
|
||||
break;
|
||||
default:
|
||||
printf(" error, compressed data format can not be recognised. ver=%d\n ", version);
|
||||
return SZ_ABS;
|
||||
}
|
||||
|
||||
int same = sameRByte & 0x01;
|
||||
(*this)->ifAdtFse = (sameRByte & 0x02)>>1; //0000,0010
|
||||
(*this)->isLossless = (sameRByte & 0x10)>>4;
|
||||
pde_exe->SZ_SIZE_TYPE = ((sameRByte & 0x40)>>6)==1?8:4;
|
||||
//pde_params->protectValueRange = (sameRByte & 0x04)>>2;
|
||||
pde_params->accelerate_pw_rel_compression = (sameRByte & 0x08) >> 3;
|
||||
int errorBoundMode = SZ_ABS;
|
||||
|
||||
convertBytesToSZParams(&(flatBytes[index]), pde_params, pde_exe);
|
||||
|
||||
index += MetaDataByteLength_double;
|
||||
|
||||
int isRegression = (sameRByte >> 7) & 0x01;
|
||||
|
||||
unsigned char dsLengthBytes[8];
|
||||
for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++)
|
||||
dsLengthBytes[i] = flatBytes[index++];
|
||||
(*this)->dataSeriesLength = bytesToSize(dsLengthBytes, pde_exe->SZ_SIZE_TYPE);
|
||||
|
||||
if((*this)->isLossless==1)
|
||||
{
|
||||
//(*this)->exactMidBytes = flatBytes+8;
|
||||
return errorBoundMode;
|
||||
}
|
||||
else if(same==1)
|
||||
{
|
||||
(*this)->allSameData = 1;
|
||||
(*this)->exactMidBytes = &(flatBytes[index]);
|
||||
return errorBoundMode;
|
||||
}
|
||||
else
|
||||
(*this)->allSameData = 0;
|
||||
|
||||
if(isRegression == 1)
|
||||
{
|
||||
(*this)->raBytes_size = flatBytesLength - 3 - 1 - MetaDataByteLength_double - pde_exe->SZ_SIZE_TYPE;
|
||||
(*this)->raBytes = &(flatBytes[index]);
|
||||
return errorBoundMode;
|
||||
}
|
||||
|
||||
unsigned char byteBuf[8];
|
||||
|
||||
for (i = 0; i < 4; i++)
|
||||
byteBuf[i] = flatBytes[index++];
|
||||
int max_quant_intervals = bytesToInt_bigEndian(byteBuf);// 4
|
||||
|
||||
pde_params->maxRangeRadius = max_quant_intervals/2;
|
||||
|
||||
for (i = 0; i < 4; i++)
|
||||
byteBuf[i] = flatBytes[index++];
|
||||
(*this)->intervals = bytesToInt_bigEndian(byteBuf);// 4
|
||||
|
||||
for (i = 0; i < 8; i++)
|
||||
byteBuf[i] = flatBytes[index++];
|
||||
(*this)->medianValue = bytesToDouble(byteBuf);//8
|
||||
|
||||
(*this)->reqLength = flatBytes[index++]; //1
|
||||
|
||||
for (i = 0; i < 8; i++)
|
||||
byteBuf[i] = flatBytes[index++];
|
||||
(*this)->realPrecision = bytesToDouble(byteBuf);//8
|
||||
|
||||
if ((*this)->ifAdtFse == 0) {
|
||||
for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++)
|
||||
byteBuf[i] = flatBytes[index++];
|
||||
(*this)->typeArray_size = bytesToSize(byteBuf, pde_exe->SZ_SIZE_TYPE);
|
||||
} else {
|
||||
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
|
||||
byteBuf[i] = flatBytes[index++];
|
||||
(*this)->FseCode_size = bytesToSize(byteBuf, pde_exe->SZ_SIZE_TYPE);// 4
|
||||
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
|
||||
byteBuf[i] = flatBytes[index++];
|
||||
(*this)->transCodeBits_size = bytesToSize(byteBuf, pde_exe->SZ_SIZE_TYPE);// 4
|
||||
}
|
||||
|
||||
for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++)
|
||||
byteBuf[i] = flatBytes[index++];
|
||||
(*this)->exactDataNum = bytesToSize(byteBuf, pde_exe->SZ_SIZE_TYPE);// ST
|
||||
|
||||
for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++)
|
||||
byteBuf[i] = flatBytes[index++];
|
||||
(*this)->exactMidBytes_size = bytesToSize(byteBuf, pde_exe->SZ_SIZE_TYPE);// ST
|
||||
|
||||
size_t logicLeadNumBitsNum = (*this)->exactDataNum * 2;
|
||||
if (logicLeadNumBitsNum % 8 == 0)
|
||||
{
|
||||
(*this)->leadNumArray_size = logicLeadNumBitsNum >> 3;
|
||||
}
|
||||
else
|
||||
{
|
||||
(*this)->leadNumArray_size = (logicLeadNumBitsNum >> 3) + 1;
|
||||
}
|
||||
|
||||
if ((*this)->ifAdtFse == 0) {
|
||||
(*this)->typeArray = &flatBytes[index];
|
||||
//retrieve the number of states (i.e., stateNum)
|
||||
(*this)->allNodes = bytesToInt_bigEndian((*this)->typeArray); //the first 4 bytes store the stateNum
|
||||
(*this)->stateNum = ((*this)->allNodes+1)/2;
|
||||
index+=(*this)->typeArray_size;
|
||||
} else {
|
||||
(*this)->FseCode = &flatBytes[index];
|
||||
index+=(*this)->FseCode_size;
|
||||
|
||||
(*this)->transCodeBits = &flatBytes[index];
|
||||
index+=(*this)->transCodeBits_size;
|
||||
}
|
||||
|
||||
// todo need check length
|
||||
(*this)->residualMidBits_size = flatBytesLength - 1 - 1 - MetaDataByteLength - pde_exe->SZ_SIZE_TYPE - 4 - 4 - 4 - 1 - 8
|
||||
- pde_exe->SZ_SIZE_TYPE - pde_exe->SZ_SIZE_TYPE
|
||||
- (*this)->leadNumArray_size - (*this)->exactMidBytes_size;
|
||||
if ((*this)->ifAdtFse == 0) {
|
||||
(*this)->residualMidBits_size = (*this)->residualMidBits_size - (*this)->typeArray_size - pde_exe->SZ_SIZE_TYPE ;
|
||||
} else {
|
||||
(*this)->residualMidBits_size = (*this)->residualMidBits_size - (*this)->FseCode_size - (*this)->transCodeBits_size - pde_exe->SZ_SIZE_TYPE - pde_exe->SZ_SIZE_TYPE;
|
||||
}
|
||||
|
||||
|
||||
(*this)->leadNumArray = &flatBytes[index];
|
||||
|
||||
index+=(*this)->leadNumArray_size;
|
||||
|
||||
(*this)->exactMidBytes = &flatBytes[index];
|
||||
|
||||
index+=(*this)->exactMidBytes_size;
|
||||
|
||||
(*this)->residualMidBits = &flatBytes[index];
|
||||
|
||||
|
||||
return errorBoundMode;
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* type's length == dataSeriesLength
|
||||
* exactMidBytes's length == exactMidBytes_size
|
||||
* leadNumIntArray's length == exactDataNum
|
||||
* escBytes's length == escBytes_size
|
||||
* resiBitLength's length == resiBitLengthSize
|
||||
* */
|
||||
void new_TightDataPointStorageD(TightDataPointStorageD **this,
|
||||
size_t dataSeriesLength, size_t exactDataNum,
|
||||
int* type, unsigned char* exactMidBytes, size_t exactMidBytes_size,
|
||||
unsigned char* leadNumIntArray, //leadNumIntArray contains readable numbers....
|
||||
unsigned char* resiMidBits, size_t resiMidBits_size,
|
||||
unsigned char resiBitLength,
|
||||
double realPrecision, double medianValue, char reqLength, unsigned int intervals,
|
||||
unsigned char radExpo)
|
||||
{
|
||||
//int i = 0;
|
||||
*this = (TightDataPointStorageD *)malloc(sizeof(TightDataPointStorageD));
|
||||
memset(*this, 0, sizeof(TightDataPointStorageD));
|
||||
(*this)->ifAdtFse = confparams_cpr->ifAdtFse;
|
||||
(*this)->allSameData = 0;
|
||||
(*this)->realPrecision = realPrecision;
|
||||
(*this)->medianValue = medianValue;
|
||||
(*this)->reqLength = reqLength;
|
||||
|
||||
(*this)->dataSeriesLength = dataSeriesLength;
|
||||
(*this)->exactDataNum = exactDataNum;
|
||||
|
||||
if ((*this)->ifAdtFse == 0) {
|
||||
// huffman
|
||||
int stateNum = 2 * intervals;
|
||||
HuffmanTree* huffmanTree = createHuffmanTree(stateNum);
|
||||
encode_withTree(huffmanTree, type, dataSeriesLength, &(*this)->typeArray, &(*this)->typeArray_size);
|
||||
SZ_ReleaseHuffman(huffmanTree);
|
||||
}
|
||||
else {
|
||||
// fse
|
||||
encode_with_fse(type, dataSeriesLength, intervals, &((*this)->FseCode), &((*this)->FseCode_size),
|
||||
&((*this)->transCodeBits), &((*this)->transCodeBits_size));
|
||||
}
|
||||
|
||||
(*this)->exactMidBytes = exactMidBytes;
|
||||
(*this)->exactMidBytes_size = exactMidBytes_size;
|
||||
|
||||
(*this)->leadNumArray_size = convertIntArray2ByteArray_fast_2b(leadNumIntArray, exactDataNum, &((*this)->leadNumArray));
|
||||
|
||||
(*this)->residualMidBits_size = convertIntArray2ByteArray_fast_dynamic(resiMidBits, resiBitLength, exactDataNum, &((*this)->residualMidBits));
|
||||
|
||||
(*this)->intervals = intervals;
|
||||
|
||||
(*this)->isLossless = 0;
|
||||
|
||||
(*this)->radExpo = radExpo;
|
||||
}
|
||||
|
||||
void convertTDPStoBytes_double(TightDataPointStorageD* tdps, unsigned char* bytes, unsigned char* dsLengthBytes, unsigned char sameByte)
|
||||
{
|
||||
size_t i, k = 0;
|
||||
unsigned char intervalsBytes[4];
|
||||
unsigned char typeArrayLengthBytes[8];
|
||||
unsigned char exactLengthBytes[8];
|
||||
unsigned char exactMidBytesLength[8];
|
||||
unsigned char realPrecisionBytes[8];
|
||||
unsigned char fsecodeLengthBytes[8];
|
||||
unsigned char transcodeLengthBytes[8];
|
||||
|
||||
unsigned char medianValueBytes[8];
|
||||
unsigned char max_quant_intervals_Bytes[4];
|
||||
|
||||
bytes[k++] = versionNumber;
|
||||
bytes[k++] = sameByte; //1 byte
|
||||
|
||||
convertSZParamsToBytes(confparams_cpr, &(bytes[k]), exe_params->optQuantMode);
|
||||
k = k + MetaDataByteLength_double;
|
||||
|
||||
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST: 4 or 8 bytes
|
||||
bytes[k++] = dsLengthBytes[i];
|
||||
intToBytes_bigEndian(max_quant_intervals_Bytes, confparams_cpr->max_quant_intervals);
|
||||
for(i = 0;i<4;i++)//4
|
||||
bytes[k++] = max_quant_intervals_Bytes[i];
|
||||
|
||||
intToBytes_bigEndian(intervalsBytes, tdps->intervals);
|
||||
for(i = 0;i<4;i++)//4
|
||||
bytes[k++] = intervalsBytes[i];
|
||||
|
||||
doubleToBytes(medianValueBytes, tdps->medianValue);
|
||||
for (i = 0; i < 8; i++)// 8
|
||||
bytes[k++] = medianValueBytes[i];
|
||||
|
||||
bytes[k++] = tdps->reqLength; //1 byte
|
||||
|
||||
doubleToBytes(realPrecisionBytes, tdps->realPrecision);
|
||||
for (i = 0; i < 8; i++)// 8
|
||||
bytes[k++] = realPrecisionBytes[i];
|
||||
|
||||
if (tdps->ifAdtFse == 0) {
|
||||
sizeToBytes(typeArrayLengthBytes, tdps->typeArray_size, exe_params->SZ_SIZE_TYPE);
|
||||
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
|
||||
bytes[k++] = typeArrayLengthBytes[i];
|
||||
} else {
|
||||
sizeToBytes(fsecodeLengthBytes, tdps->FseCode_size, exe_params->SZ_SIZE_TYPE);
|
||||
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
|
||||
bytes[k++] = fsecodeLengthBytes[i];
|
||||
|
||||
sizeToBytes(transcodeLengthBytes, tdps->transCodeBits_size, exe_params->SZ_SIZE_TYPE);
|
||||
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
|
||||
bytes[k++] = transcodeLengthBytes[i];
|
||||
}
|
||||
|
||||
sizeToBytes(exactLengthBytes, tdps->exactDataNum, exe_params->SZ_SIZE_TYPE);
|
||||
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
|
||||
bytes[k++] = exactLengthBytes[i];
|
||||
|
||||
sizeToBytes(exactMidBytesLength, tdps->exactMidBytes_size, exe_params->SZ_SIZE_TYPE);
|
||||
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
|
||||
bytes[k++] = exactMidBytesLength[i];
|
||||
|
||||
if(confparams_cpr->errorBoundMode>=PW_REL)
|
||||
{
|
||||
doubleToBytes(exactMidBytesLength, tdps->minLogValue);
|
||||
for(i = 0;i < 8; i++)
|
||||
bytes[k++] = exactMidBytesLength[i];
|
||||
}
|
||||
|
||||
// copy data
|
||||
if (tdps->ifAdtFse == 0) {
|
||||
memcpy(&(bytes[k]), tdps->typeArray, tdps->typeArray_size);
|
||||
k += tdps->typeArray_size;
|
||||
} else {
|
||||
memcpy(&(bytes[k]), tdps->FseCode, tdps->FseCode_size);
|
||||
k += tdps->FseCode_size;
|
||||
memcpy(&(bytes[k]), tdps->transCodeBits, tdps->transCodeBits_size);
|
||||
k += tdps->transCodeBits_size;
|
||||
}
|
||||
memcpy(&(bytes[k]), tdps->leadNumArray, tdps->leadNumArray_size);
|
||||
k += tdps->leadNumArray_size;
|
||||
memcpy(&(bytes[k]), tdps->exactMidBytes, tdps->exactMidBytes_size);
|
||||
k += tdps->exactMidBytes_size;
|
||||
|
||||
if(tdps->residualMidBits!=NULL)
|
||||
{
|
||||
memcpy(&(bytes[k]), tdps->residualMidBits, tdps->residualMidBits_size);
|
||||
k += tdps->residualMidBits_size;
|
||||
}
|
||||
}
|
||||
|
||||
//Convert TightDataPointStorageD to bytes...
|
||||
bool convertTDPStoFlatBytes_double(TightDataPointStorageD *tdps, unsigned char* bytes, size_t *size)
|
||||
{
|
||||
size_t i, k = 0;
|
||||
unsigned char dsLengthBytes[8];
|
||||
|
||||
if(exe_params->SZ_SIZE_TYPE==4)
|
||||
intToBytes_bigEndian(dsLengthBytes, tdps->dataSeriesLength);//4
|
||||
else
|
||||
longToBytes_bigEndian(dsLengthBytes, tdps->dataSeriesLength);//8
|
||||
|
||||
unsigned char sameByte = tdps->allSameData==1?(unsigned char)1:(unsigned char)0;
|
||||
//sameByte = sameByte | (confparams_cpr->szMode << 1);
|
||||
sameByte = sameByte | (tdps->ifAdtFse << 1); //0000,0010
|
||||
if(tdps->isLossless)
|
||||
sameByte = (unsigned char) (sameByte | 0x10);
|
||||
if(confparams_cpr->errorBoundMode>=PW_REL)
|
||||
sameByte = (unsigned char) (sameByte | 0x20); // 00100000, the 5th bit
|
||||
if(exe_params->SZ_SIZE_TYPE==8)
|
||||
sameByte = (unsigned char) (sameByte | 0x40); // 01000000, the 6th bit
|
||||
if(confparams_cpr->errorBoundMode == PW_REL && confparams_cpr->accelerate_pw_rel_compression)
|
||||
sameByte = (unsigned char) (sameByte | 0x08);
|
||||
|
||||
if(tdps->allSameData==1)
|
||||
{
|
||||
size_t totalByteLength = 1 + 1 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + tdps->exactMidBytes_size;
|
||||
//bytes = (unsigned char *)malloc(sizeof(unsigned char)*totalByteLength); // comment by tickduan
|
||||
if(totalByteLength >= tdps->dataSeriesLength * sizeof(double))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
bytes[k++] = versionNumber;
|
||||
bytes[k++] = sameByte;
|
||||
|
||||
convertSZParamsToBytes(confparams_cpr, &(bytes[k]), exe_params->optQuantMode);
|
||||
k = k + MetaDataByteLength_double;
|
||||
|
||||
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
|
||||
bytes[k++] = dsLengthBytes[i];
|
||||
|
||||
for (i = 0; i < tdps->exactMidBytes_size; i++)
|
||||
bytes[k++] = tdps->exactMidBytes[i];
|
||||
|
||||
*size = totalByteLength;
|
||||
}
|
||||
else
|
||||
{
|
||||
size_t residualMidBitsLength = tdps->residualMidBits == NULL ? 0 : tdps->residualMidBits_size;
|
||||
size_t totalByteLength = 1 + 1 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + 4 + 4 + 8 + 1 + 8
|
||||
+ exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE
|
||||
+ tdps->leadNumArray_size
|
||||
+ tdps->exactMidBytes_size
|
||||
+ residualMidBitsLength;
|
||||
|
||||
if (tdps->ifAdtFse == 0) {
|
||||
totalByteLength += tdps->typeArray_size + exe_params->SZ_SIZE_TYPE;
|
||||
} else {
|
||||
totalByteLength += tdps->FseCode_size + tdps->transCodeBits_size + exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE;
|
||||
}
|
||||
|
||||
//*bytes = (unsigned char *)malloc(sizeof(unsigned char)*totalByteLength); comment by tickduan
|
||||
if(totalByteLength >= tdps->dataSeriesLength * sizeof(double))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
convertTDPStoBytes_double(tdps, bytes, dsLengthBytes, sameByte);
|
||||
*size = totalByteLength;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
void free_TightDataPointStorageD(TightDataPointStorageD *tdps)
|
||||
{
|
||||
if(tdps->leadNumArray!=NULL)
|
||||
free(tdps->leadNumArray);
|
||||
if(tdps->FseCode!=NULL)
|
||||
free(tdps->FseCode);
|
||||
if(tdps->transCodeBits!=NULL)
|
||||
free(tdps->transCodeBits);
|
||||
if(tdps->exactMidBytes!=NULL)
|
||||
free(tdps->exactMidBytes);
|
||||
if(tdps->residualMidBits!=NULL)
|
||||
free(tdps->residualMidBits);
|
||||
if(tdps->typeArray)
|
||||
free(tdps->typeArray);
|
||||
free(tdps);
|
||||
}
|
||||
|
||||
/**
|
||||
* to free the memory used in the decompression
|
||||
* */
|
||||
void free_TightDataPointStorageD2(TightDataPointStorageD *tdps)
|
||||
{
|
||||
free(tdps);
|
||||
}
|
||||
|
|
@ -0,0 +1,435 @@
|
|||
/**
|
||||
* @file TightPointDataStorageF.c
|
||||
* @author Sheng Di and Dingwen Tao
|
||||
* @date Aug, 2016
|
||||
* @brief The functions used to construct the tightPointDataStorage element for storing compressed bytes.
|
||||
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
|
||||
* See COPYRIGHT in top-level directory.
|
||||
*/
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include "TightDataPointStorageF.h"
|
||||
#include "sz.h"
|
||||
#include "Huffman.h"
|
||||
#include "transcode.h"
|
||||
#include "fse.h"
|
||||
#include <sys/time.h>
|
||||
|
||||
void new_TightDataPointStorageF_Empty(TightDataPointStorageF **this)
|
||||
{
|
||||
TightDataPointStorageF* tdpf = (TightDataPointStorageF*)malloc(sizeof(TightDataPointStorageF));
|
||||
memset(tdpf, 0, sizeof(TightDataPointStorageF));
|
||||
*this = tdpf;
|
||||
}
|
||||
|
||||
int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsigned char* flatBytes, size_t flatBytesLength, sz_exedata* pde_exe, sz_params* pde_params)
|
||||
{
|
||||
new_TightDataPointStorageF_Empty(this);
|
||||
size_t i, index = 0;
|
||||
|
||||
//
|
||||
// parse tdps
|
||||
//
|
||||
|
||||
// 1 version(1)
|
||||
unsigned char version = flatBytes[index++]; //1
|
||||
unsigned char sameRByte = flatBytes[index++]; //1
|
||||
|
||||
// parse data format
|
||||
switch (version)
|
||||
{
|
||||
case DATA_FROMAT_VER1:
|
||||
break;
|
||||
default:
|
||||
printf(" error, float compressed data format can not be recognised. ver=%d\n ", version);
|
||||
return SZ_ABS;
|
||||
}
|
||||
|
||||
// 2 same(1) //note that 1000,0000 is reserved for regression tag.
|
||||
int same = sameRByte & 0x01; //0000,0001
|
||||
(*this)->ifAdtFse = (sameRByte & 0x02)>>1; //0000,0110
|
||||
(*this)->isLossless = (sameRByte & 0x10)>>4; //0001,0000 //0010,0000
|
||||
pde_exe->SZ_SIZE_TYPE = ((sameRByte & 0x40)>>6)==1?8:4; //0100,0000
|
||||
int errorBoundMode = SZ_ABS;
|
||||
// 3 meta(2)
|
||||
convertBytesToSZParams(&(flatBytes[index]), pde_params, pde_exe);
|
||||
index += MetaDataByteLength;
|
||||
// 4 element count(4)
|
||||
unsigned char dsLengthBytes[8];
|
||||
for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++)
|
||||
dsLengthBytes[i] = flatBytes[index++];
|
||||
(*this)->dataSeriesLength = bytesToSize(dsLengthBytes, pde_exe->SZ_SIZE_TYPE);// 4 or 8
|
||||
if((*this)->isLossless==1)
|
||||
{
|
||||
//(*this)->exactMidBytes = flatBytes+8;
|
||||
return errorBoundMode;
|
||||
}
|
||||
else if(same==1)
|
||||
{
|
||||
(*this)->allSameData = 1;
|
||||
(*this)->exactMidBytes = &(flatBytes[index]);
|
||||
return errorBoundMode;
|
||||
}
|
||||
else
|
||||
(*this)->allSameData = 0;
|
||||
// regression
|
||||
int isRegression = (sameRByte >> 7) & 0x01;
|
||||
if(isRegression == 1)
|
||||
{
|
||||
(*this)->raBytes_size = flatBytesLength - 1 - 1 - MetaDataByteLength - pde_exe->SZ_SIZE_TYPE;
|
||||
(*this)->raBytes = &(flatBytes[index]);
|
||||
return errorBoundMode;
|
||||
}
|
||||
// 5 quant intervals(4)
|
||||
unsigned char byteBuf[8];
|
||||
for (i = 0; i < 4; i++)
|
||||
byteBuf[i] = flatBytes[index++];
|
||||
int max_quant_intervals = bytesToInt_bigEndian(byteBuf);// 4
|
||||
pde_params->maxRangeRadius = max_quant_intervals/2;
|
||||
// 6 intervals
|
||||
for (i = 0; i < 4; i++)
|
||||
byteBuf[i] = flatBytes[index++];
|
||||
(*this)->intervals = bytesToInt_bigEndian(byteBuf);// 4
|
||||
// 7 median
|
||||
for (i = 0; i < 4; i++)
|
||||
byteBuf[i] = flatBytes[index++];
|
||||
(*this)->medianValue = bytesToFloat(byteBuf); //4
|
||||
// 8 reqLength
|
||||
(*this)->reqLength = flatBytes[index++]; //1
|
||||
// 9 realPrecision(8)
|
||||
for (i = 0; i < 8; i++)
|
||||
byteBuf[i] = flatBytes[index++];
|
||||
(*this)->realPrecision = bytesToDouble(byteBuf);//8
|
||||
// 10 typeArray_size
|
||||
if ((*this)->ifAdtFse == 0) {
|
||||
for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++)
|
||||
byteBuf[i] = flatBytes[index++];
|
||||
(*this)->typeArray_size = bytesToSize(byteBuf, pde_exe->SZ_SIZE_TYPE);// 4
|
||||
} else {
|
||||
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
|
||||
byteBuf[i] = flatBytes[index++];
|
||||
(*this)->FseCode_size = bytesToSize(byteBuf, pde_exe->SZ_SIZE_TYPE);// 4
|
||||
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
|
||||
byteBuf[i] = flatBytes[index++];
|
||||
(*this)->transCodeBits_size = bytesToSize(byteBuf, pde_exe->SZ_SIZE_TYPE);// 4
|
||||
}
|
||||
|
||||
// 11 exactNum
|
||||
for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++)
|
||||
byteBuf[i] = flatBytes[index++];
|
||||
(*this)->exactDataNum = bytesToSize(byteBuf, pde_exe->SZ_SIZE_TYPE);// ST
|
||||
// 12 mid size
|
||||
for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++)
|
||||
byteBuf[i] = flatBytes[index++];
|
||||
(*this)->exactMidBytes_size = bytesToSize(byteBuf, pde_exe->SZ_SIZE_TYPE);// STqq
|
||||
|
||||
// calc leadNumArray_size
|
||||
size_t logicLeadNumBitsNum = (*this)->exactDataNum * 2;
|
||||
if (logicLeadNumBitsNum % 8 == 0)
|
||||
{
|
||||
(*this)->leadNumArray_size = logicLeadNumBitsNum >> 3;
|
||||
}
|
||||
else
|
||||
{
|
||||
(*this)->leadNumArray_size = (logicLeadNumBitsNum >> 3) + 1;
|
||||
}
|
||||
|
||||
// 13 typeArray
|
||||
if ((*this)->ifAdtFse == 0) {
|
||||
(*this)->typeArray = &flatBytes[index];
|
||||
//retrieve the number of states (i.e., stateNum)
|
||||
(*this)->allNodes = bytesToInt_bigEndian((*this)->typeArray); //the first 4 bytes store the stateNum
|
||||
(*this)->stateNum = ((*this)->allNodes+1)/2;
|
||||
index+=(*this)->typeArray_size;
|
||||
} else {
|
||||
(*this)->FseCode = &flatBytes[index];
|
||||
index+=(*this)->FseCode_size;
|
||||
|
||||
(*this)->transCodeBits = &flatBytes[index];
|
||||
index+=(*this)->transCodeBits_size;
|
||||
}
|
||||
|
||||
// 14 leadNumArray
|
||||
(*this)->leadNumArray = &flatBytes[index];
|
||||
index += (*this)->leadNumArray_size;
|
||||
// 15 exactMidBytes
|
||||
(*this)->exactMidBytes = &flatBytes[index];
|
||||
index+=(*this)->exactMidBytes_size;
|
||||
// 16 residualMidBits
|
||||
(*this)->residualMidBits = &flatBytes[index];
|
||||
|
||||
// calc residualMidBits_size
|
||||
(*this)->residualMidBits_size = flatBytesLength - 1 - 1 - MetaDataByteLength - pde_exe->SZ_SIZE_TYPE - 4 - 4 - 4 - 1 - 8
|
||||
- pde_exe->SZ_SIZE_TYPE - pde_exe->SZ_SIZE_TYPE
|
||||
- (*this)->leadNumArray_size - (*this)->exactMidBytes_size;
|
||||
if ((*this)->ifAdtFse == 0) {
|
||||
(*this)->residualMidBits_size = (*this)->residualMidBits_size - (*this)->typeArray_size - pde_exe->SZ_SIZE_TYPE ;
|
||||
} else {
|
||||
(*this)->residualMidBits_size = (*this)->residualMidBits_size - (*this)->FseCode_size - (*this)->transCodeBits_size - pde_exe->SZ_SIZE_TYPE - pde_exe->SZ_SIZE_TYPE;
|
||||
}
|
||||
|
||||
// printTDPS(*this);
|
||||
return errorBoundMode;
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* type's length == dataSeriesLength
|
||||
* exactMidBytes's length == exactMidBytes_size
|
||||
* leadNumIntArray's length == exactDataNum
|
||||
* escBytes's length == escBytes_size
|
||||
* resiBitLength's length == resiBitLengthSize
|
||||
* */
|
||||
void new_TightDataPointStorageF(TightDataPointStorageF **this,
|
||||
size_t dataSeriesLength, size_t exactDataNum,
|
||||
int* type, unsigned char* exactMidBytes, size_t exactMidBytes_size,
|
||||
unsigned char* leadNumIntArray, //leadNumIntArray contains readable numbers....
|
||||
unsigned char* resiMidBits, size_t resiMidBits_size,
|
||||
unsigned char resiBitLength,
|
||||
double realPrecision, float medianValue, char reqLength, unsigned int intervals,
|
||||
unsigned char radExpo) {
|
||||
|
||||
*this = (TightDataPointStorageF *)malloc(sizeof(TightDataPointStorageF));
|
||||
memset(*this, 0, sizeof(TightDataPointStorageF));
|
||||
(*this)->ifAdtFse = confparams_cpr->ifAdtFse;
|
||||
(*this)->allSameData = 0;
|
||||
(*this)->realPrecision = realPrecision;
|
||||
(*this)->medianValue = medianValue;
|
||||
(*this)->reqLength = reqLength;
|
||||
|
||||
(*this)->dataSeriesLength = dataSeriesLength;
|
||||
(*this)->exactDataNum = exactDataNum;
|
||||
|
||||
if ((*this)->ifAdtFse == 0) {
|
||||
// huffman
|
||||
int stateNum = 2 * intervals;
|
||||
HuffmanTree* huffmanTree = createHuffmanTree(stateNum);
|
||||
encode_withTree(huffmanTree, type, dataSeriesLength, &(*this)->typeArray, &(*this)->typeArray_size);
|
||||
SZ_ReleaseHuffman(huffmanTree);
|
||||
}
|
||||
else {
|
||||
// fse
|
||||
encode_with_fse(type, dataSeriesLength, intervals, &((*this)->FseCode), &((*this)->FseCode_size),
|
||||
&((*this)->transCodeBits), &((*this)->transCodeBits_size));
|
||||
}
|
||||
|
||||
(*this)->exactMidBytes = exactMidBytes;
|
||||
(*this)->exactMidBytes_size = exactMidBytes_size;
|
||||
|
||||
(*this)->leadNumArray_size = convertIntArray2ByteArray_fast_2b(leadNumIntArray, exactDataNum, &((*this)->leadNumArray));
|
||||
|
||||
(*this)->residualMidBits_size = convertIntArray2ByteArray_fast_dynamic(resiMidBits, resiBitLength, exactDataNum, &((*this)->residualMidBits));
|
||||
|
||||
(*this)->intervals = intervals;
|
||||
|
||||
(*this)->isLossless = 0;
|
||||
|
||||
(*this)->radExpo = radExpo;
|
||||
}
|
||||
|
||||
void convertTDPStoBytes_float(TightDataPointStorageF* tdps, unsigned char* bytes, unsigned char* dsLengthBytes, unsigned char sameByte)
|
||||
{
|
||||
size_t i, k = 0;
|
||||
unsigned char intervalsBytes[4];
|
||||
unsigned char typeArrayLengthBytes[8];
|
||||
unsigned char exactLengthBytes[8];
|
||||
unsigned char exactMidBytesLength[8];
|
||||
unsigned char realPrecisionBytes[8];
|
||||
unsigned char fsecodeLengthBytes[8];
|
||||
unsigned char transcodeLengthBytes[8];
|
||||
unsigned char medianValueBytes[4];
|
||||
unsigned char max_quant_intervals_Bytes[4];
|
||||
|
||||
// 1 version
|
||||
bytes[k++] = versionNumber;
|
||||
// 2 same
|
||||
bytes[k++] = sameByte; //1 byte
|
||||
// 3 meta
|
||||
convertSZParamsToBytes(confparams_cpr, &(bytes[k]), exe_params->optQuantMode);
|
||||
k = k + MetaDataByteLength;
|
||||
// 4 element count
|
||||
for(i = 0; i < exe_params->SZ_SIZE_TYPE; i++)//ST: 4 or 8 bytes
|
||||
bytes[k++] = dsLengthBytes[i];
|
||||
intToBytes_bigEndian(max_quant_intervals_Bytes, confparams_cpr->max_quant_intervals);
|
||||
// 5 max_quant_intervals length
|
||||
for(i = 0;i<4;i++)//4
|
||||
bytes[k++] = max_quant_intervals_Bytes[i];
|
||||
// 6 intervals
|
||||
intToBytes_bigEndian(intervalsBytes, tdps->intervals);
|
||||
for(i = 0;i<4;i++)//4
|
||||
bytes[k++] = intervalsBytes[i];
|
||||
// 7 median
|
||||
floatToBytes(medianValueBytes, tdps->medianValue);
|
||||
for (i = 0; i < 4; i++)// 4
|
||||
bytes[k++] = medianValueBytes[i];
|
||||
// 8 reqLength
|
||||
bytes[k++] = tdps->reqLength; //1 byte
|
||||
// 9 realPrecision
|
||||
doubleToBytes(realPrecisionBytes, tdps->realPrecision);
|
||||
for (i = 0; i < 8; i++)// 8
|
||||
bytes[k++] = realPrecisionBytes[i];
|
||||
// 10 typeArray size
|
||||
if (tdps->ifAdtFse == 0) {
|
||||
sizeToBytes(typeArrayLengthBytes, tdps->typeArray_size, exe_params->SZ_SIZE_TYPE);
|
||||
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
|
||||
bytes[k++] = typeArrayLengthBytes[i];
|
||||
} else {
|
||||
sizeToBytes(fsecodeLengthBytes, tdps->FseCode_size, exe_params->SZ_SIZE_TYPE);
|
||||
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
|
||||
bytes[k++] = fsecodeLengthBytes[i];
|
||||
|
||||
sizeToBytes(transcodeLengthBytes, tdps->transCodeBits_size, exe_params->SZ_SIZE_TYPE);
|
||||
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
|
||||
bytes[k++] = transcodeLengthBytes[i];
|
||||
}
|
||||
// 11 exactDataNum leadNum calc by this , so not save leadNum
|
||||
sizeToBytes(exactLengthBytes, tdps->exactDataNum, exe_params->SZ_SIZE_TYPE);
|
||||
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
|
||||
bytes[k++] = exactLengthBytes[i];
|
||||
// 12 Mid size
|
||||
sizeToBytes(exactMidBytesLength, tdps->exactMidBytes_size, exe_params->SZ_SIZE_TYPE);
|
||||
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
|
||||
bytes[k++] = exactMidBytesLength[i];
|
||||
// 13 typeArray
|
||||
if (tdps->ifAdtFse == 0) {
|
||||
memcpy(&(bytes[k]), tdps->typeArray, tdps->typeArray_size);
|
||||
k += tdps->typeArray_size;
|
||||
} else {
|
||||
memcpy(&(bytes[k]), tdps->FseCode, tdps->FseCode_size);
|
||||
k += tdps->FseCode_size;
|
||||
memcpy(&(bytes[k]), tdps->transCodeBits, tdps->transCodeBits_size);
|
||||
k += tdps->transCodeBits_size;
|
||||
}
|
||||
// 14 leadNumArray_size
|
||||
memcpy(&(bytes[k]), tdps->leadNumArray, tdps->leadNumArray_size);
|
||||
k += tdps->leadNumArray_size;
|
||||
// 15 mid data
|
||||
memcpy(&(bytes[k]), tdps->exactMidBytes, tdps->exactMidBytes_size);
|
||||
k += tdps->exactMidBytes_size;
|
||||
// 16 residualMidBits
|
||||
if(tdps->residualMidBits!=NULL)
|
||||
{
|
||||
memcpy(&(bytes[k]), tdps->residualMidBits, tdps->residualMidBits_size);
|
||||
k += tdps->residualMidBits_size;
|
||||
}
|
||||
}
|
||||
|
||||
//convert TightDataPointStorageD to bytes...
|
||||
bool convertTDPStoFlatBytes_float(TightDataPointStorageF *tdps, unsigned char* bytes, size_t *size)
|
||||
{
|
||||
// printTDPS(tdps);
|
||||
size_t i, k = 0;
|
||||
unsigned char dsLengthBytes[8];
|
||||
|
||||
if(exe_params->SZ_SIZE_TYPE==4)
|
||||
intToBytes_bigEndian(dsLengthBytes, tdps->dataSeriesLength);//4
|
||||
else
|
||||
longToBytes_bigEndian(dsLengthBytes, tdps->dataSeriesLength);//8
|
||||
|
||||
unsigned char sameByte = tdps->allSameData==1?(unsigned char)1:(unsigned char)0; //0000,0001
|
||||
//sameByte = sameByte | (confparams_cpr->szMode << 1); //0000,0110 (no need because of convertSZParamsToBytes
|
||||
sameByte = sameByte | (tdps->ifAdtFse << 1); // 0000,0010
|
||||
if(tdps->isLossless)
|
||||
sameByte = (unsigned char) (sameByte | 0x10); // 0001,0000
|
||||
if(confparams_cpr->errorBoundMode>=PW_REL)
|
||||
sameByte = (unsigned char) (sameByte | 0x20); // 0010,0000, the 5th bit
|
||||
if(exe_params->SZ_SIZE_TYPE==8)
|
||||
sameByte = (unsigned char) (sameByte | 0x40); // 0100,0000, the 6th bit
|
||||
if(confparams_cpr->errorBoundMode == PW_REL && confparams_cpr->accelerate_pw_rel_compression)
|
||||
sameByte = (unsigned char) (sameByte | 0x08); //0000,1000
|
||||
//if(confparams_cpr->protectValueRange)
|
||||
// sameByte = (unsigned char) (sameByte | 0x04); //0000,0100
|
||||
|
||||
if(tdps->allSameData == 1 )
|
||||
{
|
||||
//
|
||||
// same format
|
||||
//
|
||||
size_t totalByteLength = 1 + 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + tdps->exactMidBytes_size;
|
||||
//*bytes = (unsigned char *)malloc(sizeof(unsigned char)*totalByteLength); // not need malloc comment by tickduan
|
||||
// check output buffer enough
|
||||
if(totalByteLength >= tdps->dataSeriesLength * sizeof(float) )
|
||||
{
|
||||
*size = 0;
|
||||
return false;
|
||||
}
|
||||
|
||||
// 1 version 1 byte
|
||||
bytes[k++] = versionNumber;
|
||||
// 2 same flag 1 bytes
|
||||
bytes[k++] = sameByte;
|
||||
// 3 metaData 26 bytes
|
||||
convertSZParamsToBytes(confparams_cpr, &(bytes[k]), exe_params->optQuantMode);
|
||||
k = k + MetaDataByteLength;
|
||||
// 4 data Length 4 or 8 bytes
|
||||
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
|
||||
bytes[k++] = dsLengthBytes[i];
|
||||
// 5 exactMidBytes exactMidBytes_size bytes
|
||||
for (i = 0; i < tdps->exactMidBytes_size; i++)
|
||||
bytes[k++] = tdps->exactMidBytes[i];
|
||||
|
||||
*size = totalByteLength;
|
||||
}
|
||||
else
|
||||
{
|
||||
//
|
||||
// not same format
|
||||
//
|
||||
size_t residualMidBitsLength = tdps->residualMidBits == NULL ? 0 : tdps->residualMidBits_size;
|
||||
|
||||
// version(1) + samebyte(1)
|
||||
size_t totalByteLength = 1 + 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 4 + 4 + 4 + 1 + 8
|
||||
+ exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE
|
||||
+ tdps->leadNumArray_size
|
||||
+ tdps->exactMidBytes_size
|
||||
+ residualMidBitsLength;
|
||||
if (tdps->ifAdtFse == 0) {
|
||||
totalByteLength += tdps->typeArray_size + exe_params->SZ_SIZE_TYPE;
|
||||
} else {
|
||||
totalByteLength += tdps->FseCode_size + tdps->transCodeBits_size + exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE;
|
||||
}
|
||||
|
||||
//*bytes = (unsigned char *)malloc(sizeof(unsigned char)*totalByteLength); // comment by tickduan
|
||||
if(totalByteLength >= tdps->dataSeriesLength * sizeof(float))
|
||||
{
|
||||
*size = 0;
|
||||
return false;
|
||||
}
|
||||
|
||||
convertTDPStoBytes_float(tdps, bytes, dsLengthBytes, sameByte);
|
||||
*size = totalByteLength;
|
||||
return true;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* to free the memory used in the compression
|
||||
* */
|
||||
void free_TightDataPointStorageF(TightDataPointStorageF *tdps)
|
||||
{
|
||||
if(tdps->leadNumArray!=NULL)
|
||||
free(tdps->leadNumArray);
|
||||
if(tdps->FseCode!=NULL)
|
||||
free(tdps->FseCode);
|
||||
if(tdps->transCodeBits!=NULL)
|
||||
free(tdps->transCodeBits);
|
||||
if(tdps->exactMidBytes!=NULL)
|
||||
free(tdps->exactMidBytes);
|
||||
if(tdps->residualMidBits!=NULL)
|
||||
free(tdps->residualMidBits);
|
||||
if(tdps->typeArray)
|
||||
free(tdps->typeArray);
|
||||
free(tdps);
|
||||
}
|
||||
|
||||
/**
|
||||
* to free the memory used in the decompression
|
||||
* */
|
||||
void free_TightDataPointStorageF2(TightDataPointStorageF *tdps)
|
||||
{
|
||||
free(tdps);
|
||||
}
|
||||
|
|
@ -0,0 +1,203 @@
|
|||
/**
|
||||
* @file TypeManager.c
|
||||
* @author Sheng Di
|
||||
* @date May, 2016
|
||||
* @brief TypeManager is used to manage the type array: parsing of the bytes and other types in between.
|
||||
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
|
||||
* See COPYRIGHT in top-level directory.
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include "DynamicByteArray.h"
|
||||
#include "sz.h"
|
||||
|
||||
//int convertIntArray2ByteArray_fast_8b()
|
||||
|
||||
size_t convertIntArray2ByteArray_fast_1b(unsigned char* intArray, size_t intArrayLength, unsigned char **result)
|
||||
{
|
||||
size_t byteLength = 0;
|
||||
size_t i, j;
|
||||
if(intArrayLength%8==0)
|
||||
byteLength = intArrayLength/8;
|
||||
else
|
||||
byteLength = intArrayLength/8+1;
|
||||
|
||||
if(byteLength>0)
|
||||
*result = (unsigned char*)malloc(byteLength*sizeof(unsigned char));
|
||||
else
|
||||
*result = NULL;
|
||||
size_t n = 0;
|
||||
int tmp, type;
|
||||
for(i = 0;i<byteLength;i++)
|
||||
{
|
||||
tmp = 0;
|
||||
for(j = 0;j<8&&n<intArrayLength;j++)
|
||||
{
|
||||
type = intArray[n];
|
||||
if(type == 1)
|
||||
tmp = (tmp | (1 << (7-j)));
|
||||
n++;
|
||||
}
|
||||
(*result)[i] = (unsigned char)tmp;
|
||||
}
|
||||
return byteLength;
|
||||
}
|
||||
|
||||
size_t convertIntArray2ByteArray_fast_1b_to_result(unsigned char* intArray, size_t intArrayLength, unsigned char *result)
|
||||
{
|
||||
size_t byteLength = 0;
|
||||
size_t i, j;
|
||||
if(intArrayLength%8==0)
|
||||
byteLength = intArrayLength/8;
|
||||
else
|
||||
byteLength = intArrayLength/8+1;
|
||||
|
||||
size_t n = 0;
|
||||
int tmp, type;
|
||||
for(i = 0;i<byteLength;i++)
|
||||
{
|
||||
tmp = 0;
|
||||
for(j = 0;j<8&&n<intArrayLength;j++)
|
||||
{
|
||||
type = intArray[n];
|
||||
if(type == 1)
|
||||
tmp = (tmp | (1 << (7-j)));
|
||||
n++;
|
||||
}
|
||||
result[i] = (unsigned char)tmp;
|
||||
}
|
||||
return byteLength;
|
||||
}
|
||||
|
||||
|
||||
void convertByteArray2IntArray_fast_2b(size_t stepLength, unsigned char* byteArray, size_t byteArrayLength, unsigned char **intArray)
|
||||
{
|
||||
if(stepLength > byteArrayLength*4)
|
||||
{
|
||||
printf("Error: stepLength > byteArray.length*4\n");
|
||||
printf("stepLength=%zu, byteArray.length=%zu\n", stepLength, byteArrayLength);
|
||||
exit(0);
|
||||
}
|
||||
if(stepLength>0)
|
||||
*intArray = (unsigned char*)malloc(stepLength*sizeof(unsigned char));
|
||||
else
|
||||
*intArray = NULL;
|
||||
size_t i, n = 0;
|
||||
|
||||
for (i = 0; i < byteArrayLength; i++) {
|
||||
unsigned char tmp = byteArray[i];
|
||||
(*intArray)[n++] = (tmp & 0xC0) >> 6;
|
||||
if(n==stepLength)
|
||||
break;
|
||||
(*intArray)[n++] = (tmp & 0x30) >> 4;
|
||||
if(n==stepLength)
|
||||
break;
|
||||
(*intArray)[n++] = (tmp & 0x0C) >> 2;
|
||||
if(n==stepLength)
|
||||
break;
|
||||
(*intArray)[n++] = tmp & 0x03;
|
||||
if(n==stepLength)
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* little endian
|
||||
* [01|10|11|00|....]-->[01|10|11|00][....]
|
||||
* @param timeStepType
|
||||
* @return
|
||||
*/
|
||||
size_t convertIntArray2ByteArray_fast_2b(unsigned char* timeStepType, size_t timeStepTypeLength, unsigned char **result)
|
||||
{
|
||||
size_t i, j, byteLength = 0;
|
||||
if(timeStepTypeLength%4==0)
|
||||
byteLength = timeStepTypeLength*2/8;
|
||||
else
|
||||
byteLength = timeStepTypeLength*2/8+1;
|
||||
if(byteLength>0)
|
||||
*result = (unsigned char*)malloc(byteLength*sizeof(unsigned char));
|
||||
else
|
||||
*result = NULL;
|
||||
size_t n = 0;
|
||||
for(i = 0;i<byteLength;i++)
|
||||
{
|
||||
int tmp = 0;
|
||||
for(j = 0;j<4&&n<timeStepTypeLength;j++)
|
||||
{
|
||||
int type = timeStepType[n];
|
||||
switch(type)
|
||||
{
|
||||
case 0:
|
||||
|
||||
break;
|
||||
case 1:
|
||||
tmp = (tmp | (1 << (6-j*2)));
|
||||
break;
|
||||
case 2:
|
||||
tmp = (tmp | (2 << (6-j*2)));
|
||||
break;
|
||||
case 3:
|
||||
tmp = (tmp | (3 << (6-j*2)));
|
||||
break;
|
||||
default:
|
||||
printf("Error: wrong timestep type...: type[%zu]=%d\n", n, type);
|
||||
exit(0);
|
||||
}
|
||||
n++;
|
||||
}
|
||||
(*result)[i] = (unsigned char)tmp;
|
||||
}
|
||||
return byteLength;
|
||||
}
|
||||
|
||||
INLINE int getLeftMovingSteps(size_t k, unsigned char resiBitLength)
|
||||
{
|
||||
return 8 - k%8 - resiBitLength;
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param timeStepType is the resiMidBits
|
||||
* @param resiBitLength is the length of resiMidBits for each element, (the number of resiBitLength == the # of unpredictable elements
|
||||
* @return
|
||||
*/
|
||||
size_t convertIntArray2ByteArray_fast_dynamic(unsigned char* timeStepType, unsigned char resiBitLength, size_t nbEle, unsigned char **bytes)
|
||||
{
|
||||
size_t i = 0, j = 0, k = 0;
|
||||
int value;
|
||||
DynamicByteArray* dba;
|
||||
new_DBA(&dba, 1024);
|
||||
int tmp = 0, leftMovSteps = 0;
|
||||
for(j = 0;j<nbEle;j++)
|
||||
{
|
||||
if(resiBitLength==0)
|
||||
continue;
|
||||
value = timeStepType[i];
|
||||
leftMovSteps = getLeftMovingSteps(k, resiBitLength);
|
||||
if(leftMovSteps < 0)
|
||||
{
|
||||
tmp = tmp | (value >> (-leftMovSteps));
|
||||
addDBA_Data(dba, (unsigned char)tmp);
|
||||
tmp = 0 | (value << (8+leftMovSteps));
|
||||
}
|
||||
else if(leftMovSteps > 0)
|
||||
{
|
||||
tmp = tmp | (value << leftMovSteps);
|
||||
}
|
||||
else //==0
|
||||
{
|
||||
tmp = tmp | value;
|
||||
addDBA_Data(dba, (unsigned char)tmp);
|
||||
tmp = 0;
|
||||
}
|
||||
i++;
|
||||
k += resiBitLength;
|
||||
}
|
||||
if(leftMovSteps != 0)
|
||||
addDBA_Data(dba, (unsigned char)tmp);
|
||||
convertDBAtoBytes(dba, bytes);
|
||||
size_t size = dba->size;
|
||||
free_DBA(dba);
|
||||
return size;
|
||||
}
|
||||
|
|
@ -0,0 +1,355 @@
|
|||
/**
|
||||
* @file conf.c
|
||||
* @author Sheng Di (sdi1@anl.gov or disheng222@gmail.com)
|
||||
* @date 2015.
|
||||
* @brief Configuration loading functions for the SZ library.
|
||||
* (C) 2015 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
|
||||
* See COPYRIGHT in top-level directory.
|
||||
*/
|
||||
|
||||
#include <math.h>
|
||||
#include "string.h"
|
||||
#include "sz.h"
|
||||
#include "iniparser.h"
|
||||
#include "Huffman.h"
|
||||
|
||||
//
|
||||
// set default value
|
||||
//
|
||||
void setDefaulParams(sz_exedata* exedata, sz_params* params)
|
||||
{
|
||||
// sz_params
|
||||
if(params)
|
||||
{
|
||||
// first important
|
||||
params->errorBoundMode = SZ_ABS;
|
||||
params->absErrBound = 1E-8;
|
||||
params->absErrBoundDouble = 1E-16;
|
||||
params->max_quant_intervals = 500;
|
||||
params->quantization_intervals = 100;
|
||||
params->losslessCompressor = ZSTD_COMPRESSOR; //other option: GZIP_COMPRESSOR;
|
||||
|
||||
// second important
|
||||
params->sol_ID = SZ;
|
||||
params->maxRangeRadius = params->max_quant_intervals/2;
|
||||
params->predThreshold = 0.99;
|
||||
params->sampleDistance = 100;
|
||||
params->szMode = SZ_BEST_COMPRESSION;
|
||||
|
||||
// other
|
||||
params->psnr = 90;
|
||||
params->relBoundRatio = 1E-8;
|
||||
params->accelerate_pw_rel_compression = 1;
|
||||
params->pw_relBoundRatio = 1E-3;
|
||||
params->segment_size = 36;
|
||||
params->pwr_type = SZ_PWR_MIN_TYPE;
|
||||
params->snapshotCmprStep = 5;
|
||||
params->withRegression = SZ_WITH_LINEAR_REGRESSION;
|
||||
params->randomAccess = 0; //0: no random access , 1: support random access
|
||||
params->protectValueRange = 0;
|
||||
params->plus_bits = 3;
|
||||
}
|
||||
|
||||
// sz_exedata
|
||||
if(exedata)
|
||||
{
|
||||
exedata->optQuantMode = 1;
|
||||
exedata->SZ_SIZE_TYPE = 4;
|
||||
if(params)
|
||||
{
|
||||
exedata->intvCapacity = params->maxRangeRadius*2;
|
||||
exedata->intvRadius = params->maxRangeRadius;
|
||||
}
|
||||
else
|
||||
{
|
||||
exedata->intvCapacity = 500;
|
||||
exedata->intvRadius = 200;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
unsigned int roundUpToPowerOf2(unsigned int base)
|
||||
{
|
||||
base -= 1;
|
||||
|
||||
base = base | (base >> 1);
|
||||
base = base | (base >> 2);
|
||||
base = base | (base >> 4);
|
||||
base = base | (base >> 8);
|
||||
base = base | (base >> 16);
|
||||
|
||||
return base + 1;
|
||||
}
|
||||
|
||||
void updateQuantizationInfo(int quant_intervals)
|
||||
{
|
||||
exe_params->intvCapacity = quant_intervals;
|
||||
exe_params->intvRadius = quant_intervals/2;
|
||||
}
|
||||
|
||||
double computeABSErrBoundFromPSNR(double psnr, double threshold, double value_range)
|
||||
{
|
||||
double v1 = psnr + 10 * log10(1-2.0/3.0*threshold);
|
||||
double v2 = v1/(-20);
|
||||
double v3 = pow(10, v2);
|
||||
return value_range * v3;
|
||||
}
|
||||
|
||||
double computeABSErrBoundFromNORM_ERR(double normErr, size_t nbEle)
|
||||
{
|
||||
return sqrt(3.0/nbEle)*normErr;
|
||||
}
|
||||
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
*
|
||||
*
|
||||
* @return the status of loading conf. file: 1 (success) or 0 (error code);
|
||||
* */
|
||||
int SZ_ReadConf(const char* sz_cfgFile) {
|
||||
// Check access to SZ configuration file and load dictionary
|
||||
//record the setting in confparams_cpr
|
||||
if(confparams_cpr == NULL)
|
||||
confparams_cpr = (sz_params*)malloc(sizeof(sz_params));
|
||||
if(exe_params == NULL)
|
||||
exe_params = (sz_exedata*)malloc(sizeof(sz_exedata));
|
||||
|
||||
int x = 1;
|
||||
char sol_name[256];
|
||||
char *modeBuf;
|
||||
char *errBoundMode;
|
||||
char *endianTypeString;
|
||||
dictionary *ini;
|
||||
char *par;
|
||||
|
||||
char *y = (char*)&x;
|
||||
|
||||
if(*y==1)
|
||||
sysEndianType = LITTLE_ENDIAN_SYSTEM;
|
||||
else //=0
|
||||
sysEndianType = BIG_ENDIAN_SYSTEM;
|
||||
|
||||
|
||||
// default option
|
||||
if(sz_cfgFile == NULL)
|
||||
{
|
||||
dataEndianType = LITTLE_ENDIAN_DATA;
|
||||
setDefaulParams(exe_params, confparams_cpr);
|
||||
|
||||
return SZ_SUCCESS;
|
||||
}
|
||||
|
||||
//printf("[SZ] Reading SZ configuration file (%s) ...\n", sz_cfgFile);
|
||||
ini = iniparser_load(sz_cfgFile);
|
||||
if (ini == NULL)
|
||||
{
|
||||
printf("[SZ] Iniparser failed to parse the conf. file.\n");
|
||||
return SZ_FAILED;
|
||||
}
|
||||
|
||||
endianTypeString = iniparser_getstring(ini, "ENV:dataEndianType", "LITTLE_ENDIAN_DATA");
|
||||
if(strcmp(endianTypeString, "LITTLE_ENDIAN_DATA")==0)
|
||||
dataEndianType = LITTLE_ENDIAN_DATA;
|
||||
else if(strcmp(endianTypeString, "BIG_ENDIAN_DATA")==0)
|
||||
dataEndianType = BIG_ENDIAN_DATA;
|
||||
else
|
||||
{
|
||||
printf("Error: Wrong dataEndianType: please set it correctly in sz.config.\n");
|
||||
iniparser_freedict(ini);
|
||||
return SZ_FAILED;
|
||||
}
|
||||
|
||||
// Reading/setting detection parameters
|
||||
|
||||
par = iniparser_getstring(ini, "ENV:sol_name", NULL);
|
||||
snprintf(sol_name, 256, "%s", par);
|
||||
|
||||
if(strcmp(sol_name, "SZ")==0)
|
||||
confparams_cpr->sol_ID = SZ;
|
||||
else if(strcmp(sol_name, "PASTRI")==0)
|
||||
confparams_cpr->sol_ID = PASTRI;
|
||||
else if(strcmp(sol_name, "SZ_Transpose")==0)
|
||||
confparams_cpr->sol_ID = SZ_Transpose;
|
||||
else{
|
||||
printf("[SZ] Error: wrong solution name (please check sz.config file), sol=%s\n", sol_name);
|
||||
iniparser_freedict(ini);
|
||||
return SZ_FAILED;
|
||||
}
|
||||
|
||||
if(confparams_cpr->sol_ID==SZ || confparams_cpr->sol_ID==SZ_Transpose)
|
||||
{
|
||||
int max_quant_intervals = iniparser_getint(ini, "PARAMETER:max_quant_intervals", 65536);
|
||||
confparams_cpr->max_quant_intervals = max_quant_intervals;
|
||||
|
||||
int quantization_intervals = (int)iniparser_getint(ini, "PARAMETER:quantization_intervals", 0);
|
||||
confparams_cpr->quantization_intervals = quantization_intervals;
|
||||
if(quantization_intervals>0)
|
||||
{
|
||||
updateQuantizationInfo(quantization_intervals);
|
||||
confparams_cpr->max_quant_intervals = max_quant_intervals = quantization_intervals;
|
||||
exe_params->optQuantMode = 0;
|
||||
}
|
||||
else //==0
|
||||
{
|
||||
confparams_cpr->maxRangeRadius = max_quant_intervals/2;
|
||||
|
||||
exe_params->intvCapacity = confparams_cpr->maxRangeRadius*2;
|
||||
exe_params->intvRadius = confparams_cpr->maxRangeRadius;
|
||||
|
||||
exe_params->optQuantMode = 1;
|
||||
}
|
||||
|
||||
if(quantization_intervals%2!=0)
|
||||
{
|
||||
printf("Error: quantization_intervals must be an even number!\n");
|
||||
iniparser_freedict(ini);
|
||||
return SZ_FAILED;
|
||||
}
|
||||
|
||||
confparams_cpr->predThreshold = (float)iniparser_getdouble(ini, "PARAMETER:predThreshold", 0);
|
||||
confparams_cpr->sampleDistance = (int)iniparser_getint(ini, "PARAMETER:sampleDistance", 0);
|
||||
|
||||
modeBuf = iniparser_getstring(ini, "PARAMETER:szMode", NULL);
|
||||
if(modeBuf==NULL)
|
||||
{
|
||||
printf("[SZ] Error: Null szMode setting (please check sz.config file)\n");
|
||||
iniparser_freedict(ini);
|
||||
return SZ_FAILED;
|
||||
}
|
||||
else if(strcmp(modeBuf, "SZ_BEST_SPEED")==0)
|
||||
confparams_cpr->szMode = SZ_BEST_SPEED;
|
||||
else if(strcmp(modeBuf, "SZ_DEFAULT_COMPRESSION")==0)
|
||||
confparams_cpr->szMode = SZ_DEFAULT_COMPRESSION;
|
||||
else if(strcmp(modeBuf, "SZ_BEST_COMPRESSION")==0)
|
||||
confparams_cpr->szMode = SZ_BEST_COMPRESSION;
|
||||
else
|
||||
{
|
||||
printf("[SZ] Error: Wrong szMode setting (please check sz.config file)\n");
|
||||
iniparser_freedict(ini);
|
||||
return SZ_FAILED;
|
||||
}
|
||||
|
||||
modeBuf = iniparser_getstring(ini, "PARAMETER:losslessCompressor", "ZSTD_COMPRESSOR");
|
||||
if(strcmp(modeBuf, "GZIP_COMPRESSOR")==0)
|
||||
confparams_cpr->losslessCompressor = GZIP_COMPRESSOR;
|
||||
else if(strcmp(modeBuf, "ZSTD_COMPRESSOR")==0)
|
||||
confparams_cpr->losslessCompressor = ZSTD_COMPRESSOR;
|
||||
else
|
||||
{
|
||||
printf("[SZ] Error: Wrong losslessCompressor setting (please check sz.config file)\n");\
|
||||
printf("No Such a lossless compressor: %s\n", modeBuf);
|
||||
iniparser_freedict(ini);
|
||||
return SZ_FAILED;
|
||||
}
|
||||
|
||||
modeBuf = iniparser_getstring(ini, "PARAMETER:withLinearRegression", "YES");
|
||||
if(strcmp(modeBuf, "YES")==0 || strcmp(modeBuf, "yes")==0)
|
||||
confparams_cpr->withRegression = SZ_WITH_LINEAR_REGRESSION;
|
||||
else
|
||||
confparams_cpr->withRegression = SZ_NO_REGRESSION;
|
||||
|
||||
modeBuf = iniparser_getstring(ini, "PARAMETER:protectValueRange", "YES");
|
||||
if(strcmp(modeBuf, "YES")==0)
|
||||
confparams_cpr->protectValueRange = 1;
|
||||
else
|
||||
confparams_cpr->protectValueRange = 0;
|
||||
|
||||
confparams_cpr->randomAccess = (int)iniparser_getint(ini, "PARAMETER:randomAccess", 0);
|
||||
|
||||
//TODO
|
||||
confparams_cpr->snapshotCmprStep = (int)iniparser_getint(ini, "PARAMETER:snapshotCmprStep", 5);
|
||||
|
||||
errBoundMode = iniparser_getstring(ini, "PARAMETER:errorBoundMode", NULL);
|
||||
if(errBoundMode==NULL)
|
||||
{
|
||||
printf("[SZ] Error: Null error bound setting (please check sz.config file)\n");
|
||||
iniparser_freedict(ini);
|
||||
return SZ_FAILED;
|
||||
}
|
||||
else if(strcmp(errBoundMode,"ABS")==0||strcmp(errBoundMode,"abs")==0)
|
||||
confparams_cpr->errorBoundMode=SZ_ABS;
|
||||
else if(strcmp(errBoundMode, "REL")==0||strcmp(errBoundMode,"rel")==0)
|
||||
confparams_cpr->errorBoundMode=REL;
|
||||
else if(strcmp(errBoundMode, "VR_REL")==0||strcmp(errBoundMode, "vr_rel")==0)
|
||||
confparams_cpr->errorBoundMode=REL;
|
||||
else if(strcmp(errBoundMode, "ABS_AND_REL")==0||strcmp(errBoundMode, "abs_and_rel")==0)
|
||||
confparams_cpr->errorBoundMode=ABS_AND_REL;
|
||||
else if(strcmp(errBoundMode, "ABS_OR_REL")==0||strcmp(errBoundMode, "abs_or_rel")==0)
|
||||
confparams_cpr->errorBoundMode=ABS_OR_REL;
|
||||
else if(strcmp(errBoundMode, "PW_REL")==0||strcmp(errBoundMode, "pw_rel")==0)
|
||||
confparams_cpr->errorBoundMode=PW_REL;
|
||||
else if(strcmp(errBoundMode, "PSNR")==0||strcmp(errBoundMode, "psnr")==0)
|
||||
confparams_cpr->errorBoundMode=PSNR;
|
||||
else if(strcmp(errBoundMode, "ABS_AND_PW_REL")==0||strcmp(errBoundMode, "abs_and_pw_rel")==0)
|
||||
confparams_cpr->errorBoundMode=ABS_AND_PW_REL;
|
||||
else if(strcmp(errBoundMode, "ABS_OR_PW_REL")==0||strcmp(errBoundMode, "abs_or_pw_rel")==0)
|
||||
confparams_cpr->errorBoundMode=ABS_OR_PW_REL;
|
||||
else if(strcmp(errBoundMode, "REL_AND_PW_REL")==0||strcmp(errBoundMode, "rel_and_pw_rel")==0)
|
||||
confparams_cpr->errorBoundMode=REL_AND_PW_REL;
|
||||
else if(strcmp(errBoundMode, "REL_OR_PW_REL")==0||strcmp(errBoundMode, "rel_or_pw_rel")==0)
|
||||
confparams_cpr->errorBoundMode=REL_OR_PW_REL;
|
||||
else if(strcmp(errBoundMode, "NORM")==0||strcmp(errBoundMode, "norm")==0)
|
||||
confparams_cpr->errorBoundMode=NORM;
|
||||
else
|
||||
{
|
||||
printf("[SZ] Error: Wrong error bound mode (please check sz.config file)\n");
|
||||
iniparser_freedict(ini);
|
||||
return SZ_FAILED;
|
||||
}
|
||||
|
||||
confparams_cpr->absErrBound = (double)iniparser_getdouble(ini, "PARAMETER:absErrBound", 0);
|
||||
confparams_cpr->relBoundRatio = (double)iniparser_getdouble(ini, "PARAMETER:relBoundRatio", 0);
|
||||
confparams_cpr->psnr = (double)iniparser_getdouble(ini, "PARAMETER:psnr", 0);
|
||||
confparams_cpr->normErr = (double)iniparser_getdouble(ini, "PARAMETER:normErr", 0);
|
||||
confparams_cpr->pw_relBoundRatio = (double)iniparser_getdouble(ini, "PARAMETER:pw_relBoundRatio", 0);
|
||||
confparams_cpr->segment_size = (int)iniparser_getint(ini, "PARAMETER:segment_size", 0);
|
||||
confparams_cpr->accelerate_pw_rel_compression = (int)iniparser_getint(ini, "PARAMETER:accelerate_pw_rel_compression", 1);
|
||||
|
||||
modeBuf = iniparser_getstring(ini, "PARAMETER:pwr_type", "MIN");
|
||||
|
||||
if(strcmp(modeBuf, "MIN")==0)
|
||||
confparams_cpr->pwr_type = SZ_PWR_MIN_TYPE;
|
||||
else if(strcmp(modeBuf, "AVG")==0)
|
||||
confparams_cpr->pwr_type = SZ_PWR_AVG_TYPE;
|
||||
else if(strcmp(modeBuf, "MAX")==0)
|
||||
confparams_cpr->pwr_type = SZ_PWR_MAX_TYPE;
|
||||
else if(modeBuf!=NULL)
|
||||
{
|
||||
printf("[SZ] Error: Wrong pwr_type setting (please check sz.config file).\n");
|
||||
iniparser_freedict(ini);
|
||||
return SZ_FAILED;
|
||||
}
|
||||
else //by default
|
||||
confparams_cpr->pwr_type = SZ_PWR_AVG_TYPE;
|
||||
|
||||
//initialization for Huffman encoding
|
||||
//SZ_Reset();
|
||||
}
|
||||
|
||||
|
||||
iniparser_freedict(ini);
|
||||
return SZ_SUCCESS;
|
||||
}
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief It reads and tests the configuration given.
|
||||
@return integer 1 if successfull.
|
||||
|
||||
This function reads the configuration file. Then test that the
|
||||
configuration parameters are correct (including directories).
|
||||
|
||||
**/
|
||||
/*-------------------------------------------------------------------------*/
|
||||
int SZ_LoadConf(const char* sz_cfgFile) {
|
||||
int res = SZ_ReadConf(sz_cfgFile);
|
||||
if (res != SZ_SUCCESS)
|
||||
{
|
||||
printf("[SZ] ERROR: Impossible to read configuration.\n");
|
||||
return SZ_FAILED;
|
||||
}
|
||||
return SZ_SUCCESS;
|
||||
}
|
||||
|
|
@ -0,0 +1,653 @@
|
|||
/**
|
||||
* @file double_compression.c
|
||||
* @author Sheng Di, Dingwen Tao, Xin Liang, Xiangyu Zou, Tao Lu, Wen Xia, Xuan Wang, Weizhe Zhang
|
||||
* @date April, 2016
|
||||
* @brief Compression Technique for double array
|
||||
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
|
||||
* See COPYRIGHT in top-level directory.
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include "sz.h"
|
||||
#include "DynamicByteArray.h"
|
||||
#include "DynamicIntArray.h"
|
||||
#include "TightDataPointStorageD.h"
|
||||
#include "CompressElement.h"
|
||||
#include "dataCompression.h"
|
||||
|
||||
|
||||
|
||||
float computeRangeSize_float(float* oriData, size_t size, float* valueRangeSize, float* medianValue)
|
||||
{
|
||||
size_t i = 0;
|
||||
float min = oriData[0];
|
||||
float max = min;
|
||||
for(i=1;i<size;i++)
|
||||
{
|
||||
float data = oriData[i];
|
||||
if(min>data)
|
||||
min = data;
|
||||
else if(max<data)
|
||||
max = data;
|
||||
}
|
||||
|
||||
*valueRangeSize = max - min;
|
||||
*medianValue = min + *valueRangeSize/2;
|
||||
return min;
|
||||
}
|
||||
|
||||
double computeRangeSize_double(double* oriData, size_t size, double* valueRangeSize, double* medianValue)
|
||||
{
|
||||
size_t i = 0;
|
||||
double min = oriData[0];
|
||||
double max = min;
|
||||
for(i=1;i<size;i++)
|
||||
{
|
||||
double data = oriData[i];
|
||||
if(min>data)
|
||||
min = data;
|
||||
else if(max<data)
|
||||
max = data;
|
||||
}
|
||||
|
||||
*valueRangeSize = max - min;
|
||||
*medianValue = min + *valueRangeSize/2;
|
||||
return min;
|
||||
}
|
||||
|
||||
double min_d(double a, double b)
|
||||
{
|
||||
if(a<b)
|
||||
return a;
|
||||
else
|
||||
return b;
|
||||
}
|
||||
|
||||
double max_d(double a, double b)
|
||||
{
|
||||
if(a>b)
|
||||
return a;
|
||||
else
|
||||
return b;
|
||||
}
|
||||
|
||||
float min_f(float a, float b)
|
||||
{
|
||||
if(a<b)
|
||||
return a;
|
||||
else
|
||||
return b;
|
||||
}
|
||||
|
||||
float max_f(float a, float b)
|
||||
{
|
||||
if(a>b)
|
||||
return a;
|
||||
else
|
||||
return b;
|
||||
}
|
||||
|
||||
double getRealPrecision_double(double valueRangeSize, int errBoundMode, double absErrBound, double relBoundRatio, int *status)
|
||||
{
|
||||
int state = SZ_SUCCESS;
|
||||
double precision = 0;
|
||||
if(errBoundMode==SZ_ABS||errBoundMode==ABS_OR_PW_REL||errBoundMode==ABS_AND_PW_REL)
|
||||
precision = absErrBound;
|
||||
else if(errBoundMode==REL||errBoundMode==REL_OR_PW_REL||errBoundMode==REL_AND_PW_REL)
|
||||
precision = relBoundRatio*valueRangeSize;
|
||||
else if(errBoundMode==ABS_AND_REL)
|
||||
precision = min_d(absErrBound, relBoundRatio*valueRangeSize);
|
||||
else if(errBoundMode==ABS_OR_REL)
|
||||
precision = max_d(absErrBound, relBoundRatio*valueRangeSize);
|
||||
else if(errBoundMode==PW_REL)
|
||||
precision = 0;
|
||||
else
|
||||
{
|
||||
printf("Error: error-bound-mode is incorrect!\n");
|
||||
state = SZ_BERR;
|
||||
}
|
||||
*status = state;
|
||||
return precision;
|
||||
}
|
||||
|
||||
double getRealPrecision_float(float valueRangeSize, int errBoundMode, double absErrBound, double relBoundRatio, int *status)
|
||||
{
|
||||
int state = SZ_SUCCESS;
|
||||
double precision = 0;
|
||||
if(errBoundMode==SZ_ABS||errBoundMode==ABS_OR_PW_REL||errBoundMode==ABS_AND_PW_REL)
|
||||
precision = absErrBound;
|
||||
else if(errBoundMode==REL||errBoundMode==REL_OR_PW_REL||errBoundMode==REL_AND_PW_REL)
|
||||
precision = relBoundRatio*valueRangeSize;
|
||||
else if(errBoundMode==ABS_AND_REL)
|
||||
precision = min_f(absErrBound, relBoundRatio*valueRangeSize);
|
||||
else if(errBoundMode==ABS_OR_REL)
|
||||
precision = max_f(absErrBound, relBoundRatio*valueRangeSize);
|
||||
else if(errBoundMode==PW_REL)
|
||||
precision = 0;
|
||||
else
|
||||
{
|
||||
printf("Error: error-bound-mode is incorrect!\n");
|
||||
state = SZ_BERR;
|
||||
}
|
||||
*status = state;
|
||||
return precision;
|
||||
}
|
||||
|
||||
double getRealPrecision_int(long valueRangeSize, int errBoundMode, double absErrBound, double relBoundRatio, int *status)
|
||||
{
|
||||
int state = SZ_SUCCESS;
|
||||
double precision = 0;
|
||||
if(errBoundMode==SZ_ABS||errBoundMode==ABS_OR_PW_REL||errBoundMode==ABS_AND_PW_REL)
|
||||
precision = absErrBound;
|
||||
else if(errBoundMode==REL||errBoundMode==REL_OR_PW_REL||errBoundMode==REL_AND_PW_REL)
|
||||
precision = relBoundRatio*valueRangeSize;
|
||||
else if(errBoundMode==ABS_AND_REL)
|
||||
precision = min_f(absErrBound, relBoundRatio*valueRangeSize);
|
||||
else if(errBoundMode==ABS_OR_REL)
|
||||
precision = max_f(absErrBound, relBoundRatio*valueRangeSize);
|
||||
else if(errBoundMode==PW_REL)
|
||||
precision = -1;
|
||||
else
|
||||
{
|
||||
printf("Error: error-bound-mode is incorrect!\n");
|
||||
state = SZ_BERR;
|
||||
}
|
||||
*status = state;
|
||||
return precision;
|
||||
}
|
||||
|
||||
void symTransform_8bytes(unsigned char data[8])
|
||||
{
|
||||
unsigned char tmp = data[0];
|
||||
data[0] = data[7];
|
||||
data[7] = tmp;
|
||||
|
||||
tmp = data[1];
|
||||
data[1] = data[6];
|
||||
data[6] = tmp;
|
||||
|
||||
tmp = data[2];
|
||||
data[2] = data[5];
|
||||
data[5] = tmp;
|
||||
|
||||
tmp = data[3];
|
||||
data[3] = data[4];
|
||||
data[4] = tmp;
|
||||
}
|
||||
|
||||
INLINE void symTransform_2bytes(unsigned char data[2])
|
||||
{
|
||||
unsigned char tmp = data[0];
|
||||
data[0] = data[1];
|
||||
data[1] = tmp;
|
||||
}
|
||||
|
||||
INLINE void symTransform_4bytes(unsigned char data[4])
|
||||
{
|
||||
unsigned char tmp = data[0];
|
||||
data[0] = data[3];
|
||||
data[3] = tmp;
|
||||
|
||||
tmp = data[1];
|
||||
data[1] = data[2];
|
||||
data[2] = tmp;
|
||||
}
|
||||
|
||||
INLINE void compressSingleFloatValue(FloatValueCompressElement *vce, float oriValue, float precision, float medianValue,
|
||||
int reqLength, int reqBytesLength, int resiBitsLength)
|
||||
{
|
||||
lfloat diffVal;
|
||||
diffVal.value = oriValue - medianValue;
|
||||
|
||||
// calc ignore bit count
|
||||
int ignBitCount = 32 - reqLength;
|
||||
if(ignBitCount<0)
|
||||
ignBitCount = 0;
|
||||
|
||||
intToBytes_bigEndian(vce->curBytes, diffVal.ivalue);
|
||||
|
||||
// truncate diff value tail bit with ignBitCount
|
||||
diffVal.ivalue = (diffVal.ivalue >> ignBitCount) << ignBitCount;
|
||||
|
||||
// save to vce
|
||||
vce->data = diffVal.value + medianValue;
|
||||
vce->curValue = diffVal.ivalue;
|
||||
vce->reqBytesLength = reqBytesLength;
|
||||
vce->resiBitsLength = resiBitsLength;
|
||||
}
|
||||
|
||||
void compressSingleDoubleValue(DoubleValueCompressElement *vce, double tgtValue, double precision, double medianValue,
|
||||
int reqLength, int reqBytesLength, int resiBitsLength)
|
||||
{
|
||||
double normValue = tgtValue - medianValue;
|
||||
|
||||
ldouble lfBuf;
|
||||
lfBuf.value = normValue;
|
||||
|
||||
int ignBytesLength = 64 - reqLength;
|
||||
if(ignBytesLength<0)
|
||||
ignBytesLength = 0;
|
||||
|
||||
long tmp_long = lfBuf.lvalue;
|
||||
longToBytes_bigEndian(vce->curBytes, tmp_long);
|
||||
|
||||
lfBuf.lvalue = (lfBuf.lvalue >> ignBytesLength)<<ignBytesLength;
|
||||
|
||||
//double tmpValue = lfBuf.value;
|
||||
|
||||
vce->data = lfBuf.value+medianValue;
|
||||
vce->curValue = tmp_long;
|
||||
vce->reqBytesLength = reqBytesLength;
|
||||
vce->resiBitsLength = resiBitsLength;
|
||||
}
|
||||
|
||||
int compIdenticalLeadingBytesCount_double(unsigned char* preBytes, unsigned char* curBytes)
|
||||
{
|
||||
int i, n = 0;
|
||||
for(i=0;i<8;i++)
|
||||
if(preBytes[i]==curBytes[i])
|
||||
n++;
|
||||
else
|
||||
break;
|
||||
if(n>3) n = 3;
|
||||
return n;
|
||||
}
|
||||
|
||||
INLINE int compIdenticalLeadingBytesCount_float(unsigned char* preBytes, unsigned char* curBytes)
|
||||
{
|
||||
int i, n = 0;
|
||||
for(i=0;i<4;i++)
|
||||
if(preBytes[i]==curBytes[i])
|
||||
n++;
|
||||
else
|
||||
break;
|
||||
if(n>3) n = 3;
|
||||
return n;
|
||||
}
|
||||
|
||||
//TODO double-check the correctness...
|
||||
INLINE void addExactData(DynamicByteArray *exactMidByteArray, DynamicIntArray *exactLeadNumArray,
|
||||
DynamicIntArray *resiBitArray, LossyCompressionElement *lce)
|
||||
{
|
||||
int i;
|
||||
int leadByteLength = lce->leadingZeroBytes;
|
||||
addDIA_Data(exactLeadNumArray, leadByteLength);
|
||||
unsigned char* intMidBytes = lce->integerMidBytes;
|
||||
int integerMidBytesLength = lce->integerMidBytes_Length;
|
||||
int resMidBitsLength = lce->resMidBitsLength;
|
||||
if(intMidBytes!=NULL||resMidBitsLength!=0)
|
||||
{
|
||||
if(intMidBytes!=NULL)
|
||||
for(i = 0;i<integerMidBytesLength;i++)
|
||||
addDBA_Data(exactMidByteArray, intMidBytes[i]);
|
||||
if(resMidBitsLength!=0)
|
||||
addDIA_Data(resiBitArray, lce->residualMidBits);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @deprecated
|
||||
* @return: the length of the coefficient array.
|
||||
* */
|
||||
int getPredictionCoefficients(int layers, int dimension, int **coeff_array, int *status)
|
||||
{
|
||||
size_t size = 0;
|
||||
switch(dimension)
|
||||
{
|
||||
case 1:
|
||||
switch(layers)
|
||||
{
|
||||
case 1:
|
||||
*coeff_array = (int*)malloc(sizeof(int));
|
||||
(*coeff_array)[0] = 1;
|
||||
size = 1;
|
||||
break;
|
||||
case 2:
|
||||
*coeff_array = (int*)malloc(2*sizeof(int));
|
||||
(*coeff_array)[0] = 2;
|
||||
(*coeff_array)[1] = -1;
|
||||
size = 2;
|
||||
break;
|
||||
case 3:
|
||||
*coeff_array = (int*)malloc(3*sizeof(int));
|
||||
(*coeff_array)[0] = 3;
|
||||
(*coeff_array)[1] = -3;
|
||||
(*coeff_array)[2] = 1;
|
||||
break;
|
||||
}
|
||||
break;
|
||||
case 2:
|
||||
switch(layers)
|
||||
{
|
||||
case 1:
|
||||
|
||||
break;
|
||||
case 2:
|
||||
|
||||
break;
|
||||
case 3:
|
||||
|
||||
break;
|
||||
}
|
||||
break;
|
||||
case 3:
|
||||
switch(layers)
|
||||
{
|
||||
case 1:
|
||||
|
||||
break;
|
||||
case 2:
|
||||
|
||||
break;
|
||||
case 3:
|
||||
|
||||
break;
|
||||
}
|
||||
break;
|
||||
default:
|
||||
printf("Error: dimension must be no greater than 3 in the current version.\n");
|
||||
*status = SZ_DERR;
|
||||
}
|
||||
*status = SZ_SUCCESS;
|
||||
return size;
|
||||
}
|
||||
|
||||
int computeBlockEdgeSize_2D(int segmentSize)
|
||||
{
|
||||
int i = 1;
|
||||
for(i=1; i<segmentSize;i++)
|
||||
{
|
||||
if(i*i>segmentSize)
|
||||
break;
|
||||
}
|
||||
return i;
|
||||
//return (int)(sqrt(segmentSize)+1);
|
||||
}
|
||||
|
||||
int computeBlockEdgeSize_3D(int segmentSize)
|
||||
{
|
||||
int i = 1;
|
||||
for(i=1; i<segmentSize;i++)
|
||||
{
|
||||
if(i*i*i>segmentSize)
|
||||
break;
|
||||
}
|
||||
return i;
|
||||
//return (int)(pow(segmentSize, 1.0/3)+1);
|
||||
}
|
||||
|
||||
//The following functions are float-precision version of dealing with the unpredictable data points
|
||||
int generateLossyCoefficients_float(float* oriData, double precision, size_t nbEle, int* reqBytesLength, int* resiBitsLength, float* medianValue, float* decData)
|
||||
{
|
||||
float valueRangeSize;
|
||||
|
||||
computeRangeSize_float(oriData, nbEle, &valueRangeSize, medianValue);
|
||||
short radExpo = getExponent_float(valueRangeSize/2);
|
||||
|
||||
int reqLength;
|
||||
computeReqLength_float(precision, radExpo, &reqLength, medianValue);
|
||||
|
||||
*reqBytesLength = reqLength/8;
|
||||
*resiBitsLength = reqLength%8;
|
||||
|
||||
size_t i = 0;
|
||||
for(i = 0;i < nbEle;i++)
|
||||
{
|
||||
float normValue = oriData[i] - *medianValue;
|
||||
|
||||
lfloat lfBuf;
|
||||
lfBuf.value = normValue;
|
||||
|
||||
int ignBytesLength = 32 - reqLength;
|
||||
if(ignBytesLength<0)
|
||||
ignBytesLength = 0;
|
||||
|
||||
lfBuf.ivalue = (lfBuf.ivalue >> ignBytesLength) << ignBytesLength;
|
||||
|
||||
//float tmpValue = lfBuf.value;
|
||||
|
||||
decData[i] = lfBuf.value + *medianValue;
|
||||
}
|
||||
return reqLength;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param float* oriData: inplace argument (input / output)
|
||||
*
|
||||
* */
|
||||
int compressExactDataArray_float(float* oriData, double precision, size_t nbEle, unsigned char** leadArray, unsigned char** midArray, unsigned char** resiArray,
|
||||
int reqLength, int reqBytesLength, int resiBitsLength, float medianValue)
|
||||
{
|
||||
//allocate memory for coefficient compression arrays
|
||||
DynamicIntArray *exactLeadNumArray;
|
||||
new_DIA(&exactLeadNumArray, DynArrayInitLen);
|
||||
DynamicByteArray *exactMidByteArray;
|
||||
new_DBA(&exactMidByteArray, DynArrayInitLen);
|
||||
DynamicIntArray *resiBitArray;
|
||||
new_DIA(&resiBitArray, DynArrayInitLen);
|
||||
unsigned char preDataBytes[4] = {0,0,0,0};
|
||||
|
||||
//allocate memory for vce and lce
|
||||
FloatValueCompressElement *vce = (FloatValueCompressElement*)malloc(sizeof(FloatValueCompressElement));
|
||||
LossyCompressionElement *lce = (LossyCompressionElement*)malloc(sizeof(LossyCompressionElement));
|
||||
|
||||
size_t i = 0;
|
||||
for(i = 0;i < nbEle;i++)
|
||||
{
|
||||
compressSingleFloatValue(vce, oriData[i], precision, medianValue, reqLength, reqBytesLength, resiBitsLength);
|
||||
updateLossyCompElement_Float(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
|
||||
memcpy(preDataBytes,vce->curBytes,4);
|
||||
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
|
||||
oriData[i] = vce->data;
|
||||
}
|
||||
convertDIAtoInts(exactLeadNumArray, leadArray);
|
||||
convertDBAtoBytes(exactMidByteArray,midArray);
|
||||
convertDIAtoInts(resiBitArray, resiArray);
|
||||
|
||||
size_t midArraySize = exactMidByteArray->size;
|
||||
|
||||
free(vce);
|
||||
free(lce);
|
||||
|
||||
free_DIA(exactLeadNumArray);
|
||||
free_DBA(exactMidByteArray);
|
||||
free_DIA(resiBitArray);
|
||||
|
||||
return midArraySize;
|
||||
}
|
||||
|
||||
void decompressExactDataArray_float(unsigned char* leadNum, unsigned char* exactMidBytes, unsigned char* residualMidBits, size_t nbEle, int reqLength, float medianValue, float** decData)
|
||||
{
|
||||
*decData = (float*)malloc(nbEle*sizeof(float));
|
||||
size_t i = 0, j = 0, k = 0, l = 0, p = 0, curByteIndex = 0;
|
||||
float exactData = 0;
|
||||
unsigned char preBytes[4] = {0,0,0,0};
|
||||
unsigned char curBytes[4];
|
||||
int resiBits;
|
||||
unsigned char leadingNum;
|
||||
|
||||
int reqBytesLength = reqLength/8;
|
||||
int resiBitsLength = reqLength%8;
|
||||
|
||||
for(i = 0; i<nbEle;i++)
|
||||
{
|
||||
// compute resiBits
|
||||
resiBits = 0;
|
||||
if (resiBitsLength != 0) {
|
||||
int kMod8 = k % 8;
|
||||
int rightMovSteps = getRightMovingSteps(kMod8, resiBitsLength);
|
||||
if (rightMovSteps > 0) {
|
||||
int code = getRightMovingCode(kMod8, resiBitsLength);
|
||||
resiBits = (residualMidBits[p] & code) >> rightMovSteps;
|
||||
} else if (rightMovSteps < 0) {
|
||||
int code1 = getLeftMovingCode(kMod8);
|
||||
int code2 = getRightMovingCode(kMod8, resiBitsLength);
|
||||
int leftMovSteps = -rightMovSteps;
|
||||
rightMovSteps = 8 - leftMovSteps;
|
||||
resiBits = (residualMidBits[p] & code1) << leftMovSteps;
|
||||
p++;
|
||||
resiBits = resiBits
|
||||
| ((residualMidBits[p] & code2) >> rightMovSteps);
|
||||
} else // rightMovSteps == 0
|
||||
{
|
||||
int code = getRightMovingCode(kMod8, resiBitsLength);
|
||||
resiBits = (residualMidBits[p] & code);
|
||||
p++;
|
||||
}
|
||||
k += resiBitsLength;
|
||||
}
|
||||
|
||||
// recover the exact data
|
||||
memset(curBytes, 0, 4);
|
||||
leadingNum = leadNum[l++];
|
||||
memcpy(curBytes, preBytes, leadingNum);
|
||||
for (j = leadingNum; j < reqBytesLength; j++)
|
||||
curBytes[j] = exactMidBytes[curByteIndex++];
|
||||
if (resiBitsLength != 0) {
|
||||
unsigned char resiByte = (unsigned char) (resiBits << (8 - resiBitsLength));
|
||||
curBytes[reqBytesLength] = resiByte;
|
||||
}
|
||||
|
||||
exactData = bytesToFloat(curBytes);
|
||||
(*decData)[i] = exactData + medianValue;
|
||||
memcpy(preBytes,curBytes,4);
|
||||
}
|
||||
}
|
||||
|
||||
//double-precision version of dealing with unpredictable data points in sz 2.0
|
||||
int generateLossyCoefficients_double(double* oriData, double precision, size_t nbEle, int* reqBytesLength, int* resiBitsLength, double* medianValue, double* decData)
|
||||
{
|
||||
double valueRangeSize;
|
||||
|
||||
computeRangeSize_double(oriData, nbEle, &valueRangeSize, medianValue);
|
||||
short radExpo = getExponent_double(valueRangeSize/2);
|
||||
|
||||
int reqLength;
|
||||
computeReqLength_double(precision, radExpo, &reqLength, medianValue);
|
||||
|
||||
*reqBytesLength = reqLength/8;
|
||||
*resiBitsLength = reqLength%8;
|
||||
|
||||
size_t i = 0;
|
||||
for(i = 0;i < nbEle;i++)
|
||||
{
|
||||
double normValue = oriData[i] - *medianValue;
|
||||
|
||||
ldouble ldBuf;
|
||||
ldBuf.value = normValue;
|
||||
|
||||
int ignBytesLength = 64 - reqLength;
|
||||
if(ignBytesLength<0)
|
||||
ignBytesLength = 0;
|
||||
|
||||
ldBuf.lvalue = (ldBuf.lvalue >> ignBytesLength) << ignBytesLength;
|
||||
|
||||
decData[i] = ldBuf.value + *medianValue;
|
||||
}
|
||||
return reqLength;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param double* oriData: inplace argument (input / output)
|
||||
*
|
||||
* */
|
||||
int compressExactDataArray_double(double* oriData, double precision, size_t nbEle, unsigned char** leadArray, unsigned char** midArray, unsigned char** resiArray,
|
||||
int reqLength, int reqBytesLength, int resiBitsLength, double medianValue)
|
||||
{
|
||||
//allocate memory for coefficient compression arrays
|
||||
DynamicIntArray *exactLeadNumArray;
|
||||
new_DIA(&exactLeadNumArray, DynArrayInitLen);
|
||||
DynamicByteArray *exactMidByteArray;
|
||||
new_DBA(&exactMidByteArray, DynArrayInitLen);
|
||||
DynamicIntArray *resiBitArray;
|
||||
new_DIA(&resiBitArray, DynArrayInitLen);
|
||||
unsigned char preDataBytes[8] = {0,0,0,0,0,0,0,0};
|
||||
|
||||
//allocate memory for vce and lce
|
||||
DoubleValueCompressElement *vce = (DoubleValueCompressElement*)malloc(sizeof(DoubleValueCompressElement));
|
||||
LossyCompressionElement *lce = (LossyCompressionElement*)malloc(sizeof(LossyCompressionElement));
|
||||
|
||||
size_t i = 0;
|
||||
for(i = 0;i < nbEle;i++)
|
||||
{
|
||||
compressSingleDoubleValue(vce, oriData[i], precision, medianValue, reqLength, reqBytesLength, resiBitsLength);
|
||||
updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
|
||||
memcpy(preDataBytes,vce->curBytes,8);
|
||||
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
|
||||
oriData[i] = vce->data;
|
||||
}
|
||||
convertDIAtoInts(exactLeadNumArray, leadArray);
|
||||
convertDBAtoBytes(exactMidByteArray,midArray);
|
||||
convertDIAtoInts(resiBitArray, resiArray);
|
||||
|
||||
size_t midArraySize = exactMidByteArray->size;
|
||||
|
||||
free(vce);
|
||||
free(lce);
|
||||
|
||||
free_DIA(exactLeadNumArray);
|
||||
free_DBA(exactMidByteArray);
|
||||
free_DIA(resiBitArray);
|
||||
|
||||
return midArraySize;
|
||||
}
|
||||
|
||||
void decompressExactDataArray_double(unsigned char* leadNum, unsigned char* exactMidBytes, unsigned char* residualMidBits, size_t nbEle, int reqLength, double medianValue, double** decData)
|
||||
{
|
||||
*decData = (double*)malloc(nbEle*sizeof(double));
|
||||
size_t i = 0, j = 0, k = 0, l = 0, p = 0, curByteIndex = 0;
|
||||
double exactData = 0;
|
||||
unsigned char preBytes[8] = {0,0,0,0,0,0,0,0};
|
||||
unsigned char curBytes[8];
|
||||
int resiBits;
|
||||
unsigned char leadingNum;
|
||||
|
||||
int reqBytesLength = reqLength/8;
|
||||
int resiBitsLength = reqLength%8;
|
||||
|
||||
for(i = 0; i<nbEle;i++)
|
||||
{
|
||||
// compute resiBits
|
||||
resiBits = 0;
|
||||
if (resiBitsLength != 0) {
|
||||
int kMod8 = k % 8;
|
||||
int rightMovSteps = getRightMovingSteps(kMod8, resiBitsLength);
|
||||
if (rightMovSteps > 0) {
|
||||
int code = getRightMovingCode(kMod8, resiBitsLength);
|
||||
resiBits = (residualMidBits[p] & code) >> rightMovSteps;
|
||||
} else if (rightMovSteps < 0) {
|
||||
int code1 = getLeftMovingCode(kMod8);
|
||||
int code2 = getRightMovingCode(kMod8, resiBitsLength);
|
||||
int leftMovSteps = -rightMovSteps;
|
||||
rightMovSteps = 8 - leftMovSteps;
|
||||
resiBits = (residualMidBits[p] & code1) << leftMovSteps;
|
||||
p++;
|
||||
resiBits = resiBits
|
||||
| ((residualMidBits[p] & code2) >> rightMovSteps);
|
||||
} else // rightMovSteps == 0
|
||||
{
|
||||
int code = getRightMovingCode(kMod8, resiBitsLength);
|
||||
resiBits = (residualMidBits[p] & code);
|
||||
p++;
|
||||
}
|
||||
k += resiBitsLength;
|
||||
}
|
||||
|
||||
// recover the exact data
|
||||
memset(curBytes, 0, 8);
|
||||
leadingNum = leadNum[l++];
|
||||
memcpy(curBytes, preBytes, leadingNum);
|
||||
for (j = leadingNum; j < reqBytesLength; j++)
|
||||
curBytes[j] = exactMidBytes[curByteIndex++];
|
||||
if (resiBitsLength != 0) {
|
||||
unsigned char resiByte = (unsigned char) (resiBits << (8 - resiBitsLength));
|
||||
curBytes[reqBytesLength] = resiByte;
|
||||
}
|
||||
|
||||
exactData = bytesToDouble(curBytes);
|
||||
(*decData)[i] = exactData + medianValue;
|
||||
memcpy(preBytes,curBytes,8);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,397 @@
|
|||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@file dictionary.c
|
||||
@author N. Devillard
|
||||
@brief Implements a dictionary for string variables.
|
||||
|
||||
This module implements a simple dictionary object, i.e. a list
|
||||
of string/string associations. This object is useful to store e.g.
|
||||
informations retrieved from a configuration file (ini files).
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
|
||||
/*---------------------------------------------------------------------------
|
||||
Includes
|
||||
---------------------------------------------------------------------------*/
|
||||
#include "dictionary.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
/** Maximum value size for integers and doubles. */
|
||||
#define MAXVALSZ 1024
|
||||
|
||||
/** Minimal allocated number of entries in a dictionary */
|
||||
#define DICTMINSZ 128
|
||||
|
||||
/** Invalid key token */
|
||||
#define DICT_INVALID_KEY ((char*)-1)
|
||||
|
||||
/*---------------------------------------------------------------------------
|
||||
Private functions
|
||||
---------------------------------------------------------------------------*/
|
||||
|
||||
/* Doubles the allocated size associated to a pointer */
|
||||
/* 'size' is the current allocated size. */
|
||||
static void * mem_double(void * ptr, int size)
|
||||
{
|
||||
void * newptr ;
|
||||
|
||||
newptr = calloc(2*size, 1);
|
||||
if (newptr==NULL) {
|
||||
return NULL ;
|
||||
}
|
||||
memcpy(newptr, ptr, size);
|
||||
free(ptr);
|
||||
return newptr ;
|
||||
}
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Duplicate a string
|
||||
@param s String to duplicate
|
||||
@return Pointer to a newly allocated string, to be freed with free()
|
||||
|
||||
This is a replacement for strdup(). This implementation is provided
|
||||
for systems that do not have it.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
static char * xstrdup(const char * s)
|
||||
{
|
||||
char * t ;
|
||||
if (!s)
|
||||
return NULL ;
|
||||
t = (char*)malloc(strlen(s)+1) ;
|
||||
if (t) {
|
||||
strcpy(t,s);
|
||||
}
|
||||
return t ;
|
||||
}
|
||||
|
||||
/*---------------------------------------------------------------------------
|
||||
Function codes
|
||||
---------------------------------------------------------------------------*/
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Compute the hash key for a string.
|
||||
@param key Character string to use for key.
|
||||
@return 1 unsigned int on at least 32 bits.
|
||||
|
||||
This hash function has been taken from an Article in Dr Dobbs Journal.
|
||||
This is normally a collision-free function, distributing keys evenly.
|
||||
The key is stored anyway in the struct so that collision can be avoided
|
||||
by comparing the key itself in last resort.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
unsigned dictionary_hash(const char * key)
|
||||
{
|
||||
int len ;
|
||||
unsigned hash ;
|
||||
int i ;
|
||||
|
||||
len = strlen(key);
|
||||
for (hash=0, i=0 ; i<len ; i++) {
|
||||
hash += (unsigned)key[i] ;
|
||||
hash += (hash<<10);
|
||||
hash ^= (hash>>6) ;
|
||||
}
|
||||
hash += (hash <<3);
|
||||
hash ^= (hash >>11);
|
||||
hash += (hash <<15);
|
||||
return hash ;
|
||||
}
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Create a new dictionary object.
|
||||
@param size Optional initial size of the dictionary.
|
||||
@return 1 newly allocated dictionary objet.
|
||||
|
||||
This function allocates a new dictionary object of given size and returns
|
||||
it. If you do not know in advance (roughly) the number of entries in the
|
||||
dictionary, give size=0.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
dictionary * dictionary_new(int size)
|
||||
{
|
||||
dictionary * d ;
|
||||
|
||||
/* If no size was specified, allocate space for DICTMINSZ */
|
||||
if (size<DICTMINSZ) size=DICTMINSZ ;
|
||||
|
||||
if (!(d = (dictionary *)calloc(1, sizeof(dictionary)))) {
|
||||
return NULL;
|
||||
}
|
||||
d->size = size ;
|
||||
d->val = (char **)calloc(size, sizeof(char*));
|
||||
d->key = (char **)calloc(size, sizeof(char*));
|
||||
d->hash = (unsigned int *)calloc(size, sizeof(unsigned));
|
||||
return d ;
|
||||
}
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Delete a dictionary object
|
||||
@param d dictionary object to deallocate.
|
||||
@return void
|
||||
|
||||
Deallocate a dictionary object and all memory associated to it.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
void dictionary_del(dictionary * d)
|
||||
{
|
||||
int i ;
|
||||
|
||||
if (d==NULL) return ;
|
||||
for (i=0 ; i<d->size ; i++) {
|
||||
if (d->key[i]!=NULL)
|
||||
free(d->key[i]);
|
||||
if (d->val[i]!=NULL)
|
||||
free(d->val[i]);
|
||||
}
|
||||
free(d->val);
|
||||
free(d->key);
|
||||
free(d->hash);
|
||||
free(d);
|
||||
return ;
|
||||
}
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Get a value from a dictionary.
|
||||
@param d dictionary object to search.
|
||||
@param key Key to look for in the dictionary.
|
||||
@param def Default value to return if key not found.
|
||||
@return 1 pointer to internally allocated character string.
|
||||
|
||||
This function locates a key in a dictionary and returns a pointer to its
|
||||
value, or the passed 'def' pointer if no such key can be found in
|
||||
dictionary. The returned character pointer points to data internal to the
|
||||
dictionary object, you should not try to free it or modify it.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
char * dictionary_get(dictionary * d, const char * key, char * def)
|
||||
{
|
||||
unsigned hash ;
|
||||
int i ;
|
||||
|
||||
hash = dictionary_hash(key);
|
||||
for (i=0 ; i<d->size ; i++) {
|
||||
if (d->key[i]==NULL)
|
||||
continue ;
|
||||
/* Compare hash */
|
||||
if (hash==d->hash[i]) {
|
||||
/* Compare string, to avoid hash collisions */
|
||||
if (!strcmp(key, d->key[i])) {
|
||||
return d->val[i] ;
|
||||
}
|
||||
}
|
||||
}
|
||||
return def ;
|
||||
}
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Set a value in a dictionary.
|
||||
@param d dictionary object to modify.
|
||||
@param key Key to modify or add.
|
||||
@param val Value to add.
|
||||
@return int 0 if Ok, anything else otherwise
|
||||
|
||||
If the given key is found in the dictionary, the associated value is
|
||||
replaced by the provided one. If the key cannot be found in the
|
||||
dictionary, it is added to it.
|
||||
|
||||
It is Ok to provide a NULL value for val, but NULL values for the dictionary
|
||||
or the key are considered as errors: the function will return immediately
|
||||
in such a case.
|
||||
|
||||
Notice that if you dictionary_set a variable to NULL, a call to
|
||||
dictionary_get will return a NULL value: the variable will be found, and
|
||||
its value (NULL) is returned. In other words, setting the variable
|
||||
content to NULL is equivalent to deleting the variable from the
|
||||
dictionary. It is not possible (in this implementation) to have a key in
|
||||
the dictionary without value.
|
||||
|
||||
This function returns non-zero in case of failure.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
int dictionary_set(dictionary * d, const char * key, const char * val)
|
||||
{
|
||||
int i ;
|
||||
unsigned hash ;
|
||||
|
||||
if (d==NULL || key==NULL) return -1 ;
|
||||
|
||||
/* Compute hash for this key */
|
||||
hash = dictionary_hash(key) ;
|
||||
/* Find if value is already in dictionary */
|
||||
if (d->n>0) {
|
||||
for (i=0 ; i<d->size ; i++) {
|
||||
if (d->key[i]==NULL)
|
||||
continue ;
|
||||
if (hash==d->hash[i]) { /* Same hash value */
|
||||
if (!strcmp(key, d->key[i])) { /* Same key */
|
||||
/* Found a value: modify and return */
|
||||
if (d->val[i]!=NULL)
|
||||
free(d->val[i]);
|
||||
d->val[i] = val ? xstrdup(val) : NULL ;
|
||||
/* Value has been modified: return */
|
||||
return 0 ;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
/* Add a new value */
|
||||
/* See if dictionary needs to grow */
|
||||
if (d->n==d->size) {
|
||||
|
||||
/* Reached maximum size: reallocate dictionary */
|
||||
d->val = (char **)mem_double(d->val, d->size * sizeof(char*)) ;
|
||||
d->key = (char **)mem_double(d->key, d->size * sizeof(char*)) ;
|
||||
d->hash = (unsigned int *)mem_double(d->hash, d->size * sizeof(unsigned)) ;
|
||||
if ((d->val==NULL) || (d->key==NULL) || (d->hash==NULL)) {
|
||||
/* Cannot grow dictionary */
|
||||
return -1 ;
|
||||
}
|
||||
/* Double size */
|
||||
d->size *= 2 ;
|
||||
}
|
||||
|
||||
/* Insert key in the first empty slot. Start at d->n and wrap at
|
||||
d->size. Because d->n < d->size this will necessarily
|
||||
terminate. */
|
||||
for (i=d->n ; d->key[i] ; ) {
|
||||
if(++i == d->size) i = 0;
|
||||
}
|
||||
/* Copy key */
|
||||
d->key[i] = xstrdup(key);
|
||||
d->val[i] = val ? xstrdup(val) : NULL ;
|
||||
d->hash[i] = hash;
|
||||
d->n ++ ;
|
||||
return 0 ;
|
||||
}
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Delete a key in a dictionary
|
||||
@param d dictionary object to modify.
|
||||
@param key Key to remove.
|
||||
@return void
|
||||
|
||||
This function deletes a key in a dictionary. Nothing is done if the
|
||||
key cannot be found.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
void dictionary_unset(dictionary * d, const char * key)
|
||||
{
|
||||
unsigned hash ;
|
||||
int i ;
|
||||
|
||||
if (key == NULL) {
|
||||
return;
|
||||
}
|
||||
|
||||
hash = dictionary_hash(key);
|
||||
for (i=0 ; i<d->size ; i++) {
|
||||
if (d->key[i]==NULL)
|
||||
continue ;
|
||||
/* Compare hash */
|
||||
if (hash==d->hash[i]) {
|
||||
/* Compare string, to avoid hash collisions */
|
||||
if (!strcmp(key, d->key[i])) {
|
||||
/* Found key */
|
||||
break ;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (i>=d->size)
|
||||
/* Key not found */
|
||||
return ;
|
||||
|
||||
free(d->key[i]);
|
||||
d->key[i] = NULL ;
|
||||
if (d->val[i]!=NULL) {
|
||||
free(d->val[i]);
|
||||
d->val[i] = NULL ;
|
||||
}
|
||||
d->hash[i] = 0 ;
|
||||
d->n -- ;
|
||||
return ;
|
||||
}
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Dump a dictionary to an opened file pointer.
|
||||
@param d Dictionary to dump
|
||||
@param f Opened file pointer.
|
||||
@return void
|
||||
|
||||
Dumps a dictionary onto an opened file pointer. Key pairs are printed out
|
||||
as @c [Key]=[Value], one per line. It is Ok to provide stdout or stderr as
|
||||
output file pointers.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
void dictionary_dump(dictionary * d, FILE * out)
|
||||
{
|
||||
int i ;
|
||||
|
||||
if (d==NULL || out==NULL) return ;
|
||||
if (d->n<1) {
|
||||
fprintf(out, "empty dictionary\n");
|
||||
return ;
|
||||
}
|
||||
for (i=0 ; i<d->size ; i++) {
|
||||
if (d->key[i]) {
|
||||
fprintf(out, "%20s\t[%s]\n",
|
||||
d->key[i],
|
||||
d->val[i] ? d->val[i] : "UNDEF");
|
||||
}
|
||||
}
|
||||
return ;
|
||||
}
|
||||
|
||||
|
||||
/* Test code */
|
||||
#ifdef TESTDIC
|
||||
#define NVALS 20000
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
dictionary * d ;
|
||||
char * val ;
|
||||
int i ;
|
||||
char cval[90] ;
|
||||
|
||||
/* Allocate dictionary */
|
||||
printf("allocating...\n");
|
||||
d = dictionary_new(0);
|
||||
|
||||
/* Set values in dictionary */
|
||||
printf("setting %d values...\n", NVALS);
|
||||
for (i=0 ; i<NVALS ; i++) {
|
||||
sprintf(cval, "%04d", i);
|
||||
dictionary_set(d, cval, "salut");
|
||||
}
|
||||
printf("getting %d values...\n", NVALS);
|
||||
for (i=0 ; i<NVALS ; i++) {
|
||||
sprintf(cval, "%04d", i);
|
||||
val = dictionary_get(d, cval, DICT_INVALID_KEY);
|
||||
if (val==DICT_INVALID_KEY) {
|
||||
printf("cannot get value for key [%s]\n", cval);
|
||||
}
|
||||
}
|
||||
printf("unsetting %d values...\n", NVALS);
|
||||
for (i=0 ; i<NVALS ; i++) {
|
||||
sprintf(cval, "%04d", i);
|
||||
dictionary_unset(d, cval);
|
||||
}
|
||||
if (d->n != 0) {
|
||||
printf("error deleting values\n");
|
||||
}
|
||||
printf("deallocating...\n");
|
||||
dictionary_del(d);
|
||||
return 0 ;
|
||||
}
|
||||
#endif
|
||||
/* vim: set ts=4 et sw=4 tw=75 */
|
||||
|
|
@ -0,0 +1,774 @@
|
|||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@file iniparser.c
|
||||
@author N. Devillard
|
||||
@brief Parser for ini files.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
/*---------------------------- Includes ------------------------------------*/
|
||||
#include <ctype.h>
|
||||
#include "iniparser.h"
|
||||
|
||||
/*---------------------------- Defines -------------------------------------*/
|
||||
#define ASCIILINESZ (1024)
|
||||
#define INI_INVALID_KEY ((char*)-1)
|
||||
|
||||
/*---------------------------------------------------------------------------
|
||||
Private to this module
|
||||
---------------------------------------------------------------------------*/
|
||||
/**
|
||||
* This enum stores the status for each parsed line (internal use only).
|
||||
*/
|
||||
typedef enum _line_status_ {
|
||||
LINE_UNPROCESSED,
|
||||
LINE_ERROR,
|
||||
LINE_EMPTY,
|
||||
LINE_COMMENT,
|
||||
LINE_SECTION,
|
||||
LINE_VALUE
|
||||
} line_status ;
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Convert a string to lowercase.
|
||||
@param s String to convert.
|
||||
@return ptr to statically allocated string.
|
||||
|
||||
This function returns a pointer to a statically allocated string
|
||||
containing a lowercased version of the input string. Do not free
|
||||
or modify the returned string! Since the returned string is statically
|
||||
allocated, it will be modified at each function call (not re-entrant).
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
static char * strlwc(const char * s)
|
||||
{
|
||||
static char l[ASCIILINESZ+1];
|
||||
int i ;
|
||||
|
||||
if (s==NULL) return NULL ;
|
||||
memset(l, 0, ASCIILINESZ+1);
|
||||
i=0 ;
|
||||
while (s[i] && i<ASCIILINESZ) {
|
||||
l[i] = (char)tolower((int)s[i]);
|
||||
i++ ;
|
||||
}
|
||||
l[ASCIILINESZ]=(char)0;
|
||||
return l ;
|
||||
}
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Remove blanks at the beginning and the end of a string.
|
||||
@param s String to parse.
|
||||
@return ptr to statically allocated string.
|
||||
|
||||
This function returns a pointer to a statically allocated string,
|
||||
which is identical to the input string, except that all blank
|
||||
characters at the end and the beg. of the string have been removed.
|
||||
Do not free or modify the returned string! Since the returned string
|
||||
is statically allocated, it will be modified at each function call
|
||||
(not re-entrant).
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
static char * strstrip(const char * s)
|
||||
{
|
||||
static char l[ASCIILINESZ+1];
|
||||
char * last;
|
||||
|
||||
if (s==NULL) return NULL ;
|
||||
|
||||
while (isspace((int)*s) && *s) s++;
|
||||
memset(l, 0, ASCIILINESZ+1);
|
||||
strncpy(l, s, ASCIILINESZ);
|
||||
last = l + strlen(l);
|
||||
while (last > l) {
|
||||
if (!isspace((int)*(last-1)))
|
||||
break ;
|
||||
last -- ;
|
||||
}
|
||||
*last = (char)0;
|
||||
return (char*)l ;
|
||||
}
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Get number of sections in a dictionary
|
||||
@param d Dictionary to examine
|
||||
@return int Number of sections found in dictionary
|
||||
|
||||
This function returns the number of sections found in a dictionary.
|
||||
The test to recognize sections is done on the string stored in the
|
||||
dictionary: a section name is given as "section" whereas a key is
|
||||
stored as "section:key", thus the test looks for entries that do not
|
||||
contain a colon.
|
||||
|
||||
This clearly fails in the case a section name contains a colon, but
|
||||
this should simply be avoided.
|
||||
|
||||
This function returns -1 in case of error.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
int iniparser_getnsec(dictionary * d)
|
||||
{
|
||||
int i ;
|
||||
int nsec ;
|
||||
|
||||
if (d==NULL) return -1 ;
|
||||
nsec=0 ;
|
||||
for (i=0 ; i<d->size ; i++) {
|
||||
if (d->key[i]==NULL)
|
||||
continue ;
|
||||
if (strchr(d->key[i], ':')==NULL) {
|
||||
nsec ++ ;
|
||||
}
|
||||
}
|
||||
return nsec ;
|
||||
}
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Get name for section n in a dictionary.
|
||||
@param d Dictionary to examine
|
||||
@param n Section number (from 0 to nsec-1).
|
||||
@return Pointer to char string
|
||||
|
||||
This function locates the n-th section in a dictionary and returns
|
||||
its name as a pointer to a string statically allocated inside the
|
||||
dictionary. Do not free or modify the returned string!
|
||||
|
||||
This function returns NULL in case of error.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
char * iniparser_getsecname(dictionary * d, int n)
|
||||
{
|
||||
int i ;
|
||||
int foundsec ;
|
||||
|
||||
if (d==NULL || n<0) return NULL ;
|
||||
foundsec=0 ;
|
||||
for (i=0 ; i<d->size ; i++) {
|
||||
if (d->key[i]==NULL)
|
||||
continue ;
|
||||
if (strchr(d->key[i], ':')==NULL) {
|
||||
foundsec++ ;
|
||||
if (foundsec>n)
|
||||
break ;
|
||||
}
|
||||
}
|
||||
if (foundsec<=n) {
|
||||
return NULL ;
|
||||
}
|
||||
return d->key[i] ;
|
||||
}
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Dump a dictionary to an opened file pointer.
|
||||
@param d Dictionary to dump.
|
||||
@param f Opened file pointer to dump to.
|
||||
@return void
|
||||
|
||||
This function prints out the contents of a dictionary, one element by
|
||||
line, onto the provided file pointer. It is OK to specify @c stderr
|
||||
or @c stdout as output files. This function is meant for debugging
|
||||
purposes mostly.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
void iniparser_dump(dictionary * d, FILE * f)
|
||||
{
|
||||
int i ;
|
||||
|
||||
if (d==NULL || f==NULL) return ;
|
||||
for (i=0 ; i<d->size ; i++) {
|
||||
if (d->key[i]==NULL)
|
||||
continue ;
|
||||
if (d->val[i]!=NULL) {
|
||||
fprintf(f, "[%s]=[%s]\n", d->key[i], d->val[i]);
|
||||
} else {
|
||||
fprintf(f, "[%s]=UNDEF\n", d->key[i]);
|
||||
}
|
||||
}
|
||||
return ;
|
||||
}
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Save a dictionary to a loadable ini file
|
||||
@param d Dictionary to dump
|
||||
@param f Opened file pointer to dump to
|
||||
@return void
|
||||
|
||||
This function dumps a given dictionary into a loadable ini file.
|
||||
It is Ok to specify @c stderr or @c stdout as output files.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
void iniparser_dump_ini(dictionary * d, FILE * f)
|
||||
{
|
||||
int i ;
|
||||
int nsec ;
|
||||
char * secname ;
|
||||
|
||||
if (d==NULL || f==NULL) return ;
|
||||
|
||||
nsec = iniparser_getnsec(d);
|
||||
if (nsec<1) {
|
||||
/* No section in file: dump all keys as they are */
|
||||
for (i=0 ; i<d->size ; i++) {
|
||||
if (d->key[i]==NULL)
|
||||
continue ;
|
||||
fprintf(f, "%s = %s\n", d->key[i], d->val[i]);
|
||||
}
|
||||
return ;
|
||||
}
|
||||
for (i=0 ; i<nsec ; i++) {
|
||||
secname = iniparser_getsecname(d, i) ;
|
||||
iniparser_dumpsection_ini(d, secname, f) ;
|
||||
}
|
||||
fprintf(f, "\n");
|
||||
return ;
|
||||
}
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Save a dictionary section to a loadable ini file
|
||||
@param d Dictionary to dump
|
||||
@param s Section name of dictionary to dump
|
||||
@param f Opened file pointer to dump to
|
||||
@return void
|
||||
|
||||
This function dumps a given section of a given dictionary into a loadable ini
|
||||
file. It is Ok to specify @c stderr or @c stdout as output files.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
void iniparser_dumpsection_ini(dictionary * d, char * s, FILE * f)
|
||||
{
|
||||
int j ;
|
||||
char keym[ASCIILINESZ+1];
|
||||
int seclen ;
|
||||
|
||||
if (d==NULL || f==NULL) return ;
|
||||
if (! iniparser_find_entry(d, s)) return ;
|
||||
|
||||
seclen = (int)strlen(s);
|
||||
fprintf(f, "\n[%s]\n", s);
|
||||
sprintf(keym, "%s:", s);
|
||||
for (j=0 ; j<d->size ; j++) {
|
||||
if (d->key[j]==NULL)
|
||||
continue ;
|
||||
if (!strncmp(d->key[j], keym, seclen+1)) {
|
||||
fprintf(f,
|
||||
"%-30s = %s\n",
|
||||
d->key[j]+seclen+1,
|
||||
d->val[j] ? d->val[j] : "");
|
||||
}
|
||||
}
|
||||
fprintf(f, "\n");
|
||||
return ;
|
||||
}
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Get the number of keys in a section of a dictionary.
|
||||
@param d Dictionary to examine
|
||||
@param s Section name of dictionary to examine
|
||||
@return Number of keys in section
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
int iniparser_getsecnkeys(dictionary * d, char * s)
|
||||
{
|
||||
int seclen, nkeys ;
|
||||
char keym[ASCIILINESZ+1];
|
||||
int j ;
|
||||
|
||||
nkeys = 0;
|
||||
|
||||
if (d==NULL) return nkeys;
|
||||
if (! iniparser_find_entry(d, s)) return nkeys;
|
||||
|
||||
seclen = (int)strlen(s);
|
||||
sprintf(keym, "%s:", s);
|
||||
|
||||
for (j=0 ; j<d->size ; j++) {
|
||||
if (d->key[j]==NULL)
|
||||
continue ;
|
||||
if (!strncmp(d->key[j], keym, seclen+1))
|
||||
nkeys++;
|
||||
}
|
||||
|
||||
return nkeys;
|
||||
|
||||
}
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Get the number of keys in a section of a dictionary.
|
||||
@param d Dictionary to examine
|
||||
@param s Section name of dictionary to examine
|
||||
@return pointer to statically allocated character strings
|
||||
|
||||
This function queries a dictionary and finds all keys in a given section.
|
||||
Each pointer in the returned char pointer-to-pointer is pointing to
|
||||
a string allocated in the dictionary; do not free or modify them.
|
||||
|
||||
This function returns NULL in case of error.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
char ** iniparser_getseckeys(dictionary * d, char * s)
|
||||
{
|
||||
|
||||
char **keys;
|
||||
|
||||
int i, j ;
|
||||
char keym[ASCIILINESZ+1];
|
||||
int seclen, nkeys ;
|
||||
|
||||
keys = NULL;
|
||||
|
||||
if (d==NULL) return keys;
|
||||
if (! iniparser_find_entry(d, s)) return keys;
|
||||
|
||||
nkeys = iniparser_getsecnkeys(d, s);
|
||||
|
||||
keys = (char**) malloc(nkeys*sizeof(char*));
|
||||
|
||||
seclen = (int)strlen(s);
|
||||
sprintf(keym, "%s:", s);
|
||||
|
||||
i = 0;
|
||||
|
||||
for (j=0 ; j<d->size ; j++) {
|
||||
if (d->key[j]==NULL)
|
||||
continue ;
|
||||
if (!strncmp(d->key[j], keym, seclen+1)) {
|
||||
keys[i] = d->key[j];
|
||||
i++;
|
||||
}
|
||||
}
|
||||
|
||||
return keys;
|
||||
|
||||
}
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Get the string associated to a key
|
||||
@param d Dictionary to search
|
||||
@param key Key string to look for
|
||||
@param def Default value to return if key not found.
|
||||
@return pointer to statically allocated character string
|
||||
|
||||
This function queries a dictionary for a key. A key as read from an
|
||||
ini file is given as "section:key". If the key cannot be found,
|
||||
the pointer passed as 'def' is returned.
|
||||
The returned char pointer is pointing to a string allocated in
|
||||
the dictionary, do not free or modify it.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
char * iniparser_getstring(dictionary * d, const char * key, char * def)
|
||||
{
|
||||
char * lc_key ;
|
||||
char * sval ;
|
||||
|
||||
if (d==NULL || key==NULL)
|
||||
return def ;
|
||||
|
||||
lc_key = strlwc(key);
|
||||
sval = dictionary_get(d, lc_key, def);
|
||||
return sval ;
|
||||
}
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Get the string associated to a key, convert to an int
|
||||
@param d Dictionary to search
|
||||
@param key Key string to look for
|
||||
@param notfound Value to return in case of error
|
||||
@return integer
|
||||
|
||||
This function queries a dictionary for a key. A key as read from an
|
||||
ini file is given as "section:key". If the key cannot be found,
|
||||
the notfound value is returned.
|
||||
|
||||
Supported values for integers include the usual C notation
|
||||
so decimal, octal (starting with 0) and hexadecimal (starting with 0x)
|
||||
are supported. Examples:
|
||||
|
||||
"42" -> 42
|
||||
"042" -> 34 (octal -> decimal)
|
||||
"0x42" -> 66 (hexa -> decimal)
|
||||
|
||||
Warning: the conversion may overflow in various ways. Conversion is
|
||||
totally outsourced to strtol(), see the associated man page for overflow
|
||||
handling.
|
||||
|
||||
Credits: Thanks to A. Becker for suggesting strtol()
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
int iniparser_getint(dictionary * d, const char * key, int notfound)
|
||||
{
|
||||
char * str ;
|
||||
|
||||
str = iniparser_getstring(d, key, INI_INVALID_KEY);
|
||||
if (str==INI_INVALID_KEY) return notfound ;
|
||||
return (int)strtol(str, NULL, 0);
|
||||
}
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Get the string associated to a key, convert to a long
|
||||
@param d Dictionary to search
|
||||
@param key Key string to look for
|
||||
@param notfound Value to return in case of error
|
||||
@return long
|
||||
|
||||
Credits: This function bases completely on int iniparser_getint and was
|
||||
slightly modified to return long instead of int.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
long iniparser_getlint(dictionary * d, const char * key, int notfound)
|
||||
{
|
||||
char * str ;
|
||||
|
||||
str = iniparser_getstring(d, key, INI_INVALID_KEY);
|
||||
if (str==INI_INVALID_KEY) return notfound ;
|
||||
return strtol(str, NULL, 0);
|
||||
}
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Get the string associated to a key, convert to a double
|
||||
@param d Dictionary to search
|
||||
@param key Key string to look for
|
||||
@param notfound Value to return in case of error
|
||||
@return double
|
||||
|
||||
This function queries a dictionary for a key. A key as read from an
|
||||
ini file is given as "section:key". If the key cannot be found,
|
||||
the notfound value is returned.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
double iniparser_getdouble(dictionary * d, const char * key, double notfound)
|
||||
{
|
||||
char * str ;
|
||||
|
||||
str = iniparser_getstring(d, key, INI_INVALID_KEY);
|
||||
if (str==INI_INVALID_KEY) return notfound ;
|
||||
return atof(str);
|
||||
}
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Get the string associated to a key, convert to a boolean
|
||||
@param d Dictionary to search
|
||||
@param key Key string to look for
|
||||
@param notfound Value to return in case of error
|
||||
@return integer
|
||||
|
||||
This function queries a dictionary for a key. A key as read from an
|
||||
ini file is given as "section:key". If the key cannot be found,
|
||||
the notfound value is returned.
|
||||
|
||||
A true boolean is found if one of the following is matched:
|
||||
|
||||
- A string starting with 'y'
|
||||
- A string starting with 'Y'
|
||||
- A string starting with 't'
|
||||
- A string starting with 'T'
|
||||
- A string starting with '1'
|
||||
|
||||
A false boolean is found if one of the following is matched:
|
||||
|
||||
- A string starting with 'n'
|
||||
- A string starting with 'N'
|
||||
- A string starting with 'f'
|
||||
- A string starting with 'F'
|
||||
- A string starting with '0'
|
||||
|
||||
The notfound value returned if no boolean is identified, does not
|
||||
necessarily have to be 0 or 1.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
int iniparser_getboolean(dictionary * d, const char * key, int notfound)
|
||||
{
|
||||
char * c ;
|
||||
int ret ;
|
||||
|
||||
c = iniparser_getstring(d, key, INI_INVALID_KEY);
|
||||
if (c==INI_INVALID_KEY) return notfound ;
|
||||
if (c[0]=='y' || c[0]=='Y' || c[0]=='1' || c[0]=='t' || c[0]=='T') {
|
||||
ret = 1 ;
|
||||
} else if (c[0]=='n' || c[0]=='N' || c[0]=='0' || c[0]=='f' || c[0]=='F') {
|
||||
ret = 0 ;
|
||||
} else {
|
||||
ret = notfound ;
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Finds out if a given entry exists in a dictionary
|
||||
@param ini Dictionary to search
|
||||
@param entry Name of the entry to look for
|
||||
@return integer 1 if entry exists, 0 otherwise
|
||||
|
||||
Finds out if a given entry exists in the dictionary. Since sections
|
||||
are stored as keys with NULL associated values, this is the only way
|
||||
of querying for the presence of sections in a dictionary.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
int iniparser_find_entry(
|
||||
dictionary * ini,
|
||||
const char * entry
|
||||
)
|
||||
{
|
||||
int found=0 ;
|
||||
if (iniparser_getstring(ini, entry, INI_INVALID_KEY)!=INI_INVALID_KEY) {
|
||||
found = 1 ;
|
||||
}
|
||||
return found ;
|
||||
}
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Set an entry in a dictionary.
|
||||
@param ini Dictionary to modify.
|
||||
@param entry Entry to modify (entry name)
|
||||
@param val New value to associate to the entry.
|
||||
@return int 0 if Ok, -1 otherwise.
|
||||
|
||||
If the given entry can be found in the dictionary, it is modified to
|
||||
contain the provided value. If it cannot be found, -1 is returned.
|
||||
It is Ok to set val to NULL.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
int iniparser_set(dictionary * ini, const char * entry, const char * val)
|
||||
{
|
||||
return dictionary_set(ini, strlwc(entry), val) ;
|
||||
}
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Delete an entry in a dictionary
|
||||
@param ini Dictionary to modify
|
||||
@param entry Entry to delete (entry name)
|
||||
@return void
|
||||
|
||||
If the given entry can be found, it is deleted from the dictionary.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
void iniparser_unset(dictionary * ini, const char * entry)
|
||||
{
|
||||
dictionary_unset(ini, strlwc(entry));
|
||||
}
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Load a single line from an INI file
|
||||
@param input_line Input line, may be concatenated multi-line input
|
||||
@param section Output space to store section
|
||||
@param key Output space to store key
|
||||
@param value Output space to store value
|
||||
@return line_status value
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
static line_status iniparser_line(
|
||||
const char * input_line,
|
||||
char * section,
|
||||
char * key,
|
||||
char * value)
|
||||
{
|
||||
line_status sta ;
|
||||
char line[ASCIILINESZ+1];
|
||||
int len ;
|
||||
|
||||
memset(line, 0, ASCIILINESZ + 1);
|
||||
len = (int)strlen(strstrip(input_line));
|
||||
if (len > ASCIILINESZ)
|
||||
len = ASCIILINESZ;
|
||||
strncpy(line, strstrip(input_line), len);
|
||||
len = (int)strlen(line);
|
||||
|
||||
sta = LINE_UNPROCESSED ;
|
||||
if (len<1) {
|
||||
/* Empty line */
|
||||
sta = LINE_EMPTY ;
|
||||
} else if (line[0]=='#' || line[0]==';') {
|
||||
/* Comment line */
|
||||
sta = LINE_COMMENT ;
|
||||
} else if (line[0]=='[' && line[len-1]==']') {
|
||||
/* Section name */
|
||||
sscanf(line, "[%[^]]", section);
|
||||
strcpy(section, strstrip(section));
|
||||
strcpy(section, strlwc(section));
|
||||
sta = LINE_SECTION ;
|
||||
} else if (sscanf (line, "%[^=] = \"%[^\"]\"", key, value) == 2
|
||||
|| sscanf (line, "%[^=] = '%[^\']'", key, value) == 2
|
||||
|| sscanf (line, "%[^=] = %[^;#]", key, value) == 2) {
|
||||
/* Usual key=value, with or without comments */
|
||||
strcpy(key, strstrip(key));
|
||||
strcpy(key, strlwc(key));
|
||||
strcpy(value, strstrip(value));
|
||||
/*
|
||||
* sscanf cannot handle '' or "" as empty values
|
||||
* this is done here
|
||||
*/
|
||||
if (!strcmp(value, "\"\"") || (!strcmp(value, "''"))) {
|
||||
value[0]=0 ;
|
||||
}
|
||||
sta = LINE_VALUE ;
|
||||
} else if (sscanf(line, "%[^=] = %[;#]", key, value)==2
|
||||
|| sscanf(line, "%[^=] %[=]", key, value) == 2) {
|
||||
/*
|
||||
* Special cases:
|
||||
* key=
|
||||
* key=;
|
||||
* key=#
|
||||
*/
|
||||
strcpy(key, strstrip(key));
|
||||
strcpy(key, strlwc(key));
|
||||
value[0]=0 ;
|
||||
sta = LINE_VALUE ;
|
||||
} else {
|
||||
/* Generate syntax error */
|
||||
sta = LINE_ERROR ;
|
||||
printf("===== > %s ===> %s\n", input_line, line);
|
||||
}
|
||||
return sta ;
|
||||
}
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Parse an ini file and return an allocated dictionary object
|
||||
@param ininame Name of the ini file to read.
|
||||
@return Pointer to newly allocated dictionary
|
||||
|
||||
This is the parser for ini files. This function is called, providing
|
||||
the name of the file to be read. It returns a dictionary object that
|
||||
should not be accessed directly, but through accessor functions
|
||||
instead.
|
||||
|
||||
The returned dictionary must be freed using iniparser_freedict().
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
dictionary * iniparser_load(const char * ininame)
|
||||
{
|
||||
FILE * in ;
|
||||
|
||||
char line [ASCIILINESZ+1] ;
|
||||
char section [ASCIILINESZ+1] ;
|
||||
char key [ASCIILINESZ+1] ;
|
||||
char tmp [2*ASCIILINESZ+2] ;
|
||||
char val [ASCIILINESZ+1] ;
|
||||
|
||||
int last=0 ;
|
||||
int len ;
|
||||
int lineno=0 ;
|
||||
int errs=0;
|
||||
|
||||
dictionary * dict ;
|
||||
|
||||
if ((in=fopen(ininame, "r"))==NULL) {
|
||||
fprintf(stderr, "iniparser: cannot open %s\n", ininame);
|
||||
return NULL ;
|
||||
}
|
||||
|
||||
dict = dictionary_new(0) ;
|
||||
if (!dict) {
|
||||
fclose(in);
|
||||
return NULL ;
|
||||
}
|
||||
|
||||
memset(line, 0, ASCIILINESZ);
|
||||
memset(section, 0, ASCIILINESZ);
|
||||
memset(key, 0, ASCIILINESZ);
|
||||
memset(val, 0, ASCIILINESZ);
|
||||
last=0 ;
|
||||
|
||||
while (fgets(line+last, ASCIILINESZ-last, in)!=NULL) {
|
||||
lineno++ ;
|
||||
len = (int)strlen(line)-1;
|
||||
if (len==0)
|
||||
continue;
|
||||
/* Safety check against buffer overflows */
|
||||
if (line[len]!='\n') {
|
||||
fprintf(stderr,
|
||||
"iniparser: input line too long in %s (%d)\n",
|
||||
ininame,
|
||||
lineno);
|
||||
dictionary_del(dict);
|
||||
fclose(in);
|
||||
return NULL ;
|
||||
}
|
||||
/* Get rid of \n and spaces at end of line */
|
||||
while ((len>=0) &&
|
||||
((line[len]=='\n') || (isspace(line[len])))) {
|
||||
line[len]=0 ;
|
||||
len-- ;
|
||||
}
|
||||
/* Detect multi-line */
|
||||
if (line[len]=='\\') {
|
||||
/* Multi-line value */
|
||||
last=len ;
|
||||
continue ;
|
||||
} else {
|
||||
last=0 ;
|
||||
}
|
||||
switch (iniparser_line(line, section, key, val)) {
|
||||
case LINE_EMPTY:
|
||||
case LINE_COMMENT:
|
||||
break ;
|
||||
|
||||
case LINE_SECTION:
|
||||
errs = dictionary_set(dict, section, NULL);
|
||||
break ;
|
||||
|
||||
case LINE_VALUE:
|
||||
sprintf(tmp, "%s:%s", section, key);
|
||||
errs = dictionary_set(dict, tmp, val) ;
|
||||
break ;
|
||||
|
||||
case LINE_ERROR:
|
||||
fprintf(stderr, "iniparser: syntax error in %s (%d):\n",
|
||||
ininame,
|
||||
lineno);
|
||||
fprintf(stderr, "-> %s\n", line);
|
||||
errs++ ;
|
||||
break;
|
||||
|
||||
default:
|
||||
break ;
|
||||
}
|
||||
memset(line, 0, ASCIILINESZ);
|
||||
last=0;
|
||||
if (errs<0) {
|
||||
fprintf(stderr, "iniparser: memory allocation failure\n");
|
||||
break ;
|
||||
}
|
||||
}
|
||||
if (errs) {
|
||||
dictionary_del(dict);
|
||||
dict = NULL ;
|
||||
}
|
||||
fclose(in);
|
||||
return dict ;
|
||||
}
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Free all memory associated to an ini dictionary
|
||||
@param d Dictionary to free
|
||||
@return void
|
||||
|
||||
Free all memory associated to an ini dictionary.
|
||||
It is mandatory to call this function before the dictionary object
|
||||
gets out of the current context.
|
||||
*/
|
||||
/*--------------------------------------------------------------------------*/
|
||||
void iniparser_freedict(dictionary * d)
|
||||
{
|
||||
dictionary_del(d);
|
||||
}
|
||||
|
||||
/* vim: set ts=4 et sw=4 tw=75 */
|
||||
|
|
@ -0,0 +1,197 @@
|
|||
/**
|
||||
* @file sz.c
|
||||
* @author Sheng Di and Dingwen Tao
|
||||
* @date Aug, 2016
|
||||
* @brief SZ_Init, Compression and Decompression functions
|
||||
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
|
||||
* See COPYRIGHT in top-level directory.
|
||||
*/
|
||||
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include "sz.h"
|
||||
#include "CompressElement.h"
|
||||
#include "DynamicByteArray.h"
|
||||
#include "TightDataPointStorageD.h"
|
||||
#include "TightDataPointStorageF.h"
|
||||
#include "conf.h"
|
||||
#include "utility.h"
|
||||
|
||||
|
||||
//#include "CurveFillingCompressStorage.h"
|
||||
|
||||
unsigned char versionNumber = DATA_FROMAT_VER1;
|
||||
int SZ_SIZE_TYPE_DEFUALT = 4;
|
||||
|
||||
int dataEndianType = LITTLE_ENDIAN_DATA; //*endian type of the data read from disk
|
||||
int sysEndianType = LITTLE_ENDIAN_SYSTEM ; //*sysEndianType is actually set automatically.
|
||||
|
||||
//the confparams should be separate between compression and decopmression, in case of mutual-affection when calling compression/decompression alternatively
|
||||
sz_params *confparams_cpr = NULL; //used for compression
|
||||
sz_exedata *exe_params = NULL;
|
||||
|
||||
int SZ_Init(const char *configFilePath)
|
||||
{
|
||||
// check CPU EndianType
|
||||
int x = 1;
|
||||
char *y = (char*)&x;
|
||||
if(*y==1)
|
||||
sysEndianType = LITTLE_ENDIAN_SYSTEM;
|
||||
else //=0
|
||||
sysEndianType = BIG_ENDIAN_SYSTEM;
|
||||
|
||||
int loadFileResult = SZ_LoadConf(configFilePath);
|
||||
if(loadFileResult==SZ_FAILED)
|
||||
return SZ_FAILED;
|
||||
|
||||
exe_params->SZ_SIZE_TYPE = SZ_SIZE_TYPE_DEFUALT;
|
||||
return SZ_SUCCESS;
|
||||
}
|
||||
|
||||
int SZ_Init_Params(sz_params *params)
|
||||
{
|
||||
SZ_Init(NULL);
|
||||
|
||||
if(params->losslessCompressor!=GZIP_COMPRESSOR && params->losslessCompressor!=ZSTD_COMPRESSOR)
|
||||
params->losslessCompressor = ZSTD_COMPRESSOR;
|
||||
|
||||
if(params->max_quant_intervals > 0)
|
||||
params->maxRangeRadius = params->max_quant_intervals/2;
|
||||
|
||||
memcpy(confparams_cpr, params, sizeof(sz_params));
|
||||
|
||||
if(params->quantization_intervals%2!=0)
|
||||
{
|
||||
printf("Error: quantization_intervals must be an even number!\n");
|
||||
return SZ_FAILED;
|
||||
}
|
||||
|
||||
return SZ_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
/*-------------------------------------------------------------------------*/
|
||||
/**
|
||||
@brief Perform Compression
|
||||
@param data data to be compressed
|
||||
@param outSize the size (in bytes) after compression
|
||||
@param r5,r4,r3,r2,r1 the sizes of each dimension (supporting only 5 dimensions at most in this version.
|
||||
@return compressed data (in binary stream) or NULL(0) if any errors
|
||||
|
||||
**/
|
||||
/*-------------------------------------------------------------------------*/
|
||||
|
||||
//
|
||||
// compress output data to outData and return outSize
|
||||
//
|
||||
size_t SZ_compress_args(int dataType, void *data, size_t r1, unsigned char* outData, sz_params* params)
|
||||
{
|
||||
size_t outSize = 0;
|
||||
if(dataType==SZ_FLOAT)
|
||||
{
|
||||
SZ_compress_args_float((float *)data, r1, outData, &outSize, params);
|
||||
}
|
||||
else if(dataType==SZ_DOUBLE)
|
||||
{
|
||||
SZ_compress_args_double((double *)data, r1, outData, &outSize, params);
|
||||
}
|
||||
else
|
||||
{
|
||||
printf("Error: dataType can only be SZ_FLOAT, SZ_DOUBLE .\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
return outSize;
|
||||
}
|
||||
|
||||
//
|
||||
// decompress output data to outData and return outSize
|
||||
//
|
||||
size_t SZ_decompress(int dataType, unsigned char *bytes, size_t byteLength, size_t r1, unsigned char* outData)
|
||||
{
|
||||
sz_exedata de_exe;
|
||||
memset(&de_exe, 0, sizeof(sz_exedata));
|
||||
|
||||
sz_params de_params;
|
||||
memset(&de_params, 0, sizeof(sz_params));
|
||||
|
||||
size_t outSize = 0;
|
||||
|
||||
if(dataType == SZ_FLOAT)
|
||||
{
|
||||
int status = SZ_decompress_args_float((float*)outData, r1, bytes, byteLength, 0, NULL, &de_exe, &de_params);
|
||||
if(status == SZ_SUCCESS)
|
||||
{
|
||||
return r1*sizeof(float);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
else if(dataType == SZ_DOUBLE)
|
||||
{
|
||||
int status = SZ_decompress_args_double((double*)outData, r1, bytes, byteLength, 0, NULL, &de_exe, &de_params);
|
||||
if(status == SZ_SUCCESS)
|
||||
{
|
||||
return r1*sizeof(double);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
else
|
||||
{
|
||||
printf("Error: data type cannot be the types other than SZ_FLOAT or SZ_DOUBLE\n");
|
||||
}
|
||||
|
||||
return outSize;
|
||||
}
|
||||
|
||||
void SZ_Finalize()
|
||||
{
|
||||
if(confparams_cpr!=NULL)
|
||||
{
|
||||
free(confparams_cpr);
|
||||
confparams_cpr = NULL;
|
||||
}
|
||||
if(exe_params!=NULL)
|
||||
{
|
||||
free(exe_params);
|
||||
exe_params = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
#ifdef WINDOWS
|
||||
#include <windows.h>
|
||||
int gettimeofday(struct timeval *tv, struct timezone *tz);
|
||||
|
||||
#else
|
||||
#include <sys/time.h>
|
||||
#endif
|
||||
|
||||
struct timeval costStart; /*only used for recording the cost*/
|
||||
double totalCost = 0;
|
||||
|
||||
void cost_start()
|
||||
{
|
||||
totalCost = 0;
|
||||
gettimeofday(&costStart, NULL);
|
||||
}
|
||||
|
||||
double cost_end(const char* tag)
|
||||
{
|
||||
double elapsed;
|
||||
struct timeval costEnd;
|
||||
gettimeofday(&costEnd, NULL);
|
||||
elapsed = ((costEnd.tv_sec*1000000+costEnd.tv_usec)-(costStart.tv_sec*1000000+costStart.tv_usec))/1000000.0;
|
||||
totalCost += elapsed;
|
||||
double use_ms = totalCost*1000;
|
||||
printf(" timecost %s : %.3f ms\n", tag, use_ms);
|
||||
return use_ms;
|
||||
}
|
||||
|
||||
void show_rate(int in_len, int out_len)
|
||||
{
|
||||
float rate=100*(float)out_len/(float)in_len;
|
||||
printf(" in_len=%d out_len=%d compress rate=%.4f%%\n", in_len, out_len, rate);
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,415 @@
|
|||
/**
|
||||
* @file sz_double.c
|
||||
* @author Sheng Di, Dingwen Tao, Xin Liang, Xiangyu Zou, Tao Lu, Wen Xia, Xuan Wang, Weizhe Zhang
|
||||
* @date Aug, 2016
|
||||
* @brief SZ_Init, Compression and Decompression functions
|
||||
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
|
||||
* See COPYRIGHT in top-level directory.
|
||||
*/
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <math.h>
|
||||
#include "sz.h"
|
||||
#include "CompressElement.h"
|
||||
#include "DynamicByteArray.h"
|
||||
#include "DynamicIntArray.h"
|
||||
#include "TightDataPointStorageD.h"
|
||||
#include "sz_double.h"
|
||||
#include "szd_double.h"
|
||||
#include "utility.h"
|
||||
|
||||
unsigned char* SZ_skip_compress_double(double* data, size_t dataLength, size_t* outSize)
|
||||
{
|
||||
*outSize = dataLength*sizeof(double);
|
||||
unsigned char* out = (unsigned char*)malloc(dataLength*sizeof(double));
|
||||
memcpy(out, data, dataLength*sizeof(double));
|
||||
return out;
|
||||
}
|
||||
|
||||
INLINE void computeReqLength_double(double realPrecision, short radExpo, int* reqLength, double* medianValue)
|
||||
{
|
||||
short reqExpo = getPrecisionReqLength_double(realPrecision);
|
||||
*reqLength = 12+radExpo - reqExpo; //radExpo-reqExpo == reqMantiLength
|
||||
if(*reqLength<12)
|
||||
*reqLength = 12;
|
||||
if(*reqLength>64)
|
||||
{
|
||||
*reqLength = 64;
|
||||
*medianValue = 0;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
unsigned int optimize_intervals_double_1D(double *oriData, size_t dataLength, double realPrecision)
|
||||
{
|
||||
size_t i = 0, radiusIndex;
|
||||
double pred_value = 0, pred_err;
|
||||
size_t *intervals = (size_t*)malloc(confparams_cpr->maxRangeRadius*sizeof(size_t));
|
||||
memset(intervals, 0, confparams_cpr->maxRangeRadius*sizeof(size_t));
|
||||
size_t totalSampleSize = dataLength/confparams_cpr->sampleDistance;
|
||||
for(i=2;i<dataLength;i++)
|
||||
{
|
||||
if(i%confparams_cpr->sampleDistance==0)
|
||||
{
|
||||
//pred_value = 2*oriData[i-1] - oriData[i-2];
|
||||
pred_value = oriData[i-1];
|
||||
pred_err = fabs(pred_value - oriData[i]);
|
||||
radiusIndex = (unsigned long)((pred_err/realPrecision+1)/2);
|
||||
if(radiusIndex>=confparams_cpr->maxRangeRadius)
|
||||
radiusIndex = confparams_cpr->maxRangeRadius - 1;
|
||||
intervals[radiusIndex]++;
|
||||
}
|
||||
}
|
||||
//compute the appropriate number
|
||||
size_t targetCount = totalSampleSize*confparams_cpr->predThreshold;
|
||||
size_t sum = 0;
|
||||
for(i=0;i<confparams_cpr->maxRangeRadius;i++)
|
||||
{
|
||||
sum += intervals[i];
|
||||
if(sum>targetCount)
|
||||
break;
|
||||
}
|
||||
|
||||
if(i>=confparams_cpr->maxRangeRadius)
|
||||
i = confparams_cpr->maxRangeRadius-1;
|
||||
unsigned int accIntervals = 2*(i+1);
|
||||
unsigned int powerOf2 = roundUpToPowerOf2(accIntervals);
|
||||
|
||||
if(powerOf2<32)
|
||||
powerOf2 = 32;
|
||||
|
||||
free(intervals);
|
||||
//printf("accIntervals=%d, powerOf2=%d\n", accIntervals, powerOf2);
|
||||
return powerOf2;
|
||||
}
|
||||
|
||||
TightDataPointStorageD* SZ_compress_double_1D_MDQ(double *oriData,
|
||||
size_t dataLength, double realPrecision, double valueRangeSize, double medianValue_d)
|
||||
{
|
||||
unsigned int quantization_intervals;
|
||||
if(exe_params->optQuantMode==1)
|
||||
quantization_intervals = optimize_intervals_double_1D_opt(oriData, dataLength, realPrecision);
|
||||
else
|
||||
quantization_intervals = exe_params->intvCapacity;
|
||||
//updateQuantizationInfo(quantization_intervals);
|
||||
int intvRadius = quantization_intervals/2;
|
||||
|
||||
size_t i;
|
||||
int reqLength;
|
||||
double medianValue = medianValue_d;
|
||||
short radExpo = getExponent_double(valueRangeSize/2);
|
||||
|
||||
computeReqLength_double(realPrecision, radExpo, &reqLength, &medianValue);
|
||||
|
||||
int* type = (int*) malloc(dataLength*sizeof(int));
|
||||
double* spaceFillingValue = oriData; //
|
||||
|
||||
DynamicIntArray *exactLeadNumArray;
|
||||
new_DIA(&exactLeadNumArray, DynArrayInitLen);
|
||||
|
||||
DynamicByteArray *exactMidByteArray;
|
||||
new_DBA(&exactMidByteArray, DynArrayInitLen);
|
||||
|
||||
DynamicIntArray *resiBitArray;
|
||||
new_DIA(&resiBitArray, DynArrayInitLen);
|
||||
|
||||
unsigned char preDataBytes[8];
|
||||
longToBytes_bigEndian(preDataBytes, 0);
|
||||
|
||||
int reqBytesLength = reqLength/8;
|
||||
int resiBitsLength = reqLength%8;
|
||||
double last3CmprsData[3] = {0};
|
||||
|
||||
DoubleValueCompressElement *vce = (DoubleValueCompressElement*)malloc(sizeof(DoubleValueCompressElement));
|
||||
LossyCompressionElement *lce = (LossyCompressionElement*)malloc(sizeof(LossyCompressionElement));
|
||||
|
||||
//add the first data
|
||||
type[0] = 0;
|
||||
compressSingleDoubleValue(vce, spaceFillingValue[0], realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength);
|
||||
updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
|
||||
memcpy(preDataBytes,vce->curBytes,8);
|
||||
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
|
||||
listAdd_double(last3CmprsData, vce->data);
|
||||
|
||||
//add the second data
|
||||
type[1] = 0;
|
||||
compressSingleDoubleValue(vce, spaceFillingValue[1], realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength);
|
||||
updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
|
||||
memcpy(preDataBytes,vce->curBytes,8);
|
||||
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
|
||||
listAdd_double(last3CmprsData, vce->data);
|
||||
|
||||
int state;
|
||||
double checkRadius;
|
||||
double curData;
|
||||
double pred = last3CmprsData[0];
|
||||
double predAbsErr;
|
||||
checkRadius = (quantization_intervals-1)*realPrecision;
|
||||
double interval = 2*realPrecision;
|
||||
|
||||
double recip_realPrecision = 1/realPrecision;
|
||||
for(i=2; i < dataLength; i++)
|
||||
{
|
||||
//printf("%.30G\n",last3CmprsData[0]);
|
||||
curData = spaceFillingValue[i];
|
||||
//pred = 2*last3CmprsData[0] - last3CmprsData[1];
|
||||
//pred = last3CmprsData[0];
|
||||
predAbsErr = fabs(curData - pred);
|
||||
if(predAbsErr<checkRadius)
|
||||
{
|
||||
state = (predAbsErr*recip_realPrecision+1)*0.5;
|
||||
if(curData>=pred)
|
||||
{
|
||||
type[i] = intvRadius+state;
|
||||
pred = pred + state*interval;
|
||||
}
|
||||
else //curData<pred
|
||||
{
|
||||
type[i] = intvRadius-state;
|
||||
pred = pred - state*interval;
|
||||
}
|
||||
//listAdd_double(last3CmprsData, pred);
|
||||
continue;
|
||||
}
|
||||
|
||||
//unpredictable data processing
|
||||
type[i] = 0;
|
||||
compressSingleDoubleValue(vce, curData, realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength);
|
||||
updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
|
||||
memcpy(preDataBytes,vce->curBytes,8);
|
||||
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
|
||||
|
||||
//listAdd_double(last3CmprsData, vce->data);
|
||||
pred = vce->data;
|
||||
|
||||
}//end of for
|
||||
|
||||
size_t exactDataNum = exactLeadNumArray->size;
|
||||
|
||||
TightDataPointStorageD* tdps;
|
||||
|
||||
new_TightDataPointStorageD(&tdps, dataLength, exactDataNum,
|
||||
type, exactMidByteArray->array, exactMidByteArray->size,
|
||||
exactLeadNumArray->array,
|
||||
resiBitArray->array, resiBitArray->size,
|
||||
resiBitsLength,
|
||||
realPrecision, medianValue, (char)reqLength, quantization_intervals, 0);
|
||||
|
||||
// printf("exactDataNum=%d, expSegmentsInBytes_size=%d, exactMidByteArray->size=%d\n",
|
||||
// exactDataNum, expSegmentsInBytes_size, exactMidByteArray->size);
|
||||
|
||||
//free memory
|
||||
free_DIA(exactLeadNumArray);
|
||||
free_DIA(resiBitArray);
|
||||
free(type);
|
||||
free(vce);
|
||||
free(lce);
|
||||
free(exactMidByteArray); //exactMidByteArray->array has been released in free_TightDataPointStorageF(tdps);
|
||||
|
||||
return tdps;
|
||||
}
|
||||
|
||||
void SZ_compress_args_double_StoreOriData(double* oriData, size_t dataLength, unsigned char* newByteData, size_t *outSize)
|
||||
{
|
||||
int doubleSize = sizeof(double);
|
||||
size_t k = 0, i;
|
||||
size_t totalByteLength = 1 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + 1 + doubleSize*dataLength;
|
||||
/*No need to malloc because newByteData should always already be allocated with no less totalByteLength.*/
|
||||
//*newByteData = (unsigned char*)malloc(totalByteLength);
|
||||
|
||||
unsigned char dsLengthBytes[8];
|
||||
newByteData[k++] = versionNumber;
|
||||
|
||||
if(exe_params->SZ_SIZE_TYPE==4)//1
|
||||
newByteData[k++] = 16; //00010000
|
||||
else
|
||||
newByteData[k++] = 80; //01010000: 01000000 indicates the SZ_SIZE_TYPE=8
|
||||
|
||||
convertSZParamsToBytes(confparams_cpr, &(newByteData[k]), exe_params->optQuantMode);
|
||||
k = k + MetaDataByteLength_double;
|
||||
|
||||
sizeToBytes(dsLengthBytes,dataLength, exe_params->SZ_SIZE_TYPE);
|
||||
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)//ST: 4 or 8
|
||||
newByteData[k++] = dsLengthBytes[i];
|
||||
|
||||
if(sysEndianType==BIG_ENDIAN_SYSTEM)
|
||||
memcpy(newByteData+4+MetaDataByteLength_double+exe_params->SZ_SIZE_TYPE, oriData, dataLength*doubleSize);
|
||||
else
|
||||
{
|
||||
unsigned char* p = newByteData+4+MetaDataByteLength_double+exe_params->SZ_SIZE_TYPE;
|
||||
for(i=0;i<dataLength;i++,p+=doubleSize)
|
||||
doubleToBytes(p, oriData[i]);
|
||||
}
|
||||
*outSize = totalByteLength;
|
||||
}
|
||||
|
||||
|
||||
bool SZ_compress_args_double_NoCkRngeNoGzip_1D(unsigned char* newByteData, double *oriData,
|
||||
size_t dataLength, double realPrecision, size_t *outSize, double valueRangeSize, double medianValue_d)
|
||||
{
|
||||
TightDataPointStorageD* tdps = NULL;
|
||||
tdps = SZ_compress_double_1D_MDQ(oriData, dataLength, realPrecision, valueRangeSize, medianValue_d);
|
||||
|
||||
if(!convertTDPStoFlatBytes_double(tdps, newByteData, outSize))
|
||||
{
|
||||
free_TightDataPointStorageD(tdps);
|
||||
return false;
|
||||
}
|
||||
|
||||
//if(*outSize>3 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + 1 + sizeof(double)*dataLength)
|
||||
// SZ_compress_args_double_StoreOriData(oriData, dataLength, newByteData, outSize);
|
||||
|
||||
free_TightDataPointStorageD(tdps);
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
void SZ_compress_args_double_withinRange(unsigned char* newByteData, double *oriData, size_t dataLength, size_t *outSize)
|
||||
{
|
||||
TightDataPointStorageD* tdps = (TightDataPointStorageD*) malloc(sizeof(TightDataPointStorageD));
|
||||
memset(tdps, 0, sizeof(TightDataPointStorageD));
|
||||
tdps->leadNumArray = NULL;
|
||||
tdps->residualMidBits = NULL;
|
||||
|
||||
tdps->allSameData = 1;
|
||||
tdps->dataSeriesLength = dataLength;
|
||||
tdps->exactMidBytes = (unsigned char*)malloc(sizeof(unsigned char)*8);
|
||||
tdps->isLossless = 0;
|
||||
double value = oriData[0];
|
||||
doubleToBytes(tdps->exactMidBytes, value);
|
||||
tdps->exactMidBytes_size = 8;
|
||||
|
||||
size_t tmpOutSize;
|
||||
//unsigned char *tmpByteData;
|
||||
convertTDPStoFlatBytes_double(tdps, newByteData, &tmpOutSize);
|
||||
//convertTDPStoFlatBytes_double(tdps, &tmpByteData, &tmpOutSize);
|
||||
|
||||
//*newByteData = (unsigned char*)malloc(sizeof(unsigned char)*16); //for floating-point data (1+3+4+4)
|
||||
//memcpy(*newByteData, tmpByteData, 16);
|
||||
*outSize = tmpOutSize;//12==3+1+8(double_size)+MetaDataByteLength_double
|
||||
free_TightDataPointStorageD(tdps);
|
||||
}
|
||||
|
||||
int SZ_compress_args_double(double *oriData, size_t r1, unsigned char* newByteData, size_t *outSize, sz_params* params)
|
||||
{
|
||||
int status = SZ_SUCCESS;
|
||||
size_t dataLength = r1;
|
||||
|
||||
double valueRangeSize = 0, medianValue = 0;
|
||||
|
||||
// check at least elements count
|
||||
if(dataLength <= MIN_NUM_OF_ELEMENTS)
|
||||
{
|
||||
printf("warning, double input elements count=%d less than %d, so need not do compress.\n", (int)dataLength, MIN_NUM_OF_ELEMENTS);
|
||||
return SZ_LITTER_ELEMENT;
|
||||
}
|
||||
|
||||
|
||||
double min = computeRangeSize_double(oriData, dataLength, &valueRangeSize, &medianValue);
|
||||
double max = min+valueRangeSize;
|
||||
params->dmin = min;
|
||||
params->dmax = max;
|
||||
|
||||
double realPrecision = 0;
|
||||
|
||||
if(params->errorBoundMode==PSNR)
|
||||
{
|
||||
params->errorBoundMode = SZ_ABS;
|
||||
realPrecision = params->absErrBoundDouble = computeABSErrBoundFromPSNR(params->psnr, (double)params->predThreshold, valueRangeSize);
|
||||
}
|
||||
else if(params->errorBoundMode==NORM) //norm error = sqrt(sum((xi-xi_)^2))
|
||||
{
|
||||
params->errorBoundMode = SZ_ABS;
|
||||
realPrecision = params->absErrBoundDouble = computeABSErrBoundFromNORM_ERR(params->normErr, dataLength);
|
||||
//printf("realPrecision=%lf\n", realPrecision);
|
||||
}
|
||||
else
|
||||
{
|
||||
realPrecision = getRealPrecision_double(valueRangeSize, params->errorBoundMode, params->absErrBoundDouble, params->relBoundRatio, &status);
|
||||
params->absErrBoundDouble = realPrecision;
|
||||
}
|
||||
if(valueRangeSize <= realPrecision)
|
||||
{
|
||||
SZ_compress_args_double_withinRange(newByteData, oriData, dataLength, outSize);
|
||||
}
|
||||
else
|
||||
{
|
||||
size_t tmpOutSize = 0;
|
||||
unsigned char* tmpByteData = newByteData;
|
||||
bool twoStage = params->szMode != SZ_BEST_SPEED;
|
||||
if(twoStage)
|
||||
{
|
||||
tmpByteData = (unsigned char*)malloc(r1*sizeof(double)*1.2);
|
||||
}
|
||||
|
||||
if(!SZ_compress_args_double_NoCkRngeNoGzip_1D(tmpByteData, oriData, r1, realPrecision, &tmpOutSize, valueRangeSize, medianValue))
|
||||
{
|
||||
if(twoStage)
|
||||
free(tmpByteData);
|
||||
|
||||
return SZ_ALGORITHM_ERR;
|
||||
}
|
||||
//if(tmpOutSize>=dataLength*sizeof(double) + 3 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + 1)
|
||||
// SZ_compress_args_double_StoreOriData(oriData, dataLength, tmpByteData, &tmpOutSize);
|
||||
|
||||
//
|
||||
//Call Gzip to do the further compression.
|
||||
//
|
||||
if(twoStage)
|
||||
{
|
||||
*outSize = sz_lossless_compress(params->losslessCompressor, tmpByteData, tmpOutSize, newByteData);
|
||||
free(tmpByteData);
|
||||
}
|
||||
else
|
||||
{
|
||||
*outSize = tmpOutSize;
|
||||
}
|
||||
}
|
||||
|
||||
return status;
|
||||
}
|
||||
|
||||
unsigned int optimize_intervals_double_1D_opt(double *oriData, size_t dataLength, double realPrecision)
|
||||
{
|
||||
size_t i = 0, radiusIndex;
|
||||
double pred_value = 0, pred_err;
|
||||
size_t *intervals = (size_t*)malloc(confparams_cpr->maxRangeRadius*sizeof(size_t));
|
||||
memset(intervals, 0, confparams_cpr->maxRangeRadius*sizeof(size_t));
|
||||
size_t totalSampleSize = 0;
|
||||
|
||||
double * data_pos = oriData + 2;
|
||||
while(data_pos - oriData < dataLength){
|
||||
totalSampleSize++;
|
||||
pred_value = data_pos[-1];
|
||||
pred_err = fabs(pred_value - *data_pos);
|
||||
radiusIndex = (unsigned long)((pred_err/realPrecision+1)/2);
|
||||
if(radiusIndex>=confparams_cpr->maxRangeRadius)
|
||||
radiusIndex = confparams_cpr->maxRangeRadius - 1;
|
||||
intervals[radiusIndex]++;
|
||||
|
||||
data_pos += confparams_cpr->sampleDistance;
|
||||
}
|
||||
//compute the appropriate number
|
||||
size_t targetCount = totalSampleSize*confparams_cpr->predThreshold;
|
||||
size_t sum = 0;
|
||||
for(i=0;i<confparams_cpr->maxRangeRadius;i++)
|
||||
{
|
||||
sum += intervals[i];
|
||||
if(sum>targetCount)
|
||||
break;
|
||||
}
|
||||
if(i>=confparams_cpr->maxRangeRadius)
|
||||
i = confparams_cpr->maxRangeRadius-1;
|
||||
|
||||
unsigned int accIntervals = 2*(i+1);
|
||||
unsigned int powerOf2 = roundUpToPowerOf2(accIntervals);
|
||||
|
||||
if(powerOf2<32)
|
||||
powerOf2 = 32;
|
||||
|
||||
free(intervals);
|
||||
return powerOf2;
|
||||
}
|
||||
|
|
@ -0,0 +1,434 @@
|
|||
/**
|
||||
* @file sz_float.c
|
||||
* @author Sheng Di, Dingwen Tao, Xin Liang, Xiangyu Zou, Tao Lu, Wen Xia, Xuan Wang, Weizhe Zhang
|
||||
* @date Aug, 2016
|
||||
* @brief SZ_Init, Compression and Decompression functions
|
||||
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
|
||||
* See COPYRIGHT in top-level directory.
|
||||
*/
|
||||
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <math.h>
|
||||
#include "sz.h"
|
||||
#include "CompressElement.h"
|
||||
#include "DynamicByteArray.h"
|
||||
#include "TightDataPointStorageF.h"
|
||||
#include "sz_float.h"
|
||||
#include "szd_float.h"
|
||||
#include "utility.h"
|
||||
|
||||
# define MIN(_a, _b) (((_a) < (_b)) ? (_a) : (_b))
|
||||
|
||||
|
||||
|
||||
void computeReqLength_float(double realPrecision, short rangeExpo, int* reqLength, float* medianValue)
|
||||
{
|
||||
short realPrecExpo = getPrecisionReqLength_double(realPrecision);
|
||||
*reqLength = 9 + rangeExpo - realPrecExpo + 1; //radExpo-reqExpo == reqMantiLength
|
||||
if(*reqLength<9)
|
||||
*reqLength = 9;
|
||||
if(*reqLength>32)
|
||||
{
|
||||
*reqLength = 32;
|
||||
*medianValue = 0;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
unsigned int optimize_intervals_float_1D(float *oriData, size_t dataLength, double realPrecision)
|
||||
{
|
||||
size_t i = 0, radiusIndex;
|
||||
float pred_value = 0, pred_err;
|
||||
size_t *intervals = (size_t*)malloc(confparams_cpr->maxRangeRadius*sizeof(size_t));
|
||||
memset(intervals, 0, confparams_cpr->maxRangeRadius*sizeof(size_t));
|
||||
size_t totalSampleSize = dataLength/confparams_cpr->sampleDistance;
|
||||
for(i=2;i<dataLength;i++)
|
||||
{
|
||||
if(i%confparams_cpr->sampleDistance==0)
|
||||
{
|
||||
//pred_value = 2*oriData[i-1] - oriData[i-2];
|
||||
pred_value = oriData[i-1];
|
||||
pred_err = fabs(pred_value - oriData[i]);
|
||||
radiusIndex = (unsigned long)((pred_err/realPrecision+1)/2);
|
||||
if(radiusIndex>=confparams_cpr->maxRangeRadius)
|
||||
radiusIndex = confparams_cpr->maxRangeRadius - 1;
|
||||
intervals[radiusIndex]++;
|
||||
}
|
||||
}
|
||||
//compute the appropriate number
|
||||
size_t targetCount = totalSampleSize*confparams_cpr->predThreshold;
|
||||
size_t sum = 0;
|
||||
for(i=0;i<confparams_cpr->maxRangeRadius;i++)
|
||||
{
|
||||
sum += intervals[i];
|
||||
if(sum>targetCount)
|
||||
break;
|
||||
}
|
||||
if(i>=confparams_cpr->maxRangeRadius)
|
||||
i = confparams_cpr->maxRangeRadius-1;
|
||||
|
||||
unsigned int accIntervals = 2*(i+1);
|
||||
unsigned int powerOf2 = roundUpToPowerOf2(accIntervals);
|
||||
|
||||
if(powerOf2<32)
|
||||
powerOf2 = 32;
|
||||
|
||||
free(intervals);
|
||||
//printf("accIntervals=%d, powerOf2=%d\n", accIntervals, powerOf2);
|
||||
return powerOf2;
|
||||
}
|
||||
|
||||
TightDataPointStorageF* SZ_compress_float_1D_MDQ(float *oriData,
|
||||
size_t dataLength, float realPrecision, float valueRangeSize, float medianValue_f)
|
||||
{
|
||||
unsigned int quantization_intervals;
|
||||
if(exe_params->optQuantMode==1)
|
||||
quantization_intervals = optimize_intervals_float_1D_opt(oriData, dataLength, realPrecision);
|
||||
else
|
||||
quantization_intervals = exe_params->intvCapacity;
|
||||
//updateQuantizationInfo(quantization_intervals);
|
||||
int half_interval = quantization_intervals/2;
|
||||
|
||||
//
|
||||
// calc reqlength and need set medianValue to zero.
|
||||
//
|
||||
size_t i;
|
||||
int reqLength; // need save bits length for one float . value ragne 9~32
|
||||
float medianValue = medianValue_f;
|
||||
short rangeExpo = getExponent_float(valueRangeSize/2);
|
||||
|
||||
computeReqLength_float(realPrecision, rangeExpo, &reqLength, &medianValue);
|
||||
|
||||
//
|
||||
// malloc all
|
||||
//
|
||||
|
||||
// 1 type
|
||||
int* type = (int*) malloc(dataLength*sizeof(int));
|
||||
float* spaceFillingValue = oriData; //
|
||||
// 2 lead
|
||||
DynamicIntArray *exactLeadNumArray;
|
||||
new_DIA(&exactLeadNumArray, DynArrayInitLen);
|
||||
// 3 mid
|
||||
DynamicByteArray *exactMidByteArray;
|
||||
new_DBA(&exactMidByteArray, DynArrayInitLen);
|
||||
// 4 residual bit
|
||||
DynamicIntArray *resiBitArray;
|
||||
new_DIA(&resiBitArray, DynArrayInitLen);
|
||||
|
||||
unsigned char preDiffBytes[4];
|
||||
intToBytes_bigEndian(preDiffBytes, 0);
|
||||
|
||||
// calc save byte length and bit lengths with reqLength
|
||||
int reqBytesLength = reqLength/8;
|
||||
int resiBitsLength = reqLength%8;
|
||||
//float last3CmprsData[3] = {0};
|
||||
|
||||
FloatValueCompressElement *vce = (FloatValueCompressElement*)malloc(sizeof(FloatValueCompressElement));
|
||||
LossyCompressionElement *lce = (LossyCompressionElement*)malloc(sizeof(LossyCompressionElement));
|
||||
|
||||
//add the first data
|
||||
type[0] = 0;
|
||||
compressSingleFloatValue(vce, spaceFillingValue[0], realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength);
|
||||
// set lce
|
||||
updateLossyCompElement_Float(vce->curBytes, preDiffBytes, reqBytesLength, resiBitsLength, lce);
|
||||
memcpy(preDiffBytes, vce->curBytes, 4);
|
||||
// lce to arrays
|
||||
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
|
||||
//listAdd_float(last3CmprsData, vce->data);
|
||||
|
||||
//add the second data
|
||||
type[1] = 0;
|
||||
compressSingleFloatValue(vce, spaceFillingValue[1], realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength);
|
||||
updateLossyCompElement_Float(vce->curBytes, preDiffBytes, reqBytesLength, resiBitsLength, lce);
|
||||
memcpy(preDiffBytes, vce->curBytes, 4);
|
||||
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
|
||||
//listAdd_float(last3CmprsData, vce->data);
|
||||
|
||||
int state;
|
||||
float checkRadius;
|
||||
float oriFloat;
|
||||
float pred = vce->data;
|
||||
float diff;
|
||||
checkRadius = (quantization_intervals-1)*realPrecision;
|
||||
float double_realpreci = 2*realPrecision;
|
||||
float recip_precision = 1/realPrecision;
|
||||
|
||||
for(i=2; i < dataLength; i++)
|
||||
{
|
||||
oriFloat = spaceFillingValue[i];
|
||||
//pred = 2*last3CmprsData[0] - last3CmprsData[1];
|
||||
//pred = last3CmprsData[0];
|
||||
diff = fabsf(oriFloat - pred);
|
||||
if(diff < checkRadius)
|
||||
{
|
||||
state = ((int)( diff * recip_precision + 1))>>1;
|
||||
if(oriFloat >= pred)
|
||||
{
|
||||
type[i] = half_interval + state;
|
||||
pred = pred + state * double_realpreci;
|
||||
}
|
||||
else //curData<pred
|
||||
{
|
||||
type[i] = half_interval - state;
|
||||
pred = pred - state * double_realpreci;
|
||||
}
|
||||
|
||||
//double-check the prediction error in case of machine-epsilon impact
|
||||
if(fabs(oriFloat - pred) > realPrecision)
|
||||
{
|
||||
type[i] = 0;
|
||||
compressSingleFloatValue(vce, oriFloat, realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength);
|
||||
updateLossyCompElement_Float(vce->curBytes, preDiffBytes, reqBytesLength, resiBitsLength, lce);
|
||||
memcpy(preDiffBytes, vce->curBytes, 4);
|
||||
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
|
||||
|
||||
//listAdd_float(last3CmprsData, vce->data);
|
||||
pred = vce->data;
|
||||
}
|
||||
else
|
||||
{
|
||||
//listAdd_float(last3CmprsData, pred);
|
||||
}
|
||||
// go next
|
||||
continue;
|
||||
}
|
||||
|
||||
//unpredictable data processing
|
||||
type[i] = 0;
|
||||
compressSingleFloatValue(vce, oriFloat, realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength);
|
||||
updateLossyCompElement_Float(vce->curBytes, preDiffBytes, reqBytesLength, resiBitsLength, lce);
|
||||
memcpy(preDiffBytes, vce->curBytes, 4);
|
||||
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
|
||||
|
||||
//listAdd_float(last3CmprsData, vce->data);
|
||||
pred = vce->data;
|
||||
|
||||
}//end of for
|
||||
|
||||
// char* expSegmentsInBytes;
|
||||
// int expSegmentsInBytes_size = convertESCToBytes(esc, &expSegmentsInBytes);
|
||||
size_t exactDataNum = exactLeadNumArray->size;
|
||||
|
||||
TightDataPointStorageF* tdps = NULL;
|
||||
new_TightDataPointStorageF(&tdps, dataLength, exactDataNum,
|
||||
type, exactMidByteArray->array, exactMidByteArray->size,
|
||||
exactLeadNumArray->array,
|
||||
resiBitArray->array, resiBitArray->size,
|
||||
resiBitsLength,
|
||||
realPrecision, medianValue, (char)reqLength, quantization_intervals, 0);
|
||||
|
||||
//sdi:Debug
|
||||
/* int sum =0;
|
||||
for(i=0;i<dataLength;i++)
|
||||
if(type[i]==0) sum++;
|
||||
printf("opt_quantizations=%d, exactDataNum=%zu, sum=%d\n",quantization_intervals, exactDataNum, sum);
|
||||
*/
|
||||
//free memory
|
||||
free_DIA(exactLeadNumArray);
|
||||
free_DIA(resiBitArray);
|
||||
free(type);
|
||||
free(vce);
|
||||
free(lce);
|
||||
free(exactMidByteArray); //exactMidByteArray->array has been released in free_TightDataPointStorageF(tdps);
|
||||
|
||||
return tdps;
|
||||
}
|
||||
|
||||
// compress core algorithm if success return true else return false
|
||||
bool SZ_compress_args_float_NoCkRngeNoGzip_1D( unsigned char* newByteData, float *oriData,
|
||||
size_t dataLength, double realPrecision, size_t *outSize, float valueRangeSize, float medianValue_f)
|
||||
{
|
||||
// get tdps
|
||||
TightDataPointStorageF* tdps = NULL;
|
||||
tdps = SZ_compress_float_1D_MDQ(oriData, dataLength, realPrecision, valueRangeSize, medianValue_f);
|
||||
if(tdps == NULL)
|
||||
return false;
|
||||
|
||||
// serialize
|
||||
if(!convertTDPStoFlatBytes_float(tdps, newByteData, outSize))
|
||||
{
|
||||
free_TightDataPointStorageF(tdps);
|
||||
return false;
|
||||
}
|
||||
|
||||
// check compressed size large than original
|
||||
if(*outSize > 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(float)*dataLength)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
free_TightDataPointStorageF(tdps);
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
void SZ_compress_args_float_withinRange(unsigned char* newByteData, float *oriData, size_t dataLength, size_t *outSize)
|
||||
{
|
||||
TightDataPointStorageF* tdps = (TightDataPointStorageF*) malloc(sizeof(TightDataPointStorageF));
|
||||
memset(tdps, 0, sizeof(TightDataPointStorageF));
|
||||
tdps->leadNumArray = NULL;
|
||||
tdps->residualMidBits = NULL;
|
||||
|
||||
tdps->allSameData = 1;
|
||||
tdps->dataSeriesLength = dataLength;
|
||||
tdps->exactMidBytes = (unsigned char*)malloc(sizeof(unsigned char)*4);
|
||||
tdps->isLossless = 0;
|
||||
float value = oriData[0];
|
||||
floatToBytes(tdps->exactMidBytes, value);
|
||||
tdps->exactMidBytes_size = 4;
|
||||
|
||||
size_t tmpOutSize;
|
||||
//unsigned char *tmpByteData;
|
||||
convertTDPStoFlatBytes_float(tdps, newByteData, &tmpOutSize);
|
||||
|
||||
//*newByteData = (unsigned char*)malloc(sizeof(unsigned char)*12); //for floating-point data (1+3+4+4)
|
||||
//memcpy(*newByteData, tmpByteData, 12);
|
||||
*outSize = tmpOutSize; //8+SZ_SIZE_TYPE; //8==3+1+4(float_size)
|
||||
free_TightDataPointStorageF(tdps);
|
||||
}
|
||||
|
||||
void cost_start();
|
||||
double cost_end(const char* tag);
|
||||
void show_rate( int in_len, int out_len);
|
||||
|
||||
int SZ_compress_args_float(float *oriData, size_t r1, unsigned char* newByteData, size_t *outSize, sz_params* params)
|
||||
{
|
||||
int status = SZ_SUCCESS;
|
||||
size_t dataLength = r1;
|
||||
|
||||
//cost_start();
|
||||
// check at least elements count
|
||||
if(dataLength <= MIN_NUM_OF_ELEMENTS)
|
||||
{
|
||||
printf("warning, input elements count=%d less than %d, so need not do compress.\n", (int)dataLength, MIN_NUM_OF_ELEMENTS);
|
||||
return SZ_LITTER_ELEMENT;
|
||||
}
|
||||
|
||||
float valueRangeSize = 0, medianValue = 0;
|
||||
float min = 0;
|
||||
|
||||
min = computeRangeSize_float(oriData, dataLength, &valueRangeSize, &medianValue);
|
||||
|
||||
// calc max
|
||||
float max = min+valueRangeSize;
|
||||
params->fmin = min;
|
||||
params->fmax = max;
|
||||
|
||||
// calc precision
|
||||
double realPrecision = 0;
|
||||
if(params->errorBoundMode==PSNR)
|
||||
{
|
||||
params->errorBoundMode = SZ_ABS;
|
||||
realPrecision = params->absErrBound = computeABSErrBoundFromPSNR(params->psnr, (double)params->predThreshold, (double)valueRangeSize);
|
||||
//printf("realPrecision=%lf\n", realPrecision);
|
||||
}
|
||||
else if(params->errorBoundMode==NORM) //norm error = sqrt(sum((xi-xi_)^2))
|
||||
{
|
||||
params->errorBoundMode = SZ_ABS;
|
||||
realPrecision = params->absErrBound = computeABSErrBoundFromNORM_ERR(params->normErr, dataLength);
|
||||
//printf("realPrecision=%lf\n", realPrecision);
|
||||
}
|
||||
else
|
||||
{
|
||||
realPrecision = getRealPrecision_float(valueRangeSize, params->errorBoundMode, params->absErrBound, params->relBoundRatio, &status);
|
||||
params->absErrBound = realPrecision;
|
||||
}
|
||||
|
||||
//cost_end(" sz_pre_calc");
|
||||
|
||||
//
|
||||
// do compress
|
||||
//
|
||||
if(valueRangeSize <= realPrecision)
|
||||
{
|
||||
// special deal with same data
|
||||
SZ_compress_args_float_withinRange(newByteData, oriData, dataLength, outSize);
|
||||
return SZ_SUCCESS;
|
||||
}
|
||||
|
||||
//
|
||||
// first compress with sz
|
||||
//
|
||||
size_t tmpOutSize = 0;
|
||||
unsigned char* tmpByteData = newByteData;
|
||||
bool twoStage = params->szMode != SZ_BEST_SPEED;
|
||||
if(twoStage)
|
||||
{
|
||||
tmpByteData = (unsigned char*)malloc(r1*sizeof(float) + 1024);
|
||||
}
|
||||
|
||||
// compress core algorithm
|
||||
if(!SZ_compress_args_float_NoCkRngeNoGzip_1D(tmpByteData, oriData, r1, realPrecision, &tmpOutSize, valueRangeSize, medianValue))
|
||||
{
|
||||
*outSize = 0;
|
||||
if(twoStage)
|
||||
free(tmpByteData);
|
||||
return SZ_ALGORITHM_ERR;
|
||||
}
|
||||
|
||||
//cost_end(" sz_first_compress");
|
||||
//show_rate(r1*sizeof(float), tmpOutSize);
|
||||
//
|
||||
// second compress with Call Zstd or Gzip
|
||||
//
|
||||
//cost_start();
|
||||
if(!twoStage)
|
||||
{
|
||||
*outSize = tmpOutSize;
|
||||
}
|
||||
else
|
||||
{
|
||||
*outSize = sz_lossless_compress(params->losslessCompressor, tmpByteData, tmpOutSize, newByteData);
|
||||
free(tmpByteData);
|
||||
}
|
||||
|
||||
//cost_end(" sz_second_compress");
|
||||
//show_rate(r1*sizeof(float), *outSize);
|
||||
return SZ_SUCCESS;
|
||||
}
|
||||
|
||||
unsigned int optimize_intervals_float_1D_opt(float *oriData, size_t dataLength, double realPrecision)
|
||||
{
|
||||
size_t i = 0, radiusIndex;
|
||||
float pred_value = 0, pred_err;
|
||||
size_t *intervals = (size_t*)malloc(confparams_cpr->maxRangeRadius*sizeof(size_t));
|
||||
memset(intervals, 0, confparams_cpr->maxRangeRadius*sizeof(size_t));
|
||||
size_t totalSampleSize = 0;//dataLength/confparams_cpr->sampleDistance;
|
||||
|
||||
float * data_pos = oriData + 2;
|
||||
while(data_pos - oriData < dataLength){
|
||||
totalSampleSize++;
|
||||
pred_value = data_pos[-1];
|
||||
pred_err = fabs(pred_value - *data_pos);
|
||||
radiusIndex = (unsigned long)((pred_err/realPrecision+1)/2);
|
||||
if(radiusIndex>=confparams_cpr->maxRangeRadius)
|
||||
radiusIndex = confparams_cpr->maxRangeRadius - 1;
|
||||
intervals[radiusIndex]++;
|
||||
|
||||
data_pos += confparams_cpr->sampleDistance;
|
||||
}
|
||||
//compute the appropriate number
|
||||
size_t targetCount = totalSampleSize*confparams_cpr->predThreshold;
|
||||
size_t sum = 0;
|
||||
for(i=0;i<confparams_cpr->maxRangeRadius;i++)
|
||||
{
|
||||
sum += intervals[i];
|
||||
if(sum>targetCount)
|
||||
break;
|
||||
}
|
||||
if(i>=confparams_cpr->maxRangeRadius)
|
||||
i = confparams_cpr->maxRangeRadius-1;
|
||||
|
||||
unsigned int accIntervals = 2*(i+1);
|
||||
unsigned int powerOf2 = roundUpToPowerOf2(accIntervals);
|
||||
|
||||
if(powerOf2<32)
|
||||
powerOf2 = 32;
|
||||
|
||||
free(intervals);
|
||||
return powerOf2;
|
||||
}
|
||||
|
|
@ -0,0 +1,226 @@
|
|||
/**
|
||||
* @file szd_double.c
|
||||
* @author Sheng Di, Dingwen Tao, Xin Liang, Xiangyu Zou, Tao Lu, Wen Xia, Xuan Wang, Weizhe Zhang
|
||||
* @date Aug, 2016
|
||||
* @brief
|
||||
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
|
||||
* See COPYRIGHT in top-level directory.
|
||||
*/
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include "szd_double.h"
|
||||
#include "TightDataPointStorageD.h"
|
||||
#include "sz.h"
|
||||
#include "utility.h"
|
||||
#include "Huffman.h"
|
||||
#include "transcode.h"
|
||||
|
||||
int SZ_decompress_args_double(double* newData, size_t r1, unsigned char* cmpBytes,
|
||||
size_t cmpSize, int compressionType, double* hist_data, sz_exedata* pde_exe, sz_params* pde_params)
|
||||
{
|
||||
int status = SZ_SUCCESS;
|
||||
size_t dataLength = r1;
|
||||
|
||||
//unsigned char* tmpBytes;
|
||||
size_t targetUncompressSize = dataLength <<3; //i.e., *8
|
||||
//tmpSize must be "much" smaller than dataLength
|
||||
size_t i, tmpSize = 12+MetaDataByteLength_double+8;
|
||||
unsigned char* szTmpBytes = NULL;
|
||||
bool needFree = false;
|
||||
|
||||
if(cmpSize!=12+4+MetaDataByteLength_double && cmpSize!=12+8+MetaDataByteLength_double)
|
||||
{
|
||||
pde_params->losslessCompressor = is_lossless_compressed_data(cmpBytes, cmpSize);
|
||||
if(pde_params->szMode!=SZ_TEMPORAL_COMPRESSION)
|
||||
{
|
||||
if(pde_params->losslessCompressor!=-1)
|
||||
pde_params->szMode = SZ_BEST_COMPRESSION;
|
||||
else
|
||||
pde_params->szMode = SZ_BEST_SPEED;
|
||||
}
|
||||
|
||||
|
||||
if(pde_params->szMode==SZ_BEST_SPEED)
|
||||
{
|
||||
tmpSize = cmpSize;
|
||||
szTmpBytes = cmpBytes;
|
||||
}
|
||||
else
|
||||
{
|
||||
if(targetUncompressSize<MIN_ZLIB_DEC_ALLOMEM_BYTES) //Considering the minimum size
|
||||
targetUncompressSize = MIN_ZLIB_DEC_ALLOMEM_BYTES;
|
||||
tmpSize = sz_lossless_decompress(pde_params->losslessCompressor, cmpBytes, (unsigned long)cmpSize, &szTmpBytes, (unsigned long)targetUncompressSize+4+MetaDataByteLength_double+8);
|
||||
needFree = true;
|
||||
}
|
||||
}
|
||||
else
|
||||
szTmpBytes = cmpBytes;
|
||||
|
||||
// calc postion
|
||||
//pde_params->sol_ID = szTmpBytes[1+3-2+14-4]; //szTmpBytes: version(3bytes), samebyte(1byte), [14]:sol_ID=SZ or SZ_Transpose
|
||||
//TODO: convert szTmpBytes to double array.
|
||||
TightDataPointStorageD* tdps = NULL;
|
||||
int errBoundMode = new_TightDataPointStorageD_fromFlatBytes(&tdps, szTmpBytes, tmpSize, pde_exe, pde_params);
|
||||
if(tdps == NULL)
|
||||
{
|
||||
return SZ_FORMAT_ERR;
|
||||
}
|
||||
|
||||
int doubleSize = sizeof(double);
|
||||
if(tdps->isLossless)
|
||||
{
|
||||
// *newData = (double*)malloc(doubleSize*dataLength); comment by tickduan
|
||||
if(sysEndianType==BIG_ENDIAN_SYSTEM)
|
||||
{
|
||||
memcpy(newData, szTmpBytes+4+MetaDataByteLength_double+pde_exe->SZ_SIZE_TYPE, dataLength*doubleSize);
|
||||
}
|
||||
else
|
||||
{
|
||||
unsigned char* p = szTmpBytes+4+MetaDataByteLength_double+pde_exe->SZ_SIZE_TYPE;
|
||||
for(i=0;i<dataLength;i++,p+=doubleSize)
|
||||
newData[i] = bytesToDouble(p);
|
||||
}
|
||||
}
|
||||
else if(pde_params->sol_ID==SZ_Transpose)
|
||||
{
|
||||
getSnapshotData_double_1D(newData,dataLength,tdps, errBoundMode, 0, hist_data, pde_params);
|
||||
}
|
||||
else //pde_params->sol_ID==SZ
|
||||
{
|
||||
if(tdps->raBytes_size > 0) //v2.0
|
||||
{
|
||||
getSnapshotData_double_1D(newData,r1,tdps, errBoundMode, 0, hist_data, pde_params);
|
||||
}
|
||||
else //1.4.13 or time-based compression
|
||||
{
|
||||
getSnapshotData_double_1D(newData,r1,tdps, errBoundMode, compressionType, hist_data, pde_params);
|
||||
}
|
||||
}
|
||||
|
||||
free_TightDataPointStorageD2(tdps);
|
||||
if(szTmpBytes && needFree)
|
||||
free(szTmpBytes);
|
||||
return status;
|
||||
}
|
||||
|
||||
void decompressDataSeries_double_1D(double* data, size_t dataSeriesLength, double* hist_data, TightDataPointStorageD* tdps)
|
||||
{
|
||||
//updateQuantizationInfo(tdps->intervals);
|
||||
int intvRadius = tdps->intervals/2;
|
||||
size_t i, j, k = 0, p = 0, l = 0; // k is to track the location of residual_bit
|
||||
// in resiMidBits, p is to track the
|
||||
// byte_index of resiMidBits, l is for
|
||||
// leadNum
|
||||
unsigned char* leadNum;
|
||||
double interval = tdps->realPrecision*2;
|
||||
|
||||
convertByteArray2IntArray_fast_2b(tdps->exactDataNum, tdps->leadNumArray, tdps->leadNumArray_size, &leadNum);
|
||||
//*data = (double*)malloc(sizeof(double)*dataSeriesLength); comment by tickduan
|
||||
|
||||
int* type = (int*)malloc(dataSeriesLength*sizeof(int));
|
||||
|
||||
if (tdps->ifAdtFse == false) {
|
||||
HuffmanTree* huffmanTree = createHuffmanTree(tdps->stateNum);
|
||||
decode_withTree(huffmanTree, tdps->typeArray, dataSeriesLength, type);
|
||||
SZ_ReleaseHuffman(huffmanTree);
|
||||
}
|
||||
else {
|
||||
decode_with_fse(type, dataSeriesLength, tdps->intervals, tdps->FseCode, tdps->FseCode_size,
|
||||
tdps->transCodeBits, tdps->transCodeBits_size);
|
||||
}
|
||||
|
||||
unsigned char preBytes[8];
|
||||
unsigned char curBytes[8];
|
||||
|
||||
memset(preBytes, 0, 8);
|
||||
|
||||
size_t curByteIndex = 0;
|
||||
int reqBytesLength, resiBitsLength, resiBits;
|
||||
unsigned char leadingNum;
|
||||
double medianValue, exactData, predValue;
|
||||
|
||||
reqBytesLength = tdps->reqLength/8;
|
||||
resiBitsLength = tdps->reqLength%8;
|
||||
medianValue = tdps->medianValue;
|
||||
|
||||
|
||||
int type_;
|
||||
for (i = 0; i < dataSeriesLength; i++)
|
||||
{
|
||||
type_ = type[i];
|
||||
switch (type_) {
|
||||
case 0:
|
||||
// compute resiBits
|
||||
resiBits = 0;
|
||||
if (resiBitsLength != 0) {
|
||||
int kMod8 = k % 8;
|
||||
int rightMovSteps = getRightMovingSteps(kMod8, resiBitsLength);
|
||||
if (rightMovSteps > 0) {
|
||||
int code = getRightMovingCode(kMod8, resiBitsLength);
|
||||
resiBits = (tdps->residualMidBits[p] & code) >> rightMovSteps;
|
||||
} else if (rightMovSteps < 0) {
|
||||
int code1 = getLeftMovingCode(kMod8);
|
||||
int code2 = getRightMovingCode(kMod8, resiBitsLength);
|
||||
int leftMovSteps = -rightMovSteps;
|
||||
rightMovSteps = 8 - leftMovSteps;
|
||||
resiBits = (tdps->residualMidBits[p] & code1) << leftMovSteps;
|
||||
p++;
|
||||
resiBits = resiBits
|
||||
| ((tdps->residualMidBits[p] & code2) >> rightMovSteps);
|
||||
} else // rightMovSteps == 0
|
||||
{
|
||||
int code = getRightMovingCode(kMod8, resiBitsLength);
|
||||
resiBits = (tdps->residualMidBits[p] & code);
|
||||
p++;
|
||||
}
|
||||
k += resiBitsLength;
|
||||
}
|
||||
|
||||
// recover the exact data
|
||||
memset(curBytes, 0, 8);
|
||||
leadingNum = leadNum[l++];
|
||||
memcpy(curBytes, preBytes, leadingNum);
|
||||
for (j = leadingNum; j < reqBytesLength; j++)
|
||||
curBytes[j] = tdps->exactMidBytes[curByteIndex++];
|
||||
if (resiBitsLength != 0) {
|
||||
unsigned char resiByte = (unsigned char) (resiBits << (8 - resiBitsLength));
|
||||
curBytes[reqBytesLength] = resiByte;
|
||||
}
|
||||
|
||||
exactData = bytesToDouble(curBytes);
|
||||
data[i] = exactData + medianValue;
|
||||
memcpy(preBytes,curBytes,8);
|
||||
break;
|
||||
default:
|
||||
//predValue = 2 * data[i-1] - data[i-2];
|
||||
predValue = data[i-1];
|
||||
data[i] = predValue + (type_-intvRadius)*interval;
|
||||
break;
|
||||
}
|
||||
//printf("%.30G\n",data[i]);
|
||||
}
|
||||
|
||||
free(leadNum);
|
||||
free(type);
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
void getSnapshotData_double_1D(double* data, size_t dataSeriesLength, TightDataPointStorageD* tdps, int errBoundMode, int compressionType, double* hist_data, sz_params* pde_params)
|
||||
{
|
||||
size_t i;
|
||||
if (tdps->allSameData) {
|
||||
double value = bytesToDouble(tdps->exactMidBytes);
|
||||
|
||||
//*data = (double*)malloc(sizeof(double)*dataSeriesLength); comment by tickduan
|
||||
for (i = 0; i < dataSeriesLength; i++)
|
||||
data[i] = value;
|
||||
}
|
||||
else
|
||||
{
|
||||
decompressDataSeries_double_1D(data, dataSeriesLength, hist_data, tdps);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,234 @@
|
|||
/**
|
||||
* @file szd_float.c
|
||||
* @author Sheng Di, Dingwen Tao, Xin Liang, Xiangyu Zou, Tao Lu, Wen Xia, Xuan Wang, Weizhe Zhang
|
||||
* @date Aug, 2018
|
||||
* @brief
|
||||
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
|
||||
* See COPYRIGHT in top-level directory.
|
||||
*/
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include "szd_float.h"
|
||||
#include "TightDataPointStorageF.h"
|
||||
#include "sz.h"
|
||||
#include "utility.h"
|
||||
#include "Huffman.h"
|
||||
#include "transcode.h"
|
||||
|
||||
|
||||
/**
|
||||
*
|
||||
* int compressionType: 1 (time-based compression) ; 0 (space-based compression)
|
||||
* hist_data: only valid when compressionType==1, hist_data is the historical dataset such as the data in previous time step
|
||||
*
|
||||
* @return status SUCCESSFUL (SZ_SUCCESS) or not (other error codes) f
|
||||
* */
|
||||
int SZ_decompress_args_float(float* newData, size_t r1, unsigned char* cmpBytes,
|
||||
size_t cmpSize, int compressionType, float* hist_data, sz_exedata* pde_exe, sz_params* pde_params)
|
||||
{
|
||||
int status = SZ_SUCCESS;
|
||||
size_t dataLength = r1;
|
||||
|
||||
//unsigned char* tmpBytes;
|
||||
size_t targetUncompressSize = dataLength <<2; //i.e., *4
|
||||
//tmpSize must be "much" smaller than dataLength
|
||||
size_t i, tmpSize = 8+MetaDataByteLength+pde_exe->SZ_SIZE_TYPE;
|
||||
unsigned char* szTmpBytes = NULL;
|
||||
bool needFree = false;
|
||||
|
||||
if(cmpSize!=8+4+MetaDataByteLength && cmpSize!=8+8+MetaDataByteLength) //4,8 means two posibilities of SZ_SIZE_TYPE
|
||||
{
|
||||
pde_params->losslessCompressor = is_lossless_compressed_data(cmpBytes, cmpSize);
|
||||
if(pde_params->szMode!=SZ_TEMPORAL_COMPRESSION)
|
||||
{
|
||||
if(pde_params->losslessCompressor!=-1)
|
||||
pde_params->szMode = SZ_BEST_COMPRESSION;
|
||||
else
|
||||
pde_params->szMode = SZ_BEST_SPEED;
|
||||
}
|
||||
|
||||
if(pde_params->szMode==SZ_BEST_SPEED)
|
||||
{
|
||||
tmpSize = cmpSize;
|
||||
szTmpBytes = cmpBytes;
|
||||
}
|
||||
else
|
||||
{
|
||||
if(targetUncompressSize<MIN_ZLIB_DEC_ALLOMEM_BYTES) //Considering the minimum size
|
||||
targetUncompressSize = MIN_ZLIB_DEC_ALLOMEM_BYTES;
|
||||
tmpSize = sz_lossless_decompress(pde_params->losslessCompressor, cmpBytes, (unsigned long)cmpSize, &szTmpBytes, (unsigned long)targetUncompressSize+4+MetaDataByteLength+8);
|
||||
needFree = true;
|
||||
}
|
||||
}
|
||||
else
|
||||
szTmpBytes = cmpBytes;
|
||||
|
||||
// calc sol_ID
|
||||
//pde_params->sol_ID = szTmpBytes[1+3-2+14-4]; //szTmpBytes: version(1bytes), samebyte(1byte), [14-4]:sol_ID=SZ or SZ_Transpose
|
||||
|
||||
//TODO: convert szTmpBytes to data array.
|
||||
TightDataPointStorageF* tdps = NULL;
|
||||
int errBoundMode = new_TightDataPointStorageF_fromFlatBytes(&tdps, szTmpBytes, tmpSize, pde_exe, pde_params);
|
||||
if(tdps == NULL)
|
||||
{
|
||||
return SZ_FORMAT_ERR;
|
||||
}
|
||||
|
||||
int floatSize = sizeof(float);
|
||||
if(tdps->isLossless)
|
||||
{
|
||||
//*newData = (float*)malloc(floatSize*dataLength); comment by tickduan
|
||||
if(sysEndianType==BIG_ENDIAN_SYSTEM)
|
||||
{
|
||||
memcpy(newData, szTmpBytes+4+MetaDataByteLength+pde_exe->SZ_SIZE_TYPE, dataLength*floatSize);
|
||||
}
|
||||
else
|
||||
{
|
||||
unsigned char* p = szTmpBytes+4+MetaDataByteLength+pde_exe->SZ_SIZE_TYPE;
|
||||
for(i=0;i<dataLength;i++,p+=floatSize)
|
||||
newData[i] = bytesToFloat(p);
|
||||
}
|
||||
}
|
||||
else //pde_params->sol_ID==SZ
|
||||
{
|
||||
if(tdps->raBytes_size > 0) //v2.0
|
||||
{
|
||||
getSnapshotData_float_1D(newData,r1,tdps, errBoundMode, 0, hist_data, pde_params);
|
||||
}
|
||||
else //1.4.13 or time-based compression
|
||||
{
|
||||
getSnapshotData_float_1D(newData,r1,tdps, errBoundMode, compressionType, hist_data, pde_params);
|
||||
}
|
||||
}
|
||||
|
||||
//cost_start_();
|
||||
//cost_end_();
|
||||
//printf("totalCost_=%f\n", totalCost_);
|
||||
free_TightDataPointStorageF2(tdps);
|
||||
if(szTmpBytes && needFree)
|
||||
free(szTmpBytes);
|
||||
return status;
|
||||
}
|
||||
|
||||
void decompressDataSeries_float_1D(float* data, size_t dataSeriesLength, float* hist_data, TightDataPointStorageF* tdps)
|
||||
{
|
||||
//updateQuantizationInfo(tdps->intervals);
|
||||
int intvRadius = tdps->intervals/2;
|
||||
size_t i, j, k = 0, p = 0, l = 0; // k is to track the location of residual_bit
|
||||
// in resiMidBits, p is to track the
|
||||
// byte_index of resiMidBits, l is for
|
||||
// leadNum
|
||||
unsigned char* leadNum;
|
||||
float interval = tdps->realPrecision*2;
|
||||
|
||||
convertByteArray2IntArray_fast_2b(tdps->exactDataNum, tdps->leadNumArray, tdps->leadNumArray_size, &leadNum);
|
||||
//data = (float*)malloc(sizeof(float)*dataSeriesLength); // comment by tickduan
|
||||
|
||||
int* types = (int*)malloc(dataSeriesLength*sizeof(int));
|
||||
|
||||
if (tdps->ifAdtFse == false) {
|
||||
// type tree
|
||||
HuffmanTree* huffmanTree = createHuffmanTree(tdps->stateNum);
|
||||
decode_withTree(huffmanTree, tdps->typeArray, dataSeriesLength, types);
|
||||
SZ_ReleaseHuffman(huffmanTree);
|
||||
}
|
||||
else {
|
||||
decode_with_fse(types, dataSeriesLength, tdps->intervals, tdps->FseCode, tdps->FseCode_size,
|
||||
tdps->transCodeBits, tdps->transCodeBits_size);
|
||||
}
|
||||
|
||||
unsigned char preBytes[4];
|
||||
unsigned char curBytes[4];
|
||||
|
||||
memset(preBytes, 0, 4);
|
||||
|
||||
size_t curByteIndex = 0;
|
||||
int reqBytesLength, resiBitsLength, resiBits;
|
||||
unsigned char leadingNum;
|
||||
float medianValue, exactData, predValue;
|
||||
|
||||
reqBytesLength = tdps->reqLength/8;
|
||||
resiBitsLength = tdps->reqLength%8;
|
||||
medianValue = tdps->medianValue;
|
||||
|
||||
// decompress core
|
||||
int type;
|
||||
for (i = 0; i < dataSeriesLength; i++)
|
||||
{
|
||||
type = types[i];
|
||||
switch (type)
|
||||
{
|
||||
case 0:
|
||||
// compute resiBits
|
||||
resiBits = 0;
|
||||
if (resiBitsLength != 0) {
|
||||
int kMod8 = k % 8;
|
||||
int rightMovSteps = getRightMovingSteps(kMod8, resiBitsLength);
|
||||
if (rightMovSteps > 0) {
|
||||
int code = getRightMovingCode(kMod8, resiBitsLength);
|
||||
resiBits = (tdps->residualMidBits[p] & code) >> rightMovSteps;
|
||||
} else if (rightMovSteps < 0) {
|
||||
int code1 = getLeftMovingCode(kMod8);
|
||||
int code2 = getRightMovingCode(kMod8, resiBitsLength);
|
||||
int leftMovSteps = -rightMovSteps;
|
||||
rightMovSteps = 8 - leftMovSteps;
|
||||
resiBits = (tdps->residualMidBits[p] & code1) << leftMovSteps;
|
||||
p++;
|
||||
resiBits = resiBits
|
||||
| ((tdps->residualMidBits[p] & code2) >> rightMovSteps);
|
||||
} else // rightMovSteps == 0
|
||||
{
|
||||
int code = getRightMovingCode(kMod8, resiBitsLength);
|
||||
resiBits = (tdps->residualMidBits[p] & code);
|
||||
p++;
|
||||
}
|
||||
k += resiBitsLength;
|
||||
}
|
||||
|
||||
// recover the exact data
|
||||
memset(curBytes, 0, 4);
|
||||
leadingNum = leadNum[l++];
|
||||
memcpy(curBytes, preBytes, leadingNum);
|
||||
for (j = leadingNum; j < reqBytesLength; j++)
|
||||
curBytes[j] = tdps->exactMidBytes[curByteIndex++];
|
||||
if (resiBitsLength != 0) {
|
||||
unsigned char resiByte = (unsigned char) (resiBits << (8 - resiBitsLength));
|
||||
curBytes[reqBytesLength] = resiByte;
|
||||
}
|
||||
|
||||
exactData = bytesToFloat(curBytes);
|
||||
data[i] = exactData + medianValue;
|
||||
memcpy(preBytes,curBytes,4);
|
||||
break;
|
||||
default:
|
||||
//predValue = 2 * data[i-1] - data[i-2];
|
||||
predValue = data[i-1];
|
||||
data[i] = predValue + (float)(type - intvRadius) * interval;
|
||||
break;
|
||||
}
|
||||
//printf("%.30G\n",data[i]);
|
||||
}
|
||||
|
||||
free(leadNum);
|
||||
free(types);
|
||||
return;
|
||||
}
|
||||
|
||||
void getSnapshotData_float_1D(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps, int errBoundMode, int compressionType, float* hist_data, sz_params* pde_params)
|
||||
{
|
||||
size_t i;
|
||||
|
||||
if (tdps->allSameData)
|
||||
{
|
||||
float value = bytesToFloat(tdps->exactMidBytes);
|
||||
//*data = (float*)malloc(sizeof(float)*dataSeriesLength); commnet by tickduan
|
||||
for (i = 0; i < dataSeriesLength; i++)
|
||||
data[i] = value;
|
||||
}
|
||||
else
|
||||
{
|
||||
decompressDataSeries_float_1D(data, dataSeriesLength, hist_data, tdps);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,89 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "td_sz.h"
|
||||
#include "sz.h"
|
||||
#include "conf.h"
|
||||
|
||||
//
|
||||
// Init success return 1 else 0
|
||||
//
|
||||
void tdszInit(double fPrecision, double dPrecision, unsigned int maxIntervals, unsigned int intervals, int ifAdtFse, const char* compressor)
|
||||
{
|
||||
// need malloc
|
||||
if(confparams_cpr == NULL)
|
||||
confparams_cpr = (sz_params*)malloc(sizeof(sz_params));
|
||||
if(exe_params == NULL)
|
||||
exe_params = (sz_exedata*)malloc(sizeof(sz_exedata));
|
||||
|
||||
// set default
|
||||
setDefaulParams(exe_params, confparams_cpr);
|
||||
|
||||
// overwrite with args
|
||||
confparams_cpr->absErrBound = fPrecision;
|
||||
confparams_cpr->absErrBoundDouble = dPrecision;
|
||||
confparams_cpr->max_quant_intervals = maxIntervals;
|
||||
confparams_cpr->quantization_intervals = intervals;
|
||||
confparams_cpr->ifAdtFse = ifAdtFse;
|
||||
if(strcmp(compressor, "GZIP_COMPRESSOR")==0)
|
||||
confparams_cpr->losslessCompressor = GZIP_COMPRESSOR;
|
||||
else if(strcmp(compressor, "ZSTD_COMPRESSOR")==0)
|
||||
confparams_cpr->losslessCompressor = ZSTD_COMPRESSOR;
|
||||
|
||||
return ;
|
||||
}
|
||||
|
||||
|
||||
//
|
||||
// compress interface to tdengine return value is count of output with bytes
|
||||
//
|
||||
int tdszCompress(int type, const char * input, const int nelements, const char * output)
|
||||
{
|
||||
// check valid
|
||||
sz_params comp_params = *confparams_cpr;
|
||||
|
||||
size_t outSize = SZ_compress_args(type, (void*)input, (size_t)nelements, (unsigned char*)output, &comp_params);
|
||||
return (int)outSize;
|
||||
}
|
||||
|
||||
//
|
||||
// decompress interface to tdengine return value is count of output with bytes
|
||||
//
|
||||
int tdszDecompress(int type, const char * input, int compressedSize, const int nelements, const char * output)
|
||||
{
|
||||
size_t outSize = SZ_decompress(type, (void*)input, compressedSize, (size_t)nelements, (unsigned char*)output);
|
||||
return (int)outSize;
|
||||
}
|
||||
|
||||
//
|
||||
// tdszExit
|
||||
//
|
||||
void tdszExit()
|
||||
{
|
||||
if(confparams_cpr!=NULL)
|
||||
{
|
||||
free(confparams_cpr);
|
||||
confparams_cpr = NULL;
|
||||
}
|
||||
if(exe_params!=NULL)
|
||||
{
|
||||
free(exe_params);
|
||||
exe_params = NULL;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,159 @@
|
|||
/**
|
||||
* @file transcode.c
|
||||
* @author Yu Zhong (lx1zhong@qq.com)
|
||||
* @brief Source file for ADT-FSE algorithm
|
||||
* @date 2022-08-29
|
||||
*
|
||||
* @copyright Copyright (c) 2022
|
||||
*
|
||||
*/
|
||||
|
||||
#include "transcode.h"
|
||||
#include "fse.h"
|
||||
#include "bitstream.h"
|
||||
#include <stdlib.h>
|
||||
|
||||
/**
|
||||
* @brief transform type array to FseCode & tranCodeBits
|
||||
* [type] ---minus md---> [factor] ---transcode---> [tp_code] + [bitstream of diff]
|
||||
*
|
||||
*/
|
||||
void encode_with_fse(int *type, size_t dataSeriesLength, unsigned int intervals,
|
||||
unsigned char **FseCode, size_t *FseCode_size,
|
||||
unsigned char **transCodeBits, size_t *transCodeBits_size)
|
||||
{
|
||||
int nbits = 0;
|
||||
int dstCapacity = dataSeriesLength * sizeof(int);
|
||||
uint8_t *tp_code = (uint8_t *)malloc(dataSeriesLength);
|
||||
BIT_CStream_t transCodeStream;
|
||||
(*transCodeBits) = (unsigned char*)malloc(dstCapacity);
|
||||
BIT_initCStream(&transCodeStream, (void *)(*transCodeBits), dstCapacity);
|
||||
|
||||
// transcoding
|
||||
int md = intervals / 2;
|
||||
uint8_t type2code[intervals];
|
||||
unsigned int diff[intervals];
|
||||
|
||||
for (int i = md; i < intervals; i++)
|
||||
{
|
||||
type2code[i] = (uint8_t)Int2code(i - md);
|
||||
diff[i] = i - md - code2int[type2code[i]][0];
|
||||
}
|
||||
|
||||
for (int i = md - 1; i > 0; i--)
|
||||
{
|
||||
type2code[i] = 67 - type2code[2 * md - i];
|
||||
diff[i] = diff[2*md-i];
|
||||
}
|
||||
|
||||
for (int i = 0; i < dataSeriesLength; i++)
|
||||
{
|
||||
if (type[i] == 0)
|
||||
{
|
||||
// unpredictable data
|
||||
tp_code[i] = 67;
|
||||
nbits = 0;
|
||||
}
|
||||
else
|
||||
{
|
||||
tp_code[i] = type2code[type[i]];
|
||||
nbits = code2int[tp_code[i]][1];
|
||||
BIT_addBitsFast(&transCodeStream, diff[type[i]], nbits);
|
||||
BIT_flushBitsFast(&transCodeStream);
|
||||
}
|
||||
}
|
||||
|
||||
(*FseCode) = (unsigned char*)malloc(2 * dataSeriesLength);
|
||||
size_t fse_size = FSE_compress((*FseCode), 2 * dataSeriesLength, tp_code, dataSeriesLength);
|
||||
if (FSE_isError(fse_size))
|
||||
{
|
||||
printf("ADT-FSE: FSE_compress error!\n");
|
||||
exit(1);
|
||||
}
|
||||
(*FseCode_size) = fse_size;
|
||||
|
||||
size_t const streamSize = BIT_closeCStream(&transCodeStream);
|
||||
(*transCodeBits_size) = streamSize;
|
||||
if (streamSize == 0)
|
||||
{
|
||||
printf("ADT-FSE: transCodeBits_size too small!\n");
|
||||
exit(1);
|
||||
}
|
||||
free(tp_code);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief transform FseCode & tranCodeBits to type array
|
||||
*
|
||||
*/
|
||||
void decode_with_fse(int *type, size_t dataSeriesLength, unsigned int intervals,
|
||||
unsigned char *FseCode, size_t FseCode_size,
|
||||
unsigned char *transCodeBits, size_t transCodeBits_size)
|
||||
{
|
||||
uint8_t *tp_code = (uint8_t *)malloc(dataSeriesLength);
|
||||
|
||||
if (FseCode_size <= 1)
|
||||
{
|
||||
// all zeros
|
||||
memset((void *)type, 0, sizeof(int) * dataSeriesLength);
|
||||
return;
|
||||
}
|
||||
size_t fse_size = FSE_decompress(tp_code, dataSeriesLength, FseCode, FseCode_size);
|
||||
if (FSE_isError(fse_size))
|
||||
{
|
||||
printf("ADT-FSE: FSE_decompress error!\n");
|
||||
exit(1);
|
||||
}
|
||||
if (fse_size != dataSeriesLength)
|
||||
{
|
||||
printf("ADT-FSE: FSE_decompress error! fse_size(%lu) != dataSeriesLength(%lu)!\n", fse_size, dataSeriesLength);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
BIT_DStream_t transCodeStream;
|
||||
size_t stream_size = BIT_initDStream(&transCodeStream, transCodeBits, transCodeBits_size);
|
||||
if (stream_size == 0)
|
||||
{
|
||||
printf("ADT-FSE: transcode stream empty!\n");
|
||||
exit(1);
|
||||
}
|
||||
|
||||
int md = intervals / 2;
|
||||
int code2type[67];
|
||||
for (int i = 0; i < 67; i++)
|
||||
{
|
||||
code2type[i] = code2int[i][0] + md;
|
||||
}
|
||||
|
||||
int nbits;
|
||||
size_t diff;
|
||||
for (int i = dataSeriesLength-1; i >= 0; i--)
|
||||
{
|
||||
if (tp_code[i] == 67) {
|
||||
type[i] = 0;
|
||||
continue;
|
||||
}
|
||||
nbits = code2int[tp_code[i]][1];
|
||||
|
||||
if (nbits >= 1)
|
||||
diff = BIT_readBitsFast(&transCodeStream, nbits);
|
||||
else
|
||||
diff =0;
|
||||
BIT_reloadDStream(&transCodeStream);
|
||||
|
||||
if (tp_code[i] < 34) {
|
||||
// positive number
|
||||
type[i] = code2type[tp_code[i]] + diff;
|
||||
}
|
||||
else {
|
||||
// negative number
|
||||
type[i] = code2type[tp_code[i]] - diff;
|
||||
}
|
||||
}
|
||||
free(tp_code);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
/**
|
||||
* @file utility.c
|
||||
* @author Sheng Di, Sihuan Li
|
||||
* @date Aug, 2018
|
||||
* @brief
|
||||
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
|
||||
* See COPYRIGHT in top-level directory.
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <math.h>
|
||||
#include "utility.h"
|
||||
#include "sz.h"
|
||||
#include "zstd.h"
|
||||
|
||||
|
||||
int is_lossless_compressed_data(unsigned char* compressedBytes, size_t cmpSize)
|
||||
{
|
||||
#if ZSTD_VERSION_NUMBER >= 10300
|
||||
unsigned long long frameContentSize = ZSTD_getFrameContentSize(compressedBytes, cmpSize);
|
||||
if(frameContentSize != ZSTD_CONTENTSIZE_ERROR)
|
||||
return ZSTD_COMPRESSOR;
|
||||
#else
|
||||
unsigned long long frameContentSize = ZSTD_getDecompressedSize(compressedBytes, cmpSize);
|
||||
if(frameContentSize != 0)
|
||||
return ZSTD_COMPRESSOR;
|
||||
#endif
|
||||
|
||||
return -1; //fast mode (without GZIP or ZSTD)
|
||||
}
|
||||
|
||||
unsigned long sz_lossless_compress(int losslessCompressor, unsigned char* data, unsigned long dataLength, unsigned char* compressBytes)
|
||||
{
|
||||
unsigned long outSize = 0;
|
||||
int level = 3 ; // fast mode
|
||||
size_t estimatedCompressedSize = 0;
|
||||
switch(losslessCompressor)
|
||||
{
|
||||
case ZSTD_COMPRESSOR:
|
||||
if(dataLength < 100)
|
||||
estimatedCompressedSize = 200;
|
||||
else
|
||||
estimatedCompressedSize = dataLength*1.2;
|
||||
//*compressBytes = (unsigned char*)malloc(estimatedCompressedSize); // comment by tickduan
|
||||
outSize = ZSTD_compress(compressBytes, estimatedCompressedSize, data, dataLength, level); //default setting of level is 3
|
||||
break;
|
||||
default:
|
||||
printf("Error: Unrecognized lossless compressor in sz_lossless_compress()\n");
|
||||
}
|
||||
return outSize;
|
||||
}
|
||||
|
||||
unsigned long sz_lossless_decompress(int losslessCompressor, unsigned char* compressBytes, unsigned long cmpSize, unsigned char** oriData, unsigned long targetOriSize)
|
||||
{
|
||||
unsigned long outSize = 0;
|
||||
switch(losslessCompressor)
|
||||
{
|
||||
case ZSTD_COMPRESSOR:
|
||||
*oriData = (unsigned char*)malloc(targetOriSize);
|
||||
outSize = ZSTD_decompress(*oriData, targetOriSize, compressBytes, cmpSize);
|
||||
break;
|
||||
default:
|
||||
printf("Error: Unrecognized lossless compressor in sz_lossless_decompress()\n");
|
||||
}
|
||||
return outSize;
|
||||
}
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
BSD License
|
||||
|
||||
For Zstandard software
|
||||
|
||||
Copyright (c) 2016-present, Facebook, Inc. All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without modification,
|
||||
are permitted provided that the following conditions are met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright notice, this
|
||||
list of conditions and the following disclaimer.
|
||||
|
||||
* Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
|
||||
* Neither the name Facebook nor the names of its contributors may be used to
|
||||
endorse or promote products derived from this software without specific
|
||||
prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
|
||||
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
|
||||
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
|
@ -0,0 +1,119 @@
|
|||
Zstandard library files
|
||||
================================
|
||||
|
||||
The __lib__ directory is split into several sub-directories,
|
||||
in order to make it easier to select or exclude features.
|
||||
|
||||
|
||||
#### Building
|
||||
|
||||
`Makefile` script is provided, supporting all standard [Makefile conventions](https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html#Makefile-Conventions),
|
||||
including commands variables, staged install, directory variables and standard targets.
|
||||
- `make` : generates both static and dynamic libraries
|
||||
- `make install` : install libraries in default system directories
|
||||
|
||||
`libzstd` default scope includes compression, decompression, dictionary building,
|
||||
and decoding support for legacy formats >= v0.4.0.
|
||||
|
||||
|
||||
#### API
|
||||
|
||||
Zstandard's stable API is exposed within [lib/zstd.h](zstd.h).
|
||||
|
||||
|
||||
#### Advanced API
|
||||
|
||||
Optional advanced features are exposed via :
|
||||
|
||||
- `lib/common/zstd_errors.h` : translates `size_t` function results
|
||||
into an `ZSTD_ErrorCode`, for accurate error handling.
|
||||
- `ZSTD_STATIC_LINKING_ONLY` : if this macro is defined _before_ including `zstd.h`,
|
||||
it unlocks access to advanced experimental API,
|
||||
exposed in second part of `zstd.h`.
|
||||
These APIs are not "stable", their definition may change in the future.
|
||||
As a consequence, it shall ___never be used with dynamic library___ !
|
||||
Only static linking is allowed.
|
||||
|
||||
|
||||
#### Modular build
|
||||
|
||||
It's possible to compile only a limited set of features.
|
||||
|
||||
- Directory `lib/common` is always required, for all variants.
|
||||
- Compression source code lies in `lib/compress`
|
||||
- Decompression source code lies in `lib/decompress`
|
||||
- It's possible to include only `compress` or only `decompress`, they don't depend on each other.
|
||||
- `lib/dictBuilder` : makes it possible to generate dictionaries from a set of samples.
|
||||
The API is exposed in `lib/dictBuilder/zdict.h`.
|
||||
This module depends on both `lib/common` and `lib/compress` .
|
||||
- `lib/legacy` : source code to decompress legacy zstd formats, starting from `v0.1.0`.
|
||||
This module depends on `lib/common` and `lib/decompress`.
|
||||
To enable this feature, it's required to define `ZSTD_LEGACY_SUPPORT` during compilation.
|
||||
Typically, with `gcc`, add argument `-DZSTD_LEGACY_SUPPORT=1`.
|
||||
Using higher number limits versions supported.
|
||||
For example, `ZSTD_LEGACY_SUPPORT=2` means : "support legacy formats >= v0.2.0".
|
||||
`ZSTD_LEGACY_SUPPORT=3` means : "support legacy formats >= v0.3.0", and so on.
|
||||
Starting v0.8.0, all versions of `zstd` produce frames compliant with specification.
|
||||
As a consequence, `ZSTD_LEGACY_SUPPORT=8` (or more) doesn't trigger legacy support.
|
||||
Also, `ZSTD_LEGACY_SUPPORT=0` means "do __not__ support legacy formats".
|
||||
Once enabled, this capability is transparently triggered within decompression functions.
|
||||
It's also possible to invoke directly legacy API, as exposed in `lib/legacy/zstd_legacy.h`.
|
||||
Each version also provides an additional dedicated set of advanced API.
|
||||
For example, advanced API for version `v0.4` is exposed in `lib/legacy/zstd_v04.h` .
|
||||
Note : `lib/legacy` only supports _decoding_ legacy formats.
|
||||
- Similarly, you can define `ZSTD_LIB_COMPRESSION, ZSTD_LIB_DECOMPRESSION`, `ZSTD_LIB_DICTBUILDER`,
|
||||
and `ZSTD_LIB_DEPRECATED` as 0 to forgo compilation of the corresponding features. This will
|
||||
also disable compilation of all dependencies (eg. `ZSTD_LIB_COMPRESSION=0` will also disable
|
||||
dictBuilder).
|
||||
|
||||
|
||||
#### Multithreading support
|
||||
|
||||
Multithreading is disabled by default when building with `make`.
|
||||
Enabling multithreading requires 2 conditions :
|
||||
- set macro `ZSTD_MULTITHREAD`
|
||||
- on POSIX systems : compile with pthread (`-pthread` compilation flag for `gcc`)
|
||||
|
||||
Both conditions are automatically triggered by invoking `make lib-mt` target.
|
||||
Note that, when linking a POSIX program with a multithreaded version of `libzstd`,
|
||||
it's necessary to trigger `-pthread` flag during link stage.
|
||||
|
||||
Multithreading capabilities are exposed
|
||||
via [advanced API `ZSTD_compress_generic()` defined in `lib/zstd.h`](https://github.com/facebook/zstd/blob/dev/lib/zstd.h#L919).
|
||||
This API is still considered experimental,
|
||||
but is expected to become "stable" at some point in the future.
|
||||
|
||||
|
||||
#### Windows : using MinGW+MSYS to create DLL
|
||||
|
||||
DLL can be created using MinGW+MSYS with the `make libzstd` command.
|
||||
This command creates `dll\libzstd.dll` and the import library `dll\libzstd.lib`.
|
||||
The import library is only required with Visual C++.
|
||||
The header file `zstd.h` and the dynamic library `dll\libzstd.dll` are required to
|
||||
compile a project using gcc/MinGW.
|
||||
The dynamic library has to be added to linking options.
|
||||
It means that if a project that uses ZSTD consists of a single `test-dll.c`
|
||||
file it should be linked with `dll\libzstd.dll`. For example:
|
||||
```
|
||||
gcc $(CFLAGS) -Iinclude/ test-dll.c -o test-dll dll\libzstd.dll
|
||||
```
|
||||
The compiled executable will require ZSTD DLL which is available at `dll\libzstd.dll`.
|
||||
|
||||
|
||||
#### Deprecated API
|
||||
|
||||
Obsolete API on their way out are stored in directory `lib/deprecated`.
|
||||
At this stage, it contains older streaming prototypes, in `lib/deprecated/zbuff.h`.
|
||||
These prototypes will be removed in some future version.
|
||||
Consider migrating code towards supported streaming API exposed in `zstd.h`.
|
||||
|
||||
|
||||
#### Miscellaneous
|
||||
|
||||
The other files are not source code. There are :
|
||||
|
||||
- `LICENSE` : contains the BSD license text
|
||||
- `Makefile` : `make` script to build and install zstd library (static and dynamic)
|
||||
- `BUCK` : support for `buck` build system (https://buckbuild.com/)
|
||||
- `libzstd.pc.in` : for `pkg-config` (used in `make install`)
|
||||
- `README.md` : this file
|
||||
|
|
@ -0,0 +1,458 @@
|
|||
/* ******************************************************************
|
||||
bitstream
|
||||
Part of FSE library
|
||||
Copyright (C) 2013-present, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
****************************************************************** */
|
||||
#ifndef BITSTREAM_H_MODULE
|
||||
#define BITSTREAM_H_MODULE
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/*
|
||||
* This API consists of small unitary functions, which must be inlined for best performance.
|
||||
* Since link-time-optimization is not available for all compilers,
|
||||
* these functions are defined into a .h to be included.
|
||||
*/
|
||||
|
||||
/*-****************************************
|
||||
* Dependencies
|
||||
******************************************/
|
||||
#include "mem.h" /* unaligned access routines */
|
||||
#include "debug.h" /* assert(), DEBUGLOG(), RAWLOG() */
|
||||
#include "error_private.h" /* error codes and messages */
|
||||
|
||||
|
||||
/*=========================================
|
||||
* Target specific
|
||||
=========================================*/
|
||||
#if defined(__BMI__) && defined(__GNUC__)
|
||||
# include <immintrin.h> /* support for bextr (experimental) */
|
||||
#endif
|
||||
|
||||
#define STREAM_ACCUMULATOR_MIN_32 25
|
||||
#define STREAM_ACCUMULATOR_MIN_64 57
|
||||
#define STREAM_ACCUMULATOR_MIN ((U32)(MEM_32bits() ? STREAM_ACCUMULATOR_MIN_32 : STREAM_ACCUMULATOR_MIN_64))
|
||||
|
||||
|
||||
/*-******************************************
|
||||
* bitStream encoding API (write forward)
|
||||
********************************************/
|
||||
/* bitStream can mix input from multiple sources.
|
||||
* A critical property of these streams is that they encode and decode in **reverse** direction.
|
||||
* So the first bit sequence you add will be the last to be read, like a LIFO stack.
|
||||
*/
|
||||
typedef struct {
|
||||
size_t bitContainer;
|
||||
unsigned bitPos;
|
||||
char* startPtr;
|
||||
char* ptr;
|
||||
char* endPtr;
|
||||
} BIT_CStream_t;
|
||||
|
||||
MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* dstBuffer, size_t dstCapacity);
|
||||
MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits);
|
||||
MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC);
|
||||
MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC);
|
||||
|
||||
/* Start with initCStream, providing the size of buffer to write into.
|
||||
* bitStream will never write outside of this buffer.
|
||||
* `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code.
|
||||
*
|
||||
* bits are first added to a local register.
|
||||
* Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems.
|
||||
* Writing data into memory is an explicit operation, performed by the flushBits function.
|
||||
* Hence keep track how many bits are potentially stored into local register to avoid register overflow.
|
||||
* After a flushBits, a maximum of 7 bits might still be stored into local register.
|
||||
*
|
||||
* Avoid storing elements of more than 24 bits if you want compatibility with 32-bits bitstream readers.
|
||||
*
|
||||
* Last operation is to close the bitStream.
|
||||
* The function returns the final size of CStream in bytes.
|
||||
* If data couldn't fit into `dstBuffer`, it will return a 0 ( == not storable)
|
||||
*/
|
||||
|
||||
|
||||
/*-********************************************
|
||||
* bitStream decoding API (read backward)
|
||||
**********************************************/
|
||||
typedef struct {
|
||||
size_t bitContainer;
|
||||
unsigned bitsConsumed;
|
||||
const char* ptr;
|
||||
const char* start;
|
||||
const char* limitPtr;
|
||||
} BIT_DStream_t;
|
||||
|
||||
typedef enum { BIT_DStream_unfinished = 0,
|
||||
BIT_DStream_endOfBuffer = 1,
|
||||
BIT_DStream_completed = 2,
|
||||
BIT_DStream_overflow = 3 } BIT_DStream_status; /* result of BIT_reloadDStream() */
|
||||
/* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */
|
||||
|
||||
MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize);
|
||||
MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits);
|
||||
MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD);
|
||||
MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
|
||||
|
||||
|
||||
/* Start by invoking BIT_initDStream().
|
||||
* A chunk of the bitStream is then stored into a local register.
|
||||
* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
|
||||
* You can then retrieve bitFields stored into the local register, **in reverse order**.
|
||||
* Local register is explicitly reloaded from memory by the BIT_reloadDStream() method.
|
||||
* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished.
|
||||
* Otherwise, it can be less than that, so proceed accordingly.
|
||||
* Checking if DStream has reached its end can be performed with BIT_endOfDStream().
|
||||
*/
|
||||
|
||||
|
||||
/*-****************************************
|
||||
* unsafe API
|
||||
******************************************/
|
||||
MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits);
|
||||
/* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */
|
||||
|
||||
MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC);
|
||||
/* unsafe version; does not check buffer overflow */
|
||||
|
||||
MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
|
||||
/* faster, but works only if nbBits >= 1 */
|
||||
|
||||
|
||||
|
||||
/*-**************************************************************
|
||||
* Internal functions
|
||||
****************************************************************/
|
||||
MEM_STATIC unsigned BIT_highbit32 (U32 val)
|
||||
{
|
||||
assert(val != 0);
|
||||
{
|
||||
# if defined(_MSC_VER) /* Visual */
|
||||
unsigned long r=0;
|
||||
_BitScanReverse ( &r, val );
|
||||
return (unsigned) r;
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
|
||||
return 31 - __builtin_clz (val);
|
||||
# else /* Software version */
|
||||
static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29,
|
||||
11, 14, 16, 18, 22, 25, 3, 30,
|
||||
8, 12, 20, 28, 15, 17, 24, 7,
|
||||
19, 27, 23, 6, 26, 5, 4, 31 };
|
||||
U32 v = val;
|
||||
v |= v >> 1;
|
||||
v |= v >> 2;
|
||||
v |= v >> 4;
|
||||
v |= v >> 8;
|
||||
v |= v >> 16;
|
||||
return DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27];
|
||||
# endif
|
||||
}
|
||||
}
|
||||
|
||||
/*===== Local Constants =====*/
|
||||
static const unsigned BIT_mask[] = {
|
||||
0, 1, 3, 7, 0xF, 0x1F,
|
||||
0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF,
|
||||
0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF,
|
||||
0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF,
|
||||
0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF, 0x7FFFFFF, 0xFFFFFFF, 0x1FFFFFFF,
|
||||
0x3FFFFFFF, 0x7FFFFFFF}; /* up to 31 bits */
|
||||
#define BIT_MASK_SIZE (sizeof(BIT_mask) / sizeof(BIT_mask[0]))
|
||||
|
||||
/*-**************************************************************
|
||||
* bitStream encoding
|
||||
****************************************************************/
|
||||
/*! BIT_initCStream() :
|
||||
* `dstCapacity` must be > sizeof(size_t)
|
||||
* @return : 0 if success,
|
||||
* otherwise an error code (can be tested using ERR_isError()) */
|
||||
MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC,
|
||||
void* startPtr, size_t dstCapacity)
|
||||
{
|
||||
bitC->bitContainer = 0;
|
||||
bitC->bitPos = 0;
|
||||
bitC->startPtr = (char*)startPtr;
|
||||
bitC->ptr = bitC->startPtr;
|
||||
bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->bitContainer);
|
||||
if (dstCapacity <= sizeof(bitC->bitContainer)) return ERROR(dstSize_tooSmall);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*! BIT_addBits() :
|
||||
* can add up to 31 bits into `bitC`.
|
||||
* Note : does not check for register overflow ! */
|
||||
MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC,
|
||||
size_t value, unsigned nbBits)
|
||||
{
|
||||
MEM_STATIC_ASSERT(BIT_MASK_SIZE == 32);
|
||||
assert(nbBits < BIT_MASK_SIZE);
|
||||
assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8);
|
||||
bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos;
|
||||
bitC->bitPos += nbBits;
|
||||
}
|
||||
|
||||
/*! BIT_addBitsFast() :
|
||||
* works only if `value` is _clean_,
|
||||
* meaning all high bits above nbBits are 0 */
|
||||
MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC,
|
||||
size_t value, unsigned nbBits)
|
||||
{
|
||||
assert((value>>nbBits) == 0);
|
||||
assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8);
|
||||
bitC->bitContainer |= value << bitC->bitPos;
|
||||
bitC->bitPos += nbBits;
|
||||
}
|
||||
|
||||
/*! BIT_flushBitsFast() :
|
||||
* assumption : bitContainer has not overflowed
|
||||
* unsafe version; does not check buffer overflow */
|
||||
MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC)
|
||||
{
|
||||
size_t const nbBytes = bitC->bitPos >> 3;
|
||||
assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8);
|
||||
MEM_writeLEST(bitC->ptr, bitC->bitContainer);
|
||||
bitC->ptr += nbBytes;
|
||||
assert(bitC->ptr <= bitC->endPtr);
|
||||
bitC->bitPos &= 7;
|
||||
bitC->bitContainer >>= nbBytes*8;
|
||||
}
|
||||
|
||||
/*! BIT_flushBits() :
|
||||
* assumption : bitContainer has not overflowed
|
||||
* safe version; check for buffer overflow, and prevents it.
|
||||
* note : does not signal buffer overflow.
|
||||
* overflow will be revealed later on using BIT_closeCStream() */
|
||||
MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC)
|
||||
{
|
||||
size_t const nbBytes = bitC->bitPos >> 3;
|
||||
assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8);
|
||||
MEM_writeLEST(bitC->ptr, bitC->bitContainer);
|
||||
bitC->ptr += nbBytes;
|
||||
if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr;
|
||||
bitC->bitPos &= 7;
|
||||
bitC->bitContainer >>= nbBytes*8;
|
||||
}
|
||||
|
||||
/*! BIT_closeCStream() :
|
||||
* @return : size of CStream, in bytes,
|
||||
* or 0 if it could not fit into dstBuffer */
|
||||
MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC)
|
||||
{
|
||||
BIT_addBitsFast(bitC, 1, 1); /* endMark */
|
||||
BIT_flushBits(bitC);
|
||||
if (bitC->ptr >= bitC->endPtr) return 0; /* overflow detected */
|
||||
return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0);
|
||||
}
|
||||
|
||||
|
||||
/*-********************************************************
|
||||
* bitStream decoding
|
||||
**********************************************************/
|
||||
/*! BIT_initDStream() :
|
||||
* Initialize a BIT_DStream_t.
|
||||
* `bitD` : a pointer to an already allocated BIT_DStream_t structure.
|
||||
* `srcSize` must be the *exact* size of the bitStream, in bytes.
|
||||
* @return : size of stream (== srcSize), or an errorCode if a problem is detected
|
||||
*/
|
||||
MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize)
|
||||
{
|
||||
if (srcSize < 1) { memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); }
|
||||
|
||||
bitD->start = (const char*)srcBuffer;
|
||||
bitD->limitPtr = bitD->start + sizeof(bitD->bitContainer);
|
||||
|
||||
if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */
|
||||
bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer);
|
||||
bitD->bitContainer = MEM_readLEST(bitD->ptr);
|
||||
{ BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
|
||||
bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */
|
||||
if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ }
|
||||
} else {
|
||||
bitD->ptr = bitD->start;
|
||||
bitD->bitContainer = *(const BYTE*)(bitD->start);
|
||||
switch(srcSize)
|
||||
{
|
||||
case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16);
|
||||
/* fall-through */
|
||||
|
||||
case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24);
|
||||
/* fall-through */
|
||||
|
||||
case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32);
|
||||
/* fall-through */
|
||||
|
||||
case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24;
|
||||
/* fall-through */
|
||||
|
||||
case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16;
|
||||
/* fall-through */
|
||||
|
||||
case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8;
|
||||
/* fall-through */
|
||||
|
||||
default: break;
|
||||
}
|
||||
{ BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
|
||||
bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0;
|
||||
if (lastByte == 0) return ERROR(corruption_detected); /* endMark not present */
|
||||
}
|
||||
bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8;
|
||||
}
|
||||
|
||||
return srcSize;
|
||||
}
|
||||
|
||||
MEM_STATIC size_t BIT_getUpperBits(size_t bitContainer, U32 const start)
|
||||
{
|
||||
return bitContainer >> start;
|
||||
}
|
||||
|
||||
MEM_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits)
|
||||
{
|
||||
#if defined(__BMI__) && defined(__GNUC__) && __GNUC__*1000+__GNUC_MINOR__ >= 4008 /* experimental */
|
||||
# if defined(__x86_64__)
|
||||
if (sizeof(bitContainer)==8)
|
||||
return _bextr_u64(bitContainer, start, nbBits);
|
||||
else
|
||||
# endif
|
||||
return _bextr_u32(bitContainer, start, nbBits);
|
||||
#else
|
||||
assert(nbBits < BIT_MASK_SIZE);
|
||||
return (bitContainer >> start) & BIT_mask[nbBits];
|
||||
#endif
|
||||
}
|
||||
|
||||
MEM_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits)
|
||||
{
|
||||
assert(nbBits < BIT_MASK_SIZE);
|
||||
return bitContainer & BIT_mask[nbBits];
|
||||
}
|
||||
|
||||
/*! BIT_lookBits() :
|
||||
* Provides next n bits from local register.
|
||||
* local register is not modified.
|
||||
* On 32-bits, maxNbBits==24.
|
||||
* On 64-bits, maxNbBits==56.
|
||||
* @return : value extracted */
|
||||
MEM_STATIC size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits)
|
||||
{
|
||||
#if defined(__BMI__) && defined(__GNUC__) /* experimental; fails if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8 */
|
||||
return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits);
|
||||
#else
|
||||
U32 const regMask = sizeof(bitD->bitContainer)*8 - 1;
|
||||
return ((bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> 1) >> ((regMask-nbBits) & regMask);
|
||||
#endif
|
||||
}
|
||||
|
||||
/*! BIT_lookBitsFast() :
|
||||
* unsafe version; only works if nbBits >= 1 */
|
||||
MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits)
|
||||
{
|
||||
U32 const regMask = sizeof(bitD->bitContainer)*8 - 1;
|
||||
assert(nbBits >= 1);
|
||||
return (bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> (((regMask+1)-nbBits) & regMask);
|
||||
}
|
||||
|
||||
MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
|
||||
{
|
||||
bitD->bitsConsumed += nbBits;
|
||||
}
|
||||
|
||||
/*! BIT_readBits() :
|
||||
* Read (consume) next n bits from local register and update.
|
||||
* Pay attention to not read more than nbBits contained into local register.
|
||||
* @return : extracted value. */
|
||||
MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
|
||||
{
|
||||
size_t const value = BIT_lookBits(bitD, nbBits);
|
||||
BIT_skipBits(bitD, nbBits);
|
||||
return value;
|
||||
}
|
||||
|
||||
/*! BIT_readBitsFast() :
|
||||
* unsafe version; only works only if nbBits >= 1 */
|
||||
MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
|
||||
{
|
||||
size_t const value = BIT_lookBitsFast(bitD, nbBits);
|
||||
assert(nbBits >= 1);
|
||||
BIT_skipBits(bitD, nbBits);
|
||||
return value;
|
||||
}
|
||||
|
||||
/*! BIT_reloadDStream() :
|
||||
* Refill `bitD` from buffer previously set in BIT_initDStream() .
|
||||
* This function is safe, it guarantees it will not read beyond src buffer.
|
||||
* @return : status of `BIT_DStream_t` internal register.
|
||||
* when status == BIT_DStream_unfinished, internal register is filled with at least 25 or 57 bits */
|
||||
MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
|
||||
{
|
||||
if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* overflow detected, like end of stream */
|
||||
return BIT_DStream_overflow;
|
||||
|
||||
if (bitD->ptr >= bitD->limitPtr) {
|
||||
bitD->ptr -= bitD->bitsConsumed >> 3;
|
||||
bitD->bitsConsumed &= 7;
|
||||
bitD->bitContainer = MEM_readLEST(bitD->ptr);
|
||||
return BIT_DStream_unfinished;
|
||||
}
|
||||
if (bitD->ptr == bitD->start) {
|
||||
if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer;
|
||||
return BIT_DStream_completed;
|
||||
}
|
||||
/* start < ptr < limitPtr */
|
||||
{ U32 nbBytes = bitD->bitsConsumed >> 3;
|
||||
BIT_DStream_status result = BIT_DStream_unfinished;
|
||||
if (bitD->ptr - nbBytes < bitD->start) {
|
||||
nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */
|
||||
result = BIT_DStream_endOfBuffer;
|
||||
}
|
||||
bitD->ptr -= nbBytes;
|
||||
bitD->bitsConsumed -= nbBytes*8;
|
||||
bitD->bitContainer = MEM_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD->bitContainer), otherwise bitD->ptr == bitD->start */
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
/*! BIT_endOfDStream() :
|
||||
* @return : 1 if DStream has _exactly_ reached its end (all bits consumed).
|
||||
*/
|
||||
MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream)
|
||||
{
|
||||
return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8));
|
||||
}
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* BITSTREAM_H_MODULE */
|
||||
|
|
@ -0,0 +1,111 @@
|
|||
/*
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#ifndef ZSTD_COMPILER_H
|
||||
#define ZSTD_COMPILER_H
|
||||
|
||||
/*-*******************************************************
|
||||
* Compiler specifics
|
||||
*********************************************************/
|
||||
/* force inlining */
|
||||
#if defined (__GNUC__) || defined(__cplusplus) || defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
|
||||
# define INLINE_KEYWORD inline
|
||||
#else
|
||||
# define INLINE_KEYWORD
|
||||
#endif
|
||||
|
||||
#if defined(__GNUC__)
|
||||
# define FORCE_INLINE_ATTR __attribute__((always_inline))
|
||||
#elif defined(_MSC_VER)
|
||||
# define FORCE_INLINE_ATTR __forceinline
|
||||
#else
|
||||
# define FORCE_INLINE_ATTR
|
||||
#endif
|
||||
|
||||
/**
|
||||
* FORCE_INLINE_TEMPLATE is used to define C "templates", which take constant
|
||||
* parameters. They must be inlined for the compiler to elimininate the constant
|
||||
* branches.
|
||||
*/
|
||||
#define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR
|
||||
/**
|
||||
* HINT_INLINE is used to help the compiler generate better code. It is *not*
|
||||
* used for "templates", so it can be tweaked based on the compilers
|
||||
* performance.
|
||||
*
|
||||
* gcc-4.8 and gcc-4.9 have been shown to benefit from leaving off the
|
||||
* always_inline attribute.
|
||||
*
|
||||
* clang up to 5.0.0 (trunk) benefit tremendously from the always_inline
|
||||
* attribute.
|
||||
*/
|
||||
#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 4 && __GNUC_MINOR__ >= 8 && __GNUC__ < 5
|
||||
# define HINT_INLINE static INLINE_KEYWORD
|
||||
#else
|
||||
# define HINT_INLINE static INLINE_KEYWORD FORCE_INLINE_ATTR
|
||||
#endif
|
||||
|
||||
/* force no inlining */
|
||||
#ifdef _MSC_VER
|
||||
# define FORCE_NOINLINE static __declspec(noinline)
|
||||
#else
|
||||
# ifdef __GNUC__
|
||||
# define FORCE_NOINLINE static __attribute__((__noinline__))
|
||||
# else
|
||||
# define FORCE_NOINLINE static
|
||||
# endif
|
||||
#endif
|
||||
|
||||
/* target attribute */
|
||||
#ifndef __has_attribute
|
||||
#define __has_attribute(x) 0 /* Compatibility with non-clang compilers. */
|
||||
#endif
|
||||
#if defined(__GNUC__)
|
||||
# define TARGET_ATTRIBUTE(target) __attribute__((__target__(target)))
|
||||
#else
|
||||
# define TARGET_ATTRIBUTE(target)
|
||||
#endif
|
||||
|
||||
/* Enable runtime BMI2 dispatch based on the CPU.
|
||||
* Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default.
|
||||
*/
|
||||
#ifndef DYNAMIC_BMI2
|
||||
#if ((defined(__clang__) && __has_attribute(__target__)) \
|
||||
|| (defined(__GNUC__) \
|
||||
&& (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) \
|
||||
&& (defined(__x86_64__) || defined(_M_X86)) \
|
||||
&& !defined(__BMI2__)
|
||||
# define DYNAMIC_BMI2 1
|
||||
#else
|
||||
# define DYNAMIC_BMI2 0
|
||||
#endif
|
||||
#endif
|
||||
|
||||
/* prefetch */
|
||||
#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) /* _mm_prefetch() is not defined outside of x86/x64 */
|
||||
# include <mmintrin.h> /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
|
||||
# define PREFETCH(ptr) _mm_prefetch((const char*)ptr, _MM_HINT_T0)
|
||||
#elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) )
|
||||
# define PREFETCH(ptr) __builtin_prefetch(ptr, 0, 0)
|
||||
#else
|
||||
# define PREFETCH(ptr) /* disabled */
|
||||
#endif
|
||||
|
||||
/* disable warnings */
|
||||
#ifdef _MSC_VER /* Visual Studio */
|
||||
# include <intrin.h> /* For Visual 2005 */
|
||||
# pragma warning(disable : 4100) /* disable: C4100: unreferenced formal parameter */
|
||||
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
|
||||
# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
|
||||
# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */
|
||||
# pragma warning(disable : 4324) /* disable: C4324: padded structure */
|
||||
#endif
|
||||
|
||||
#endif /* ZSTD_COMPILER_H */
|
||||
|
|
@ -0,0 +1,215 @@
|
|||
/*
|
||||
* Copyright (c) 2018-present, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#ifndef ZSTD_COMMON_CPU_H
|
||||
#define ZSTD_COMMON_CPU_H
|
||||
|
||||
/**
|
||||
* Implementation taken from folly/CpuId.h
|
||||
* https://github.com/facebook/folly/blob/master/folly/CpuId.h
|
||||
*/
|
||||
|
||||
#include <string.h>
|
||||
|
||||
#include "mem.h"
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#include <intrin.h>
|
||||
#endif
|
||||
|
||||
typedef struct {
|
||||
U32 f1c;
|
||||
U32 f1d;
|
||||
U32 f7b;
|
||||
U32 f7c;
|
||||
} ZSTD_cpuid_t;
|
||||
|
||||
MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) {
|
||||
U32 f1c = 0;
|
||||
U32 f1d = 0;
|
||||
U32 f7b = 0;
|
||||
U32 f7c = 0;
|
||||
#ifdef _MSC_VER
|
||||
int reg[4];
|
||||
__cpuid((int*)reg, 0);
|
||||
{
|
||||
int const n = reg[0];
|
||||
if (n >= 1) {
|
||||
__cpuid((int*)reg, 1);
|
||||
f1c = (U32)reg[2];
|
||||
f1d = (U32)reg[3];
|
||||
}
|
||||
if (n >= 7) {
|
||||
__cpuidex((int*)reg, 7, 0);
|
||||
f7b = (U32)reg[1];
|
||||
f7c = (U32)reg[2];
|
||||
}
|
||||
}
|
||||
#elif defined(__i386__) && defined(__PIC__) && !defined(__clang__) && defined(__GNUC__)
|
||||
/* The following block like the normal cpuid branch below, but gcc
|
||||
* reserves ebx for use of its pic register so we must specially
|
||||
* handle the save and restore to avoid clobbering the register
|
||||
*/
|
||||
U32 n;
|
||||
__asm__(
|
||||
"pushl %%ebx\n\t"
|
||||
"cpuid\n\t"
|
||||
"popl %%ebx\n\t"
|
||||
: "=a"(n)
|
||||
: "a"(0)
|
||||
: "ecx", "edx");
|
||||
if (n >= 1) {
|
||||
U32 f1a;
|
||||
__asm__(
|
||||
"pushl %%ebx\n\t"
|
||||
"cpuid\n\t"
|
||||
"popl %%ebx\n\t"
|
||||
: "=a"(f1a), "=c"(f1c), "=d"(f1d)
|
||||
: "a"(1));
|
||||
}
|
||||
if (n >= 7) {
|
||||
__asm__(
|
||||
"pushl %%ebx\n\t"
|
||||
"cpuid\n\t"
|
||||
"movl %%ebx, %%eax\n\r"
|
||||
"popl %%ebx"
|
||||
: "=a"(f7b), "=c"(f7c)
|
||||
: "a"(7), "c"(0)
|
||||
: "edx");
|
||||
}
|
||||
#elif defined(__x86_64__) || defined(_M_X64) || defined(__i386__)
|
||||
U32 n;
|
||||
__asm__("cpuid" : "=a"(n) : "a"(0) : "ebx", "ecx", "edx");
|
||||
if (n >= 1) {
|
||||
U32 f1a;
|
||||
__asm__("cpuid" : "=a"(f1a), "=c"(f1c), "=d"(f1d) : "a"(1) : "ebx");
|
||||
}
|
||||
if (n >= 7) {
|
||||
U32 f7a;
|
||||
__asm__("cpuid"
|
||||
: "=a"(f7a), "=b"(f7b), "=c"(f7c)
|
||||
: "a"(7), "c"(0)
|
||||
: "edx");
|
||||
}
|
||||
#endif
|
||||
{
|
||||
ZSTD_cpuid_t cpuid;
|
||||
cpuid.f1c = f1c;
|
||||
cpuid.f1d = f1d;
|
||||
cpuid.f7b = f7b;
|
||||
cpuid.f7c = f7c;
|
||||
return cpuid;
|
||||
}
|
||||
}
|
||||
|
||||
#define X(name, r, bit) \
|
||||
MEM_STATIC int ZSTD_cpuid_##name(ZSTD_cpuid_t const cpuid) { \
|
||||
return ((cpuid.r) & (1U << bit)) != 0; \
|
||||
}
|
||||
|
||||
/* cpuid(1): Processor Info and Feature Bits. */
|
||||
#define C(name, bit) X(name, f1c, bit)
|
||||
C(sse3, 0)
|
||||
C(pclmuldq, 1)
|
||||
C(dtes64, 2)
|
||||
C(monitor, 3)
|
||||
C(dscpl, 4)
|
||||
C(vmx, 5)
|
||||
C(smx, 6)
|
||||
C(eist, 7)
|
||||
C(tm2, 8)
|
||||
C(ssse3, 9)
|
||||
C(cnxtid, 10)
|
||||
C(fma, 12)
|
||||
C(cx16, 13)
|
||||
C(xtpr, 14)
|
||||
C(pdcm, 15)
|
||||
C(pcid, 17)
|
||||
C(dca, 18)
|
||||
C(sse41, 19)
|
||||
C(sse42, 20)
|
||||
C(x2apic, 21)
|
||||
C(movbe, 22)
|
||||
C(popcnt, 23)
|
||||
C(tscdeadline, 24)
|
||||
C(aes, 25)
|
||||
C(xsave, 26)
|
||||
C(osxsave, 27)
|
||||
C(avx, 28)
|
||||
C(f16c, 29)
|
||||
C(rdrand, 30)
|
||||
#undef C
|
||||
#define D(name, bit) X(name, f1d, bit)
|
||||
D(fpu, 0)
|
||||
D(vme, 1)
|
||||
D(de, 2)
|
||||
D(pse, 3)
|
||||
D(tsc, 4)
|
||||
D(msr, 5)
|
||||
D(pae, 6)
|
||||
D(mce, 7)
|
||||
D(cx8, 8)
|
||||
D(apic, 9)
|
||||
D(sep, 11)
|
||||
D(mtrr, 12)
|
||||
D(pge, 13)
|
||||
D(mca, 14)
|
||||
D(cmov, 15)
|
||||
D(pat, 16)
|
||||
D(pse36, 17)
|
||||
D(psn, 18)
|
||||
D(clfsh, 19)
|
||||
D(ds, 21)
|
||||
D(acpi, 22)
|
||||
D(mmx, 23)
|
||||
D(fxsr, 24)
|
||||
D(sse, 25)
|
||||
D(sse2, 26)
|
||||
D(ss, 27)
|
||||
D(htt, 28)
|
||||
D(tm, 29)
|
||||
D(pbe, 31)
|
||||
#undef D
|
||||
|
||||
/* cpuid(7): Extended Features. */
|
||||
#define B(name, bit) X(name, f7b, bit)
|
||||
B(bmi1, 3)
|
||||
B(hle, 4)
|
||||
B(avx2, 5)
|
||||
B(smep, 7)
|
||||
B(bmi2, 8)
|
||||
B(erms, 9)
|
||||
B(invpcid, 10)
|
||||
B(rtm, 11)
|
||||
B(mpx, 14)
|
||||
B(avx512f, 16)
|
||||
B(avx512dq, 17)
|
||||
B(rdseed, 18)
|
||||
B(adx, 19)
|
||||
B(smap, 20)
|
||||
B(avx512ifma, 21)
|
||||
B(pcommit, 22)
|
||||
B(clflushopt, 23)
|
||||
B(clwb, 24)
|
||||
B(avx512pf, 26)
|
||||
B(avx512er, 27)
|
||||
B(avx512cd, 28)
|
||||
B(sha, 29)
|
||||
B(avx512bw, 30)
|
||||
B(avx512vl, 31)
|
||||
#undef B
|
||||
#define C(name, bit) X(name, f7c, bit)
|
||||
C(prefetchwt1, 0)
|
||||
C(avx512vbmi, 1)
|
||||
#undef C
|
||||
|
||||
#undef X
|
||||
|
||||
#endif /* ZSTD_COMMON_CPU_H */
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
/* ******************************************************************
|
||||
debug
|
||||
Part of FSE library
|
||||
Copyright (C) 2013-present, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
****************************************************************** */
|
||||
|
||||
|
||||
/*
|
||||
* This module only hosts one global variable
|
||||
* which can be used to dynamically influence the verbosity of traces,
|
||||
* such as DEBUGLOG and RAWLOG
|
||||
*/
|
||||
|
||||
#include "debug.h"
|
||||
|
||||
int g_debuglevel = DEBUGLEVEL;
|
||||
|
|
@ -0,0 +1,123 @@
|
|||
/* ******************************************************************
|
||||
debug
|
||||
Part of FSE library
|
||||
Copyright (C) 2013-present, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
****************************************************************** */
|
||||
|
||||
|
||||
/*
|
||||
* The purpose of this header is to enable debug functions.
|
||||
* They regroup assert(), DEBUGLOG() and RAWLOG() for run-time,
|
||||
* and DEBUG_STATIC_ASSERT() for compile-time.
|
||||
*
|
||||
* By default, DEBUGLEVEL==0, which means run-time debug is disabled.
|
||||
*
|
||||
* Level 1 enables assert() only.
|
||||
* Starting level 2, traces can be generated and pushed to stderr.
|
||||
* The higher the level, the more verbose the traces.
|
||||
*
|
||||
* It's possible to dynamically adjust level using variable g_debug_level,
|
||||
* which is only declared if DEBUGLEVEL>=2,
|
||||
* and is a global variable, not multi-thread protected (use with care)
|
||||
*/
|
||||
|
||||
#ifndef DEBUG_H_12987983217
|
||||
#define DEBUG_H_12987983217
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
/* static assert is triggered at compile time, leaving no runtime artefact,
|
||||
* but can only work with compile-time constants.
|
||||
* This variant can only be used inside a function. */
|
||||
#define DEBUG_STATIC_ASSERT(c) (void)sizeof(char[(c) ? 1 : -1])
|
||||
|
||||
|
||||
/* DEBUGLEVEL is expected to be defined externally,
|
||||
* typically through compiler command line.
|
||||
* Value must be a number. */
|
||||
#ifndef DEBUGLEVEL
|
||||
# define DEBUGLEVEL 0
|
||||
#endif
|
||||
|
||||
/* recommended values for DEBUGLEVEL :
|
||||
* 0 : no debug, all run-time functions disabled
|
||||
* 1 : no display, enables assert() only
|
||||
* 2 : reserved, for currently active debug path
|
||||
* 3 : events once per object lifetime (CCtx, CDict, etc.)
|
||||
* 4 : events once per frame
|
||||
* 5 : events once per block
|
||||
* 6 : events once per sequence (verbose)
|
||||
* 7+: events at every position (*very* verbose)
|
||||
*
|
||||
* It's generally inconvenient to output traces > 5.
|
||||
* In which case, it's possible to selectively enable higher verbosity levels
|
||||
* by modifying g_debug_level.
|
||||
*/
|
||||
|
||||
#if (DEBUGLEVEL>=1)
|
||||
# include <assert.h>
|
||||
#else
|
||||
# ifndef assert /* assert may be already defined, due to prior #include <assert.h> */
|
||||
# define assert(condition) ((void)0) /* disable assert (default) */
|
||||
# endif
|
||||
#endif
|
||||
|
||||
#if (DEBUGLEVEL>=2)
|
||||
# include <stdio.h>
|
||||
extern int g_debuglevel; /* here, this variable is only declared,
|
||||
it actually lives in debug.c,
|
||||
and is shared by the whole process.
|
||||
It's typically used to enable very verbose levels
|
||||
on selective conditions (such as position in src) */
|
||||
|
||||
# define RAWLOG(l, ...) { \
|
||||
if (l<=g_debuglevel) { \
|
||||
fprintf(stderr, __VA_ARGS__); \
|
||||
} }
|
||||
# define DEBUGLOG(l, ...) { \
|
||||
if (l<=g_debuglevel) { \
|
||||
fprintf(stderr, __FILE__ ": " __VA_ARGS__); \
|
||||
fprintf(stderr, " \n"); \
|
||||
} }
|
||||
#else
|
||||
# define RAWLOG(l, ...) {} /* disabled */
|
||||
# define DEBUGLOG(l, ...) {} /* disabled */
|
||||
#endif
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* DEBUG_H_12987983217 */
|
||||
|
|
@ -0,0 +1,236 @@
|
|||
/*
|
||||
Common functions of New Generation Entropy library
|
||||
Copyright (C) 2016, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
*************************************************************************** */
|
||||
|
||||
/* *************************************
|
||||
* Dependencies
|
||||
***************************************/
|
||||
#include "mem.h"
|
||||
#include "error_private.h" /* ERR_*, ERROR */
|
||||
#define FSE_STATIC_LINKING_ONLY /* FSE_MIN_TABLELOG */
|
||||
#include "fse.h"
|
||||
#define HUF_STATIC_LINKING_ONLY /* HUF_TABLELOG_ABSOLUTEMAX */
|
||||
#include "huf.h"
|
||||
|
||||
|
||||
/*=== Version ===*/
|
||||
unsigned FSE_versionNumber(void) { return FSE_VERSION_NUMBER; }
|
||||
|
||||
|
||||
/*=== Error Management ===*/
|
||||
unsigned FSE_isError(size_t code) { return ERR_isError(code); }
|
||||
const char* FSE_getErrorName(size_t code) { return ERR_getErrorName(code); }
|
||||
|
||||
unsigned HUF_isError(size_t code) { return ERR_isError(code); }
|
||||
const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); }
|
||||
|
||||
|
||||
/*-**************************************************************
|
||||
* FSE NCount encoding-decoding
|
||||
****************************************************************/
|
||||
size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
|
||||
const void* headerBuffer, size_t hbSize)
|
||||
{
|
||||
const BYTE* const istart = (const BYTE*) headerBuffer;
|
||||
const BYTE* const iend = istart + hbSize;
|
||||
const BYTE* ip = istart;
|
||||
int nbBits;
|
||||
int remaining;
|
||||
int threshold;
|
||||
U32 bitStream;
|
||||
int bitCount;
|
||||
unsigned charnum = 0;
|
||||
int previous0 = 0;
|
||||
|
||||
if (hbSize < 4) {
|
||||
/* This function only works when hbSize >= 4 */
|
||||
char buffer[4];
|
||||
memset(buffer, 0, sizeof(buffer));
|
||||
memcpy(buffer, headerBuffer, hbSize);
|
||||
{ size_t const countSize = FSE_readNCount(normalizedCounter, maxSVPtr, tableLogPtr,
|
||||
buffer, sizeof(buffer));
|
||||
if (FSE_isError(countSize)) return countSize;
|
||||
if (countSize > hbSize) return ERROR(corruption_detected);
|
||||
return countSize;
|
||||
} }
|
||||
assert(hbSize >= 4);
|
||||
|
||||
/* init */
|
||||
memset(normalizedCounter, 0, (*maxSVPtr+1) * sizeof(normalizedCounter[0])); /* all symbols not present in NCount have a frequency of 0 */
|
||||
bitStream = MEM_readLE32(ip);
|
||||
nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */
|
||||
if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge);
|
||||
bitStream >>= 4;
|
||||
bitCount = 4;
|
||||
*tableLogPtr = nbBits;
|
||||
remaining = (1<<nbBits)+1;
|
||||
threshold = 1<<nbBits;
|
||||
nbBits++;
|
||||
|
||||
while ((remaining>1) & (charnum<=*maxSVPtr)) {
|
||||
if (previous0) {
|
||||
unsigned n0 = charnum;
|
||||
while ((bitStream & 0xFFFF) == 0xFFFF) {
|
||||
n0 += 24;
|
||||
if (ip < iend-5) {
|
||||
ip += 2;
|
||||
bitStream = MEM_readLE32(ip) >> bitCount;
|
||||
} else {
|
||||
bitStream >>= 16;
|
||||
bitCount += 16;
|
||||
} }
|
||||
while ((bitStream & 3) == 3) {
|
||||
n0 += 3;
|
||||
bitStream >>= 2;
|
||||
bitCount += 2;
|
||||
}
|
||||
n0 += bitStream & 3;
|
||||
bitCount += 2;
|
||||
if (n0 > *maxSVPtr) return ERROR(maxSymbolValue_tooSmall);
|
||||
while (charnum < n0) normalizedCounter[charnum++] = 0;
|
||||
if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) {
|
||||
assert((bitCount >> 3) <= 3); /* For first condition to work */
|
||||
ip += bitCount>>3;
|
||||
bitCount &= 7;
|
||||
bitStream = MEM_readLE32(ip) >> bitCount;
|
||||
} else {
|
||||
bitStream >>= 2;
|
||||
} }
|
||||
{ int const max = (2*threshold-1) - remaining;
|
||||
int count;
|
||||
|
||||
if ((bitStream & (threshold-1)) < (U32)max) {
|
||||
count = bitStream & (threshold-1);
|
||||
bitCount += nbBits-1;
|
||||
} else {
|
||||
count = bitStream & (2*threshold-1);
|
||||
if (count >= threshold) count -= max;
|
||||
bitCount += nbBits;
|
||||
}
|
||||
|
||||
count--; /* extra accuracy */
|
||||
remaining -= count < 0 ? -count : count; /* -1 means +1 */
|
||||
normalizedCounter[charnum++] = (short)count;
|
||||
previous0 = !count;
|
||||
while (remaining < threshold) {
|
||||
nbBits--;
|
||||
threshold >>= 1;
|
||||
}
|
||||
|
||||
if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) {
|
||||
ip += bitCount>>3;
|
||||
bitCount &= 7;
|
||||
} else {
|
||||
bitCount -= (int)(8 * (iend - 4 - ip));
|
||||
ip = iend - 4;
|
||||
}
|
||||
bitStream = MEM_readLE32(ip) >> (bitCount & 31);
|
||||
} } /* while ((remaining>1) & (charnum<=*maxSVPtr)) */
|
||||
if (remaining != 1) return ERROR(corruption_detected);
|
||||
if (bitCount > 32) return ERROR(corruption_detected);
|
||||
*maxSVPtr = charnum-1;
|
||||
|
||||
ip += (bitCount+7)>>3;
|
||||
return ip-istart;
|
||||
}
|
||||
|
||||
|
||||
/*! HUF_readStats() :
|
||||
Read compact Huffman tree, saved by HUF_writeCTable().
|
||||
`huffWeight` is destination buffer.
|
||||
`rankStats` is assumed to be a table of at least HUF_TABLELOG_MAX U32.
|
||||
@return : size read from `src` , or an error Code .
|
||||
Note : Needed by HUF_readCTable() and HUF_readDTableX?() .
|
||||
*/
|
||||
size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
|
||||
U32* nbSymbolsPtr, U32* tableLogPtr,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
U32 weightTotal;
|
||||
const BYTE* ip = (const BYTE*) src;
|
||||
size_t iSize;
|
||||
size_t oSize;
|
||||
|
||||
if (!srcSize) return ERROR(srcSize_wrong);
|
||||
iSize = ip[0];
|
||||
/* memset(huffWeight, 0, hwSize); *//* is not necessary, even though some analyzer complain ... */
|
||||
|
||||
if (iSize >= 128) { /* special header */
|
||||
oSize = iSize - 127;
|
||||
iSize = ((oSize+1)/2);
|
||||
if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
|
||||
if (oSize >= hwSize) return ERROR(corruption_detected);
|
||||
ip += 1;
|
||||
{ U32 n;
|
||||
for (n=0; n<oSize; n+=2) {
|
||||
huffWeight[n] = ip[n/2] >> 4;
|
||||
huffWeight[n+1] = ip[n/2] & 15;
|
||||
} } }
|
||||
else { /* header compressed with FSE (normal case) */
|
||||
FSE_DTable fseWorkspace[FSE_DTABLE_SIZE_U32(6)]; /* 6 is max possible tableLog for HUF header (maybe even 5, to be tested) */
|
||||
if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
|
||||
oSize = FSE_decompress_wksp(huffWeight, hwSize-1, ip+1, iSize, fseWorkspace, 6); /* max (hwSize-1) values decoded, as last one is implied */
|
||||
if (FSE_isError(oSize)) return oSize;
|
||||
}
|
||||
|
||||
/* collect weight stats */
|
||||
memset(rankStats, 0, (HUF_TABLELOG_MAX + 1) * sizeof(U32));
|
||||
weightTotal = 0;
|
||||
{ U32 n; for (n=0; n<oSize; n++) {
|
||||
if (huffWeight[n] >= HUF_TABLELOG_MAX) return ERROR(corruption_detected);
|
||||
rankStats[huffWeight[n]]++;
|
||||
weightTotal += (1 << huffWeight[n]) >> 1;
|
||||
} }
|
||||
if (weightTotal == 0) return ERROR(corruption_detected);
|
||||
|
||||
/* get last non-null symbol weight (implied, total must be 2^n) */
|
||||
{ U32 const tableLog = BIT_highbit32(weightTotal) + 1;
|
||||
if (tableLog > HUF_TABLELOG_MAX) return ERROR(corruption_detected);
|
||||
*tableLogPtr = tableLog;
|
||||
/* determine last weight */
|
||||
{ U32 const total = 1 << tableLog;
|
||||
U32 const rest = total - weightTotal;
|
||||
U32 const verif = 1 << BIT_highbit32(rest);
|
||||
U32 const lastWeight = BIT_highbit32(rest) + 1;
|
||||
if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */
|
||||
huffWeight[oSize] = (BYTE)lastWeight;
|
||||
rankStats[lastWeight]++;
|
||||
} }
|
||||
|
||||
/* check tree construction validity */
|
||||
if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */
|
||||
|
||||
/* results */
|
||||
*nbSymbolsPtr = (U32)(oSize+1);
|
||||
return iSize+1;
|
||||
}
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
/*
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
/* The purpose of this file is to have a single list of error strings embedded in binary */
|
||||
|
||||
#include "error_private.h"
|
||||
|
||||
const char* ERR_getErrorString(ERR_enum code)
|
||||
{
|
||||
static const char* const notErrorCode = "Unspecified error code";
|
||||
switch( code )
|
||||
{
|
||||
case PREFIX(no_error): return "No error detected";
|
||||
case PREFIX(GENERIC): return "Error (generic)";
|
||||
case PREFIX(prefix_unknown): return "Unknown frame descriptor";
|
||||
case PREFIX(version_unsupported): return "Version not supported";
|
||||
case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter";
|
||||
case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding";
|
||||
case PREFIX(corruption_detected): return "Corrupted block detected";
|
||||
case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";
|
||||
case PREFIX(parameter_unsupported): return "Unsupported parameter";
|
||||
case PREFIX(parameter_outOfBound): return "Parameter is out of bound";
|
||||
case PREFIX(init_missing): return "Context should be init first";
|
||||
case PREFIX(memory_allocation): return "Allocation error : not enough memory";
|
||||
case PREFIX(workSpace_tooSmall): return "workSpace buffer is not large enough";
|
||||
case PREFIX(stage_wrong): return "Operation not authorized at current processing stage";
|
||||
case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
|
||||
case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
|
||||
case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
|
||||
case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
|
||||
case PREFIX(dictionary_wrong): return "Dictionary mismatch";
|
||||
case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples";
|
||||
case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
|
||||
case PREFIX(srcSize_wrong): return "Src size is incorrect";
|
||||
/* following error codes are not stable and may be removed or changed in a future version */
|
||||
case PREFIX(frameIndex_tooLarge): return "Frame index is too large";
|
||||
case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking";
|
||||
case PREFIX(maxCode):
|
||||
default: return notErrorCode;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,76 @@
|
|||
/*
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
/* Note : this module is expected to remain private, do not expose it */
|
||||
|
||||
#ifndef ERROR_H_MODULE
|
||||
#define ERROR_H_MODULE
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
/* ****************************************
|
||||
* Dependencies
|
||||
******************************************/
|
||||
#include <stddef.h> /* size_t */
|
||||
#include "zstd_errors.h" /* enum list */
|
||||
|
||||
|
||||
/* ****************************************
|
||||
* Compiler-specific
|
||||
******************************************/
|
||||
#if defined(__GNUC__)
|
||||
# define ERR_STATIC static __attribute__((unused))
|
||||
#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
|
||||
# define ERR_STATIC static inline
|
||||
#elif defined(_MSC_VER)
|
||||
# define ERR_STATIC static __inline
|
||||
#else
|
||||
# define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
|
||||
#endif
|
||||
|
||||
|
||||
/*-****************************************
|
||||
* Customization (error_public.h)
|
||||
******************************************/
|
||||
typedef ZSTD_ErrorCode ERR_enum;
|
||||
#define PREFIX(name) ZSTD_error_##name
|
||||
|
||||
|
||||
/*-****************************************
|
||||
* Error codes handling
|
||||
******************************************/
|
||||
#undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */
|
||||
#define ERROR(name) ZSTD_ERROR(name)
|
||||
#define ZSTD_ERROR(name) ((size_t)-PREFIX(name))
|
||||
|
||||
ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); }
|
||||
|
||||
ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); }
|
||||
|
||||
|
||||
/*-****************************************
|
||||
* Error Strings
|
||||
******************************************/
|
||||
|
||||
const char* ERR_getErrorString(ERR_enum code); /* error_private.c */
|
||||
|
||||
ERR_STATIC const char* ERR_getErrorName(size_t code)
|
||||
{
|
||||
return ERR_getErrorString(ERR_getErrorCode(code));
|
||||
}
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ERROR_H_MODULE */
|
||||
|
|
@ -0,0 +1,708 @@
|
|||
/* ******************************************************************
|
||||
FSE : Finite State Entropy codec
|
||||
Public Prototypes declaration
|
||||
Copyright (C) 2013-2016, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
****************************************************************** */
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#ifndef FSE_H
|
||||
#define FSE_H
|
||||
|
||||
|
||||
/*-*****************************************
|
||||
* Dependencies
|
||||
******************************************/
|
||||
#include <stddef.h> /* size_t, ptrdiff_t */
|
||||
|
||||
|
||||
/*-*****************************************
|
||||
* FSE_PUBLIC_API : control library symbols visibility
|
||||
******************************************/
|
||||
#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4)
|
||||
# define FSE_PUBLIC_API __attribute__ ((visibility ("default")))
|
||||
#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) /* Visual expected */
|
||||
# define FSE_PUBLIC_API __declspec(dllexport)
|
||||
#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1)
|
||||
# define FSE_PUBLIC_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
|
||||
#else
|
||||
# define FSE_PUBLIC_API
|
||||
#endif
|
||||
|
||||
/*------ Version ------*/
|
||||
#define FSE_VERSION_MAJOR 0
|
||||
#define FSE_VERSION_MINOR 9
|
||||
#define FSE_VERSION_RELEASE 0
|
||||
|
||||
#define FSE_LIB_VERSION FSE_VERSION_MAJOR.FSE_VERSION_MINOR.FSE_VERSION_RELEASE
|
||||
#define FSE_QUOTE(str) #str
|
||||
#define FSE_EXPAND_AND_QUOTE(str) FSE_QUOTE(str)
|
||||
#define FSE_VERSION_STRING FSE_EXPAND_AND_QUOTE(FSE_LIB_VERSION)
|
||||
|
||||
#define FSE_VERSION_NUMBER (FSE_VERSION_MAJOR *100*100 + FSE_VERSION_MINOR *100 + FSE_VERSION_RELEASE)
|
||||
FSE_PUBLIC_API unsigned FSE_versionNumber(void); /**< library version number; to be used when checking dll version */
|
||||
|
||||
|
||||
/*-****************************************
|
||||
* FSE simple functions
|
||||
******************************************/
|
||||
/*! FSE_compress() :
|
||||
Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'.
|
||||
'dst' buffer must be already allocated. Compression runs faster is dstCapacity >= FSE_compressBound(srcSize).
|
||||
@return : size of compressed data (<= dstCapacity).
|
||||
Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
|
||||
if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead.
|
||||
if FSE_isError(return), compression failed (more details using FSE_getErrorName())
|
||||
*/
|
||||
FSE_PUBLIC_API size_t FSE_compress(void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize);
|
||||
|
||||
/*! FSE_decompress():
|
||||
Decompress FSE data from buffer 'cSrc', of size 'cSrcSize',
|
||||
into already allocated destination buffer 'dst', of size 'dstCapacity'.
|
||||
@return : size of regenerated data (<= maxDstSize),
|
||||
or an error code, which can be tested using FSE_isError() .
|
||||
|
||||
** Important ** : FSE_decompress() does not decompress non-compressible nor RLE data !!!
|
||||
Why ? : making this distinction requires a header.
|
||||
Header management is intentionally delegated to the user layer, which can better manage special cases.
|
||||
*/
|
||||
FSE_PUBLIC_API size_t FSE_decompress(void* dst, size_t dstCapacity,
|
||||
const void* cSrc, size_t cSrcSize);
|
||||
|
||||
|
||||
/*-*****************************************
|
||||
* Tool functions
|
||||
******************************************/
|
||||
FSE_PUBLIC_API size_t FSE_compressBound(size_t size); /* maximum compressed size */
|
||||
|
||||
/* Error Management */
|
||||
FSE_PUBLIC_API unsigned FSE_isError(size_t code); /* tells if a return value is an error code */
|
||||
FSE_PUBLIC_API const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */
|
||||
|
||||
|
||||
/*-*****************************************
|
||||
* FSE advanced functions
|
||||
******************************************/
|
||||
/*! FSE_compress2() :
|
||||
Same as FSE_compress(), but allows the selection of 'maxSymbolValue' and 'tableLog'
|
||||
Both parameters can be defined as '0' to mean : use default value
|
||||
@return : size of compressed data
|
||||
Special values : if return == 0, srcData is not compressible => Nothing is stored within cSrc !!!
|
||||
if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression.
|
||||
if FSE_isError(return), it's an error code.
|
||||
*/
|
||||
FSE_PUBLIC_API size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
|
||||
|
||||
|
||||
/*-*****************************************
|
||||
* FSE detailed API
|
||||
******************************************/
|
||||
/*!
|
||||
FSE_compress() does the following:
|
||||
1. count symbol occurrence from source[] into table count[] (see hist.h)
|
||||
2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog)
|
||||
3. save normalized counters to memory buffer using writeNCount()
|
||||
4. build encoding table 'CTable' from normalized counters
|
||||
5. encode the data stream using encoding table 'CTable'
|
||||
|
||||
FSE_decompress() does the following:
|
||||
1. read normalized counters with readNCount()
|
||||
2. build decoding table 'DTable' from normalized counters
|
||||
3. decode the data stream using decoding table 'DTable'
|
||||
|
||||
The following API allows targeting specific sub-functions for advanced tasks.
|
||||
For example, it's possible to compress several blocks using the same 'CTable',
|
||||
or to save and provide normalized distribution using external method.
|
||||
*/
|
||||
|
||||
/* *** COMPRESSION *** */
|
||||
|
||||
/*! FSE_optimalTableLog():
|
||||
dynamically downsize 'tableLog' when conditions are met.
|
||||
It saves CPU time, by using smaller tables, while preserving or even improving compression ratio.
|
||||
@return : recommended tableLog (necessarily <= 'maxTableLog') */
|
||||
FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
|
||||
|
||||
/*! FSE_normalizeCount():
|
||||
normalize counts so that sum(count[]) == Power_of_2 (2^tableLog)
|
||||
'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1).
|
||||
@return : tableLog,
|
||||
or an errorCode, which can be tested using FSE_isError() */
|
||||
FSE_PUBLIC_API size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog,
|
||||
const unsigned* count, size_t srcSize, unsigned maxSymbolValue);
|
||||
|
||||
/*! FSE_NCountWriteBound():
|
||||
Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'.
|
||||
Typically useful for allocation purpose. */
|
||||
FSE_PUBLIC_API size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog);
|
||||
|
||||
/*! FSE_writeNCount():
|
||||
Compactly save 'normalizedCounter' into 'buffer'.
|
||||
@return : size of the compressed table,
|
||||
or an errorCode, which can be tested using FSE_isError(). */
|
||||
FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize,
|
||||
const short* normalizedCounter,
|
||||
unsigned maxSymbolValue, unsigned tableLog);
|
||||
|
||||
/*! Constructor and Destructor of FSE_CTable.
|
||||
Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */
|
||||
typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */
|
||||
FSE_PUBLIC_API FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog);
|
||||
FSE_PUBLIC_API void FSE_freeCTable (FSE_CTable* ct);
|
||||
|
||||
/*! FSE_buildCTable():
|
||||
Builds `ct`, which must be already allocated, using FSE_createCTable().
|
||||
@return : 0, or an errorCode, which can be tested using FSE_isError() */
|
||||
FSE_PUBLIC_API size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
|
||||
|
||||
/*! FSE_compress_usingCTable():
|
||||
Compress `src` using `ct` into `dst` which must be already allocated.
|
||||
@return : size of compressed data (<= `dstCapacity`),
|
||||
or 0 if compressed data could not fit into `dst`,
|
||||
or an errorCode, which can be tested using FSE_isError() */
|
||||
FSE_PUBLIC_API size_t FSE_compress_usingCTable (void* dst, size_t dstCapacity, const void* src, size_t srcSize, const FSE_CTable* ct);
|
||||
|
||||
/*!
|
||||
Tutorial :
|
||||
----------
|
||||
The first step is to count all symbols. FSE_count() does this job very fast.
|
||||
Result will be saved into 'count', a table of unsigned int, which must be already allocated, and have 'maxSymbolValuePtr[0]+1' cells.
|
||||
'src' is a table of bytes of size 'srcSize'. All values within 'src' MUST be <= maxSymbolValuePtr[0]
|
||||
maxSymbolValuePtr[0] will be updated, with its real value (necessarily <= original value)
|
||||
FSE_count() will return the number of occurrence of the most frequent symbol.
|
||||
This can be used to know if there is a single symbol within 'src', and to quickly evaluate its compressibility.
|
||||
If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).
|
||||
|
||||
The next step is to normalize the frequencies.
|
||||
FSE_normalizeCount() will ensure that sum of frequencies is == 2 ^'tableLog'.
|
||||
It also guarantees a minimum of 1 to any Symbol with frequency >= 1.
|
||||
You can use 'tableLog'==0 to mean "use default tableLog value".
|
||||
If you are unsure of which tableLog value to use, you can ask FSE_optimalTableLog(),
|
||||
which will provide the optimal valid tableLog given sourceSize, maxSymbolValue, and a user-defined maximum (0 means "default").
|
||||
|
||||
The result of FSE_normalizeCount() will be saved into a table,
|
||||
called 'normalizedCounter', which is a table of signed short.
|
||||
'normalizedCounter' must be already allocated, and have at least 'maxSymbolValue+1' cells.
|
||||
The return value is tableLog if everything proceeded as expected.
|
||||
It is 0 if there is a single symbol within distribution.
|
||||
If there is an error (ex: invalid tableLog value), the function will return an ErrorCode (which can be tested using FSE_isError()).
|
||||
|
||||
'normalizedCounter' can be saved in a compact manner to a memory area using FSE_writeNCount().
|
||||
'buffer' must be already allocated.
|
||||
For guaranteed success, buffer size must be at least FSE_headerBound().
|
||||
The result of the function is the number of bytes written into 'buffer'.
|
||||
If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError(); ex : buffer size too small).
|
||||
|
||||
'normalizedCounter' can then be used to create the compression table 'CTable'.
|
||||
The space required by 'CTable' must be already allocated, using FSE_createCTable().
|
||||
You can then use FSE_buildCTable() to fill 'CTable'.
|
||||
If there is an error, both functions will return an ErrorCode (which can be tested using FSE_isError()).
|
||||
|
||||
'CTable' can then be used to compress 'src', with FSE_compress_usingCTable().
|
||||
Similar to FSE_count(), the convention is that 'src' is assumed to be a table of char of size 'srcSize'
|
||||
The function returns the size of compressed data (without header), necessarily <= `dstCapacity`.
|
||||
If it returns '0', compressed data could not fit into 'dst'.
|
||||
If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).
|
||||
*/
|
||||
|
||||
|
||||
/* *** DECOMPRESSION *** */
|
||||
|
||||
/*! FSE_readNCount():
|
||||
Read compactly saved 'normalizedCounter' from 'rBuffer'.
|
||||
@return : size read from 'rBuffer',
|
||||
or an errorCode, which can be tested using FSE_isError().
|
||||
maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */
|
||||
FSE_PUBLIC_API size_t FSE_readNCount (short* normalizedCounter,
|
||||
unsigned* maxSymbolValuePtr, unsigned* tableLogPtr,
|
||||
const void* rBuffer, size_t rBuffSize);
|
||||
|
||||
/*! Constructor and Destructor of FSE_DTable.
|
||||
Note that its size depends on 'tableLog' */
|
||||
typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */
|
||||
FSE_PUBLIC_API FSE_DTable* FSE_createDTable(unsigned tableLog);
|
||||
FSE_PUBLIC_API void FSE_freeDTable(FSE_DTable* dt);
|
||||
|
||||
/*! FSE_buildDTable():
|
||||
Builds 'dt', which must be already allocated, using FSE_createDTable().
|
||||
return : 0, or an errorCode, which can be tested using FSE_isError() */
|
||||
FSE_PUBLIC_API size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
|
||||
|
||||
/*! FSE_decompress_usingDTable():
|
||||
Decompress compressed source `cSrc` of size `cSrcSize` using `dt`
|
||||
into `dst` which must be already allocated.
|
||||
@return : size of regenerated data (necessarily <= `dstCapacity`),
|
||||
or an errorCode, which can be tested using FSE_isError() */
|
||||
FSE_PUBLIC_API size_t FSE_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt);
|
||||
|
||||
/*!
|
||||
Tutorial :
|
||||
----------
|
||||
(Note : these functions only decompress FSE-compressed blocks.
|
||||
If block is uncompressed, use memcpy() instead
|
||||
If block is a single repeated byte, use memset() instead )
|
||||
|
||||
The first step is to obtain the normalized frequencies of symbols.
|
||||
This can be performed by FSE_readNCount() if it was saved using FSE_writeNCount().
|
||||
'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short.
|
||||
In practice, that means it's necessary to know 'maxSymbolValue' beforehand,
|
||||
or size the table to handle worst case situations (typically 256).
|
||||
FSE_readNCount() will provide 'tableLog' and 'maxSymbolValue'.
|
||||
The result of FSE_readNCount() is the number of bytes read from 'rBuffer'.
|
||||
Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that.
|
||||
If there is an error, the function will return an error code, which can be tested using FSE_isError().
|
||||
|
||||
The next step is to build the decompression tables 'FSE_DTable' from 'normalizedCounter'.
|
||||
This is performed by the function FSE_buildDTable().
|
||||
The space required by 'FSE_DTable' must be already allocated using FSE_createDTable().
|
||||
If there is an error, the function will return an error code, which can be tested using FSE_isError().
|
||||
|
||||
`FSE_DTable` can then be used to decompress `cSrc`, with FSE_decompress_usingDTable().
|
||||
`cSrcSize` must be strictly correct, otherwise decompression will fail.
|
||||
FSE_decompress_usingDTable() result will tell how many bytes were regenerated (<=`dstCapacity`).
|
||||
If there is an error, the function will return an error code, which can be tested using FSE_isError(). (ex: dst buffer too small)
|
||||
*/
|
||||
|
||||
#endif /* FSE_H */
|
||||
|
||||
#if defined(FSE_STATIC_LINKING_ONLY) && !defined(FSE_H_FSE_STATIC_LINKING_ONLY)
|
||||
#define FSE_H_FSE_STATIC_LINKING_ONLY
|
||||
|
||||
/* *** Dependency *** */
|
||||
#include "bitstream.h"
|
||||
|
||||
|
||||
/* *****************************************
|
||||
* Static allocation
|
||||
*******************************************/
|
||||
/* FSE buffer bounds */
|
||||
#define FSE_NCOUNTBOUND 512
|
||||
#define FSE_BLOCKBOUND(size) (size + (size>>7))
|
||||
#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
|
||||
|
||||
/* It is possible to statically allocate FSE CTable/DTable as a table of FSE_CTable/FSE_DTable using below macros */
|
||||
#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<(maxTableLog-1)) + ((maxSymbolValue+1)*2))
|
||||
#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<maxTableLog))
|
||||
|
||||
/* or use the size to malloc() space directly. Pay attention to alignment restrictions though */
|
||||
#define FSE_CTABLE_SIZE(maxTableLog, maxSymbolValue) (FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(FSE_CTable))
|
||||
#define FSE_DTABLE_SIZE(maxTableLog) (FSE_DTABLE_SIZE_U32(maxTableLog) * sizeof(FSE_DTable))
|
||||
|
||||
|
||||
/* *****************************************
|
||||
* FSE advanced API
|
||||
***************************************** */
|
||||
|
||||
unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus);
|
||||
/**< same as FSE_optimalTableLog(), which used `minus==2` */
|
||||
|
||||
/* FSE_compress_wksp() :
|
||||
* Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`).
|
||||
* FSE_WKSP_SIZE_U32() provides the minimum size required for `workSpace` as a table of FSE_CTable.
|
||||
*/
|
||||
#define FSE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ( FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) + ((maxTableLog > 12) ? (1 << (maxTableLog - 2)) : 1024) )
|
||||
size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
|
||||
|
||||
size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits);
|
||||
/**< build a fake FSE_CTable, designed for a flat distribution, where each symbol uses nbBits */
|
||||
|
||||
size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue);
|
||||
/**< build a fake FSE_CTable, designed to compress always the same symbolValue */
|
||||
|
||||
/* FSE_buildCTable_wksp() :
|
||||
* Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`).
|
||||
* `wkspSize` must be >= `(1<<tableLog)`.
|
||||
*/
|
||||
size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
|
||||
|
||||
size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits);
|
||||
/**< build a fake FSE_DTable, designed to read a flat distribution where each symbol uses nbBits */
|
||||
|
||||
size_t FSE_buildDTable_rle (FSE_DTable* dt, unsigned char symbolValue);
|
||||
/**< build a fake FSE_DTable, designed to always generate the same symbolValue */
|
||||
|
||||
size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, FSE_DTable* workSpace, unsigned maxLog);
|
||||
/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DTABLE_SIZE_U32(maxLog)` */
|
||||
|
||||
typedef enum {
|
||||
FSE_repeat_none, /**< Cannot use the previous table */
|
||||
FSE_repeat_check, /**< Can use the previous table but it must be checked */
|
||||
FSE_repeat_valid /**< Can use the previous table and it is asumed to be valid */
|
||||
} FSE_repeat;
|
||||
|
||||
/* *****************************************
|
||||
* FSE symbol compression API
|
||||
*******************************************/
|
||||
/*!
|
||||
This API consists of small unitary functions, which highly benefit from being inlined.
|
||||
Hence their body are included in next section.
|
||||
*/
|
||||
typedef struct {
|
||||
ptrdiff_t value;
|
||||
const void* stateTable;
|
||||
const void* symbolTT;
|
||||
unsigned stateLog;
|
||||
} FSE_CState_t;
|
||||
|
||||
static void FSE_initCState(FSE_CState_t* CStatePtr, const FSE_CTable* ct);
|
||||
|
||||
static void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* CStatePtr, unsigned symbol);
|
||||
|
||||
static void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* CStatePtr);
|
||||
|
||||
/**<
|
||||
These functions are inner components of FSE_compress_usingCTable().
|
||||
They allow the creation of custom streams, mixing multiple tables and bit sources.
|
||||
|
||||
A key property to keep in mind is that encoding and decoding are done **in reverse direction**.
|
||||
So the first symbol you will encode is the last you will decode, like a LIFO stack.
|
||||
|
||||
You will need a few variables to track your CStream. They are :
|
||||
|
||||
FSE_CTable ct; // Provided by FSE_buildCTable()
|
||||
BIT_CStream_t bitStream; // bitStream tracking structure
|
||||
FSE_CState_t state; // State tracking structure (can have several)
|
||||
|
||||
|
||||
The first thing to do is to init bitStream and state.
|
||||
size_t errorCode = BIT_initCStream(&bitStream, dstBuffer, maxDstSize);
|
||||
FSE_initCState(&state, ct);
|
||||
|
||||
Note that BIT_initCStream() can produce an error code, so its result should be tested, using FSE_isError();
|
||||
You can then encode your input data, byte after byte.
|
||||
FSE_encodeSymbol() outputs a maximum of 'tableLog' bits at a time.
|
||||
Remember decoding will be done in reverse direction.
|
||||
FSE_encodeByte(&bitStream, &state, symbol);
|
||||
|
||||
At any time, you can also add any bit sequence.
|
||||
Note : maximum allowed nbBits is 25, for compatibility with 32-bits decoders
|
||||
BIT_addBits(&bitStream, bitField, nbBits);
|
||||
|
||||
The above methods don't commit data to memory, they just store it into local register, for speed.
|
||||
Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
|
||||
Writing data to memory is a manual operation, performed by the flushBits function.
|
||||
BIT_flushBits(&bitStream);
|
||||
|
||||
Your last FSE encoding operation shall be to flush your last state value(s).
|
||||
FSE_flushState(&bitStream, &state);
|
||||
|
||||
Finally, you must close the bitStream.
|
||||
The function returns the size of CStream in bytes.
|
||||
If data couldn't fit into dstBuffer, it will return a 0 ( == not compressible)
|
||||
If there is an error, it returns an errorCode (which can be tested using FSE_isError()).
|
||||
size_t size = BIT_closeCStream(&bitStream);
|
||||
*/
|
||||
|
||||
|
||||
/* *****************************************
|
||||
* FSE symbol decompression API
|
||||
*******************************************/
|
||||
typedef struct {
|
||||
size_t state;
|
||||
const void* table; /* precise table may vary, depending on U16 */
|
||||
} FSE_DState_t;
|
||||
|
||||
|
||||
static void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt);
|
||||
|
||||
static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD);
|
||||
|
||||
static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr);
|
||||
|
||||
/**<
|
||||
Let's now decompose FSE_decompress_usingDTable() into its unitary components.
|
||||
You will decode FSE-encoded symbols from the bitStream,
|
||||
and also any other bitFields you put in, **in reverse order**.
|
||||
|
||||
You will need a few variables to track your bitStream. They are :
|
||||
|
||||
BIT_DStream_t DStream; // Stream context
|
||||
FSE_DState_t DState; // State context. Multiple ones are possible
|
||||
FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable()
|
||||
|
||||
The first thing to do is to init the bitStream.
|
||||
errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize);
|
||||
|
||||
You should then retrieve your initial state(s)
|
||||
(in reverse flushing order if you have several ones) :
|
||||
errorCode = FSE_initDState(&DState, &DStream, DTablePtr);
|
||||
|
||||
You can then decode your data, symbol after symbol.
|
||||
For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'.
|
||||
Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
|
||||
unsigned char symbol = FSE_decodeSymbol(&DState, &DStream);
|
||||
|
||||
You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
|
||||
Note : maximum allowed nbBits is 25, for 32-bits compatibility
|
||||
size_t bitField = BIT_readBits(&DStream, nbBits);
|
||||
|
||||
All above operations only read from local register (which size depends on size_t).
|
||||
Refueling the register from memory is manually performed by the reload method.
|
||||
endSignal = FSE_reloadDStream(&DStream);
|
||||
|
||||
BIT_reloadDStream() result tells if there is still some more data to read from DStream.
|
||||
BIT_DStream_unfinished : there is still some data left into the DStream.
|
||||
BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
|
||||
BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
|
||||
BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
|
||||
|
||||
When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
|
||||
to properly detect the exact end of stream.
|
||||
After each decoded symbol, check if DStream is fully consumed using this simple test :
|
||||
BIT_reloadDStream(&DStream) >= BIT_DStream_completed
|
||||
|
||||
When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
|
||||
Checking if DStream has reached its end is performed by :
|
||||
BIT_endOfDStream(&DStream);
|
||||
Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
|
||||
FSE_endOfDState(&DState);
|
||||
*/
|
||||
|
||||
|
||||
/* *****************************************
|
||||
* FSE unsafe API
|
||||
*******************************************/
|
||||
static unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD);
|
||||
/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */
|
||||
|
||||
|
||||
/* *****************************************
|
||||
* Implementation of inlined functions
|
||||
*******************************************/
|
||||
typedef struct {
|
||||
int deltaFindState;
|
||||
U32 deltaNbBits;
|
||||
} FSE_symbolCompressionTransform; /* total 8 bytes */
|
||||
|
||||
MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct)
|
||||
{
|
||||
const void* ptr = ct;
|
||||
const U16* u16ptr = (const U16*) ptr;
|
||||
const U32 tableLog = MEM_read16(ptr);
|
||||
statePtr->value = (ptrdiff_t)1<<tableLog;
|
||||
statePtr->stateTable = u16ptr+2;
|
||||
statePtr->symbolTT = ((const U32*)ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1));
|
||||
statePtr->stateLog = tableLog;
|
||||
}
|
||||
|
||||
|
||||
/*! FSE_initCState2() :
|
||||
* Same as FSE_initCState(), but the first symbol to include (which will be the last to be read)
|
||||
* uses the smallest state value possible, saving the cost of this symbol */
|
||||
MEM_STATIC void FSE_initCState2(FSE_CState_t* statePtr, const FSE_CTable* ct, U32 symbol)
|
||||
{
|
||||
FSE_initCState(statePtr, ct);
|
||||
{ const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
|
||||
const U16* stateTable = (const U16*)(statePtr->stateTable);
|
||||
U32 nbBitsOut = (U32)((symbolTT.deltaNbBits + (1<<15)) >> 16);
|
||||
statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits;
|
||||
statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
|
||||
}
|
||||
}
|
||||
|
||||
MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, U32 symbol)
|
||||
{
|
||||
FSE_symbolCompressionTransform const symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
|
||||
const U16* const stateTable = (const U16*)(statePtr->stateTable);
|
||||
U32 const nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16);
|
||||
BIT_addBits(bitC, statePtr->value, nbBitsOut);
|
||||
statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
|
||||
}
|
||||
|
||||
MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePtr)
|
||||
{
|
||||
BIT_addBits(bitC, statePtr->value, statePtr->stateLog);
|
||||
BIT_flushBits(bitC);
|
||||
}
|
||||
|
||||
|
||||
/* FSE_getMaxNbBits() :
|
||||
* Approximate maximum cost of a symbol, in bits.
|
||||
* Fractional get rounded up (i.e : a symbol with a normalized frequency of 3 gives the same result as a frequency of 2)
|
||||
* note 1 : assume symbolValue is valid (<= maxSymbolValue)
|
||||
* note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */
|
||||
MEM_STATIC U32 FSE_getMaxNbBits(const void* symbolTTPtr, U32 symbolValue)
|
||||
{
|
||||
const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*) symbolTTPtr;
|
||||
return (symbolTT[symbolValue].deltaNbBits + ((1<<16)-1)) >> 16;
|
||||
}
|
||||
|
||||
/* FSE_bitCost() :
|
||||
* Approximate symbol cost, as fractional value, using fixed-point format (accuracyLog fractional bits)
|
||||
* note 1 : assume symbolValue is valid (<= maxSymbolValue)
|
||||
* note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */
|
||||
MEM_STATIC U32 FSE_bitCost(const void* symbolTTPtr, U32 tableLog, U32 symbolValue, U32 accuracyLog)
|
||||
{
|
||||
const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*) symbolTTPtr;
|
||||
U32 const minNbBits = symbolTT[symbolValue].deltaNbBits >> 16;
|
||||
U32 const threshold = (minNbBits+1) << 16;
|
||||
assert(tableLog < 16);
|
||||
assert(accuracyLog < 31-tableLog); /* ensure enough room for renormalization double shift */
|
||||
{ U32 const tableSize = 1 << tableLog;
|
||||
U32 const deltaFromThreshold = threshold - (symbolTT[symbolValue].deltaNbBits + tableSize);
|
||||
U32 const normalizedDeltaFromThreshold = (deltaFromThreshold << accuracyLog) >> tableLog; /* linear interpolation (very approximate) */
|
||||
U32 const bitMultiplier = 1 << accuracyLog;
|
||||
assert(symbolTT[symbolValue].deltaNbBits + tableSize <= threshold);
|
||||
assert(normalizedDeltaFromThreshold <= bitMultiplier);
|
||||
return (minNbBits+1)*bitMultiplier - normalizedDeltaFromThreshold;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/* ====== Decompression ====== */
|
||||
|
||||
typedef struct {
|
||||
U16 tableLog;
|
||||
U16 fastMode;
|
||||
} FSE_DTableHeader; /* sizeof U32 */
|
||||
|
||||
typedef struct
|
||||
{
|
||||
unsigned short newState;
|
||||
unsigned char symbol;
|
||||
unsigned char nbBits;
|
||||
} FSE_decode_t; /* size == U32 */
|
||||
|
||||
MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt)
|
||||
{
|
||||
const void* ptr = dt;
|
||||
const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr;
|
||||
DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog);
|
||||
BIT_reloadDStream(bitD);
|
||||
DStatePtr->table = dt + 1;
|
||||
}
|
||||
|
||||
MEM_STATIC BYTE FSE_peekSymbol(const FSE_DState_t* DStatePtr)
|
||||
{
|
||||
FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
|
||||
return DInfo.symbol;
|
||||
}
|
||||
|
||||
MEM_STATIC void FSE_updateState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
|
||||
{
|
||||
FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
|
||||
U32 const nbBits = DInfo.nbBits;
|
||||
size_t const lowBits = BIT_readBits(bitD, nbBits);
|
||||
DStatePtr->state = DInfo.newState + lowBits;
|
||||
}
|
||||
|
||||
MEM_STATIC BYTE FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
|
||||
{
|
||||
FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
|
||||
U32 const nbBits = DInfo.nbBits;
|
||||
BYTE const symbol = DInfo.symbol;
|
||||
size_t const lowBits = BIT_readBits(bitD, nbBits);
|
||||
|
||||
DStatePtr->state = DInfo.newState + lowBits;
|
||||
return symbol;
|
||||
}
|
||||
|
||||
/*! FSE_decodeSymbolFast() :
|
||||
unsafe, only works if no symbol has a probability > 50% */
|
||||
MEM_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
|
||||
{
|
||||
FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
|
||||
U32 const nbBits = DInfo.nbBits;
|
||||
BYTE const symbol = DInfo.symbol;
|
||||
size_t const lowBits = BIT_readBitsFast(bitD, nbBits);
|
||||
|
||||
DStatePtr->state = DInfo.newState + lowBits;
|
||||
return symbol;
|
||||
}
|
||||
|
||||
MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
|
||||
{
|
||||
return DStatePtr->state == 0;
|
||||
}
|
||||
|
||||
|
||||
|
||||
#ifndef FSE_COMMONDEFS_ONLY
|
||||
|
||||
/* **************************************************************
|
||||
* Tuning parameters
|
||||
****************************************************************/
|
||||
/*!MEMORY_USAGE :
|
||||
* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
|
||||
* Increasing memory usage improves compression ratio
|
||||
* Reduced memory usage can improve speed, due to cache effect
|
||||
* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */
|
||||
#ifndef FSE_MAX_MEMORY_USAGE
|
||||
# define FSE_MAX_MEMORY_USAGE 14
|
||||
#endif
|
||||
#ifndef FSE_DEFAULT_MEMORY_USAGE
|
||||
# define FSE_DEFAULT_MEMORY_USAGE 13
|
||||
#endif
|
||||
|
||||
/*!FSE_MAX_SYMBOL_VALUE :
|
||||
* Maximum symbol value authorized.
|
||||
* Required for proper stack allocation */
|
||||
#ifndef FSE_MAX_SYMBOL_VALUE
|
||||
# define FSE_MAX_SYMBOL_VALUE 255
|
||||
#endif
|
||||
|
||||
/* **************************************************************
|
||||
* template functions type & suffix
|
||||
****************************************************************/
|
||||
#define FSE_FUNCTION_TYPE BYTE
|
||||
#define FSE_FUNCTION_EXTENSION
|
||||
#define FSE_DECODE_TYPE FSE_decode_t
|
||||
|
||||
|
||||
#endif /* !FSE_COMMONDEFS_ONLY */
|
||||
|
||||
|
||||
/* ***************************************************************
|
||||
* Constants
|
||||
*****************************************************************/
|
||||
#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2)
|
||||
#define FSE_MAX_TABLESIZE (1U<<FSE_MAX_TABLELOG)
|
||||
#define FSE_MAXTABLESIZE_MASK (FSE_MAX_TABLESIZE-1)
|
||||
#define FSE_DEFAULT_TABLELOG (FSE_DEFAULT_MEMORY_USAGE-2)
|
||||
#define FSE_MIN_TABLELOG 5
|
||||
|
||||
#define FSE_TABLELOG_ABSOLUTE_MAX 15
|
||||
#if FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX
|
||||
# error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported"
|
||||
#endif
|
||||
|
||||
#define FSE_TABLESTEP(tableSize) ((tableSize>>1) + (tableSize>>3) + 3)
|
||||
|
||||
|
||||
#endif /* FSE_STATIC_LINKING_ONLY */
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
|
@ -0,0 +1,309 @@
|
|||
/* ******************************************************************
|
||||
FSE : Finite State Entropy decoder
|
||||
Copyright (C) 2013-2015, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
****************************************************************** */
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* Includes
|
||||
****************************************************************/
|
||||
#include <stdlib.h> /* malloc, free, qsort */
|
||||
#include <string.h> /* memcpy, memset */
|
||||
#include "bitstream.h"
|
||||
#include "compiler.h"
|
||||
#define FSE_STATIC_LINKING_ONLY
|
||||
#include "fse.h"
|
||||
#include "error_private.h"
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* Error Management
|
||||
****************************************************************/
|
||||
#define FSE_isError ERR_isError
|
||||
#define FSE_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) /* use only *after* variable declarations */
|
||||
|
||||
/* check and forward error code */
|
||||
#define CHECK_F(f) { size_t const e = f; if (FSE_isError(e)) return e; }
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* Templates
|
||||
****************************************************************/
|
||||
/*
|
||||
designed to be included
|
||||
for type-specific functions (template emulation in C)
|
||||
Objective is to write these functions only once, for improved maintenance
|
||||
*/
|
||||
|
||||
/* safety checks */
|
||||
#ifndef FSE_FUNCTION_EXTENSION
|
||||
# error "FSE_FUNCTION_EXTENSION must be defined"
|
||||
#endif
|
||||
#ifndef FSE_FUNCTION_TYPE
|
||||
# error "FSE_FUNCTION_TYPE must be defined"
|
||||
#endif
|
||||
|
||||
/* Function names */
|
||||
#define FSE_CAT(X,Y) X##Y
|
||||
#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y)
|
||||
#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y)
|
||||
|
||||
|
||||
/* Function templates */
|
||||
FSE_DTable* FSE_createDTable (unsigned tableLog)
|
||||
{
|
||||
if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX;
|
||||
return (FSE_DTable*)malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) );
|
||||
}
|
||||
|
||||
void FSE_freeDTable (FSE_DTable* dt)
|
||||
{
|
||||
free(dt);
|
||||
}
|
||||
|
||||
size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
|
||||
{
|
||||
void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */
|
||||
FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr);
|
||||
U16 symbolNext[FSE_MAX_SYMBOL_VALUE+1];
|
||||
|
||||
U32 const maxSV1 = maxSymbolValue + 1;
|
||||
U32 const tableSize = 1 << tableLog;
|
||||
U32 highThreshold = tableSize-1;
|
||||
|
||||
/* Sanity Checks */
|
||||
if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge);
|
||||
if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
|
||||
|
||||
/* Init, lay down lowprob symbols */
|
||||
{ FSE_DTableHeader DTableH;
|
||||
DTableH.tableLog = (U16)tableLog;
|
||||
DTableH.fastMode = 1;
|
||||
{ S16 const largeLimit= (S16)(1 << (tableLog-1));
|
||||
U32 s;
|
||||
for (s=0; s<maxSV1; s++) {
|
||||
if (normalizedCounter[s]==-1) {
|
||||
tableDecode[highThreshold--].symbol = (FSE_FUNCTION_TYPE)s;
|
||||
symbolNext[s] = 1;
|
||||
} else {
|
||||
if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0;
|
||||
symbolNext[s] = normalizedCounter[s];
|
||||
} } }
|
||||
memcpy(dt, &DTableH, sizeof(DTableH));
|
||||
}
|
||||
|
||||
/* Spread symbols */
|
||||
{ U32 const tableMask = tableSize-1;
|
||||
U32 const step = FSE_TABLESTEP(tableSize);
|
||||
U32 s, position = 0;
|
||||
for (s=0; s<maxSV1; s++) {
|
||||
int i;
|
||||
for (i=0; i<normalizedCounter[s]; i++) {
|
||||
tableDecode[position].symbol = (FSE_FUNCTION_TYPE)s;
|
||||
position = (position + step) & tableMask;
|
||||
while (position > highThreshold) position = (position + step) & tableMask; /* lowprob area */
|
||||
} }
|
||||
if (position!=0) return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */
|
||||
}
|
||||
|
||||
/* Build Decoding table */
|
||||
{ U32 u;
|
||||
for (u=0; u<tableSize; u++) {
|
||||
FSE_FUNCTION_TYPE const symbol = (FSE_FUNCTION_TYPE)(tableDecode[u].symbol);
|
||||
U32 const nextState = symbolNext[symbol]++;
|
||||
tableDecode[u].nbBits = (BYTE) (tableLog - BIT_highbit32(nextState) );
|
||||
tableDecode[u].newState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize);
|
||||
} }
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
#ifndef FSE_COMMONDEFS_ONLY
|
||||
|
||||
/*-*******************************************************
|
||||
* Decompression (Byte symbols)
|
||||
*********************************************************/
|
||||
size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue)
|
||||
{
|
||||
void* ptr = dt;
|
||||
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;
|
||||
void* dPtr = dt + 1;
|
||||
FSE_decode_t* const cell = (FSE_decode_t*)dPtr;
|
||||
|
||||
DTableH->tableLog = 0;
|
||||
DTableH->fastMode = 0;
|
||||
|
||||
cell->newState = 0;
|
||||
cell->symbol = symbolValue;
|
||||
cell->nbBits = 0;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits)
|
||||
{
|
||||
void* ptr = dt;
|
||||
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;
|
||||
void* dPtr = dt + 1;
|
||||
FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr;
|
||||
const unsigned tableSize = 1 << nbBits;
|
||||
const unsigned tableMask = tableSize - 1;
|
||||
const unsigned maxSV1 = tableMask+1;
|
||||
unsigned s;
|
||||
|
||||
/* Sanity checks */
|
||||
if (nbBits < 1) return ERROR(GENERIC); /* min size */
|
||||
|
||||
/* Build Decoding Table */
|
||||
DTableH->tableLog = (U16)nbBits;
|
||||
DTableH->fastMode = 1;
|
||||
for (s=0; s<maxSV1; s++) {
|
||||
dinfo[s].newState = 0;
|
||||
dinfo[s].symbol = (BYTE)s;
|
||||
dinfo[s].nbBits = (BYTE)nbBits;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
FORCE_INLINE_TEMPLATE size_t FSE_decompress_usingDTable_generic(
|
||||
void* dst, size_t maxDstSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
const FSE_DTable* dt, const unsigned fast)
|
||||
{
|
||||
BYTE* const ostart = (BYTE*) dst;
|
||||
BYTE* op = ostart;
|
||||
BYTE* const omax = op + maxDstSize;
|
||||
BYTE* const olimit = omax-3;
|
||||
|
||||
BIT_DStream_t bitD;
|
||||
FSE_DState_t state1;
|
||||
FSE_DState_t state2;
|
||||
|
||||
/* Init */
|
||||
CHECK_F(BIT_initDStream(&bitD, cSrc, cSrcSize));
|
||||
|
||||
FSE_initDState(&state1, &bitD, dt);
|
||||
FSE_initDState(&state2, &bitD, dt);
|
||||
|
||||
#define FSE_GETSYMBOL(statePtr) fast ? FSE_decodeSymbolFast(statePtr, &bitD) : FSE_decodeSymbol(statePtr, &bitD)
|
||||
|
||||
/* 4 symbols per loop */
|
||||
for ( ; (BIT_reloadDStream(&bitD)==BIT_DStream_unfinished) & (op<olimit) ; op+=4) {
|
||||
op[0] = FSE_GETSYMBOL(&state1);
|
||||
|
||||
if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */
|
||||
BIT_reloadDStream(&bitD);
|
||||
|
||||
op[1] = FSE_GETSYMBOL(&state2);
|
||||
|
||||
if (FSE_MAX_TABLELOG*4+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */
|
||||
{ if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) { op+=2; break; } }
|
||||
|
||||
op[2] = FSE_GETSYMBOL(&state1);
|
||||
|
||||
if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */
|
||||
BIT_reloadDStream(&bitD);
|
||||
|
||||
op[3] = FSE_GETSYMBOL(&state2);
|
||||
}
|
||||
|
||||
/* tail */
|
||||
/* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */
|
||||
while (1) {
|
||||
if (op>(omax-2)) return ERROR(dstSize_tooSmall);
|
||||
*op++ = FSE_GETSYMBOL(&state1);
|
||||
if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) {
|
||||
*op++ = FSE_GETSYMBOL(&state2);
|
||||
break;
|
||||
}
|
||||
|
||||
if (op>(omax-2)) return ERROR(dstSize_tooSmall);
|
||||
*op++ = FSE_GETSYMBOL(&state2);
|
||||
if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) {
|
||||
*op++ = FSE_GETSYMBOL(&state1);
|
||||
break;
|
||||
} }
|
||||
|
||||
return op-ostart;
|
||||
}
|
||||
|
||||
|
||||
size_t FSE_decompress_usingDTable(void* dst, size_t originalSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
const FSE_DTable* dt)
|
||||
{
|
||||
const void* ptr = dt;
|
||||
const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr;
|
||||
const U32 fastMode = DTableH->fastMode;
|
||||
|
||||
/* select fast mode (static) */
|
||||
if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1);
|
||||
return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0);
|
||||
}
|
||||
|
||||
|
||||
size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, FSE_DTable* workSpace, unsigned maxLog)
|
||||
{
|
||||
const BYTE* const istart = (const BYTE*)cSrc;
|
||||
const BYTE* ip = istart;
|
||||
short counting[FSE_MAX_SYMBOL_VALUE+1];
|
||||
unsigned tableLog;
|
||||
unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
|
||||
|
||||
/* normal FSE decoding mode */
|
||||
size_t const NCountLength = FSE_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize);
|
||||
if (FSE_isError(NCountLength)) return NCountLength;
|
||||
//if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size; supposed to be already checked in NCountLength, only remaining case : NCountLength==cSrcSize */
|
||||
if (tableLog > maxLog) return ERROR(tableLog_tooLarge);
|
||||
ip += NCountLength;
|
||||
cSrcSize -= NCountLength;
|
||||
|
||||
CHECK_F( FSE_buildDTable (workSpace, counting, maxSymbolValue, tableLog) );
|
||||
|
||||
return FSE_decompress_usingDTable (dst, dstCapacity, ip, cSrcSize, workSpace); /* always return, even if it is an error code */
|
||||
}
|
||||
|
||||
|
||||
typedef FSE_DTable DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)];
|
||||
|
||||
size_t FSE_decompress(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
DTable_max_t dt; /* Static analyzer seems unable to understand this table will be properly initialized later */
|
||||
return FSE_decompress_wksp(dst, dstCapacity, cSrc, cSrcSize, dt, FSE_MAX_TABLELOG);
|
||||
}
|
||||
|
||||
|
||||
|
||||
#endif /* FSE_COMMONDEFS_ONLY */
|
||||
|
|
@ -0,0 +1,334 @@
|
|||
/* ******************************************************************
|
||||
huff0 huffman codec,
|
||||
part of Finite State Entropy library
|
||||
Copyright (C) 2013-present, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
****************************************************************** */
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#ifndef HUF_H_298734234
|
||||
#define HUF_H_298734234
|
||||
|
||||
/* *** Dependencies *** */
|
||||
#include <stddef.h> /* size_t */
|
||||
|
||||
|
||||
/* *** library symbols visibility *** */
|
||||
/* Note : when linking with -fvisibility=hidden on gcc, or by default on Visual,
|
||||
* HUF symbols remain "private" (internal symbols for library only).
|
||||
* Set macro FSE_DLL_EXPORT to 1 if you want HUF symbols visible on DLL interface */
|
||||
#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4)
|
||||
# define HUF_PUBLIC_API __attribute__ ((visibility ("default")))
|
||||
#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) /* Visual expected */
|
||||
# define HUF_PUBLIC_API __declspec(dllexport)
|
||||
#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1)
|
||||
# define HUF_PUBLIC_API __declspec(dllimport) /* not required, just to generate faster code (saves a function pointer load from IAT and an indirect jump) */
|
||||
#else
|
||||
# define HUF_PUBLIC_API
|
||||
#endif
|
||||
|
||||
|
||||
/* ========================== */
|
||||
/* *** simple functions *** */
|
||||
/* ========================== */
|
||||
|
||||
/** HUF_compress() :
|
||||
* Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'.
|
||||
* 'dst' buffer must be already allocated.
|
||||
* Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize).
|
||||
* `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB.
|
||||
* @return : size of compressed data (<= `dstCapacity`).
|
||||
* Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
|
||||
* if HUF_isError(return), compression failed (more details using HUF_getErrorName())
|
||||
*/
|
||||
HUF_PUBLIC_API size_t HUF_compress(void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize);
|
||||
|
||||
/** HUF_decompress() :
|
||||
* Decompress HUF data from buffer 'cSrc', of size 'cSrcSize',
|
||||
* into already allocated buffer 'dst', of minimum size 'dstSize'.
|
||||
* `originalSize` : **must** be the ***exact*** size of original (uncompressed) data.
|
||||
* Note : in contrast with FSE, HUF_decompress can regenerate
|
||||
* RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data,
|
||||
* because it knows size to regenerate (originalSize).
|
||||
* @return : size of regenerated data (== originalSize),
|
||||
* or an error code, which can be tested using HUF_isError()
|
||||
*/
|
||||
HUF_PUBLIC_API size_t HUF_decompress(void* dst, size_t originalSize,
|
||||
const void* cSrc, size_t cSrcSize);
|
||||
|
||||
|
||||
/* *** Tool functions *** */
|
||||
#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */
|
||||
HUF_PUBLIC_API size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */
|
||||
|
||||
/* Error Management */
|
||||
HUF_PUBLIC_API unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */
|
||||
HUF_PUBLIC_API const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */
|
||||
|
||||
|
||||
/* *** Advanced function *** */
|
||||
|
||||
/** HUF_compress2() :
|
||||
* Same as HUF_compress(), but offers control over `maxSymbolValue` and `tableLog`.
|
||||
* `maxSymbolValue` must be <= HUF_SYMBOLVALUE_MAX .
|
||||
* `tableLog` must be `<= HUF_TABLELOG_MAX` . */
|
||||
HUF_PUBLIC_API size_t HUF_compress2 (void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
unsigned maxSymbolValue, unsigned tableLog);
|
||||
|
||||
/** HUF_compress4X_wksp() :
|
||||
* Same as HUF_compress2(), but uses externally allocated `workSpace`.
|
||||
* `workspace` must have minimum alignment of 4, and be at least as large as HUF_WORKSPACE_SIZE */
|
||||
#define HUF_WORKSPACE_SIZE (6 << 10)
|
||||
#define HUF_WORKSPACE_SIZE_U32 (HUF_WORKSPACE_SIZE / sizeof(U32))
|
||||
HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
unsigned maxSymbolValue, unsigned tableLog,
|
||||
void* workSpace, size_t wkspSize);
|
||||
|
||||
#endif /* HUF_H_298734234 */
|
||||
|
||||
/* ******************************************************************
|
||||
* WARNING !!
|
||||
* The following section contains advanced and experimental definitions
|
||||
* which shall never be used in the context of a dynamic library,
|
||||
* because they are not guaranteed to remain stable in the future.
|
||||
* Only consider them in association with static linking.
|
||||
* *****************************************************************/
|
||||
#if defined(HUF_STATIC_LINKING_ONLY) && !defined(HUF_H_HUF_STATIC_LINKING_ONLY)
|
||||
#define HUF_H_HUF_STATIC_LINKING_ONLY
|
||||
|
||||
/* *** Dependencies *** */
|
||||
#include "mem.h" /* U32 */
|
||||
|
||||
|
||||
/* *** Constants *** */
|
||||
#define HUF_TABLELOG_MAX 12 /* max runtime value of tableLog (due to static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */
|
||||
#define HUF_TABLELOG_DEFAULT 11 /* default tableLog value when none specified */
|
||||
#define HUF_SYMBOLVALUE_MAX 255
|
||||
|
||||
#define HUF_TABLELOG_ABSOLUTEMAX 15 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
|
||||
#if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX)
|
||||
# error "HUF_TABLELOG_MAX is too large !"
|
||||
#endif
|
||||
|
||||
|
||||
/* ****************************************
|
||||
* Static allocation
|
||||
******************************************/
|
||||
/* HUF buffer bounds */
|
||||
#define HUF_CTABLEBOUND 129
|
||||
#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true when incompressible is pre-filtered with fast heuristic */
|
||||
#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
|
||||
|
||||
/* static allocation of HUF's Compression Table */
|
||||
#define HUF_CTABLE_SIZE_U32(maxSymbolValue) ((maxSymbolValue)+1) /* Use tables of U32, for proper alignment */
|
||||
#define HUF_CTABLE_SIZE(maxSymbolValue) (HUF_CTABLE_SIZE_U32(maxSymbolValue) * sizeof(U32))
|
||||
#define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \
|
||||
U32 name##hb[HUF_CTABLE_SIZE_U32(maxSymbolValue)]; \
|
||||
void* name##hv = &(name##hb); \
|
||||
HUF_CElt* name = (HUF_CElt*)(name##hv) /* no final ; */
|
||||
|
||||
/* static allocation of HUF's DTable */
|
||||
typedef U32 HUF_DTable;
|
||||
#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog)))
|
||||
#define HUF_CREATE_STATIC_DTABLEX1(DTable, maxTableLog) \
|
||||
HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1) * 0x01000001) }
|
||||
#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \
|
||||
HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog) * 0x01000001) }
|
||||
|
||||
|
||||
/* ****************************************
|
||||
* Advanced decompression functions
|
||||
******************************************/
|
||||
size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
|
||||
size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
|
||||
|
||||
size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */
|
||||
size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */
|
||||
size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< considers RLE and uncompressed as errors */
|
||||
size_t HUF_decompress4X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
|
||||
size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */
|
||||
size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
|
||||
size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */
|
||||
|
||||
|
||||
/* ****************************************
|
||||
* HUF detailed API
|
||||
* ****************************************/
|
||||
|
||||
/*! HUF_compress() does the following:
|
||||
* 1. count symbol occurrence from source[] into table count[] using FSE_count() (exposed within "fse.h")
|
||||
* 2. (optional) refine tableLog using HUF_optimalTableLog()
|
||||
* 3. build Huffman table from count using HUF_buildCTable()
|
||||
* 4. save Huffman table to memory buffer using HUF_writeCTable()
|
||||
* 5. encode the data stream using HUF_compress4X_usingCTable()
|
||||
*
|
||||
* The following API allows targeting specific sub-functions for advanced tasks.
|
||||
* For example, it's possible to compress several blocks using the same 'CTable',
|
||||
* or to save and regenerate 'CTable' using external methods.
|
||||
*/
|
||||
unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
|
||||
typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */
|
||||
size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits); /* @return : maxNbBits; CTable and count can overlap. In which case, CTable will overwrite count content */
|
||||
size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog);
|
||||
size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
|
||||
|
||||
typedef enum {
|
||||
HUF_repeat_none, /**< Cannot use the previous table */
|
||||
HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat */
|
||||
HUF_repeat_valid /**< Can use the previous table and it is assumed to be valid */
|
||||
} HUF_repeat;
|
||||
/** HUF_compress4X_repeat() :
|
||||
* Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
|
||||
* If it uses hufTable it does not modify hufTable or repeat.
|
||||
* If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
|
||||
* If preferRepeat then the old table will always be used if valid. */
|
||||
size_t HUF_compress4X_repeat(void* dst, size_t dstSize,
|
||||
const void* src, size_t srcSize,
|
||||
unsigned maxSymbolValue, unsigned tableLog,
|
||||
void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */
|
||||
HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2);
|
||||
|
||||
/** HUF_buildCTable_wksp() :
|
||||
* Same as HUF_buildCTable(), but using externally allocated scratch buffer.
|
||||
* `workSpace` must be aligned on 4-bytes boundaries, and its size must be >= HUF_CTABLE_WORKSPACE_SIZE.
|
||||
*/
|
||||
#define HUF_CTABLE_WORKSPACE_SIZE_U32 (2*HUF_SYMBOLVALUE_MAX +1 +1)
|
||||
#define HUF_CTABLE_WORKSPACE_SIZE (HUF_CTABLE_WORKSPACE_SIZE_U32 * sizeof(unsigned))
|
||||
size_t HUF_buildCTable_wksp (HUF_CElt* tree,
|
||||
const U32* count, U32 maxSymbolValue, U32 maxNbBits,
|
||||
void* workSpace, size_t wkspSize);
|
||||
|
||||
/*! HUF_readStats() :
|
||||
* Read compact Huffman tree, saved by HUF_writeCTable().
|
||||
* `huffWeight` is destination buffer.
|
||||
* @return : size read from `src` , or an error Code .
|
||||
* Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */
|
||||
size_t HUF_readStats(BYTE* huffWeight, size_t hwSize,
|
||||
U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr,
|
||||
const void* src, size_t srcSize);
|
||||
|
||||
/** HUF_readCTable() :
|
||||
* Loading a CTable saved with HUF_writeCTable() */
|
||||
size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
|
||||
|
||||
/** HUF_getNbBits() :
|
||||
* Read nbBits from CTable symbolTable, for symbol `symbolValue` presumed <= HUF_SYMBOLVALUE_MAX
|
||||
* Note 1 : is not inlined, as HUF_CElt definition is private
|
||||
* Note 2 : const void* used, so that it can provide a statically allocated table as argument (which uses type U32) */
|
||||
U32 HUF_getNbBits(const void* symbolTable, U32 symbolValue);
|
||||
|
||||
/*
|
||||
* HUF_decompress() does the following:
|
||||
* 1. select the decompression algorithm (X1, X2) based on pre-computed heuristics
|
||||
* 2. build Huffman table from save, using HUF_readDTableX?()
|
||||
* 3. decode 1 or 4 segments in parallel using HUF_decompress?X?_usingDTable()
|
||||
*/
|
||||
|
||||
/** HUF_selectDecoder() :
|
||||
* Tells which decoder is likely to decode faster,
|
||||
* based on a set of pre-computed metrics.
|
||||
* @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 .
|
||||
* Assumption : 0 < dstSize <= 128 KB */
|
||||
U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize);
|
||||
|
||||
/**
|
||||
* The minimum workspace size for the `workSpace` used in
|
||||
* HUF_readDTableX1_wksp() and HUF_readDTableX2_wksp().
|
||||
*
|
||||
* The space used depends on HUF_TABLELOG_MAX, ranging from ~1500 bytes when
|
||||
* HUF_TABLE_LOG_MAX=12 to ~1850 bytes when HUF_TABLE_LOG_MAX=15.
|
||||
* Buffer overflow errors may potentially occur if code modifications result in
|
||||
* a required workspace size greater than that specified in the following
|
||||
* macro.
|
||||
*/
|
||||
#define HUF_DECOMPRESS_WORKSPACE_SIZE (2 << 10)
|
||||
#define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32))
|
||||
|
||||
size_t HUF_readDTableX1 (HUF_DTable* DTable, const void* src, size_t srcSize);
|
||||
size_t HUF_readDTableX1_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
|
||||
size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize);
|
||||
size_t HUF_readDTableX2_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
|
||||
|
||||
size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
|
||||
size_t HUF_decompress4X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
|
||||
size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
|
||||
|
||||
|
||||
/* ====================== */
|
||||
/* single stream variants */
|
||||
/* ====================== */
|
||||
|
||||
size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
|
||||
size_t HUF_compress1X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */
|
||||
size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
|
||||
/** HUF_compress1X_repeat() :
|
||||
* Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
|
||||
* If it uses hufTable it does not modify hufTable or repeat.
|
||||
* If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
|
||||
* If preferRepeat then the old table will always be used if valid. */
|
||||
size_t HUF_compress1X_repeat(void* dst, size_t dstSize,
|
||||
const void* src, size_t srcSize,
|
||||
unsigned maxSymbolValue, unsigned tableLog,
|
||||
void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */
|
||||
HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2);
|
||||
|
||||
size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
|
||||
size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */
|
||||
|
||||
size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
|
||||
size_t HUF_decompress1X_DCtx_wksp (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize);
|
||||
size_t HUF_decompress1X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
|
||||
size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */
|
||||
size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
|
||||
size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */
|
||||
|
||||
size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); /**< automatic selection of sing or double symbol decoder, based on DTable */
|
||||
size_t HUF_decompress1X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
|
||||
size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
|
||||
|
||||
/* BMI2 variants.
|
||||
* If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0.
|
||||
*/
|
||||
size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2);
|
||||
size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);
|
||||
size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2);
|
||||
size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);
|
||||
|
||||
#endif /* HUF_STATIC_LINKING_ONLY */
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
|
@ -0,0 +1,362 @@
|
|||
/*
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#ifndef MEM_H_MODULE
|
||||
#define MEM_H_MODULE
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/*-****************************************
|
||||
* Dependencies
|
||||
******************************************/
|
||||
#include <stddef.h> /* size_t, ptrdiff_t */
|
||||
#include <string.h> /* memcpy */
|
||||
|
||||
|
||||
/*-****************************************
|
||||
* Compiler specifics
|
||||
******************************************/
|
||||
#if defined(_MSC_VER) /* Visual Studio */
|
||||
# include <stdlib.h> /* _byteswap_ulong */
|
||||
# include <intrin.h> /* _byteswap_* */
|
||||
#endif
|
||||
#if defined(__GNUC__)
|
||||
# define MEM_STATIC static __inline __attribute__((unused))
|
||||
#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
|
||||
# define MEM_STATIC static inline
|
||||
#elif defined(_MSC_VER)
|
||||
# define MEM_STATIC static __inline
|
||||
#else
|
||||
# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
|
||||
#endif
|
||||
|
||||
/* code only tested on 32 and 64 bits systems */
|
||||
#define MEM_STATIC_ASSERT(c) { enum { MEM_static_assert = 1/(int)(!!(c)) }; }
|
||||
MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); }
|
||||
|
||||
|
||||
/*-**************************************************************
|
||||
* Basic Types
|
||||
*****************************************************************/
|
||||
#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
|
||||
# include <stdint.h>
|
||||
typedef uint8_t BYTE;
|
||||
typedef uint16_t U16;
|
||||
typedef int16_t S16;
|
||||
typedef uint32_t U32;
|
||||
typedef int32_t S32;
|
||||
typedef uint64_t U64;
|
||||
typedef int64_t S64;
|
||||
#else
|
||||
typedef unsigned char BYTE;
|
||||
typedef unsigned short U16;
|
||||
typedef signed short S16;
|
||||
typedef unsigned int U32;
|
||||
typedef signed int S32;
|
||||
typedef unsigned long long U64;
|
||||
typedef signed long long S64;
|
||||
#endif
|
||||
|
||||
|
||||
/*-**************************************************************
|
||||
* Memory I/O
|
||||
*****************************************************************/
|
||||
/* MEM_FORCE_MEMORY_ACCESS :
|
||||
* By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
|
||||
* Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
|
||||
* The below switch allow to select different access method for improved performance.
|
||||
* Method 0 (default) : use `memcpy()`. Safe and portable.
|
||||
* Method 1 : `__packed` statement. It depends on compiler extension (i.e., not portable).
|
||||
* This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
|
||||
* Method 2 : direct access. This method is portable but violate C standard.
|
||||
* It can generate buggy code on targets depending on alignment.
|
||||
* In some circumstances, it's the only known way to get the most performance (i.e. GCC + ARMv6)
|
||||
* See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
|
||||
* Prefer these methods in priority order (0 > 1 > 2)
|
||||
*/
|
||||
#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
|
||||
# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
|
||||
# define MEM_FORCE_MEMORY_ACCESS 2
|
||||
# elif defined(__INTEL_COMPILER) || defined(__GNUC__)
|
||||
# define MEM_FORCE_MEMORY_ACCESS 1
|
||||
# endif
|
||||
#endif
|
||||
|
||||
MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; }
|
||||
MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; }
|
||||
|
||||
MEM_STATIC unsigned MEM_isLittleEndian(void)
|
||||
{
|
||||
const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */
|
||||
return one.c[0];
|
||||
}
|
||||
|
||||
#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2)
|
||||
|
||||
/* violates C standard, by lying on structure alignment.
|
||||
Only use if no other choice to achieve best performance on target platform */
|
||||
MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }
|
||||
MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
|
||||
MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
|
||||
MEM_STATIC size_t MEM_readST(const void* memPtr) { return *(const size_t*) memPtr; }
|
||||
|
||||
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
|
||||
MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }
|
||||
MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; }
|
||||
|
||||
#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
|
||||
|
||||
/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
|
||||
/* currently only defined for gcc and icc */
|
||||
#if defined(_MSC_VER) || (defined(__INTEL_COMPILER) && defined(WIN32))
|
||||
__pragma( pack(push, 1) )
|
||||
typedef struct { U16 v; } unalign16;
|
||||
typedef struct { U32 v; } unalign32;
|
||||
typedef struct { U64 v; } unalign64;
|
||||
typedef struct { size_t v; } unalignArch;
|
||||
__pragma( pack(pop) )
|
||||
#else
|
||||
typedef struct { U16 v; } __attribute__((packed)) unalign16;
|
||||
typedef struct { U32 v; } __attribute__((packed)) unalign32;
|
||||
typedef struct { U64 v; } __attribute__((packed)) unalign64;
|
||||
typedef struct { size_t v; } __attribute__((packed)) unalignArch;
|
||||
#endif
|
||||
|
||||
MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign16*)ptr)->v; }
|
||||
MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign32*)ptr)->v; }
|
||||
MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign64*)ptr)->v; }
|
||||
MEM_STATIC size_t MEM_readST(const void* ptr) { return ((const unalignArch*)ptr)->v; }
|
||||
|
||||
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign16*)memPtr)->v = value; }
|
||||
MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign32*)memPtr)->v = value; }
|
||||
MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign64*)memPtr)->v = value; }
|
||||
|
||||
#else
|
||||
|
||||
/* default method, safe and standard.
|
||||
can sometimes prove slower */
|
||||
|
||||
MEM_STATIC U16 MEM_read16(const void* memPtr)
|
||||
{
|
||||
U16 val; memcpy(&val, memPtr, sizeof(val)); return val;
|
||||
}
|
||||
|
||||
MEM_STATIC U32 MEM_read32(const void* memPtr)
|
||||
{
|
||||
U32 val; memcpy(&val, memPtr, sizeof(val)); return val;
|
||||
}
|
||||
|
||||
MEM_STATIC U64 MEM_read64(const void* memPtr)
|
||||
{
|
||||
U64 val; memcpy(&val, memPtr, sizeof(val)); return val;
|
||||
}
|
||||
|
||||
MEM_STATIC size_t MEM_readST(const void* memPtr)
|
||||
{
|
||||
size_t val; memcpy(&val, memPtr, sizeof(val)); return val;
|
||||
}
|
||||
|
||||
MEM_STATIC void MEM_write16(void* memPtr, U16 value)
|
||||
{
|
||||
memcpy(memPtr, &value, sizeof(value));
|
||||
}
|
||||
|
||||
MEM_STATIC void MEM_write32(void* memPtr, U32 value)
|
||||
{
|
||||
memcpy(memPtr, &value, sizeof(value));
|
||||
}
|
||||
|
||||
MEM_STATIC void MEM_write64(void* memPtr, U64 value)
|
||||
{
|
||||
memcpy(memPtr, &value, sizeof(value));
|
||||
}
|
||||
|
||||
#endif /* MEM_FORCE_MEMORY_ACCESS */
|
||||
|
||||
MEM_STATIC U32 MEM_swap32(U32 in)
|
||||
{
|
||||
#if defined(_MSC_VER) /* Visual Studio */
|
||||
return _byteswap_ulong(in);
|
||||
#elif defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)
|
||||
return __builtin_bswap32(in);
|
||||
#else
|
||||
return ((in << 24) & 0xff000000 ) |
|
||||
((in << 8) & 0x00ff0000 ) |
|
||||
((in >> 8) & 0x0000ff00 ) |
|
||||
((in >> 24) & 0x000000ff );
|
||||
#endif
|
||||
}
|
||||
|
||||
MEM_STATIC U64 MEM_swap64(U64 in)
|
||||
{
|
||||
#if defined(_MSC_VER) /* Visual Studio */
|
||||
return _byteswap_uint64(in);
|
||||
#elif defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)
|
||||
return __builtin_bswap64(in);
|
||||
#else
|
||||
return ((in << 56) & 0xff00000000000000ULL) |
|
||||
((in << 40) & 0x00ff000000000000ULL) |
|
||||
((in << 24) & 0x0000ff0000000000ULL) |
|
||||
((in << 8) & 0x000000ff00000000ULL) |
|
||||
((in >> 8) & 0x00000000ff000000ULL) |
|
||||
((in >> 24) & 0x0000000000ff0000ULL) |
|
||||
((in >> 40) & 0x000000000000ff00ULL) |
|
||||
((in >> 56) & 0x00000000000000ffULL);
|
||||
#endif
|
||||
}
|
||||
|
||||
MEM_STATIC size_t MEM_swapST(size_t in)
|
||||
{
|
||||
if (MEM_32bits())
|
||||
return (size_t)MEM_swap32((U32)in);
|
||||
else
|
||||
return (size_t)MEM_swap64((U64)in);
|
||||
}
|
||||
|
||||
/*=== Little endian r/w ===*/
|
||||
|
||||
MEM_STATIC U16 MEM_readLE16(const void* memPtr)
|
||||
{
|
||||
if (MEM_isLittleEndian())
|
||||
return MEM_read16(memPtr);
|
||||
else {
|
||||
const BYTE* p = (const BYTE*)memPtr;
|
||||
return (U16)(p[0] + (p[1]<<8));
|
||||
}
|
||||
}
|
||||
|
||||
MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val)
|
||||
{
|
||||
if (MEM_isLittleEndian()) {
|
||||
MEM_write16(memPtr, val);
|
||||
} else {
|
||||
BYTE* p = (BYTE*)memPtr;
|
||||
p[0] = (BYTE)val;
|
||||
p[1] = (BYTE)(val>>8);
|
||||
}
|
||||
}
|
||||
|
||||
MEM_STATIC U32 MEM_readLE24(const void* memPtr)
|
||||
{
|
||||
return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16);
|
||||
}
|
||||
|
||||
MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val)
|
||||
{
|
||||
MEM_writeLE16(memPtr, (U16)val);
|
||||
((BYTE*)memPtr)[2] = (BYTE)(val>>16);
|
||||
}
|
||||
|
||||
MEM_STATIC U32 MEM_readLE32(const void* memPtr)
|
||||
{
|
||||
if (MEM_isLittleEndian())
|
||||
return MEM_read32(memPtr);
|
||||
else
|
||||
return MEM_swap32(MEM_read32(memPtr));
|
||||
}
|
||||
|
||||
MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32)
|
||||
{
|
||||
if (MEM_isLittleEndian())
|
||||
MEM_write32(memPtr, val32);
|
||||
else
|
||||
MEM_write32(memPtr, MEM_swap32(val32));
|
||||
}
|
||||
|
||||
MEM_STATIC U64 MEM_readLE64(const void* memPtr)
|
||||
{
|
||||
if (MEM_isLittleEndian())
|
||||
return MEM_read64(memPtr);
|
||||
else
|
||||
return MEM_swap64(MEM_read64(memPtr));
|
||||
}
|
||||
|
||||
MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64)
|
||||
{
|
||||
if (MEM_isLittleEndian())
|
||||
MEM_write64(memPtr, val64);
|
||||
else
|
||||
MEM_write64(memPtr, MEM_swap64(val64));
|
||||
}
|
||||
|
||||
MEM_STATIC size_t MEM_readLEST(const void* memPtr)
|
||||
{
|
||||
if (MEM_32bits())
|
||||
return (size_t)MEM_readLE32(memPtr);
|
||||
else
|
||||
return (size_t)MEM_readLE64(memPtr);
|
||||
}
|
||||
|
||||
MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val)
|
||||
{
|
||||
if (MEM_32bits())
|
||||
MEM_writeLE32(memPtr, (U32)val);
|
||||
else
|
||||
MEM_writeLE64(memPtr, (U64)val);
|
||||
}
|
||||
|
||||
/*=== Big endian r/w ===*/
|
||||
|
||||
MEM_STATIC U32 MEM_readBE32(const void* memPtr)
|
||||
{
|
||||
if (MEM_isLittleEndian())
|
||||
return MEM_swap32(MEM_read32(memPtr));
|
||||
else
|
||||
return MEM_read32(memPtr);
|
||||
}
|
||||
|
||||
MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32)
|
||||
{
|
||||
if (MEM_isLittleEndian())
|
||||
MEM_write32(memPtr, MEM_swap32(val32));
|
||||
else
|
||||
MEM_write32(memPtr, val32);
|
||||
}
|
||||
|
||||
MEM_STATIC U64 MEM_readBE64(const void* memPtr)
|
||||
{
|
||||
if (MEM_isLittleEndian())
|
||||
return MEM_swap64(MEM_read64(memPtr));
|
||||
else
|
||||
return MEM_read64(memPtr);
|
||||
}
|
||||
|
||||
MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64)
|
||||
{
|
||||
if (MEM_isLittleEndian())
|
||||
MEM_write64(memPtr, MEM_swap64(val64));
|
||||
else
|
||||
MEM_write64(memPtr, val64);
|
||||
}
|
||||
|
||||
MEM_STATIC size_t MEM_readBEST(const void* memPtr)
|
||||
{
|
||||
if (MEM_32bits())
|
||||
return (size_t)MEM_readBE32(memPtr);
|
||||
else
|
||||
return (size_t)MEM_readBE64(memPtr);
|
||||
}
|
||||
|
||||
MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val)
|
||||
{
|
||||
if (MEM_32bits())
|
||||
MEM_writeBE32(memPtr, (U32)val);
|
||||
else
|
||||
MEM_writeBE64(memPtr, (U64)val);
|
||||
}
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* MEM_H_MODULE */
|
||||
|
|
@ -0,0 +1,340 @@
|
|||
/*
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
|
||||
/* ====== Dependencies ======= */
|
||||
#include <stddef.h> /* size_t */
|
||||
#include "debug.h" /* assert */
|
||||
#include "zstd_internal.h" /* ZSTD_malloc, ZSTD_free */
|
||||
#include "pool.h"
|
||||
|
||||
/* ====== Compiler specifics ====== */
|
||||
#if defined(_MSC_VER)
|
||||
# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
|
||||
#endif
|
||||
|
||||
|
||||
#ifdef ZSTD_MULTITHREAD
|
||||
|
||||
#include "threading.h" /* pthread adaptation */
|
||||
|
||||
/* A job is a function and an opaque argument */
|
||||
typedef struct POOL_job_s {
|
||||
POOL_function function;
|
||||
void *opaque;
|
||||
} POOL_job;
|
||||
|
||||
struct POOL_ctx_s {
|
||||
ZSTD_customMem customMem;
|
||||
/* Keep track of the threads */
|
||||
ZSTD_pthread_t* threads;
|
||||
size_t threadCapacity;
|
||||
size_t threadLimit;
|
||||
|
||||
/* The queue is a circular buffer */
|
||||
POOL_job *queue;
|
||||
size_t queueHead;
|
||||
size_t queueTail;
|
||||
size_t queueSize;
|
||||
|
||||
/* The number of threads working on jobs */
|
||||
size_t numThreadsBusy;
|
||||
/* Indicates if the queue is empty */
|
||||
int queueEmpty;
|
||||
|
||||
/* The mutex protects the queue */
|
||||
ZSTD_pthread_mutex_t queueMutex;
|
||||
/* Condition variable for pushers to wait on when the queue is full */
|
||||
ZSTD_pthread_cond_t queuePushCond;
|
||||
/* Condition variables for poppers to wait on when the queue is empty */
|
||||
ZSTD_pthread_cond_t queuePopCond;
|
||||
/* Indicates if the queue is shutting down */
|
||||
int shutdown;
|
||||
};
|
||||
|
||||
/* POOL_thread() :
|
||||
* Work thread for the thread pool.
|
||||
* Waits for jobs and executes them.
|
||||
* @returns : NULL on failure else non-null.
|
||||
*/
|
||||
static void* POOL_thread(void* opaque) {
|
||||
POOL_ctx* const ctx = (POOL_ctx*)opaque;
|
||||
if (!ctx) { return NULL; }
|
||||
for (;;) {
|
||||
/* Lock the mutex and wait for a non-empty queue or until shutdown */
|
||||
ZSTD_pthread_mutex_lock(&ctx->queueMutex);
|
||||
|
||||
while ( ctx->queueEmpty
|
||||
|| (ctx->numThreadsBusy >= ctx->threadLimit) ) {
|
||||
if (ctx->shutdown) {
|
||||
/* even if !queueEmpty, (possible if numThreadsBusy >= threadLimit),
|
||||
* a few threads will be shutdown while !queueEmpty,
|
||||
* but enough threads will remain active to finish the queue */
|
||||
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
||||
return opaque;
|
||||
}
|
||||
ZSTD_pthread_cond_wait(&ctx->queuePopCond, &ctx->queueMutex);
|
||||
}
|
||||
/* Pop a job off the queue */
|
||||
{ POOL_job const job = ctx->queue[ctx->queueHead];
|
||||
ctx->queueHead = (ctx->queueHead + 1) % ctx->queueSize;
|
||||
ctx->numThreadsBusy++;
|
||||
ctx->queueEmpty = ctx->queueHead == ctx->queueTail;
|
||||
/* Unlock the mutex, signal a pusher, and run the job */
|
||||
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
||||
ZSTD_pthread_cond_signal(&ctx->queuePushCond);
|
||||
|
||||
job.function(job.opaque);
|
||||
|
||||
/* If the intended queue size was 0, signal after finishing job */
|
||||
ZSTD_pthread_mutex_lock(&ctx->queueMutex);
|
||||
ctx->numThreadsBusy--;
|
||||
if (ctx->queueSize == 1) {
|
||||
ZSTD_pthread_cond_signal(&ctx->queuePushCond);
|
||||
}
|
||||
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
||||
}
|
||||
} /* for (;;) */
|
||||
assert(0); /* Unreachable */
|
||||
}
|
||||
|
||||
POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) {
|
||||
return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem);
|
||||
}
|
||||
|
||||
POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize,
|
||||
ZSTD_customMem customMem) {
|
||||
POOL_ctx* ctx;
|
||||
/* Check parameters */
|
||||
if (!numThreads) { return NULL; }
|
||||
/* Allocate the context and zero initialize */
|
||||
ctx = (POOL_ctx*)ZSTD_calloc(sizeof(POOL_ctx), customMem);
|
||||
if (!ctx) { return NULL; }
|
||||
/* Initialize the job queue.
|
||||
* It needs one extra space since one space is wasted to differentiate
|
||||
* empty and full queues.
|
||||
*/
|
||||
ctx->queueSize = queueSize + 1;
|
||||
ctx->queue = (POOL_job*)ZSTD_malloc(ctx->queueSize * sizeof(POOL_job), customMem);
|
||||
ctx->queueHead = 0;
|
||||
ctx->queueTail = 0;
|
||||
ctx->numThreadsBusy = 0;
|
||||
ctx->queueEmpty = 1;
|
||||
(void)ZSTD_pthread_mutex_init(&ctx->queueMutex, NULL);
|
||||
(void)ZSTD_pthread_cond_init(&ctx->queuePushCond, NULL);
|
||||
(void)ZSTD_pthread_cond_init(&ctx->queuePopCond, NULL);
|
||||
ctx->shutdown = 0;
|
||||
/* Allocate space for the thread handles */
|
||||
ctx->threads = (ZSTD_pthread_t*)ZSTD_malloc(numThreads * sizeof(ZSTD_pthread_t), customMem);
|
||||
ctx->threadCapacity = 0;
|
||||
ctx->customMem = customMem;
|
||||
/* Check for errors */
|
||||
if (!ctx->threads || !ctx->queue) { POOL_free(ctx); return NULL; }
|
||||
/* Initialize the threads */
|
||||
{ size_t i;
|
||||
for (i = 0; i < numThreads; ++i) {
|
||||
if (ZSTD_pthread_create(&ctx->threads[i], NULL, &POOL_thread, ctx)) {
|
||||
ctx->threadCapacity = i;
|
||||
POOL_free(ctx);
|
||||
return NULL;
|
||||
} }
|
||||
ctx->threadCapacity = numThreads;
|
||||
ctx->threadLimit = numThreads;
|
||||
}
|
||||
return ctx;
|
||||
}
|
||||
|
||||
/*! POOL_join() :
|
||||
Shutdown the queue, wake any sleeping threads, and join all of the threads.
|
||||
*/
|
||||
static void POOL_join(POOL_ctx* ctx) {
|
||||
/* Shut down the queue */
|
||||
ZSTD_pthread_mutex_lock(&ctx->queueMutex);
|
||||
ctx->shutdown = 1;
|
||||
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
||||
/* Wake up sleeping threads */
|
||||
ZSTD_pthread_cond_broadcast(&ctx->queuePushCond);
|
||||
ZSTD_pthread_cond_broadcast(&ctx->queuePopCond);
|
||||
/* Join all of the threads */
|
||||
{ size_t i;
|
||||
for (i = 0; i < ctx->threadCapacity; ++i) {
|
||||
ZSTD_pthread_join(ctx->threads[i], NULL); /* note : could fail */
|
||||
} }
|
||||
}
|
||||
|
||||
void POOL_free(POOL_ctx *ctx) {
|
||||
if (!ctx) { return; }
|
||||
POOL_join(ctx);
|
||||
ZSTD_pthread_mutex_destroy(&ctx->queueMutex);
|
||||
ZSTD_pthread_cond_destroy(&ctx->queuePushCond);
|
||||
ZSTD_pthread_cond_destroy(&ctx->queuePopCond);
|
||||
ZSTD_free(ctx->queue, ctx->customMem);
|
||||
ZSTD_free(ctx->threads, ctx->customMem);
|
||||
ZSTD_free(ctx, ctx->customMem);
|
||||
}
|
||||
|
||||
|
||||
|
||||
size_t POOL_sizeof(POOL_ctx *ctx) {
|
||||
if (ctx==NULL) return 0; /* supports sizeof NULL */
|
||||
return sizeof(*ctx)
|
||||
+ ctx->queueSize * sizeof(POOL_job)
|
||||
+ ctx->threadCapacity * sizeof(ZSTD_pthread_t);
|
||||
}
|
||||
|
||||
|
||||
/* @return : 0 on success, 1 on error */
|
||||
static int POOL_resize_internal(POOL_ctx* ctx, size_t numThreads)
|
||||
{
|
||||
if (numThreads <= ctx->threadCapacity) {
|
||||
if (!numThreads) return 1;
|
||||
ctx->threadLimit = numThreads;
|
||||
return 0;
|
||||
}
|
||||
/* numThreads > threadCapacity */
|
||||
{ ZSTD_pthread_t* const threadPool = (ZSTD_pthread_t*)ZSTD_malloc(numThreads * sizeof(ZSTD_pthread_t), ctx->customMem);
|
||||
if (!threadPool) return 1;
|
||||
/* replace existing thread pool */
|
||||
memcpy(threadPool, ctx->threads, ctx->threadCapacity * sizeof(*threadPool));
|
||||
ZSTD_free(ctx->threads, ctx->customMem);
|
||||
ctx->threads = threadPool;
|
||||
/* Initialize additional threads */
|
||||
{ size_t threadId;
|
||||
for (threadId = ctx->threadCapacity; threadId < numThreads; ++threadId) {
|
||||
if (ZSTD_pthread_create(&threadPool[threadId], NULL, &POOL_thread, ctx)) {
|
||||
ctx->threadCapacity = threadId;
|
||||
return 1;
|
||||
} }
|
||||
} }
|
||||
/* successfully expanded */
|
||||
ctx->threadCapacity = numThreads;
|
||||
ctx->threadLimit = numThreads;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* @return : 0 on success, 1 on error */
|
||||
int POOL_resize(POOL_ctx* ctx, size_t numThreads)
|
||||
{
|
||||
int result;
|
||||
if (ctx==NULL) return 1;
|
||||
ZSTD_pthread_mutex_lock(&ctx->queueMutex);
|
||||
result = POOL_resize_internal(ctx, numThreads);
|
||||
ZSTD_pthread_cond_broadcast(&ctx->queuePopCond);
|
||||
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns 1 if the queue is full and 0 otherwise.
|
||||
*
|
||||
* When queueSize is 1 (pool was created with an intended queueSize of 0),
|
||||
* then a queue is empty if there is a thread free _and_ no job is waiting.
|
||||
*/
|
||||
static int isQueueFull(POOL_ctx const* ctx) {
|
||||
if (ctx->queueSize > 1) {
|
||||
return ctx->queueHead == ((ctx->queueTail + 1) % ctx->queueSize);
|
||||
} else {
|
||||
return (ctx->numThreadsBusy == ctx->threadLimit) ||
|
||||
!ctx->queueEmpty;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static void POOL_add_internal(POOL_ctx* ctx, POOL_function function, void *opaque)
|
||||
{
|
||||
POOL_job const job = {function, opaque};
|
||||
assert(ctx != NULL);
|
||||
if (ctx->shutdown) return;
|
||||
|
||||
ctx->queueEmpty = 0;
|
||||
ctx->queue[ctx->queueTail] = job;
|
||||
ctx->queueTail = (ctx->queueTail + 1) % ctx->queueSize;
|
||||
ZSTD_pthread_cond_signal(&ctx->queuePopCond);
|
||||
}
|
||||
|
||||
void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque)
|
||||
{
|
||||
assert(ctx != NULL);
|
||||
ZSTD_pthread_mutex_lock(&ctx->queueMutex);
|
||||
/* Wait until there is space in the queue for the new job */
|
||||
while (isQueueFull(ctx) && (!ctx->shutdown)) {
|
||||
ZSTD_pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex);
|
||||
}
|
||||
POOL_add_internal(ctx, function, opaque);
|
||||
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
||||
}
|
||||
|
||||
|
||||
int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque)
|
||||
{
|
||||
assert(ctx != NULL);
|
||||
ZSTD_pthread_mutex_lock(&ctx->queueMutex);
|
||||
if (isQueueFull(ctx)) {
|
||||
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
||||
return 0;
|
||||
}
|
||||
POOL_add_internal(ctx, function, opaque);
|
||||
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
||||
#else /* ZSTD_MULTITHREAD not defined */
|
||||
|
||||
/* ========================== */
|
||||
/* No multi-threading support */
|
||||
/* ========================== */
|
||||
|
||||
|
||||
/* We don't need any data, but if it is empty, malloc() might return NULL. */
|
||||
struct POOL_ctx_s {
|
||||
int dummy;
|
||||
};
|
||||
static POOL_ctx g_ctx;
|
||||
|
||||
POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) {
|
||||
return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem);
|
||||
}
|
||||
|
||||
POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem) {
|
||||
(void)numThreads;
|
||||
(void)queueSize;
|
||||
(void)customMem;
|
||||
return &g_ctx;
|
||||
}
|
||||
|
||||
void POOL_free(POOL_ctx* ctx) {
|
||||
assert(!ctx || ctx == &g_ctx);
|
||||
(void)ctx;
|
||||
}
|
||||
|
||||
int POOL_resize(POOL_ctx* ctx, size_t numThreads) {
|
||||
(void)ctx; (void)numThreads;
|
||||
return 0;
|
||||
}
|
||||
|
||||
void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque) {
|
||||
(void)ctx;
|
||||
function(opaque);
|
||||
}
|
||||
|
||||
int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque) {
|
||||
(void)ctx;
|
||||
function(opaque);
|
||||
return 1;
|
||||
}
|
||||
|
||||
size_t POOL_sizeof(POOL_ctx* ctx) {
|
||||
if (ctx==NULL) return 0; /* supports sizeof NULL */
|
||||
assert(ctx == &g_ctx);
|
||||
return sizeof(*ctx);
|
||||
}
|
||||
|
||||
#endif /* ZSTD_MULTITHREAD */
|
||||
|
|
@ -0,0 +1,84 @@
|
|||
/*
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#ifndef POOL_H
|
||||
#define POOL_H
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
#include <stddef.h> /* size_t */
|
||||
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_customMem */
|
||||
#include "zstd.h"
|
||||
|
||||
typedef struct POOL_ctx_s POOL_ctx;
|
||||
|
||||
/*! POOL_create() :
|
||||
* Create a thread pool with at most `numThreads` threads.
|
||||
* `numThreads` must be at least 1.
|
||||
* The maximum number of queued jobs before blocking is `queueSize`.
|
||||
* @return : POOL_ctx pointer on success, else NULL.
|
||||
*/
|
||||
POOL_ctx* POOL_create(size_t numThreads, size_t queueSize);
|
||||
|
||||
POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize,
|
||||
ZSTD_customMem customMem);
|
||||
|
||||
/*! POOL_free() :
|
||||
* Free a thread pool returned by POOL_create().
|
||||
*/
|
||||
void POOL_free(POOL_ctx* ctx);
|
||||
|
||||
/*! POOL_resize() :
|
||||
* Expands or shrinks pool's number of threads.
|
||||
* This is more efficient than releasing + creating a new context,
|
||||
* since it tries to preserve and re-use existing threads.
|
||||
* `numThreads` must be at least 1.
|
||||
* @return : 0 when resize was successful,
|
||||
* !0 (typically 1) if there is an error.
|
||||
* note : only numThreads can be resized, queueSize remains unchanged.
|
||||
*/
|
||||
int POOL_resize(POOL_ctx* ctx, size_t numThreads);
|
||||
|
||||
/*! POOL_sizeof() :
|
||||
* @return threadpool memory usage
|
||||
* note : compatible with NULL (returns 0 in this case)
|
||||
*/
|
||||
size_t POOL_sizeof(POOL_ctx* ctx);
|
||||
|
||||
/*! POOL_function :
|
||||
* The function type that can be added to a thread pool.
|
||||
*/
|
||||
typedef void (*POOL_function)(void*);
|
||||
|
||||
/*! POOL_add() :
|
||||
* Add the job `function(opaque)` to the thread pool. `ctx` must be valid.
|
||||
* Possibly blocks until there is room in the queue.
|
||||
* Note : The function may be executed asynchronously,
|
||||
* therefore, `opaque` must live until function has been completed.
|
||||
*/
|
||||
void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque);
|
||||
|
||||
|
||||
/*! POOL_tryAdd() :
|
||||
* Add the job `function(opaque)` to thread pool _if_ a worker is available.
|
||||
* Returns immediately even if not (does not block).
|
||||
* @return : 1 if successful, 0 if not.
|
||||
*/
|
||||
int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque);
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
|
@ -0,0 +1,75 @@
|
|||
/**
|
||||
* Copyright (c) 2016 Tino Reichardt
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
*
|
||||
* You can contact the author at:
|
||||
* - zstdmt source repository: https://github.com/mcmilk/zstdmt
|
||||
*/
|
||||
|
||||
/**
|
||||
* This file will hold wrapper for systems, which do not support pthreads
|
||||
*/
|
||||
|
||||
/* create fake symbol to avoid empty trnaslation unit warning */
|
||||
int g_ZSTD_threading_useles_symbol;
|
||||
|
||||
#if defined(ZSTD_MULTITHREAD) && defined(_WIN32)
|
||||
|
||||
/**
|
||||
* Windows minimalist Pthread Wrapper, based on :
|
||||
* http://www.cse.wustl.edu/~schmidt/win32-cv-1.html
|
||||
*/
|
||||
|
||||
|
||||
/* === Dependencies === */
|
||||
#include <process.h>
|
||||
#include <errno.h>
|
||||
#include "threading.h"
|
||||
|
||||
|
||||
/* === Implementation === */
|
||||
|
||||
static unsigned __stdcall worker(void *arg)
|
||||
{
|
||||
ZSTD_pthread_t* const thread = (ZSTD_pthread_t*) arg;
|
||||
thread->arg = thread->start_routine(thread->arg);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused,
|
||||
void* (*start_routine) (void*), void* arg)
|
||||
{
|
||||
(void)unused;
|
||||
thread->arg = arg;
|
||||
thread->start_routine = start_routine;
|
||||
thread->handle = (HANDLE) _beginthreadex(NULL, 0, worker, thread, 0, NULL);
|
||||
|
||||
if (!thread->handle)
|
||||
return errno;
|
||||
else
|
||||
return 0;
|
||||
}
|
||||
|
||||
int ZSTD_pthread_join(ZSTD_pthread_t thread, void **value_ptr)
|
||||
{
|
||||
DWORD result;
|
||||
|
||||
if (!thread.handle) return 0;
|
||||
|
||||
result = WaitForSingleObject(thread.handle, INFINITE);
|
||||
switch (result) {
|
||||
case WAIT_OBJECT_0:
|
||||
if (value_ptr) *value_ptr = thread.arg;
|
||||
return 0;
|
||||
case WAIT_ABANDONED:
|
||||
return EINVAL;
|
||||
default:
|
||||
return GetLastError();
|
||||
}
|
||||
}
|
||||
|
||||
#endif /* ZSTD_MULTITHREAD */
|
||||
|
|
@ -0,0 +1,123 @@
|
|||
/**
|
||||
* Copyright (c) 2016 Tino Reichardt
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
*
|
||||
* You can contact the author at:
|
||||
* - zstdmt source repository: https://github.com/mcmilk/zstdmt
|
||||
*/
|
||||
|
||||
#ifndef THREADING_H_938743
|
||||
#define THREADING_H_938743
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#if defined(ZSTD_MULTITHREAD) && defined(_WIN32)
|
||||
|
||||
/**
|
||||
* Windows minimalist Pthread Wrapper, based on :
|
||||
* http://www.cse.wustl.edu/~schmidt/win32-cv-1.html
|
||||
*/
|
||||
#ifdef WINVER
|
||||
# undef WINVER
|
||||
#endif
|
||||
#define WINVER 0x0600
|
||||
|
||||
#ifdef _WIN32_WINNT
|
||||
# undef _WIN32_WINNT
|
||||
#endif
|
||||
#define _WIN32_WINNT 0x0600
|
||||
|
||||
#ifndef WIN32_LEAN_AND_MEAN
|
||||
# define WIN32_LEAN_AND_MEAN
|
||||
#endif
|
||||
|
||||
#undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */
|
||||
#include <windows.h>
|
||||
#undef ERROR
|
||||
#define ERROR(name) ZSTD_ERROR(name)
|
||||
|
||||
|
||||
/* mutex */
|
||||
#define ZSTD_pthread_mutex_t CRITICAL_SECTION
|
||||
#define ZSTD_pthread_mutex_init(a, b) ((void)(b), InitializeCriticalSection((a)), 0)
|
||||
#define ZSTD_pthread_mutex_destroy(a) DeleteCriticalSection((a))
|
||||
#define ZSTD_pthread_mutex_lock(a) EnterCriticalSection((a))
|
||||
#define ZSTD_pthread_mutex_unlock(a) LeaveCriticalSection((a))
|
||||
|
||||
/* condition variable */
|
||||
#define ZSTD_pthread_cond_t CONDITION_VARIABLE
|
||||
#define ZSTD_pthread_cond_init(a, b) ((void)(b), InitializeConditionVariable((a)), 0)
|
||||
#define ZSTD_pthread_cond_destroy(a) ((void)(a))
|
||||
#define ZSTD_pthread_cond_wait(a, b) SleepConditionVariableCS((a), (b), INFINITE)
|
||||
#define ZSTD_pthread_cond_signal(a) WakeConditionVariable((a))
|
||||
#define ZSTD_pthread_cond_broadcast(a) WakeAllConditionVariable((a))
|
||||
|
||||
/* ZSTD_pthread_create() and ZSTD_pthread_join() */
|
||||
typedef struct {
|
||||
HANDLE handle;
|
||||
void* (*start_routine)(void*);
|
||||
void* arg;
|
||||
} ZSTD_pthread_t;
|
||||
|
||||
int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused,
|
||||
void* (*start_routine) (void*), void* arg);
|
||||
|
||||
int ZSTD_pthread_join(ZSTD_pthread_t thread, void** value_ptr);
|
||||
|
||||
/**
|
||||
* add here more wrappers as required
|
||||
*/
|
||||
|
||||
|
||||
#elif defined(ZSTD_MULTITHREAD) /* posix assumed ; need a better detection method */
|
||||
/* === POSIX Systems === */
|
||||
# include <pthread.h>
|
||||
|
||||
#define ZSTD_pthread_mutex_t pthread_mutex_t
|
||||
#define ZSTD_pthread_mutex_init(a, b) pthread_mutex_init((a), (b))
|
||||
#define ZSTD_pthread_mutex_destroy(a) pthread_mutex_destroy((a))
|
||||
#define ZSTD_pthread_mutex_lock(a) pthread_mutex_lock((a))
|
||||
#define ZSTD_pthread_mutex_unlock(a) pthread_mutex_unlock((a))
|
||||
|
||||
#define ZSTD_pthread_cond_t pthread_cond_t
|
||||
#define ZSTD_pthread_cond_init(a, b) pthread_cond_init((a), (b))
|
||||
#define ZSTD_pthread_cond_destroy(a) pthread_cond_destroy((a))
|
||||
#define ZSTD_pthread_cond_wait(a, b) pthread_cond_wait((a), (b))
|
||||
#define ZSTD_pthread_cond_signal(a) pthread_cond_signal((a))
|
||||
#define ZSTD_pthread_cond_broadcast(a) pthread_cond_broadcast((a))
|
||||
|
||||
#define ZSTD_pthread_t pthread_t
|
||||
#define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d))
|
||||
#define ZSTD_pthread_join(a, b) pthread_join((a),(b))
|
||||
|
||||
#else /* ZSTD_MULTITHREAD not defined */
|
||||
/* No multithreading support */
|
||||
|
||||
typedef int ZSTD_pthread_mutex_t;
|
||||
#define ZSTD_pthread_mutex_init(a, b) ((void)(a), (void)(b), 0)
|
||||
#define ZSTD_pthread_mutex_destroy(a) ((void)(a))
|
||||
#define ZSTD_pthread_mutex_lock(a) ((void)(a))
|
||||
#define ZSTD_pthread_mutex_unlock(a) ((void)(a))
|
||||
|
||||
typedef int ZSTD_pthread_cond_t;
|
||||
#define ZSTD_pthread_cond_init(a, b) ((void)(a), (void)(b), 0)
|
||||
#define ZSTD_pthread_cond_destroy(a) ((void)(a))
|
||||
#define ZSTD_pthread_cond_wait(a, b) ((void)(a), (void)(b))
|
||||
#define ZSTD_pthread_cond_signal(a) ((void)(a))
|
||||
#define ZSTD_pthread_cond_broadcast(a) ((void)(a))
|
||||
|
||||
/* do not use ZSTD_pthread_t */
|
||||
|
||||
#endif /* ZSTD_MULTITHREAD */
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* THREADING_H_938743 */
|
||||
|
|
@ -0,0 +1,875 @@
|
|||
/*
|
||||
* xxHash - Fast Hash algorithm
|
||||
* Copyright (C) 2012-2016, Yann Collet
|
||||
*
|
||||
* BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are
|
||||
* met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above
|
||||
* copyright notice, this list of conditions and the following disclaimer
|
||||
* in the documentation and/or other materials provided with the
|
||||
* distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
* You can contact the author at :
|
||||
* - xxHash homepage: http://www.xxhash.com
|
||||
* - xxHash source repository : https://github.com/Cyan4973/xxHash
|
||||
*/
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Tuning parameters
|
||||
***************************************/
|
||||
/*!XXH_FORCE_MEMORY_ACCESS :
|
||||
* By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
|
||||
* Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
|
||||
* The below switch allow to select different access method for improved performance.
|
||||
* Method 0 (default) : use `memcpy()`. Safe and portable.
|
||||
* Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
|
||||
* This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
|
||||
* Method 2 : direct access. This method doesn't depend on compiler but violate C standard.
|
||||
* It can generate buggy code on targets which do not support unaligned memory accesses.
|
||||
* But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
|
||||
* See http://stackoverflow.com/a/32095106/646947 for details.
|
||||
* Prefer these methods in priority order (0 > 1 > 2)
|
||||
*/
|
||||
#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
|
||||
# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
|
||||
# define XXH_FORCE_MEMORY_ACCESS 2
|
||||
# elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \
|
||||
(defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
|
||||
# define XXH_FORCE_MEMORY_ACCESS 1
|
||||
# endif
|
||||
#endif
|
||||
|
||||
/*!XXH_ACCEPT_NULL_INPUT_POINTER :
|
||||
* If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer.
|
||||
* When this option is enabled, xxHash output for null input pointers will be the same as a null-length input.
|
||||
* By default, this option is disabled. To enable it, uncomment below define :
|
||||
*/
|
||||
/* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */
|
||||
|
||||
/*!XXH_FORCE_NATIVE_FORMAT :
|
||||
* By default, xxHash library provides endian-independant Hash values, based on little-endian convention.
|
||||
* Results are therefore identical for little-endian and big-endian CPU.
|
||||
* This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format.
|
||||
* Should endian-independance be of no importance for your application, you may set the #define below to 1,
|
||||
* to improve speed for Big-endian CPU.
|
||||
* This option has no impact on Little_Endian CPU.
|
||||
*/
|
||||
#ifndef XXH_FORCE_NATIVE_FORMAT /* can be defined externally */
|
||||
# define XXH_FORCE_NATIVE_FORMAT 0
|
||||
#endif
|
||||
|
||||
/*!XXH_FORCE_ALIGN_CHECK :
|
||||
* This is a minor performance trick, only useful with lots of very small keys.
|
||||
* It means : check for aligned/unaligned input.
|
||||
* The check costs one initial branch per hash; set to 0 when the input data
|
||||
* is guaranteed to be aligned.
|
||||
*/
|
||||
#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */
|
||||
# if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64)
|
||||
# define XXH_FORCE_ALIGN_CHECK 0
|
||||
# else
|
||||
# define XXH_FORCE_ALIGN_CHECK 1
|
||||
# endif
|
||||
#endif
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Includes & Memory related functions
|
||||
***************************************/
|
||||
/* Modify the local functions below should you wish to use some other memory routines */
|
||||
/* for malloc(), free() */
|
||||
#include <stdlib.h>
|
||||
static void* XXH_malloc(size_t s) { return malloc(s); }
|
||||
static void XXH_free (void* p) { free(p); }
|
||||
/* for memcpy() */
|
||||
#include <string.h>
|
||||
static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); }
|
||||
|
||||
#ifndef XXH_STATIC_LINKING_ONLY
|
||||
# define XXH_STATIC_LINKING_ONLY
|
||||
#endif
|
||||
#include "xxhash.h"
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Compiler Specific Options
|
||||
***************************************/
|
||||
#if defined (__GNUC__) || defined(__cplusplus) || defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
|
||||
# define INLINE_KEYWORD inline
|
||||
#else
|
||||
# define INLINE_KEYWORD
|
||||
#endif
|
||||
|
||||
#if defined(__GNUC__)
|
||||
# define FORCE_INLINE_ATTR __attribute__((always_inline))
|
||||
#elif defined(_MSC_VER)
|
||||
# define FORCE_INLINE_ATTR __forceinline
|
||||
#else
|
||||
# define FORCE_INLINE_ATTR
|
||||
#endif
|
||||
|
||||
#define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR
|
||||
|
||||
|
||||
#ifdef _MSC_VER
|
||||
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
|
||||
#endif
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Basic Types
|
||||
***************************************/
|
||||
#ifndef MEM_MODULE
|
||||
# define MEM_MODULE
|
||||
# if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
|
||||
# include <stdint.h>
|
||||
typedef uint8_t BYTE;
|
||||
typedef uint16_t U16;
|
||||
typedef uint32_t U32;
|
||||
typedef int32_t S32;
|
||||
typedef uint64_t U64;
|
||||
# else
|
||||
typedef unsigned char BYTE;
|
||||
typedef unsigned short U16;
|
||||
typedef unsigned int U32;
|
||||
typedef signed int S32;
|
||||
typedef unsigned long long U64; /* if your compiler doesn't support unsigned long long, replace by another 64-bit type here. Note that xxhash.h will also need to be updated. */
|
||||
# endif
|
||||
#endif
|
||||
|
||||
|
||||
#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))
|
||||
|
||||
/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */
|
||||
static U32 XXH_read32(const void* memPtr) { return *(const U32*) memPtr; }
|
||||
static U64 XXH_read64(const void* memPtr) { return *(const U64*) memPtr; }
|
||||
|
||||
#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))
|
||||
|
||||
/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
|
||||
/* currently only defined for gcc and icc */
|
||||
typedef union { U32 u32; U64 u64; } __attribute__((packed)) unalign;
|
||||
|
||||
static U32 XXH_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
|
||||
static U64 XXH_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
|
||||
|
||||
#else
|
||||
|
||||
/* portable and safe solution. Generally efficient.
|
||||
* see : http://stackoverflow.com/a/32095106/646947
|
||||
*/
|
||||
|
||||
static U32 XXH_read32(const void* memPtr)
|
||||
{
|
||||
U32 val;
|
||||
memcpy(&val, memPtr, sizeof(val));
|
||||
return val;
|
||||
}
|
||||
|
||||
static U64 XXH_read64(const void* memPtr)
|
||||
{
|
||||
U64 val;
|
||||
memcpy(&val, memPtr, sizeof(val));
|
||||
return val;
|
||||
}
|
||||
|
||||
#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */
|
||||
|
||||
|
||||
/* ****************************************
|
||||
* Compiler-specific Functions and Macros
|
||||
******************************************/
|
||||
#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
|
||||
|
||||
/* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */
|
||||
#if defined(_MSC_VER)
|
||||
# define XXH_rotl32(x,r) _rotl(x,r)
|
||||
# define XXH_rotl64(x,r) _rotl64(x,r)
|
||||
#else
|
||||
# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r)))
|
||||
# define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r)))
|
||||
#endif
|
||||
|
||||
#if defined(_MSC_VER) /* Visual Studio */
|
||||
# define XXH_swap32 _byteswap_ulong
|
||||
# define XXH_swap64 _byteswap_uint64
|
||||
#elif GCC_VERSION >= 403
|
||||
# define XXH_swap32 __builtin_bswap32
|
||||
# define XXH_swap64 __builtin_bswap64
|
||||
#else
|
||||
static U32 XXH_swap32 (U32 x)
|
||||
{
|
||||
return ((x << 24) & 0xff000000 ) |
|
||||
((x << 8) & 0x00ff0000 ) |
|
||||
((x >> 8) & 0x0000ff00 ) |
|
||||
((x >> 24) & 0x000000ff );
|
||||
}
|
||||
static U64 XXH_swap64 (U64 x)
|
||||
{
|
||||
return ((x << 56) & 0xff00000000000000ULL) |
|
||||
((x << 40) & 0x00ff000000000000ULL) |
|
||||
((x << 24) & 0x0000ff0000000000ULL) |
|
||||
((x << 8) & 0x000000ff00000000ULL) |
|
||||
((x >> 8) & 0x00000000ff000000ULL) |
|
||||
((x >> 24) & 0x0000000000ff0000ULL) |
|
||||
((x >> 40) & 0x000000000000ff00ULL) |
|
||||
((x >> 56) & 0x00000000000000ffULL);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Architecture Macros
|
||||
***************************************/
|
||||
typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess;
|
||||
|
||||
/* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example on the compiler command line */
|
||||
#ifndef XXH_CPU_LITTLE_ENDIAN
|
||||
static const int g_one = 1;
|
||||
# define XXH_CPU_LITTLE_ENDIAN (*(const char*)(&g_one))
|
||||
#endif
|
||||
|
||||
|
||||
/* ***************************
|
||||
* Memory reads
|
||||
*****************************/
|
||||
typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment;
|
||||
|
||||
FORCE_INLINE_TEMPLATE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
|
||||
{
|
||||
if (align==XXH_unaligned)
|
||||
return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr));
|
||||
else
|
||||
return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr);
|
||||
}
|
||||
|
||||
FORCE_INLINE_TEMPLATE U32 XXH_readLE32(const void* ptr, XXH_endianess endian)
|
||||
{
|
||||
return XXH_readLE32_align(ptr, endian, XXH_unaligned);
|
||||
}
|
||||
|
||||
static U32 XXH_readBE32(const void* ptr)
|
||||
{
|
||||
return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr);
|
||||
}
|
||||
|
||||
FORCE_INLINE_TEMPLATE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
|
||||
{
|
||||
if (align==XXH_unaligned)
|
||||
return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr));
|
||||
else
|
||||
return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr);
|
||||
}
|
||||
|
||||
FORCE_INLINE_TEMPLATE U64 XXH_readLE64(const void* ptr, XXH_endianess endian)
|
||||
{
|
||||
return XXH_readLE64_align(ptr, endian, XXH_unaligned);
|
||||
}
|
||||
|
||||
static U64 XXH_readBE64(const void* ptr)
|
||||
{
|
||||
return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr);
|
||||
}
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Macros
|
||||
***************************************/
|
||||
#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Constants
|
||||
***************************************/
|
||||
static const U32 PRIME32_1 = 2654435761U;
|
||||
static const U32 PRIME32_2 = 2246822519U;
|
||||
static const U32 PRIME32_3 = 3266489917U;
|
||||
static const U32 PRIME32_4 = 668265263U;
|
||||
static const U32 PRIME32_5 = 374761393U;
|
||||
|
||||
static const U64 PRIME64_1 = 11400714785074694791ULL;
|
||||
static const U64 PRIME64_2 = 14029467366897019727ULL;
|
||||
static const U64 PRIME64_3 = 1609587929392839161ULL;
|
||||
static const U64 PRIME64_4 = 9650029242287828579ULL;
|
||||
static const U64 PRIME64_5 = 2870177450012600261ULL;
|
||||
|
||||
XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; }
|
||||
|
||||
|
||||
/* **************************
|
||||
* Utils
|
||||
****************************/
|
||||
XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* restrict dstState, const XXH32_state_t* restrict srcState)
|
||||
{
|
||||
memcpy(dstState, srcState, sizeof(*dstState));
|
||||
}
|
||||
|
||||
XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* restrict dstState, const XXH64_state_t* restrict srcState)
|
||||
{
|
||||
memcpy(dstState, srcState, sizeof(*dstState));
|
||||
}
|
||||
|
||||
|
||||
/* ***************************
|
||||
* Simple Hash Functions
|
||||
*****************************/
|
||||
|
||||
static U32 XXH32_round(U32 seed, U32 input)
|
||||
{
|
||||
seed += input * PRIME32_2;
|
||||
seed = XXH_rotl32(seed, 13);
|
||||
seed *= PRIME32_1;
|
||||
return seed;
|
||||
}
|
||||
|
||||
FORCE_INLINE_TEMPLATE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align)
|
||||
{
|
||||
const BYTE* p = (const BYTE*)input;
|
||||
const BYTE* bEnd = p + len;
|
||||
U32 h32;
|
||||
#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align)
|
||||
|
||||
#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
|
||||
if (p==NULL) {
|
||||
len=0;
|
||||
bEnd=p=(const BYTE*)(size_t)16;
|
||||
}
|
||||
#endif
|
||||
|
||||
if (len>=16) {
|
||||
const BYTE* const limit = bEnd - 16;
|
||||
U32 v1 = seed + PRIME32_1 + PRIME32_2;
|
||||
U32 v2 = seed + PRIME32_2;
|
||||
U32 v3 = seed + 0;
|
||||
U32 v4 = seed - PRIME32_1;
|
||||
|
||||
do {
|
||||
v1 = XXH32_round(v1, XXH_get32bits(p)); p+=4;
|
||||
v2 = XXH32_round(v2, XXH_get32bits(p)); p+=4;
|
||||
v3 = XXH32_round(v3, XXH_get32bits(p)); p+=4;
|
||||
v4 = XXH32_round(v4, XXH_get32bits(p)); p+=4;
|
||||
} while (p<=limit);
|
||||
|
||||
h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18);
|
||||
} else {
|
||||
h32 = seed + PRIME32_5;
|
||||
}
|
||||
|
||||
h32 += (U32) len;
|
||||
|
||||
while (p+4<=bEnd) {
|
||||
h32 += XXH_get32bits(p) * PRIME32_3;
|
||||
h32 = XXH_rotl32(h32, 17) * PRIME32_4 ;
|
||||
p+=4;
|
||||
}
|
||||
|
||||
while (p<bEnd) {
|
||||
h32 += (*p) * PRIME32_5;
|
||||
h32 = XXH_rotl32(h32, 11) * PRIME32_1 ;
|
||||
p++;
|
||||
}
|
||||
|
||||
h32 ^= h32 >> 15;
|
||||
h32 *= PRIME32_2;
|
||||
h32 ^= h32 >> 13;
|
||||
h32 *= PRIME32_3;
|
||||
h32 ^= h32 >> 16;
|
||||
|
||||
return h32;
|
||||
}
|
||||
|
||||
|
||||
XXH_PUBLIC_API unsigned int XXH32 (const void* input, size_t len, unsigned int seed)
|
||||
{
|
||||
#if 0
|
||||
/* Simple version, good for code maintenance, but unfortunately slow for small inputs */
|
||||
XXH32_CREATESTATE_STATIC(state);
|
||||
XXH32_reset(state, seed);
|
||||
XXH32_update(state, input, len);
|
||||
return XXH32_digest(state);
|
||||
#else
|
||||
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
|
||||
|
||||
if (XXH_FORCE_ALIGN_CHECK) {
|
||||
if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */
|
||||
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
|
||||
return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
|
||||
else
|
||||
return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
|
||||
} }
|
||||
|
||||
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
|
||||
return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);
|
||||
else
|
||||
return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned);
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
static U64 XXH64_round(U64 acc, U64 input)
|
||||
{
|
||||
acc += input * PRIME64_2;
|
||||
acc = XXH_rotl64(acc, 31);
|
||||
acc *= PRIME64_1;
|
||||
return acc;
|
||||
}
|
||||
|
||||
static U64 XXH64_mergeRound(U64 acc, U64 val)
|
||||
{
|
||||
val = XXH64_round(0, val);
|
||||
acc ^= val;
|
||||
acc = acc * PRIME64_1 + PRIME64_4;
|
||||
return acc;
|
||||
}
|
||||
|
||||
FORCE_INLINE_TEMPLATE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align)
|
||||
{
|
||||
const BYTE* p = (const BYTE*)input;
|
||||
const BYTE* const bEnd = p + len;
|
||||
U64 h64;
|
||||
#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align)
|
||||
|
||||
#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
|
||||
if (p==NULL) {
|
||||
len=0;
|
||||
bEnd=p=(const BYTE*)(size_t)32;
|
||||
}
|
||||
#endif
|
||||
|
||||
if (len>=32) {
|
||||
const BYTE* const limit = bEnd - 32;
|
||||
U64 v1 = seed + PRIME64_1 + PRIME64_2;
|
||||
U64 v2 = seed + PRIME64_2;
|
||||
U64 v3 = seed + 0;
|
||||
U64 v4 = seed - PRIME64_1;
|
||||
|
||||
do {
|
||||
v1 = XXH64_round(v1, XXH_get64bits(p)); p+=8;
|
||||
v2 = XXH64_round(v2, XXH_get64bits(p)); p+=8;
|
||||
v3 = XXH64_round(v3, XXH_get64bits(p)); p+=8;
|
||||
v4 = XXH64_round(v4, XXH_get64bits(p)); p+=8;
|
||||
} while (p<=limit);
|
||||
|
||||
h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
|
||||
h64 = XXH64_mergeRound(h64, v1);
|
||||
h64 = XXH64_mergeRound(h64, v2);
|
||||
h64 = XXH64_mergeRound(h64, v3);
|
||||
h64 = XXH64_mergeRound(h64, v4);
|
||||
|
||||
} else {
|
||||
h64 = seed + PRIME64_5;
|
||||
}
|
||||
|
||||
h64 += (U64) len;
|
||||
|
||||
while (p+8<=bEnd) {
|
||||
U64 const k1 = XXH64_round(0, XXH_get64bits(p));
|
||||
h64 ^= k1;
|
||||
h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4;
|
||||
p+=8;
|
||||
}
|
||||
|
||||
if (p+4<=bEnd) {
|
||||
h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1;
|
||||
h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
|
||||
p+=4;
|
||||
}
|
||||
|
||||
while (p<bEnd) {
|
||||
h64 ^= (*p) * PRIME64_5;
|
||||
h64 = XXH_rotl64(h64, 11) * PRIME64_1;
|
||||
p++;
|
||||
}
|
||||
|
||||
h64 ^= h64 >> 33;
|
||||
h64 *= PRIME64_2;
|
||||
h64 ^= h64 >> 29;
|
||||
h64 *= PRIME64_3;
|
||||
h64 ^= h64 >> 32;
|
||||
|
||||
return h64;
|
||||
}
|
||||
|
||||
|
||||
XXH_PUBLIC_API unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed)
|
||||
{
|
||||
#if 0
|
||||
/* Simple version, good for code maintenance, but unfortunately slow for small inputs */
|
||||
XXH64_CREATESTATE_STATIC(state);
|
||||
XXH64_reset(state, seed);
|
||||
XXH64_update(state, input, len);
|
||||
return XXH64_digest(state);
|
||||
#else
|
||||
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
|
||||
|
||||
if (XXH_FORCE_ALIGN_CHECK) {
|
||||
if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */
|
||||
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
|
||||
return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
|
||||
else
|
||||
return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
|
||||
} }
|
||||
|
||||
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
|
||||
return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);
|
||||
else
|
||||
return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned);
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
/* **************************************************
|
||||
* Advanced Hash Functions
|
||||
****************************************************/
|
||||
|
||||
XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void)
|
||||
{
|
||||
return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t));
|
||||
}
|
||||
XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr)
|
||||
{
|
||||
XXH_free(statePtr);
|
||||
return XXH_OK;
|
||||
}
|
||||
|
||||
XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void)
|
||||
{
|
||||
return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t));
|
||||
}
|
||||
XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr)
|
||||
{
|
||||
XXH_free(statePtr);
|
||||
return XXH_OK;
|
||||
}
|
||||
|
||||
|
||||
/*** Hash feed ***/
|
||||
|
||||
XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed)
|
||||
{
|
||||
XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */
|
||||
memset(&state, 0, sizeof(state)-4); /* do not write into reserved, for future removal */
|
||||
state.v1 = seed + PRIME32_1 + PRIME32_2;
|
||||
state.v2 = seed + PRIME32_2;
|
||||
state.v3 = seed + 0;
|
||||
state.v4 = seed - PRIME32_1;
|
||||
memcpy(statePtr, &state, sizeof(state));
|
||||
return XXH_OK;
|
||||
}
|
||||
|
||||
|
||||
XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed)
|
||||
{
|
||||
XXH64_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */
|
||||
memset(&state, 0, sizeof(state)-8); /* do not write into reserved, for future removal */
|
||||
state.v1 = seed + PRIME64_1 + PRIME64_2;
|
||||
state.v2 = seed + PRIME64_2;
|
||||
state.v3 = seed + 0;
|
||||
state.v4 = seed - PRIME64_1;
|
||||
memcpy(statePtr, &state, sizeof(state));
|
||||
return XXH_OK;
|
||||
}
|
||||
|
||||
|
||||
FORCE_INLINE_TEMPLATE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian)
|
||||
{
|
||||
const BYTE* p = (const BYTE*)input;
|
||||
const BYTE* const bEnd = p + len;
|
||||
|
||||
#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
|
||||
if (input==NULL) return XXH_ERROR;
|
||||
#endif
|
||||
|
||||
state->total_len_32 += (unsigned)len;
|
||||
state->large_len |= (len>=16) | (state->total_len_32>=16);
|
||||
|
||||
if (state->memsize + len < 16) { /* fill in tmp buffer */
|
||||
XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len);
|
||||
state->memsize += (unsigned)len;
|
||||
return XXH_OK;
|
||||
}
|
||||
|
||||
if (state->memsize) { /* some data left from previous update */
|
||||
XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize);
|
||||
{ const U32* p32 = state->mem32;
|
||||
state->v1 = XXH32_round(state->v1, XXH_readLE32(p32, endian)); p32++;
|
||||
state->v2 = XXH32_round(state->v2, XXH_readLE32(p32, endian)); p32++;
|
||||
state->v3 = XXH32_round(state->v3, XXH_readLE32(p32, endian)); p32++;
|
||||
state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian)); p32++;
|
||||
}
|
||||
p += 16-state->memsize;
|
||||
state->memsize = 0;
|
||||
}
|
||||
|
||||
if (p <= bEnd-16) {
|
||||
const BYTE* const limit = bEnd - 16;
|
||||
U32 v1 = state->v1;
|
||||
U32 v2 = state->v2;
|
||||
U32 v3 = state->v3;
|
||||
U32 v4 = state->v4;
|
||||
|
||||
do {
|
||||
v1 = XXH32_round(v1, XXH_readLE32(p, endian)); p+=4;
|
||||
v2 = XXH32_round(v2, XXH_readLE32(p, endian)); p+=4;
|
||||
v3 = XXH32_round(v3, XXH_readLE32(p, endian)); p+=4;
|
||||
v4 = XXH32_round(v4, XXH_readLE32(p, endian)); p+=4;
|
||||
} while (p<=limit);
|
||||
|
||||
state->v1 = v1;
|
||||
state->v2 = v2;
|
||||
state->v3 = v3;
|
||||
state->v4 = v4;
|
||||
}
|
||||
|
||||
if (p < bEnd) {
|
||||
XXH_memcpy(state->mem32, p, (size_t)(bEnd-p));
|
||||
state->memsize = (unsigned)(bEnd-p);
|
||||
}
|
||||
|
||||
return XXH_OK;
|
||||
}
|
||||
|
||||
XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len)
|
||||
{
|
||||
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
|
||||
|
||||
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
|
||||
return XXH32_update_endian(state_in, input, len, XXH_littleEndian);
|
||||
else
|
||||
return XXH32_update_endian(state_in, input, len, XXH_bigEndian);
|
||||
}
|
||||
|
||||
|
||||
|
||||
FORCE_INLINE_TEMPLATE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian)
|
||||
{
|
||||
const BYTE * p = (const BYTE*)state->mem32;
|
||||
const BYTE* const bEnd = (const BYTE*)(state->mem32) + state->memsize;
|
||||
U32 h32;
|
||||
|
||||
if (state->large_len) {
|
||||
h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18);
|
||||
} else {
|
||||
h32 = state->v3 /* == seed */ + PRIME32_5;
|
||||
}
|
||||
|
||||
h32 += state->total_len_32;
|
||||
|
||||
while (p+4<=bEnd) {
|
||||
h32 += XXH_readLE32(p, endian) * PRIME32_3;
|
||||
h32 = XXH_rotl32(h32, 17) * PRIME32_4;
|
||||
p+=4;
|
||||
}
|
||||
|
||||
while (p<bEnd) {
|
||||
h32 += (*p) * PRIME32_5;
|
||||
h32 = XXH_rotl32(h32, 11) * PRIME32_1;
|
||||
p++;
|
||||
}
|
||||
|
||||
h32 ^= h32 >> 15;
|
||||
h32 *= PRIME32_2;
|
||||
h32 ^= h32 >> 13;
|
||||
h32 *= PRIME32_3;
|
||||
h32 ^= h32 >> 16;
|
||||
|
||||
return h32;
|
||||
}
|
||||
|
||||
|
||||
XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* state_in)
|
||||
{
|
||||
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
|
||||
|
||||
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
|
||||
return XXH32_digest_endian(state_in, XXH_littleEndian);
|
||||
else
|
||||
return XXH32_digest_endian(state_in, XXH_bigEndian);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/* **** XXH64 **** */
|
||||
|
||||
FORCE_INLINE_TEMPLATE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian)
|
||||
{
|
||||
const BYTE* p = (const BYTE*)input;
|
||||
const BYTE* const bEnd = p + len;
|
||||
|
||||
#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
|
||||
if (input==NULL) return XXH_ERROR;
|
||||
#endif
|
||||
|
||||
state->total_len += len;
|
||||
|
||||
if (state->memsize + len < 32) { /* fill in tmp buffer */
|
||||
XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len);
|
||||
state->memsize += (U32)len;
|
||||
return XXH_OK;
|
||||
}
|
||||
|
||||
if (state->memsize) { /* tmp buffer is full */
|
||||
XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize);
|
||||
state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0, endian));
|
||||
state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1, endian));
|
||||
state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2, endian));
|
||||
state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3, endian));
|
||||
p += 32-state->memsize;
|
||||
state->memsize = 0;
|
||||
}
|
||||
|
||||
if (p+32 <= bEnd) {
|
||||
const BYTE* const limit = bEnd - 32;
|
||||
U64 v1 = state->v1;
|
||||
U64 v2 = state->v2;
|
||||
U64 v3 = state->v3;
|
||||
U64 v4 = state->v4;
|
||||
|
||||
do {
|
||||
v1 = XXH64_round(v1, XXH_readLE64(p, endian)); p+=8;
|
||||
v2 = XXH64_round(v2, XXH_readLE64(p, endian)); p+=8;
|
||||
v3 = XXH64_round(v3, XXH_readLE64(p, endian)); p+=8;
|
||||
v4 = XXH64_round(v4, XXH_readLE64(p, endian)); p+=8;
|
||||
} while (p<=limit);
|
||||
|
||||
state->v1 = v1;
|
||||
state->v2 = v2;
|
||||
state->v3 = v3;
|
||||
state->v4 = v4;
|
||||
}
|
||||
|
||||
if (p < bEnd) {
|
||||
XXH_memcpy(state->mem64, p, (size_t)(bEnd-p));
|
||||
state->memsize = (unsigned)(bEnd-p);
|
||||
}
|
||||
|
||||
return XXH_OK;
|
||||
}
|
||||
|
||||
XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len)
|
||||
{
|
||||
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
|
||||
|
||||
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
|
||||
return XXH64_update_endian(state_in, input, len, XXH_littleEndian);
|
||||
else
|
||||
return XXH64_update_endian(state_in, input, len, XXH_bigEndian);
|
||||
}
|
||||
|
||||
|
||||
|
||||
FORCE_INLINE_TEMPLATE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian)
|
||||
{
|
||||
const BYTE * p = (const BYTE*)state->mem64;
|
||||
const BYTE* const bEnd = (const BYTE*)state->mem64 + state->memsize;
|
||||
U64 h64;
|
||||
|
||||
if (state->total_len >= 32) {
|
||||
U64 const v1 = state->v1;
|
||||
U64 const v2 = state->v2;
|
||||
U64 const v3 = state->v3;
|
||||
U64 const v4 = state->v4;
|
||||
|
||||
h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
|
||||
h64 = XXH64_mergeRound(h64, v1);
|
||||
h64 = XXH64_mergeRound(h64, v2);
|
||||
h64 = XXH64_mergeRound(h64, v3);
|
||||
h64 = XXH64_mergeRound(h64, v4);
|
||||
} else {
|
||||
h64 = state->v3 + PRIME64_5;
|
||||
}
|
||||
|
||||
h64 += (U64) state->total_len;
|
||||
|
||||
while (p+8<=bEnd) {
|
||||
U64 const k1 = XXH64_round(0, XXH_readLE64(p, endian));
|
||||
h64 ^= k1;
|
||||
h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4;
|
||||
p+=8;
|
||||
}
|
||||
|
||||
if (p+4<=bEnd) {
|
||||
h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1;
|
||||
h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
|
||||
p+=4;
|
||||
}
|
||||
|
||||
while (p<bEnd) {
|
||||
h64 ^= (*p) * PRIME64_5;
|
||||
h64 = XXH_rotl64(h64, 11) * PRIME64_1;
|
||||
p++;
|
||||
}
|
||||
|
||||
h64 ^= h64 >> 33;
|
||||
h64 *= PRIME64_2;
|
||||
h64 ^= h64 >> 29;
|
||||
h64 *= PRIME64_3;
|
||||
h64 ^= h64 >> 32;
|
||||
|
||||
return h64;
|
||||
}
|
||||
|
||||
|
||||
XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* state_in)
|
||||
{
|
||||
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
|
||||
|
||||
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
|
||||
return XXH64_digest_endian(state_in, XXH_littleEndian);
|
||||
else
|
||||
return XXH64_digest_endian(state_in, XXH_bigEndian);
|
||||
}
|
||||
|
||||
|
||||
/* **************************
|
||||
* Canonical representation
|
||||
****************************/
|
||||
|
||||
/*! Default XXH result types are basic unsigned 32 and 64 bits.
|
||||
* The canonical representation follows human-readable write convention, aka big-endian (large digits first).
|
||||
* These functions allow transformation of hash result into and from its canonical format.
|
||||
* This way, hash values can be written into a file or buffer, and remain comparable across different systems and programs.
|
||||
*/
|
||||
|
||||
XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash)
|
||||
{
|
||||
XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t));
|
||||
if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash);
|
||||
memcpy(dst, &hash, sizeof(*dst));
|
||||
}
|
||||
|
||||
XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash)
|
||||
{
|
||||
XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t));
|
||||
if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash);
|
||||
memcpy(dst, &hash, sizeof(*dst));
|
||||
}
|
||||
|
||||
XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src)
|
||||
{
|
||||
return XXH_readBE32(src);
|
||||
}
|
||||
|
||||
XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src)
|
||||
{
|
||||
return XXH_readBE64(src);
|
||||
}
|
||||
|
|
@ -0,0 +1,305 @@
|
|||
/*
|
||||
xxHash - Extremely Fast Hash algorithm
|
||||
Header File
|
||||
Copyright (C) 2012-2016, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- xxHash source repository : https://github.com/Cyan4973/xxHash
|
||||
*/
|
||||
|
||||
/* Notice extracted from xxHash homepage :
|
||||
|
||||
xxHash is an extremely fast Hash algorithm, running at RAM speed limits.
|
||||
It also successfully passes all tests from the SMHasher suite.
|
||||
|
||||
Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz)
|
||||
|
||||
Name Speed Q.Score Author
|
||||
xxHash 5.4 GB/s 10
|
||||
CrapWow 3.2 GB/s 2 Andrew
|
||||
MumurHash 3a 2.7 GB/s 10 Austin Appleby
|
||||
SpookyHash 2.0 GB/s 10 Bob Jenkins
|
||||
SBox 1.4 GB/s 9 Bret Mulvey
|
||||
Lookup3 1.2 GB/s 9 Bob Jenkins
|
||||
SuperFastHash 1.2 GB/s 1 Paul Hsieh
|
||||
CityHash64 1.05 GB/s 10 Pike & Alakuijala
|
||||
FNV 0.55 GB/s 5 Fowler, Noll, Vo
|
||||
CRC32 0.43 GB/s 9
|
||||
MD5-32 0.33 GB/s 10 Ronald L. Rivest
|
||||
SHA1-32 0.28 GB/s 10
|
||||
|
||||
Q.Score is a measure of quality of the hash function.
|
||||
It depends on successfully passing SMHasher test set.
|
||||
10 is a perfect score.
|
||||
|
||||
A 64-bits version, named XXH64, is available since r35.
|
||||
It offers much better speed, but for 64-bits applications only.
|
||||
Name Speed on 64 bits Speed on 32 bits
|
||||
XXH64 13.8 GB/s 1.9 GB/s
|
||||
XXH32 6.8 GB/s 6.0 GB/s
|
||||
*/
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#ifndef XXHASH_H_5627135585666179
|
||||
#define XXHASH_H_5627135585666179 1
|
||||
|
||||
|
||||
/* ****************************
|
||||
* Definitions
|
||||
******************************/
|
||||
#include <stddef.h> /* size_t */
|
||||
typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode;
|
||||
|
||||
|
||||
/* ****************************
|
||||
* API modifier
|
||||
******************************/
|
||||
/** XXH_PRIVATE_API
|
||||
* This is useful if you want to include xxhash functions in `static` mode
|
||||
* in order to inline them, and remove their symbol from the public list.
|
||||
* Methodology :
|
||||
* #define XXH_PRIVATE_API
|
||||
* #include "xxhash.h"
|
||||
* `xxhash.c` is automatically included.
|
||||
* It's not useful to compile and link it as a separate module anymore.
|
||||
*/
|
||||
#ifdef XXH_PRIVATE_API
|
||||
# ifndef XXH_STATIC_LINKING_ONLY
|
||||
# define XXH_STATIC_LINKING_ONLY
|
||||
# endif
|
||||
# if defined(__GNUC__)
|
||||
# define XXH_PUBLIC_API static __inline __attribute__((unused))
|
||||
# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
|
||||
# define XXH_PUBLIC_API static inline
|
||||
# elif defined(_MSC_VER)
|
||||
# define XXH_PUBLIC_API static __inline
|
||||
# else
|
||||
# define XXH_PUBLIC_API static /* this version may generate warnings for unused static functions; disable the relevant warning */
|
||||
# endif
|
||||
#else
|
||||
# define XXH_PUBLIC_API /* do nothing */
|
||||
#endif /* XXH_PRIVATE_API */
|
||||
|
||||
/*!XXH_NAMESPACE, aka Namespace Emulation :
|
||||
|
||||
If you want to include _and expose_ xxHash functions from within your own library,
|
||||
but also want to avoid symbol collisions with another library which also includes xxHash,
|
||||
|
||||
you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library
|
||||
with the value of XXH_NAMESPACE (so avoid to keep it NULL and avoid numeric values).
|
||||
|
||||
Note that no change is required within the calling program as long as it includes `xxhash.h` :
|
||||
regular symbol name will be automatically translated by this header.
|
||||
*/
|
||||
#ifdef XXH_NAMESPACE
|
||||
# define XXH_CAT(A,B) A##B
|
||||
# define XXH_NAME2(A,B) XXH_CAT(A,B)
|
||||
# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32)
|
||||
# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64)
|
||||
# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber)
|
||||
# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState)
|
||||
# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState)
|
||||
# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState)
|
||||
# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState)
|
||||
# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset)
|
||||
# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset)
|
||||
# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update)
|
||||
# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update)
|
||||
# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest)
|
||||
# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest)
|
||||
# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState)
|
||||
# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState)
|
||||
# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash)
|
||||
# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash)
|
||||
# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical)
|
||||
# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical)
|
||||
#endif
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Version
|
||||
***************************************/
|
||||
#define XXH_VERSION_MAJOR 0
|
||||
#define XXH_VERSION_MINOR 6
|
||||
#define XXH_VERSION_RELEASE 2
|
||||
#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE)
|
||||
XXH_PUBLIC_API unsigned XXH_versionNumber (void);
|
||||
|
||||
|
||||
/* ****************************
|
||||
* Simple Hash Functions
|
||||
******************************/
|
||||
typedef unsigned int XXH32_hash_t;
|
||||
typedef unsigned long long XXH64_hash_t;
|
||||
|
||||
XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, unsigned int seed);
|
||||
XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, unsigned long long seed);
|
||||
|
||||
/*!
|
||||
XXH32() :
|
||||
Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input".
|
||||
The memory between input & input+length must be valid (allocated and read-accessible).
|
||||
"seed" can be used to alter the result predictably.
|
||||
Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s
|
||||
XXH64() :
|
||||
Calculate the 64-bits hash of sequence of length "len" stored at memory address "input".
|
||||
"seed" can be used to alter the result predictably.
|
||||
This function runs 2x faster on 64-bits systems, but slower on 32-bits systems (see benchmark).
|
||||
*/
|
||||
|
||||
|
||||
/* ****************************
|
||||
* Streaming Hash Functions
|
||||
******************************/
|
||||
typedef struct XXH32_state_s XXH32_state_t; /* incomplete type */
|
||||
typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */
|
||||
|
||||
/*! State allocation, compatible with dynamic libraries */
|
||||
|
||||
XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void);
|
||||
XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr);
|
||||
|
||||
XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void);
|
||||
XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr);
|
||||
|
||||
|
||||
/* hash streaming */
|
||||
|
||||
XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned int seed);
|
||||
XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length);
|
||||
XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr);
|
||||
|
||||
XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed);
|
||||
XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length);
|
||||
XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr);
|
||||
|
||||
/*
|
||||
These functions generate the xxHash of an input provided in multiple segments.
|
||||
Note that, for small input, they are slower than single-call functions, due to state management.
|
||||
For small input, prefer `XXH32()` and `XXH64()` .
|
||||
|
||||
XXH state must first be allocated, using XXH*_createState() .
|
||||
|
||||
Start a new hash by initializing state with a seed, using XXH*_reset().
|
||||
|
||||
Then, feed the hash state by calling XXH*_update() as many times as necessary.
|
||||
Obviously, input must be allocated and read accessible.
|
||||
The function returns an error code, with 0 meaning OK, and any other value meaning there is an error.
|
||||
|
||||
Finally, a hash value can be produced anytime, by using XXH*_digest().
|
||||
This function returns the nn-bits hash as an int or long long.
|
||||
|
||||
It's still possible to continue inserting input into the hash state after a digest,
|
||||
and generate some new hashes later on, by calling again XXH*_digest().
|
||||
|
||||
When done, free XXH state space if it was allocated dynamically.
|
||||
*/
|
||||
|
||||
|
||||
/* **************************
|
||||
* Utils
|
||||
****************************/
|
||||
#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* ! C99 */
|
||||
# define restrict /* disable restrict */
|
||||
#endif
|
||||
|
||||
XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* restrict dst_state, const XXH32_state_t* restrict src_state);
|
||||
XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* restrict dst_state, const XXH64_state_t* restrict src_state);
|
||||
|
||||
|
||||
/* **************************
|
||||
* Canonical representation
|
||||
****************************/
|
||||
/* Default result type for XXH functions are primitive unsigned 32 and 64 bits.
|
||||
* The canonical representation uses human-readable write convention, aka big-endian (large digits first).
|
||||
* These functions allow transformation of hash result into and from its canonical format.
|
||||
* This way, hash values can be written into a file / memory, and remain comparable on different systems and programs.
|
||||
*/
|
||||
typedef struct { unsigned char digest[4]; } XXH32_canonical_t;
|
||||
typedef struct { unsigned char digest[8]; } XXH64_canonical_t;
|
||||
|
||||
XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash);
|
||||
XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash);
|
||||
|
||||
XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src);
|
||||
XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src);
|
||||
|
||||
#endif /* XXHASH_H_5627135585666179 */
|
||||
|
||||
|
||||
|
||||
/* ================================================================================================
|
||||
This section contains definitions which are not guaranteed to remain stable.
|
||||
They may change in future versions, becoming incompatible with a different version of the library.
|
||||
They shall only be used with static linking.
|
||||
Never use these definitions in association with dynamic linking !
|
||||
=================================================================================================== */
|
||||
#if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXH_STATIC_H_3543687687345)
|
||||
#define XXH_STATIC_H_3543687687345
|
||||
|
||||
/* These definitions are only meant to allow allocation of XXH state
|
||||
statically, on stack, or in a struct for example.
|
||||
Do not use members directly. */
|
||||
|
||||
struct XXH32_state_s {
|
||||
unsigned total_len_32;
|
||||
unsigned large_len;
|
||||
unsigned v1;
|
||||
unsigned v2;
|
||||
unsigned v3;
|
||||
unsigned v4;
|
||||
unsigned mem32[4]; /* buffer defined as U32 for alignment */
|
||||
unsigned memsize;
|
||||
unsigned reserved; /* never read nor write, will be removed in a future version */
|
||||
}; /* typedef'd to XXH32_state_t */
|
||||
|
||||
struct XXH64_state_s {
|
||||
unsigned long long total_len;
|
||||
unsigned long long v1;
|
||||
unsigned long long v2;
|
||||
unsigned long long v3;
|
||||
unsigned long long v4;
|
||||
unsigned long long mem64[4]; /* buffer defined as U64 for alignment */
|
||||
unsigned memsize;
|
||||
unsigned reserved[2]; /* never read nor write, will be removed in a future version */
|
||||
}; /* typedef'd to XXH64_state_t */
|
||||
|
||||
|
||||
# ifdef XXH_PRIVATE_API
|
||||
# include "xxhash.c" /* include xxhash functions as `static`, for inlining */
|
||||
# endif
|
||||
|
||||
#endif /* XXH_STATIC_LINKING_ONLY && XXH_STATIC_H_3543687687345 */
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
|
@ -0,0 +1,81 @@
|
|||
/*
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Dependencies
|
||||
***************************************/
|
||||
#include <stdlib.h> /* malloc, calloc, free */
|
||||
#include <string.h> /* memset */
|
||||
#include "error_private.h"
|
||||
#include "zstd_internal.h"
|
||||
|
||||
|
||||
/*-****************************************
|
||||
* Version
|
||||
******************************************/
|
||||
unsigned ZSTD_versionNumber(void) { return ZSTD_VERSION_NUMBER; }
|
||||
|
||||
const char* ZSTD_versionString(void) { return ZSTD_VERSION_STRING; }
|
||||
|
||||
|
||||
/*-****************************************
|
||||
* ZSTD Error Management
|
||||
******************************************/
|
||||
/*! ZSTD_isError() :
|
||||
* tells if a return value is an error code */
|
||||
unsigned ZSTD_isError(size_t code) { return ERR_isError(code); }
|
||||
|
||||
/*! ZSTD_getErrorName() :
|
||||
* provides error code string from function result (useful for debugging) */
|
||||
const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); }
|
||||
|
||||
/*! ZSTD_getError() :
|
||||
* convert a `size_t` function result into a proper ZSTD_errorCode enum */
|
||||
ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); }
|
||||
|
||||
/*! ZSTD_getErrorString() :
|
||||
* provides error code string from enum */
|
||||
const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorString(code); }
|
||||
|
||||
|
||||
|
||||
/*=**************************************************************
|
||||
* Custom allocator
|
||||
****************************************************************/
|
||||
void* ZSTD_malloc(size_t size, ZSTD_customMem customMem)
|
||||
{
|
||||
if (customMem.customAlloc)
|
||||
return customMem.customAlloc(customMem.opaque, size);
|
||||
return malloc(size);
|
||||
}
|
||||
|
||||
void* ZSTD_calloc(size_t size, ZSTD_customMem customMem)
|
||||
{
|
||||
if (customMem.customAlloc) {
|
||||
/* calloc implemented as malloc+memset;
|
||||
* not as efficient as calloc, but next best guess for custom malloc */
|
||||
void* const ptr = customMem.customAlloc(customMem.opaque, size);
|
||||
memset(ptr, 0, size);
|
||||
return ptr;
|
||||
}
|
||||
return calloc(1, size);
|
||||
}
|
||||
|
||||
void ZSTD_free(void* ptr, ZSTD_customMem customMem)
|
||||
{
|
||||
if (ptr!=NULL) {
|
||||
if (customMem.customFree)
|
||||
customMem.customFree(customMem.opaque, ptr);
|
||||
else
|
||||
free(ptr);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,92 @@
|
|||
/*
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#ifndef ZSTD_ERRORS_H_398273423
|
||||
#define ZSTD_ERRORS_H_398273423
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/*===== dependency =====*/
|
||||
#include <stddef.h> /* size_t */
|
||||
|
||||
|
||||
/* ===== ZSTDERRORLIB_API : control library symbols visibility ===== */
|
||||
#ifndef ZSTDERRORLIB_VISIBILITY
|
||||
# if defined(__GNUC__) && (__GNUC__ >= 4)
|
||||
# define ZSTDERRORLIB_VISIBILITY __attribute__ ((visibility ("default")))
|
||||
# else
|
||||
# define ZSTDERRORLIB_VISIBILITY
|
||||
# endif
|
||||
#endif
|
||||
#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
|
||||
# define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBILITY
|
||||
#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
|
||||
# define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
|
||||
#else
|
||||
# define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBILITY
|
||||
#endif
|
||||
|
||||
/*-*********************************************
|
||||
* Error codes list
|
||||
*-*********************************************
|
||||
* Error codes _values_ are pinned down since v1.3.1 only.
|
||||
* Therefore, don't rely on values if you may link to any version < v1.3.1.
|
||||
*
|
||||
* Only values < 100 are considered stable.
|
||||
*
|
||||
* note 1 : this API shall be used with static linking only.
|
||||
* dynamic linking is not yet officially supported.
|
||||
* note 2 : Prefer relying on the enum than on its value whenever possible
|
||||
* This is the only supported way to use the error list < v1.3.1
|
||||
* note 3 : ZSTD_isError() is always correct, whatever the library version.
|
||||
**********************************************/
|
||||
typedef enum {
|
||||
ZSTD_error_no_error = 0,
|
||||
ZSTD_error_GENERIC = 1,
|
||||
ZSTD_error_prefix_unknown = 10,
|
||||
ZSTD_error_version_unsupported = 12,
|
||||
ZSTD_error_frameParameter_unsupported = 14,
|
||||
ZSTD_error_frameParameter_windowTooLarge = 16,
|
||||
ZSTD_error_corruption_detected = 20,
|
||||
ZSTD_error_checksum_wrong = 22,
|
||||
ZSTD_error_dictionary_corrupted = 30,
|
||||
ZSTD_error_dictionary_wrong = 32,
|
||||
ZSTD_error_dictionaryCreation_failed = 34,
|
||||
ZSTD_error_parameter_unsupported = 40,
|
||||
ZSTD_error_parameter_outOfBound = 42,
|
||||
ZSTD_error_tableLog_tooLarge = 44,
|
||||
ZSTD_error_maxSymbolValue_tooLarge = 46,
|
||||
ZSTD_error_maxSymbolValue_tooSmall = 48,
|
||||
ZSTD_error_stage_wrong = 60,
|
||||
ZSTD_error_init_missing = 62,
|
||||
ZSTD_error_memory_allocation = 64,
|
||||
ZSTD_error_workSpace_tooSmall= 66,
|
||||
ZSTD_error_dstSize_tooSmall = 70,
|
||||
ZSTD_error_srcSize_wrong = 72,
|
||||
/* following error codes are __NOT STABLE__, they can be removed or changed in future versions */
|
||||
ZSTD_error_frameIndex_tooLarge = 100,
|
||||
ZSTD_error_seekableIO = 102,
|
||||
ZSTD_error_maxCode = 120 /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */
|
||||
} ZSTD_ErrorCode;
|
||||
|
||||
/*! ZSTD_getErrorCode() :
|
||||
convert a `size_t` function result into a `ZSTD_ErrorCode` enum type,
|
||||
which can be used to compare with enum list published above */
|
||||
ZSTDERRORLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
|
||||
ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code); /**< Same as ZSTD_getErrorName, but using a `ZSTD_ErrorCode` enum argument */
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ZSTD_ERRORS_H_398273423 */
|
||||
|
|
@ -0,0 +1,256 @@
|
|||
/*
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#ifndef ZSTD_CCOMMON_H_MODULE
|
||||
#define ZSTD_CCOMMON_H_MODULE
|
||||
|
||||
/* this module contains definitions which must be identical
|
||||
* across compression, decompression and dictBuilder.
|
||||
* It also contains a few functions useful to at least 2 of them
|
||||
* and which benefit from being inlined */
|
||||
|
||||
/*-*************************************
|
||||
* Dependencies
|
||||
***************************************/
|
||||
#include "compiler.h"
|
||||
#include "mem.h"
|
||||
#include "debug.h" /* assert, DEBUGLOG, RAWLOG, g_debuglevel */
|
||||
#include "error_private.h"
|
||||
#define ZSTD_STATIC_LINKING_ONLY
|
||||
#include "zstd.h"
|
||||
#define FSE_STATIC_LINKING_ONLY
|
||||
#include "fse.h"
|
||||
#define HUF_STATIC_LINKING_ONLY
|
||||
#include "huf.h"
|
||||
#ifndef XXH_STATIC_LINKING_ONLY
|
||||
# define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
|
||||
#endif
|
||||
#include "xxhash.h" /* XXH_reset, update, digest */
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* ---- static assert (debug) --- */
|
||||
#define ZSTD_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c)
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* shared macros
|
||||
***************************************/
|
||||
#undef MIN
|
||||
#undef MAX
|
||||
#define MIN(a,b) ((a)<(b) ? (a) : (b))
|
||||
#define MAX(a,b) ((a)>(b) ? (a) : (b))
|
||||
#define CHECK_F(f) { size_t const errcod = f; if (ERR_isError(errcod)) return errcod; } /* check and Forward error code */
|
||||
#define CHECK_E(f, e) { size_t const errcod = f; if (ERR_isError(errcod)) return ERROR(e); } /* check and send Error code */
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Common constants
|
||||
***************************************/
|
||||
#define ZSTD_OPT_NUM (1<<12)
|
||||
|
||||
#define ZSTD_REP_NUM 3 /* number of repcodes */
|
||||
#define ZSTD_REP_MOVE (ZSTD_REP_NUM-1)
|
||||
static const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 };
|
||||
|
||||
#define KB *(1 <<10)
|
||||
#define MB *(1 <<20)
|
||||
#define GB *(1U<<30)
|
||||
|
||||
#define BIT7 128
|
||||
#define BIT6 64
|
||||
#define BIT5 32
|
||||
#define BIT4 16
|
||||
#define BIT1 2
|
||||
#define BIT0 1
|
||||
|
||||
#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10
|
||||
#define ZSTD_WINDOWLOG_DEFAULTMAX 27 /* Default maximum allowed window log */
|
||||
static const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 };
|
||||
static const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 };
|
||||
|
||||
#define ZSTD_FRAMEIDSIZE 4
|
||||
static const size_t ZSTD_frameIdSize = ZSTD_FRAMEIDSIZE; /* magic number size */
|
||||
|
||||
#define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */
|
||||
static const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE;
|
||||
typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e;
|
||||
|
||||
#define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */
|
||||
#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */ + MIN_SEQUENCES_SIZE /* nbSeq==0 */) /* for a non-null block */
|
||||
|
||||
#define HufLog 12
|
||||
typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingType_e;
|
||||
|
||||
#define LONGNBSEQ 0x7F00
|
||||
|
||||
#define MINMATCH 3
|
||||
|
||||
#define Litbits 8
|
||||
#define MaxLit ((1<<Litbits) - 1)
|
||||
#define MaxML 52
|
||||
#define MaxLL 35
|
||||
#define DefaultMaxOff 28
|
||||
#define MaxOff 31
|
||||
#define MaxSeq MAX(MaxLL, MaxML) /* Assumption : MaxOff < MaxLL,MaxML */
|
||||
#define MLFSELog 9
|
||||
#define LLFSELog 9
|
||||
#define OffFSELog 8
|
||||
#define MaxFSELog MAX(MAX(MLFSELog, LLFSELog), OffFSELog)
|
||||
|
||||
static const U32 LL_bits[MaxLL+1] = { 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
1, 1, 1, 1, 2, 2, 3, 3,
|
||||
4, 6, 7, 8, 9,10,11,12,
|
||||
13,14,15,16 };
|
||||
static const S16 LL_defaultNorm[MaxLL+1] = { 4, 3, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 1, 1, 1,
|
||||
2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 3, 2, 1, 1, 1, 1, 1,
|
||||
-1,-1,-1,-1 };
|
||||
#define LL_DEFAULTNORMLOG 6 /* for static allocation */
|
||||
static const U32 LL_defaultNormLog = LL_DEFAULTNORMLOG;
|
||||
|
||||
static const U32 ML_bits[MaxML+1] = { 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
1, 1, 1, 1, 2, 2, 3, 3,
|
||||
4, 4, 5, 7, 8, 9,10,11,
|
||||
12,13,14,15,16 };
|
||||
static const S16 ML_defaultNorm[MaxML+1] = { 1, 4, 3, 2, 2, 2, 2, 2,
|
||||
2, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1,-1,-1,
|
||||
-1,-1,-1,-1,-1 };
|
||||
#define ML_DEFAULTNORMLOG 6 /* for static allocation */
|
||||
static const U32 ML_defaultNormLog = ML_DEFAULTNORMLOG;
|
||||
|
||||
static const S16 OF_defaultNorm[DefaultMaxOff+1] = { 1, 1, 1, 1, 1, 1, 2, 2,
|
||||
2, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1,
|
||||
-1,-1,-1,-1,-1 };
|
||||
#define OF_DEFAULTNORMLOG 5 /* for static allocation */
|
||||
static const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG;
|
||||
|
||||
|
||||
/*-*******************************************
|
||||
* Shared functions to include for inlining
|
||||
*********************************************/
|
||||
static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
|
||||
#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
|
||||
|
||||
/*! ZSTD_wildcopy() :
|
||||
* custom version of memcpy(), can overwrite up to WILDCOPY_OVERLENGTH bytes (if length==0) */
|
||||
#define WILDCOPY_OVERLENGTH 8
|
||||
MEM_STATIC void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
BYTE* op = (BYTE*)dst;
|
||||
BYTE* const oend = op + length;
|
||||
do
|
||||
COPY8(op, ip)
|
||||
while (op < oend);
|
||||
}
|
||||
|
||||
MEM_STATIC void ZSTD_wildcopy_e(void* dst, const void* src, void* dstEnd) /* should be faster for decoding, but strangely, not verified on all platform */
|
||||
{
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
BYTE* op = (BYTE*)dst;
|
||||
BYTE* const oend = (BYTE*)dstEnd;
|
||||
do
|
||||
COPY8(op, ip)
|
||||
while (op < oend);
|
||||
}
|
||||
|
||||
|
||||
/*-*******************************************
|
||||
* Private declarations
|
||||
*********************************************/
|
||||
typedef struct seqDef_s {
|
||||
U32 offset;
|
||||
U16 litLength;
|
||||
U16 matchLength;
|
||||
} seqDef;
|
||||
|
||||
typedef struct {
|
||||
seqDef* sequencesStart;
|
||||
seqDef* sequences;
|
||||
BYTE* litStart;
|
||||
BYTE* lit;
|
||||
BYTE* llCode;
|
||||
BYTE* mlCode;
|
||||
BYTE* ofCode;
|
||||
U32 longLengthID; /* 0 == no longLength; 1 == Lit.longLength; 2 == Match.longLength; */
|
||||
U32 longLengthPos;
|
||||
} seqStore_t;
|
||||
|
||||
const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); /* compress & dictBuilder */
|
||||
void ZSTD_seqToCodes(const seqStore_t* seqStorePtr); /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */
|
||||
|
||||
/* custom memory allocation functions */
|
||||
void* ZSTD_malloc(size_t size, ZSTD_customMem customMem);
|
||||
void* ZSTD_calloc(size_t size, ZSTD_customMem customMem);
|
||||
void ZSTD_free(void* ptr, ZSTD_customMem customMem);
|
||||
|
||||
|
||||
MEM_STATIC U32 ZSTD_highbit32(U32 val) /* compress, dictBuilder, decodeCorpus */
|
||||
{
|
||||
assert(val != 0);
|
||||
{
|
||||
# if defined(_MSC_VER) /* Visual */
|
||||
unsigned long r=0;
|
||||
_BitScanReverse(&r, val);
|
||||
return (unsigned)r;
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */
|
||||
return 31 - __builtin_clz(val);
|
||||
# else /* Software version */
|
||||
static const U32 DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
|
||||
U32 v = val;
|
||||
v |= v >> 1;
|
||||
v |= v >> 2;
|
||||
v |= v >> 4;
|
||||
v |= v >> 8;
|
||||
v |= v >> 16;
|
||||
return DeBruijnClz[(v * 0x07C4ACDDU) >> 27];
|
||||
# endif
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/* ZSTD_invalidateRepCodes() :
|
||||
* ensures next compression will not use repcodes from previous block.
|
||||
* Note : only works with regular variant;
|
||||
* do not use with extDict variant ! */
|
||||
void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx); /* zstdmt, adaptive_compression (shouldn't get this definition from here) */
|
||||
|
||||
|
||||
typedef struct {
|
||||
blockType_e blockType;
|
||||
U32 lastBlock;
|
||||
U32 origSize;
|
||||
} blockProperties_t;
|
||||
|
||||
/*! ZSTD_getcBlockSize() :
|
||||
* Provides the size of compressed block from block header `src` */
|
||||
/* Used by: decompress, fullbench (does not get its definition from here) */
|
||||
size_t ZSTD_getcBlockSize(const void* src, size_t srcSize,
|
||||
blockProperties_t* bpPtr);
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ZSTD_CCOMMON_H_MODULE */
|
||||
|
|
@ -0,0 +1,714 @@
|
|||
/* ******************************************************************
|
||||
FSE : Finite State Entropy encoder
|
||||
Copyright (C) 2013-present, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
****************************************************************** */
|
||||
|
||||
/* **************************************************************
|
||||
* Includes
|
||||
****************************************************************/
|
||||
#include <stdlib.h> /* malloc, free, qsort */
|
||||
#include <string.h> /* memcpy, memset */
|
||||
#include "compiler.h"
|
||||
#include "mem.h" /* U32, U16, etc. */
|
||||
#include "debug.h" /* assert, DEBUGLOG */
|
||||
#include "hist.h" /* HIST_count_wksp */
|
||||
#include "bitstream.h"
|
||||
#define FSE_STATIC_LINKING_ONLY
|
||||
#include "fse.h"
|
||||
#include "error_private.h"
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* Error Management
|
||||
****************************************************************/
|
||||
#define FSE_isError ERR_isError
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* Templates
|
||||
****************************************************************/
|
||||
/*
|
||||
designed to be included
|
||||
for type-specific functions (template emulation in C)
|
||||
Objective is to write these functions only once, for improved maintenance
|
||||
*/
|
||||
|
||||
/* safety checks */
|
||||
#ifndef FSE_FUNCTION_EXTENSION
|
||||
# error "FSE_FUNCTION_EXTENSION must be defined"
|
||||
#endif
|
||||
#ifndef FSE_FUNCTION_TYPE
|
||||
# error "FSE_FUNCTION_TYPE must be defined"
|
||||
#endif
|
||||
|
||||
/* Function names */
|
||||
#define FSE_CAT(X,Y) X##Y
|
||||
#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y)
|
||||
#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y)
|
||||
|
||||
|
||||
/* Function templates */
|
||||
|
||||
/* FSE_buildCTable_wksp() :
|
||||
* Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`).
|
||||
* wkspSize should be sized to handle worst case situation, which is `1<<max_tableLog * sizeof(FSE_FUNCTION_TYPE)`
|
||||
* workSpace must also be properly aligned with FSE_FUNCTION_TYPE requirements
|
||||
*/
|
||||
size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize)
|
||||
{
|
||||
U32 const tableSize = 1 << tableLog;
|
||||
U32 const tableMask = tableSize - 1;
|
||||
void* const ptr = ct;
|
||||
U16* const tableU16 = ( (U16*) ptr) + 2;
|
||||
void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableLog ? tableSize>>1 : 1) ;
|
||||
FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT);
|
||||
U32 const step = FSE_TABLESTEP(tableSize);
|
||||
U32 cumul[FSE_MAX_SYMBOL_VALUE+2];
|
||||
|
||||
FSE_FUNCTION_TYPE* const tableSymbol = (FSE_FUNCTION_TYPE*)workSpace;
|
||||
U32 highThreshold = tableSize-1;
|
||||
|
||||
/* CTable header */
|
||||
if (((size_t)1 << tableLog) * sizeof(FSE_FUNCTION_TYPE) > wkspSize) return ERROR(tableLog_tooLarge);
|
||||
tableU16[-2] = (U16) tableLog;
|
||||
tableU16[-1] = (U16) maxSymbolValue;
|
||||
assert(tableLog < 16); /* required for the threshold strategy to work */
|
||||
|
||||
/* For explanations on how to distribute symbol values over the table :
|
||||
* http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */
|
||||
|
||||
/* symbol start positions */
|
||||
{ U32 u;
|
||||
cumul[0] = 0;
|
||||
for (u=1; u<=maxSymbolValue+1; u++) {
|
||||
if (normalizedCounter[u-1]==-1) { /* Low proba symbol */
|
||||
cumul[u] = cumul[u-1] + 1;
|
||||
tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(u-1);
|
||||
} else {
|
||||
cumul[u] = cumul[u-1] + normalizedCounter[u-1];
|
||||
} }
|
||||
cumul[maxSymbolValue+1] = tableSize+1;
|
||||
}
|
||||
|
||||
/* Spread symbols */
|
||||
{ U32 position = 0;
|
||||
U32 symbol;
|
||||
for (symbol=0; symbol<=maxSymbolValue; symbol++) {
|
||||
int nbOccurences;
|
||||
for (nbOccurences=0; nbOccurences<normalizedCounter[symbol]; nbOccurences++) {
|
||||
tableSymbol[position] = (FSE_FUNCTION_TYPE)symbol;
|
||||
position = (position + step) & tableMask;
|
||||
while (position > highThreshold) position = (position + step) & tableMask; /* Low proba area */
|
||||
} }
|
||||
|
||||
if (position!=0) return ERROR(GENERIC); /* Must have gone through all positions */
|
||||
}
|
||||
|
||||
/* Build table */
|
||||
{ U32 u; for (u=0; u<tableSize; u++) {
|
||||
FSE_FUNCTION_TYPE s = tableSymbol[u]; /* note : static analyzer may not understand tableSymbol is properly initialized */
|
||||
tableU16[cumul[s]++] = (U16) (tableSize+u); /* TableU16 : sorted by symbol order; gives next state value */
|
||||
} }
|
||||
|
||||
/* Build Symbol Transformation Table */
|
||||
{ unsigned total = 0;
|
||||
unsigned s;
|
||||
for (s=0; s<=maxSymbolValue; s++) {
|
||||
switch (normalizedCounter[s])
|
||||
{
|
||||
case 0:
|
||||
/* filling nonetheless, for compatibility with FSE_getMaxNbBits() */
|
||||
symbolTT[s].deltaNbBits = ((tableLog+1) << 16) - (1<<tableLog);
|
||||
break;
|
||||
|
||||
case -1:
|
||||
case 1:
|
||||
symbolTT[s].deltaNbBits = (tableLog << 16) - (1<<tableLog);
|
||||
symbolTT[s].deltaFindState = total - 1;
|
||||
total ++;
|
||||
break;
|
||||
default :
|
||||
{
|
||||
U32 const maxBitsOut = tableLog - BIT_highbit32 (normalizedCounter[s]-1);
|
||||
U32 const minStatePlus = normalizedCounter[s] << maxBitsOut;
|
||||
symbolTT[s].deltaNbBits = (maxBitsOut << 16) - minStatePlus;
|
||||
symbolTT[s].deltaFindState = total - normalizedCounter[s];
|
||||
total += normalizedCounter[s];
|
||||
} } } }
|
||||
|
||||
#if 0 /* debug : symbol costs */
|
||||
DEBUGLOG(5, "\n --- table statistics : ");
|
||||
{ U32 symbol;
|
||||
for (symbol=0; symbol<=maxSymbolValue; symbol++) {
|
||||
DEBUGLOG(5, "%3u: w=%3i, maxBits=%u, fracBits=%.2f",
|
||||
symbol, normalizedCounter[symbol],
|
||||
FSE_getMaxNbBits(symbolTT, symbol),
|
||||
(double)FSE_bitCost(symbolTT, tableLog, symbol, 8) / 256);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
|
||||
{
|
||||
FSE_FUNCTION_TYPE tableSymbol[FSE_MAX_TABLESIZE]; /* memset() is not necessary, even if static analyzer complain about it */
|
||||
return FSE_buildCTable_wksp(ct, normalizedCounter, maxSymbolValue, tableLog, tableSymbol, sizeof(tableSymbol));
|
||||
}
|
||||
|
||||
|
||||
|
||||
#ifndef FSE_COMMONDEFS_ONLY
|
||||
|
||||
|
||||
/*-**************************************************************
|
||||
* FSE NCount encoding
|
||||
****************************************************************/
|
||||
size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog)
|
||||
{
|
||||
size_t const maxHeaderSize = (((maxSymbolValue+1) * tableLog) >> 3) + 3;
|
||||
return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */
|
||||
}
|
||||
|
||||
static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize,
|
||||
const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog,
|
||||
unsigned writeIsSafe)
|
||||
{
|
||||
BYTE* const ostart = (BYTE*) header;
|
||||
BYTE* out = ostart;
|
||||
BYTE* const oend = ostart + headerBufferSize;
|
||||
int nbBits;
|
||||
const int tableSize = 1 << tableLog;
|
||||
int remaining;
|
||||
int threshold;
|
||||
U32 bitStream;
|
||||
int bitCount;
|
||||
unsigned charnum = 0;
|
||||
int previous0 = 0;
|
||||
|
||||
bitStream = 0;
|
||||
bitCount = 0;
|
||||
/* Table Size */
|
||||
bitStream += (tableLog-FSE_MIN_TABLELOG) << bitCount;
|
||||
bitCount += 4;
|
||||
|
||||
/* Init */
|
||||
remaining = tableSize+1; /* +1 for extra accuracy */
|
||||
threshold = tableSize;
|
||||
nbBits = tableLog+1;
|
||||
|
||||
while (remaining>1) { /* stops at 1 */
|
||||
if (previous0) {
|
||||
unsigned start = charnum;
|
||||
while (!normalizedCounter[charnum]) charnum++;
|
||||
while (charnum >= start+24) {
|
||||
start+=24;
|
||||
bitStream += 0xFFFFU << bitCount;
|
||||
if ((!writeIsSafe) && (out > oend-2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */
|
||||
out[0] = (BYTE) bitStream;
|
||||
out[1] = (BYTE)(bitStream>>8);
|
||||
out+=2;
|
||||
bitStream>>=16;
|
||||
}
|
||||
while (charnum >= start+3) {
|
||||
start+=3;
|
||||
bitStream += 3 << bitCount;
|
||||
bitCount += 2;
|
||||
}
|
||||
bitStream += (charnum-start) << bitCount;
|
||||
bitCount += 2;
|
||||
if (bitCount>16) {
|
||||
if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */
|
||||
out[0] = (BYTE)bitStream;
|
||||
out[1] = (BYTE)(bitStream>>8);
|
||||
out += 2;
|
||||
bitStream >>= 16;
|
||||
bitCount -= 16;
|
||||
} }
|
||||
{ int count = normalizedCounter[charnum++];
|
||||
int const max = (2*threshold-1)-remaining;
|
||||
remaining -= count < 0 ? -count : count;
|
||||
count++; /* +1 for extra accuracy */
|
||||
if (count>=threshold) count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */
|
||||
bitStream += count << bitCount;
|
||||
bitCount += nbBits;
|
||||
bitCount -= (count<max);
|
||||
previous0 = (count==1);
|
||||
if (remaining<1) return ERROR(GENERIC);
|
||||
while (remaining<threshold) { nbBits--; threshold>>=1; }
|
||||
}
|
||||
if (bitCount>16) {
|
||||
if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */
|
||||
out[0] = (BYTE)bitStream;
|
||||
out[1] = (BYTE)(bitStream>>8);
|
||||
out += 2;
|
||||
bitStream >>= 16;
|
||||
bitCount -= 16;
|
||||
} }
|
||||
|
||||
/* flush remaining bitStream */
|
||||
if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */
|
||||
out[0] = (BYTE)bitStream;
|
||||
out[1] = (BYTE)(bitStream>>8);
|
||||
out+= (bitCount+7) /8;
|
||||
|
||||
if (charnum > maxSymbolValue + 1) return ERROR(GENERIC);
|
||||
|
||||
return (out-ostart);
|
||||
}
|
||||
|
||||
|
||||
size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
|
||||
{
|
||||
if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported */
|
||||
if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported */
|
||||
|
||||
if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog))
|
||||
return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0);
|
||||
|
||||
return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1);
|
||||
}
|
||||
|
||||
|
||||
/*-**************************************************************
|
||||
* FSE Compression Code
|
||||
****************************************************************/
|
||||
/*! FSE_sizeof_CTable() :
|
||||
FSE_CTable is a variable size structure which contains :
|
||||
`U16 tableLog;`
|
||||
`U16 maxSymbolValue;`
|
||||
`U16 nextStateNumber[1 << tableLog];` // This size is variable
|
||||
`FSE_symbolCompressionTransform symbolTT[maxSymbolValue+1];` // This size is variable
|
||||
Allocation is manual (C standard does not support variable-size structures).
|
||||
*/
|
||||
size_t FSE_sizeof_CTable (unsigned maxSymbolValue, unsigned tableLog)
|
||||
{
|
||||
if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
|
||||
return FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32);
|
||||
}
|
||||
|
||||
FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog)
|
||||
{
|
||||
size_t size;
|
||||
if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX;
|
||||
size = FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32);
|
||||
return (FSE_CTable*)malloc(size);
|
||||
}
|
||||
|
||||
void FSE_freeCTable (FSE_CTable* ct) { free(ct); }
|
||||
|
||||
/* provides the minimum logSize to safely represent a distribution */
|
||||
static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue)
|
||||
{
|
||||
U32 minBitsSrc = BIT_highbit32((U32)(srcSize - 1)) + 1;
|
||||
U32 minBitsSymbols = BIT_highbit32(maxSymbolValue) + 2;
|
||||
U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols;
|
||||
assert(srcSize > 1); /* Not supported, RLE should be used instead */
|
||||
return minBits;
|
||||
}
|
||||
|
||||
unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus)
|
||||
{
|
||||
U32 maxBitsSrc = BIT_highbit32((U32)(srcSize - 1)) - minus;
|
||||
U32 tableLog = maxTableLog;
|
||||
U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue);
|
||||
assert(srcSize > 1); /* Not supported, RLE should be used instead */
|
||||
if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG;
|
||||
if (maxBitsSrc < tableLog) tableLog = maxBitsSrc; /* Accuracy can be reduced */
|
||||
if (minBits > tableLog) tableLog = minBits; /* Need a minimum to safely represent all symbol values */
|
||||
if (tableLog < FSE_MIN_TABLELOG) tableLog = FSE_MIN_TABLELOG;
|
||||
if (tableLog > FSE_MAX_TABLELOG) tableLog = FSE_MAX_TABLELOG;
|
||||
return tableLog;
|
||||
}
|
||||
|
||||
unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue)
|
||||
{
|
||||
return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 2);
|
||||
}
|
||||
|
||||
|
||||
/* Secondary normalization method.
|
||||
To be used when primary method fails. */
|
||||
|
||||
static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, size_t total, U32 maxSymbolValue)
|
||||
{
|
||||
short const NOT_YET_ASSIGNED = -2;
|
||||
U32 s;
|
||||
U32 distributed = 0;
|
||||
U32 ToDistribute;
|
||||
|
||||
/* Init */
|
||||
U32 const lowThreshold = (U32)(total >> tableLog);
|
||||
U32 lowOne = (U32)((total * 3) >> (tableLog + 1));
|
||||
|
||||
for (s=0; s<=maxSymbolValue; s++) {
|
||||
if (count[s] == 0) {
|
||||
norm[s]=0;
|
||||
continue;
|
||||
}
|
||||
if (count[s] <= lowThreshold) {
|
||||
norm[s] = -1;
|
||||
distributed++;
|
||||
total -= count[s];
|
||||
continue;
|
||||
}
|
||||
if (count[s] <= lowOne) {
|
||||
norm[s] = 1;
|
||||
distributed++;
|
||||
total -= count[s];
|
||||
continue;
|
||||
}
|
||||
|
||||
norm[s]=NOT_YET_ASSIGNED;
|
||||
}
|
||||
ToDistribute = (1 << tableLog) - distributed;
|
||||
|
||||
if ((total / ToDistribute) > lowOne) {
|
||||
/* risk of rounding to zero */
|
||||
lowOne = (U32)((total * 3) / (ToDistribute * 2));
|
||||
for (s=0; s<=maxSymbolValue; s++) {
|
||||
if ((norm[s] == NOT_YET_ASSIGNED) && (count[s] <= lowOne)) {
|
||||
norm[s] = 1;
|
||||
distributed++;
|
||||
total -= count[s];
|
||||
continue;
|
||||
} }
|
||||
ToDistribute = (1 << tableLog) - distributed;
|
||||
}
|
||||
|
||||
if (distributed == maxSymbolValue+1) {
|
||||
/* all values are pretty poor;
|
||||
probably incompressible data (should have already been detected);
|
||||
find max, then give all remaining points to max */
|
||||
U32 maxV = 0, maxC = 0;
|
||||
for (s=0; s<=maxSymbolValue; s++)
|
||||
if (count[s] > maxC) { maxV=s; maxC=count[s]; }
|
||||
norm[maxV] += (short)ToDistribute;
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (total == 0) {
|
||||
/* all of the symbols were low enough for the lowOne or lowThreshold */
|
||||
for (s=0; ToDistribute > 0; s = (s+1)%(maxSymbolValue+1))
|
||||
if (norm[s] > 0) { ToDistribute--; norm[s]++; }
|
||||
return 0;
|
||||
}
|
||||
|
||||
{ U64 const vStepLog = 62 - tableLog;
|
||||
U64 const mid = (1ULL << (vStepLog-1)) - 1;
|
||||
U64 const rStep = ((((U64)1<<vStepLog) * ToDistribute) + mid) / total; /* scale on remaining */
|
||||
U64 tmpTotal = mid;
|
||||
for (s=0; s<=maxSymbolValue; s++) {
|
||||
if (norm[s]==NOT_YET_ASSIGNED) {
|
||||
U64 const end = tmpTotal + (count[s] * rStep);
|
||||
U32 const sStart = (U32)(tmpTotal >> vStepLog);
|
||||
U32 const sEnd = (U32)(end >> vStepLog);
|
||||
U32 const weight = sEnd - sStart;
|
||||
if (weight < 1)
|
||||
return ERROR(GENERIC);
|
||||
norm[s] = (short)weight;
|
||||
tmpTotal = end;
|
||||
} } }
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog,
|
||||
const unsigned* count, size_t total,
|
||||
unsigned maxSymbolValue)
|
||||
{
|
||||
/* Sanity checks */
|
||||
if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG;
|
||||
if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported size */
|
||||
if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported size */
|
||||
if (tableLog < FSE_minTableLog(total, maxSymbolValue)) return ERROR(GENERIC); /* Too small tableLog, compression potentially impossible */
|
||||
|
||||
{ static U32 const rtbTable[] = { 0, 473195, 504333, 520860, 550000, 700000, 750000, 830000 };
|
||||
U64 const scale = 62 - tableLog;
|
||||
U64 const step = ((U64)1<<62) / total; /* <== here, one division ! */
|
||||
U64 const vStep = 1ULL<<(scale-20);
|
||||
int stillToDistribute = 1<<tableLog;
|
||||
unsigned s;
|
||||
unsigned largest=0;
|
||||
short largestP=0;
|
||||
U32 lowThreshold = (U32)(total >> tableLog);
|
||||
|
||||
for (s=0; s<=maxSymbolValue; s++) {
|
||||
if (count[s] == total) return 0; /* rle special case */
|
||||
if (count[s] == 0) { normalizedCounter[s]=0; continue; }
|
||||
if (count[s] <= lowThreshold) {
|
||||
normalizedCounter[s] = -1;
|
||||
stillToDistribute--;
|
||||
} else {
|
||||
short proba = (short)((count[s]*step) >> scale);
|
||||
if (proba<8) {
|
||||
U64 restToBeat = vStep * rtbTable[proba];
|
||||
proba += (count[s]*step) - ((U64)proba<<scale) > restToBeat;
|
||||
}
|
||||
if (proba > largestP) { largestP=proba; largest=s; }
|
||||
normalizedCounter[s] = proba;
|
||||
stillToDistribute -= proba;
|
||||
} }
|
||||
if (-stillToDistribute >= (normalizedCounter[largest] >> 1)) {
|
||||
/* corner case, need another normalization method */
|
||||
size_t const errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue);
|
||||
if (FSE_isError(errorCode)) return errorCode;
|
||||
}
|
||||
else normalizedCounter[largest] += (short)stillToDistribute;
|
||||
}
|
||||
|
||||
#if 0
|
||||
{ /* Print Table (debug) */
|
||||
U32 s;
|
||||
U32 nTotal = 0;
|
||||
for (s=0; s<=maxSymbolValue; s++)
|
||||
RAWLOG(2, "%3i: %4i \n", s, normalizedCounter[s]);
|
||||
for (s=0; s<=maxSymbolValue; s++)
|
||||
nTotal += abs(normalizedCounter[s]);
|
||||
if (nTotal != (1U<<tableLog))
|
||||
RAWLOG(2, "Warning !!! Total == %u != %u !!!", nTotal, 1U<<tableLog);
|
||||
getchar();
|
||||
}
|
||||
#endif
|
||||
|
||||
return tableLog;
|
||||
}
|
||||
|
||||
|
||||
/* fake FSE_CTable, for raw (uncompressed) input */
|
||||
size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits)
|
||||
{
|
||||
const unsigned tableSize = 1 << nbBits;
|
||||
const unsigned tableMask = tableSize - 1;
|
||||
const unsigned maxSymbolValue = tableMask;
|
||||
void* const ptr = ct;
|
||||
U16* const tableU16 = ( (U16*) ptr) + 2;
|
||||
void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableSize>>1); /* assumption : tableLog >= 1 */
|
||||
FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT);
|
||||
unsigned s;
|
||||
|
||||
/* Sanity checks */
|
||||
if (nbBits < 1) return ERROR(GENERIC); /* min size */
|
||||
|
||||
/* header */
|
||||
tableU16[-2] = (U16) nbBits;
|
||||
tableU16[-1] = (U16) maxSymbolValue;
|
||||
|
||||
/* Build table */
|
||||
for (s=0; s<tableSize; s++)
|
||||
tableU16[s] = (U16)(tableSize + s);
|
||||
|
||||
/* Build Symbol Transformation Table */
|
||||
{ const U32 deltaNbBits = (nbBits << 16) - (1 << nbBits);
|
||||
for (s=0; s<=maxSymbolValue; s++) {
|
||||
symbolTT[s].deltaNbBits = deltaNbBits;
|
||||
symbolTT[s].deltaFindState = s-1;
|
||||
} }
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* fake FSE_CTable, for rle input (always same symbol) */
|
||||
size_t FSE_buildCTable_rle (FSE_CTable* ct, BYTE symbolValue)
|
||||
{
|
||||
void* ptr = ct;
|
||||
U16* tableU16 = ( (U16*) ptr) + 2;
|
||||
void* FSCTptr = (U32*)ptr + 2;
|
||||
FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*) FSCTptr;
|
||||
|
||||
/* header */
|
||||
tableU16[-2] = (U16) 0;
|
||||
tableU16[-1] = (U16) symbolValue;
|
||||
|
||||
/* Build table */
|
||||
tableU16[0] = 0;
|
||||
tableU16[1] = 0; /* just in case */
|
||||
|
||||
/* Build Symbol Transformation Table */
|
||||
symbolTT[symbolValue].deltaNbBits = 0;
|
||||
symbolTT[symbolValue].deltaFindState = 0;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
static size_t FSE_compress_usingCTable_generic (void* dst, size_t dstSize,
|
||||
const void* src, size_t srcSize,
|
||||
const FSE_CTable* ct, const unsigned fast)
|
||||
{
|
||||
const BYTE* const istart = (const BYTE*) src;
|
||||
const BYTE* const iend = istart + srcSize;
|
||||
const BYTE* ip=iend;
|
||||
|
||||
BIT_CStream_t bitC;
|
||||
FSE_CState_t CState1, CState2;
|
||||
|
||||
/* init */
|
||||
if (srcSize <= 2) return 0;
|
||||
{ size_t const initError = BIT_initCStream(&bitC, dst, dstSize);
|
||||
if (FSE_isError(initError)) return 0; /* not enough space available to write a bitstream */ }
|
||||
|
||||
#define FSE_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s))
|
||||
|
||||
if (srcSize & 1) {
|
||||
FSE_initCState2(&CState1, ct, *--ip);
|
||||
FSE_initCState2(&CState2, ct, *--ip);
|
||||
FSE_encodeSymbol(&bitC, &CState1, *--ip);
|
||||
FSE_FLUSHBITS(&bitC);
|
||||
} else {
|
||||
FSE_initCState2(&CState2, ct, *--ip);
|
||||
FSE_initCState2(&CState1, ct, *--ip);
|
||||
}
|
||||
|
||||
/* join to mod 4 */
|
||||
srcSize -= 2;
|
||||
if ((sizeof(bitC.bitContainer)*8 > FSE_MAX_TABLELOG*4+7 ) && (srcSize & 2)) { /* test bit 2 */
|
||||
FSE_encodeSymbol(&bitC, &CState2, *--ip);
|
||||
FSE_encodeSymbol(&bitC, &CState1, *--ip);
|
||||
FSE_FLUSHBITS(&bitC);
|
||||
}
|
||||
|
||||
/* 2 or 4 encoding per loop */
|
||||
while ( ip>istart ) {
|
||||
|
||||
FSE_encodeSymbol(&bitC, &CState2, *--ip);
|
||||
|
||||
if (sizeof(bitC.bitContainer)*8 < FSE_MAX_TABLELOG*2+7 ) /* this test must be static */
|
||||
FSE_FLUSHBITS(&bitC);
|
||||
|
||||
FSE_encodeSymbol(&bitC, &CState1, *--ip);
|
||||
|
||||
if (sizeof(bitC.bitContainer)*8 > FSE_MAX_TABLELOG*4+7 ) { /* this test must be static */
|
||||
FSE_encodeSymbol(&bitC, &CState2, *--ip);
|
||||
FSE_encodeSymbol(&bitC, &CState1, *--ip);
|
||||
}
|
||||
|
||||
FSE_FLUSHBITS(&bitC);
|
||||
}
|
||||
|
||||
FSE_flushCState(&bitC, &CState2);
|
||||
FSE_flushCState(&bitC, &CState1);
|
||||
return BIT_closeCStream(&bitC);
|
||||
}
|
||||
|
||||
size_t FSE_compress_usingCTable (void* dst, size_t dstSize,
|
||||
const void* src, size_t srcSize,
|
||||
const FSE_CTable* ct)
|
||||
{
|
||||
unsigned const fast = (dstSize >= FSE_BLOCKBOUND(srcSize));
|
||||
|
||||
if (fast)
|
||||
return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 1);
|
||||
else
|
||||
return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 0);
|
||||
}
|
||||
|
||||
|
||||
size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); }
|
||||
|
||||
#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return e
|
||||
#define CHECK_F(f) { CHECK_V_F(_var_err__, f); }
|
||||
|
||||
/* FSE_compress_wksp() :
|
||||
* Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`).
|
||||
* `wkspSize` size must be `(1<<tableLog)`.
|
||||
*/
|
||||
size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize)
|
||||
{
|
||||
BYTE* const ostart = (BYTE*) dst;
|
||||
BYTE* op = ostart;
|
||||
BYTE* const oend = ostart + dstSize;
|
||||
|
||||
U32 count[FSE_MAX_SYMBOL_VALUE+1];
|
||||
S16 norm[FSE_MAX_SYMBOL_VALUE+1];
|
||||
FSE_CTable* CTable = (FSE_CTable*)workSpace;
|
||||
size_t const CTableSize = FSE_CTABLE_SIZE_U32(tableLog, maxSymbolValue);
|
||||
void* scratchBuffer = (void*)(CTable + CTableSize);
|
||||
size_t const scratchBufferSize = wkspSize - (CTableSize * sizeof(FSE_CTable));
|
||||
|
||||
/* init conditions */
|
||||
if (wkspSize < FSE_WKSP_SIZE_U32(tableLog, maxSymbolValue)) return ERROR(tableLog_tooLarge);
|
||||
if (srcSize <= 1) return 0; /* Not compressible */
|
||||
if (!maxSymbolValue) maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
|
||||
if (!tableLog) tableLog = FSE_DEFAULT_TABLELOG;
|
||||
|
||||
/* Scan input and build symbol stats */
|
||||
{ CHECK_V_F(maxCount, HIST_count_wksp(count, &maxSymbolValue, src, srcSize, (unsigned*)scratchBuffer) );
|
||||
if (maxCount == srcSize) return 1; /* only a single symbol in src : rle */
|
||||
if (maxCount == 1) return 0; /* each symbol present maximum once => not compressible */
|
||||
if (maxCount < (srcSize >> 7)) return 0; /* Heuristic : not compressible enough */
|
||||
}
|
||||
|
||||
tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue);
|
||||
CHECK_F( FSE_normalizeCount(norm, tableLog, count, srcSize, maxSymbolValue) );
|
||||
|
||||
/* Write table description header */
|
||||
{ CHECK_V_F(nc_err, FSE_writeNCount(op, oend-op, norm, maxSymbolValue, tableLog) );
|
||||
op += nc_err;
|
||||
}
|
||||
|
||||
/* Compress */
|
||||
CHECK_F( FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, scratchBufferSize) );
|
||||
{ CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, src, srcSize, CTable) );
|
||||
if (cSize == 0) return 0; /* not enough space for compressed data */
|
||||
op += cSize;
|
||||
}
|
||||
|
||||
/* check compressibility */
|
||||
if ( (size_t)(op-ostart) >= srcSize-1 ) return 0;
|
||||
|
||||
return op-ostart;
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
FSE_CTable CTable_max[FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)];
|
||||
BYTE scratchBuffer[1 << FSE_MAX_TABLELOG];
|
||||
} fseWkspMax_t;
|
||||
|
||||
size_t FSE_compress2 (void* dst, size_t dstCapacity, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog)
|
||||
{
|
||||
fseWkspMax_t scratchBuffer;
|
||||
DEBUG_STATIC_ASSERT(sizeof(scratchBuffer) >= FSE_WKSP_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)); /* compilation failures here means scratchBuffer is not large enough */
|
||||
if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
|
||||
return FSE_compress_wksp(dst, dstCapacity, src, srcSize, maxSymbolValue, tableLog, &scratchBuffer, sizeof(scratchBuffer));
|
||||
}
|
||||
|
||||
size_t FSE_compress (void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
||||
{
|
||||
return FSE_compress2(dst, dstCapacity, src, srcSize, FSE_MAX_SYMBOL_VALUE, FSE_DEFAULT_TABLELOG);
|
||||
}
|
||||
|
||||
|
||||
#endif /* FSE_COMMONDEFS_ONLY */
|
||||
|
|
@ -0,0 +1,195 @@
|
|||
/* ******************************************************************
|
||||
hist : Histogram functions
|
||||
part of Finite State Entropy project
|
||||
Copyright (C) 2013-present, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
****************************************************************** */
|
||||
|
||||
/* --- dependencies --- */
|
||||
#include "mem.h" /* U32, BYTE, etc. */
|
||||
#include "debug.h" /* assert, DEBUGLOG */
|
||||
#include "error_private.h" /* ERROR */
|
||||
#include "hist.h"
|
||||
|
||||
|
||||
/* --- Error management --- */
|
||||
unsigned HIST_isError(size_t code) { return ERR_isError(code); }
|
||||
|
||||
/*-**************************************************************
|
||||
* Histogram functions
|
||||
****************************************************************/
|
||||
unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
const BYTE* const end = ip + srcSize;
|
||||
unsigned maxSymbolValue = *maxSymbolValuePtr;
|
||||
unsigned largestCount=0;
|
||||
|
||||
memset(count, 0, (maxSymbolValue+1) * sizeof(*count));
|
||||
if (srcSize==0) { *maxSymbolValuePtr = 0; return 0; }
|
||||
|
||||
while (ip<end) {
|
||||
assert(*ip <= maxSymbolValue);
|
||||
count[*ip++]++;
|
||||
}
|
||||
|
||||
while (!count[maxSymbolValue]) maxSymbolValue--;
|
||||
*maxSymbolValuePtr = maxSymbolValue;
|
||||
|
||||
{ U32 s;
|
||||
for (s=0; s<=maxSymbolValue; s++)
|
||||
if (count[s] > largestCount) largestCount = count[s];
|
||||
}
|
||||
|
||||
return largestCount;
|
||||
}
|
||||
|
||||
|
||||
/* HIST_count_parallel_wksp() :
|
||||
* store histogram into 4 intermediate tables, recombined at the end.
|
||||
* this design makes better use of OoO cpus,
|
||||
* and is noticeably faster when some values are heavily repeated.
|
||||
* But it needs some additional workspace for intermediate tables.
|
||||
* `workSpace` size must be a table of size >= HIST_WKSP_SIZE_U32.
|
||||
* @return : largest histogram frequency,
|
||||
* or an error code (notably when histogram would be larger than *maxSymbolValuePtr). */
|
||||
static size_t HIST_count_parallel_wksp(
|
||||
unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
const void* source, size_t sourceSize,
|
||||
unsigned checkMax,
|
||||
unsigned* const workSpace)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*)source;
|
||||
const BYTE* const iend = ip+sourceSize;
|
||||
unsigned maxSymbolValue = *maxSymbolValuePtr;
|
||||
unsigned max=0;
|
||||
U32* const Counting1 = workSpace;
|
||||
U32* const Counting2 = Counting1 + 256;
|
||||
U32* const Counting3 = Counting2 + 256;
|
||||
U32* const Counting4 = Counting3 + 256;
|
||||
|
||||
memset(workSpace, 0, 4*256*sizeof(unsigned));
|
||||
|
||||
/* safety checks */
|
||||
if (!sourceSize) {
|
||||
memset(count, 0, maxSymbolValue + 1);
|
||||
*maxSymbolValuePtr = 0;
|
||||
return 0;
|
||||
}
|
||||
if (!maxSymbolValue) maxSymbolValue = 255; /* 0 == default */
|
||||
|
||||
/* by stripes of 16 bytes */
|
||||
{ U32 cached = MEM_read32(ip); ip += 4;
|
||||
while (ip < iend-15) {
|
||||
U32 c = cached; cached = MEM_read32(ip); ip += 4;
|
||||
Counting1[(BYTE) c ]++;
|
||||
Counting2[(BYTE)(c>>8) ]++;
|
||||
Counting3[(BYTE)(c>>16)]++;
|
||||
Counting4[ c>>24 ]++;
|
||||
c = cached; cached = MEM_read32(ip); ip += 4;
|
||||
Counting1[(BYTE) c ]++;
|
||||
Counting2[(BYTE)(c>>8) ]++;
|
||||
Counting3[(BYTE)(c>>16)]++;
|
||||
Counting4[ c>>24 ]++;
|
||||
c = cached; cached = MEM_read32(ip); ip += 4;
|
||||
Counting1[(BYTE) c ]++;
|
||||
Counting2[(BYTE)(c>>8) ]++;
|
||||
Counting3[(BYTE)(c>>16)]++;
|
||||
Counting4[ c>>24 ]++;
|
||||
c = cached; cached = MEM_read32(ip); ip += 4;
|
||||
Counting1[(BYTE) c ]++;
|
||||
Counting2[(BYTE)(c>>8) ]++;
|
||||
Counting3[(BYTE)(c>>16)]++;
|
||||
Counting4[ c>>24 ]++;
|
||||
}
|
||||
ip-=4;
|
||||
}
|
||||
|
||||
/* finish last symbols */
|
||||
while (ip<iend) Counting1[*ip++]++;
|
||||
|
||||
if (checkMax) { /* verify stats will fit into destination table */
|
||||
U32 s; for (s=255; s>maxSymbolValue; s--) {
|
||||
Counting1[s] += Counting2[s] + Counting3[s] + Counting4[s];
|
||||
if (Counting1[s]) return ERROR(maxSymbolValue_tooSmall);
|
||||
} }
|
||||
|
||||
{ U32 s;
|
||||
if (maxSymbolValue > 255) maxSymbolValue = 255;
|
||||
for (s=0; s<=maxSymbolValue; s++) {
|
||||
count[s] = Counting1[s] + Counting2[s] + Counting3[s] + Counting4[s];
|
||||
if (count[s] > max) max = count[s];
|
||||
} }
|
||||
|
||||
while (!count[maxSymbolValue]) maxSymbolValue--;
|
||||
*maxSymbolValuePtr = maxSymbolValue;
|
||||
return (size_t)max;
|
||||
}
|
||||
|
||||
/* HIST_countFast_wksp() :
|
||||
* Same as HIST_countFast(), but using an externally provided scratch buffer.
|
||||
* `workSpace` size must be table of >= HIST_WKSP_SIZE_U32 unsigned */
|
||||
size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
const void* source, size_t sourceSize,
|
||||
unsigned* workSpace)
|
||||
{
|
||||
if (sourceSize < 1500) /* heuristic threshold */
|
||||
return HIST_count_simple(count, maxSymbolValuePtr, source, sourceSize);
|
||||
return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, 0, workSpace);
|
||||
}
|
||||
|
||||
/* fast variant (unsafe : won't check if src contains values beyond count[] limit) */
|
||||
size_t HIST_countFast(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
const void* source, size_t sourceSize)
|
||||
{
|
||||
unsigned tmpCounters[HIST_WKSP_SIZE_U32];
|
||||
return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, tmpCounters);
|
||||
}
|
||||
|
||||
/* HIST_count_wksp() :
|
||||
* Same as HIST_count(), but using an externally provided scratch buffer.
|
||||
* `workSpace` size must be table of >= HIST_WKSP_SIZE_U32 unsigned */
|
||||
size_t HIST_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
const void* source, size_t sourceSize, unsigned* workSpace)
|
||||
{
|
||||
if (*maxSymbolValuePtr < 255)
|
||||
return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, 1, workSpace);
|
||||
*maxSymbolValuePtr = 255;
|
||||
return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, workSpace);
|
||||
}
|
||||
|
||||
size_t HIST_count(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
unsigned tmpCounters[HIST_WKSP_SIZE_U32];
|
||||
return HIST_count_wksp(count, maxSymbolValuePtr, src, srcSize, tmpCounters);
|
||||
}
|
||||
|
|
@ -0,0 +1,92 @@
|
|||
/* ******************************************************************
|
||||
hist : Histogram functions
|
||||
part of Finite State Entropy project
|
||||
Copyright (C) 2013-present, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
****************************************************************** */
|
||||
|
||||
/* --- dependencies --- */
|
||||
#include <stddef.h> /* size_t */
|
||||
|
||||
|
||||
/* --- simple histogram functions --- */
|
||||
|
||||
/*! HIST_count():
|
||||
* Provides the precise count of each byte within a table 'count'.
|
||||
* 'count' is a table of unsigned int, of minimum size (*maxSymbolValuePtr+1).
|
||||
* Updates *maxSymbolValuePtr with actual largest symbol value detected.
|
||||
* @return : count of the most frequent symbol (which isn't identified).
|
||||
* or an error code, which can be tested using HIST_isError().
|
||||
* note : if return == srcSize, there is only one symbol.
|
||||
*/
|
||||
size_t HIST_count(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
const void* src, size_t srcSize);
|
||||
|
||||
unsigned HIST_isError(size_t code); /*< tells if a return value is an error code */
|
||||
|
||||
|
||||
/* --- advanced histogram functions --- */
|
||||
|
||||
#define HIST_WKSP_SIZE_U32 1024
|
||||
/** HIST_count_wksp() :
|
||||
* Same as HIST_count(), but using an externally provided scratch buffer.
|
||||
* Benefit is this function will use very little stack space.
|
||||
* `workSpace` must be a table of unsigned of size >= HIST_WKSP_SIZE_U32
|
||||
*/
|
||||
size_t HIST_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
const void* src, size_t srcSize,
|
||||
unsigned* workSpace);
|
||||
|
||||
/** HIST_countFast() :
|
||||
* same as HIST_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr.
|
||||
* This function is unsafe, and will segfault if any value within `src` is `> *maxSymbolValuePtr`
|
||||
*/
|
||||
size_t HIST_countFast(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
const void* src, size_t srcSize);
|
||||
|
||||
/** HIST_countFast_wksp() :
|
||||
* Same as HIST_countFast(), but using an externally provided scratch buffer.
|
||||
* `workSpace` must be a table of unsigned of size >= HIST_WKSP_SIZE_U32
|
||||
*/
|
||||
size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
const void* src, size_t srcSize,
|
||||
unsigned* workSpace);
|
||||
|
||||
/*! HIST_count_simple() :
|
||||
* Same as HIST_countFast(), this function is unsafe,
|
||||
* and will segfault if any value within `src` is `> *maxSymbolValuePtr`.
|
||||
* It is also a bit slower for large inputs.
|
||||
* However, it does not need any additional memory (not even on stack).
|
||||
* @return : count of the most frequent symbol.
|
||||
* Note this function doesn't produce any error (i.e. it must succeed).
|
||||
*/
|
||||
unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
const void* src, size_t srcSize);
|
||||
|
|
@ -0,0 +1,796 @@
|
|||
/* ******************************************************************
|
||||
Huffman encoder, part of New Generation Entropy library
|
||||
Copyright (C) 2013-2016, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
****************************************************************** */
|
||||
|
||||
/* **************************************************************
|
||||
* Compiler specifics
|
||||
****************************************************************/
|
||||
#ifdef _MSC_VER /* Visual Studio */
|
||||
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
|
||||
#endif
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* Includes
|
||||
****************************************************************/
|
||||
#include <string.h> /* memcpy, memset */
|
||||
#include <stdio.h> /* printf (debug) */
|
||||
#include "compiler.h"
|
||||
#include "bitstream.h"
|
||||
#include "hist.h"
|
||||
#define FSE_STATIC_LINKING_ONLY /* FSE_optimalTableLog_internal */
|
||||
#include "fse.h" /* header compression */
|
||||
#define HUF_STATIC_LINKING_ONLY
|
||||
#include "huf.h"
|
||||
#include "error_private.h"
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* Error Management
|
||||
****************************************************************/
|
||||
#define HUF_isError ERR_isError
|
||||
#define HUF_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) /* use only *after* variable declarations */
|
||||
#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return e
|
||||
#define CHECK_F(f) { CHECK_V_F(_var_err__, f); }
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* Utils
|
||||
****************************************************************/
|
||||
unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue)
|
||||
{
|
||||
return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1);
|
||||
}
|
||||
|
||||
|
||||
/* *******************************************************
|
||||
* HUF : Huffman block compression
|
||||
*********************************************************/
|
||||
/* HUF_compressWeights() :
|
||||
* Same as FSE_compress(), but dedicated to huff0's weights compression.
|
||||
* The use case needs much less stack memory.
|
||||
* Note : all elements within weightTable are supposed to be <= HUF_TABLELOG_MAX.
|
||||
*/
|
||||
#define MAX_FSE_TABLELOG_FOR_HUFF_HEADER 6
|
||||
size_t HUF_compressWeights (void* dst, size_t dstSize, const void* weightTable, size_t wtSize)
|
||||
{
|
||||
BYTE* const ostart = (BYTE*) dst;
|
||||
BYTE* op = ostart;
|
||||
BYTE* const oend = ostart + dstSize;
|
||||
|
||||
U32 maxSymbolValue = HUF_TABLELOG_MAX;
|
||||
U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER;
|
||||
|
||||
FSE_CTable CTable[FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX)];
|
||||
BYTE scratchBuffer[1<<MAX_FSE_TABLELOG_FOR_HUFF_HEADER];
|
||||
|
||||
U32 count[HUF_TABLELOG_MAX+1];
|
||||
S16 norm[HUF_TABLELOG_MAX+1];
|
||||
|
||||
/* init conditions */
|
||||
if (wtSize <= 1) return 0; /* Not compressible */
|
||||
|
||||
/* Scan input and build symbol stats */
|
||||
{ unsigned const maxCount = HIST_count_simple(count, &maxSymbolValue, weightTable, wtSize); /* never fails */
|
||||
if (maxCount == wtSize) return 1; /* only a single symbol in src : rle */
|
||||
if (maxCount == 1) return 0; /* each symbol present maximum once => not compressible */
|
||||
}
|
||||
|
||||
tableLog = FSE_optimalTableLog(tableLog, wtSize, maxSymbolValue);
|
||||
CHECK_F( FSE_normalizeCount(norm, tableLog, count, wtSize, maxSymbolValue) );
|
||||
|
||||
/* Write table description header */
|
||||
{ CHECK_V_F(hSize, FSE_writeNCount(op, oend-op, norm, maxSymbolValue, tableLog) );
|
||||
op += hSize;
|
||||
}
|
||||
|
||||
/* Compress */
|
||||
CHECK_F( FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, sizeof(scratchBuffer)) );
|
||||
{ CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, weightTable, wtSize, CTable) );
|
||||
if (cSize == 0) return 0; /* not enough space for compressed data */
|
||||
op += cSize;
|
||||
}
|
||||
|
||||
return op-ostart;
|
||||
}
|
||||
|
||||
|
||||
struct HUF_CElt_s {
|
||||
U16 val;
|
||||
BYTE nbBits;
|
||||
}; /* typedef'd to HUF_CElt within "huf.h" */
|
||||
|
||||
/*! HUF_writeCTable() :
|
||||
`CTable` : Huffman tree to save, using huf representation.
|
||||
@return : size of saved CTable */
|
||||
size_t HUF_writeCTable (void* dst, size_t maxDstSize,
|
||||
const HUF_CElt* CTable, U32 maxSymbolValue, U32 huffLog)
|
||||
{
|
||||
BYTE bitsToWeight[HUF_TABLELOG_MAX + 1]; /* precomputed conversion table */
|
||||
BYTE huffWeight[HUF_SYMBOLVALUE_MAX];
|
||||
BYTE* op = (BYTE*)dst;
|
||||
U32 n;
|
||||
|
||||
/* check conditions */
|
||||
if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge);
|
||||
|
||||
/* convert to weight */
|
||||
bitsToWeight[0] = 0;
|
||||
for (n=1; n<huffLog+1; n++)
|
||||
bitsToWeight[n] = (BYTE)(huffLog + 1 - n);
|
||||
for (n=0; n<maxSymbolValue; n++)
|
||||
huffWeight[n] = bitsToWeight[CTable[n].nbBits];
|
||||
|
||||
/* attempt weights compression by FSE */
|
||||
{ CHECK_V_F(hSize, HUF_compressWeights(op+1, maxDstSize-1, huffWeight, maxSymbolValue) );
|
||||
if ((hSize>1) & (hSize < maxSymbolValue/2)) { /* FSE compressed */
|
||||
op[0] = (BYTE)hSize;
|
||||
return hSize+1;
|
||||
} }
|
||||
|
||||
/* write raw values as 4-bits (max : 15) */
|
||||
if (maxSymbolValue > (256-128)) return ERROR(GENERIC); /* should not happen : likely means source cannot be compressed */
|
||||
if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */
|
||||
op[0] = (BYTE)(128 /*special case*/ + (maxSymbolValue-1));
|
||||
huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause msan issue in final combination */
|
||||
for (n=0; n<maxSymbolValue; n+=2)
|
||||
op[(n/2)+1] = (BYTE)((huffWeight[n] << 4) + huffWeight[n+1]);
|
||||
return ((maxSymbolValue+1)/2) + 1;
|
||||
}
|
||||
|
||||
|
||||
size_t HUF_readCTable (HUF_CElt* CTable, U32* maxSymbolValuePtr, const void* src, size_t srcSize)
|
||||
{
|
||||
BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; /* init not required, even though some static analyzer may complain */
|
||||
U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */
|
||||
U32 tableLog = 0;
|
||||
U32 nbSymbols = 0;
|
||||
|
||||
/* get symbol weights */
|
||||
CHECK_V_F(readSize, HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX+1, rankVal, &nbSymbols, &tableLog, src, srcSize));
|
||||
|
||||
/* check result */
|
||||
if (tableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
|
||||
if (nbSymbols > *maxSymbolValuePtr+1) return ERROR(maxSymbolValue_tooSmall);
|
||||
|
||||
/* Prepare base value per rank */
|
||||
{ U32 n, nextRankStart = 0;
|
||||
for (n=1; n<=tableLog; n++) {
|
||||
U32 current = nextRankStart;
|
||||
nextRankStart += (rankVal[n] << (n-1));
|
||||
rankVal[n] = current;
|
||||
} }
|
||||
|
||||
/* fill nbBits */
|
||||
{ U32 n; for (n=0; n<nbSymbols; n++) {
|
||||
const U32 w = huffWeight[n];
|
||||
CTable[n].nbBits = (BYTE)(tableLog + 1 - w);
|
||||
} }
|
||||
|
||||
/* fill val */
|
||||
{ U16 nbPerRank[HUF_TABLELOG_MAX+2] = {0}; /* support w=0=>n=tableLog+1 */
|
||||
U16 valPerRank[HUF_TABLELOG_MAX+2] = {0};
|
||||
{ U32 n; for (n=0; n<nbSymbols; n++) nbPerRank[CTable[n].nbBits]++; }
|
||||
/* determine stating value per rank */
|
||||
valPerRank[tableLog+1] = 0; /* for w==0 */
|
||||
{ U16 min = 0;
|
||||
U32 n; for (n=tableLog; n>0; n--) { /* start at n=tablelog <-> w=1 */
|
||||
valPerRank[n] = min; /* get starting value within each rank */
|
||||
min += nbPerRank[n];
|
||||
min >>= 1;
|
||||
} }
|
||||
/* assign value within rank, symbol order */
|
||||
{ U32 n; for (n=0; n<nbSymbols; n++) CTable[n].val = valPerRank[CTable[n].nbBits]++; }
|
||||
}
|
||||
|
||||
*maxSymbolValuePtr = nbSymbols - 1;
|
||||
return readSize;
|
||||
}
|
||||
|
||||
U32 HUF_getNbBits(const void* symbolTable, U32 symbolValue)
|
||||
{
|
||||
const HUF_CElt* table = (const HUF_CElt*)symbolTable;
|
||||
assert(symbolValue <= HUF_SYMBOLVALUE_MAX);
|
||||
return table[symbolValue].nbBits;
|
||||
}
|
||||
|
||||
|
||||
typedef struct nodeElt_s {
|
||||
U32 count;
|
||||
U16 parent;
|
||||
BYTE byte;
|
||||
BYTE nbBits;
|
||||
} nodeElt;
|
||||
|
||||
static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
|
||||
{
|
||||
const U32 largestBits = huffNode[lastNonNull].nbBits;
|
||||
if (largestBits <= maxNbBits) return largestBits; /* early exit : no elt > maxNbBits */
|
||||
|
||||
/* there are several too large elements (at least >= 2) */
|
||||
{ int totalCost = 0;
|
||||
const U32 baseCost = 1 << (largestBits - maxNbBits);
|
||||
U32 n = lastNonNull;
|
||||
|
||||
while (huffNode[n].nbBits > maxNbBits) {
|
||||
totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits));
|
||||
huffNode[n].nbBits = (BYTE)maxNbBits;
|
||||
n --;
|
||||
} /* n stops at huffNode[n].nbBits <= maxNbBits */
|
||||
while (huffNode[n].nbBits == maxNbBits) n--; /* n end at index of smallest symbol using < maxNbBits */
|
||||
|
||||
/* renorm totalCost */
|
||||
totalCost >>= (largestBits - maxNbBits); /* note : totalCost is necessarily a multiple of baseCost */
|
||||
|
||||
/* repay normalized cost */
|
||||
{ U32 const noSymbol = 0xF0F0F0F0;
|
||||
U32 rankLast[HUF_TABLELOG_MAX+2];
|
||||
int pos;
|
||||
|
||||
/* Get pos of last (smallest) symbol per rank */
|
||||
memset(rankLast, 0xF0, sizeof(rankLast));
|
||||
{ U32 currentNbBits = maxNbBits;
|
||||
for (pos=n ; pos >= 0; pos--) {
|
||||
if (huffNode[pos].nbBits >= currentNbBits) continue;
|
||||
currentNbBits = huffNode[pos].nbBits; /* < maxNbBits */
|
||||
rankLast[maxNbBits-currentNbBits] = pos;
|
||||
} }
|
||||
|
||||
while (totalCost > 0) {
|
||||
U32 nBitsToDecrease = BIT_highbit32(totalCost) + 1;
|
||||
for ( ; nBitsToDecrease > 1; nBitsToDecrease--) {
|
||||
U32 highPos = rankLast[nBitsToDecrease];
|
||||
U32 lowPos = rankLast[nBitsToDecrease-1];
|
||||
if (highPos == noSymbol) continue;
|
||||
if (lowPos == noSymbol) break;
|
||||
{ U32 const highTotal = huffNode[highPos].count;
|
||||
U32 const lowTotal = 2 * huffNode[lowPos].count;
|
||||
if (highTotal <= lowTotal) break;
|
||||
} }
|
||||
/* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */
|
||||
/* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */
|
||||
while ((nBitsToDecrease<=HUF_TABLELOG_MAX) && (rankLast[nBitsToDecrease] == noSymbol))
|
||||
nBitsToDecrease ++;
|
||||
totalCost -= 1 << (nBitsToDecrease-1);
|
||||
if (rankLast[nBitsToDecrease-1] == noSymbol)
|
||||
rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease]; /* this rank is no longer empty */
|
||||
huffNode[rankLast[nBitsToDecrease]].nbBits ++;
|
||||
if (rankLast[nBitsToDecrease] == 0) /* special case, reached largest symbol */
|
||||
rankLast[nBitsToDecrease] = noSymbol;
|
||||
else {
|
||||
rankLast[nBitsToDecrease]--;
|
||||
if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease)
|
||||
rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */
|
||||
} } /* while (totalCost > 0) */
|
||||
|
||||
while (totalCost < 0) { /* Sometimes, cost correction overshoot */
|
||||
if (rankLast[1] == noSymbol) { /* special case : no rank 1 symbol (using maxNbBits-1); let's create one from largest rank 0 (using maxNbBits) */
|
||||
while (huffNode[n].nbBits == maxNbBits) n--;
|
||||
huffNode[n+1].nbBits--;
|
||||
rankLast[1] = n+1;
|
||||
totalCost++;
|
||||
continue;
|
||||
}
|
||||
huffNode[ rankLast[1] + 1 ].nbBits--;
|
||||
rankLast[1]++;
|
||||
totalCost ++;
|
||||
} } } /* there are several too large elements (at least >= 2) */
|
||||
|
||||
return maxNbBits;
|
||||
}
|
||||
|
||||
|
||||
typedef struct {
|
||||
U32 base;
|
||||
U32 current;
|
||||
} rankPos;
|
||||
|
||||
static void HUF_sort(nodeElt* huffNode, const U32* count, U32 maxSymbolValue)
|
||||
{
|
||||
rankPos rank[32];
|
||||
U32 n;
|
||||
|
||||
memset(rank, 0, sizeof(rank));
|
||||
for (n=0; n<=maxSymbolValue; n++) {
|
||||
U32 r = BIT_highbit32(count[n] + 1);
|
||||
rank[r].base ++;
|
||||
}
|
||||
for (n=30; n>0; n--) rank[n-1].base += rank[n].base;
|
||||
for (n=0; n<32; n++) rank[n].current = rank[n].base;
|
||||
for (n=0; n<=maxSymbolValue; n++) {
|
||||
U32 const c = count[n];
|
||||
U32 const r = BIT_highbit32(c+1) + 1;
|
||||
U32 pos = rank[r].current++;
|
||||
while ((pos > rank[r].base) && (c > huffNode[pos-1].count)) {
|
||||
huffNode[pos] = huffNode[pos-1];
|
||||
pos--;
|
||||
}
|
||||
huffNode[pos].count = c;
|
||||
huffNode[pos].byte = (BYTE)n;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/** HUF_buildCTable_wksp() :
|
||||
* Same as HUF_buildCTable(), but using externally allocated scratch buffer.
|
||||
* `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as a table of HUF_CTABLE_WORKSPACE_SIZE_U32 unsigned.
|
||||
*/
|
||||
#define STARTNODE (HUF_SYMBOLVALUE_MAX+1)
|
||||
typedef nodeElt huffNodeTable[HUF_CTABLE_WORKSPACE_SIZE_U32];
|
||||
size_t HUF_buildCTable_wksp (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits, void* workSpace, size_t wkspSize)
|
||||
{
|
||||
nodeElt* const huffNode0 = (nodeElt*)workSpace;
|
||||
nodeElt* const huffNode = huffNode0+1;
|
||||
U32 n, nonNullRank;
|
||||
int lowS, lowN;
|
||||
U16 nodeNb = STARTNODE;
|
||||
U32 nodeRoot;
|
||||
|
||||
/* safety checks */
|
||||
if (((size_t)workSpace & 3) != 0) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */
|
||||
if (wkspSize < sizeof(huffNodeTable)) return ERROR(workSpace_tooSmall);
|
||||
if (maxNbBits == 0) maxNbBits = HUF_TABLELOG_DEFAULT;
|
||||
if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge);
|
||||
memset(huffNode0, 0, sizeof(huffNodeTable));
|
||||
|
||||
/* sort, decreasing order */
|
||||
HUF_sort(huffNode, count, maxSymbolValue);
|
||||
|
||||
/* init for parents */
|
||||
nonNullRank = maxSymbolValue;
|
||||
while(huffNode[nonNullRank].count == 0) nonNullRank--;
|
||||
lowS = nonNullRank; nodeRoot = nodeNb + lowS - 1; lowN = nodeNb;
|
||||
huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS-1].count;
|
||||
huffNode[lowS].parent = huffNode[lowS-1].parent = nodeNb;
|
||||
nodeNb++; lowS-=2;
|
||||
for (n=nodeNb; n<=nodeRoot; n++) huffNode[n].count = (U32)(1U<<30);
|
||||
huffNode0[0].count = (U32)(1U<<31); /* fake entry, strong barrier */
|
||||
|
||||
/* create parents */
|
||||
while (nodeNb <= nodeRoot) {
|
||||
U32 n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++;
|
||||
U32 n2 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++;
|
||||
huffNode[nodeNb].count = huffNode[n1].count + huffNode[n2].count;
|
||||
huffNode[n1].parent = huffNode[n2].parent = nodeNb;
|
||||
nodeNb++;
|
||||
}
|
||||
|
||||
/* distribute weights (unlimited tree height) */
|
||||
huffNode[nodeRoot].nbBits = 0;
|
||||
for (n=nodeRoot-1; n>=STARTNODE; n--)
|
||||
huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1;
|
||||
for (n=0; n<=nonNullRank; n++)
|
||||
huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1;
|
||||
|
||||
/* enforce maxTableLog */
|
||||
maxNbBits = HUF_setMaxHeight(huffNode, nonNullRank, maxNbBits);
|
||||
|
||||
/* fill result into tree (val, nbBits) */
|
||||
{ U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0};
|
||||
U16 valPerRank[HUF_TABLELOG_MAX+1] = {0};
|
||||
if (maxNbBits > HUF_TABLELOG_MAX) return ERROR(GENERIC); /* check fit into table */
|
||||
for (n=0; n<=nonNullRank; n++)
|
||||
nbPerRank[huffNode[n].nbBits]++;
|
||||
/* determine stating value per rank */
|
||||
{ U16 min = 0;
|
||||
for (n=maxNbBits; n>0; n--) {
|
||||
valPerRank[n] = min; /* get starting value within each rank */
|
||||
min += nbPerRank[n];
|
||||
min >>= 1;
|
||||
} }
|
||||
for (n=0; n<=maxSymbolValue; n++)
|
||||
tree[huffNode[n].byte].nbBits = huffNode[n].nbBits; /* push nbBits per symbol, symbol order */
|
||||
for (n=0; n<=maxSymbolValue; n++)
|
||||
tree[n].val = valPerRank[tree[n].nbBits]++; /* assign value within rank, symbol order */
|
||||
}
|
||||
|
||||
return maxNbBits;
|
||||
}
|
||||
|
||||
/** HUF_buildCTable() :
|
||||
* @return : maxNbBits
|
||||
* Note : count is used before tree is written, so they can safely overlap
|
||||
*/
|
||||
size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits)
|
||||
{
|
||||
huffNodeTable nodeTable;
|
||||
return HUF_buildCTable_wksp(tree, count, maxSymbolValue, maxNbBits, nodeTable, sizeof(nodeTable));
|
||||
}
|
||||
|
||||
static size_t HUF_estimateCompressedSize(HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue)
|
||||
{
|
||||
size_t nbBits = 0;
|
||||
int s;
|
||||
for (s = 0; s <= (int)maxSymbolValue; ++s) {
|
||||
nbBits += CTable[s].nbBits * count[s];
|
||||
}
|
||||
return nbBits >> 3;
|
||||
}
|
||||
|
||||
static int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue) {
|
||||
int bad = 0;
|
||||
int s;
|
||||
for (s = 0; s <= (int)maxSymbolValue; ++s) {
|
||||
bad |= (count[s] != 0) & (CTable[s].nbBits == 0);
|
||||
}
|
||||
return !bad;
|
||||
}
|
||||
|
||||
size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); }
|
||||
|
||||
FORCE_INLINE_TEMPLATE void
|
||||
HUF_encodeSymbol(BIT_CStream_t* bitCPtr, U32 symbol, const HUF_CElt* CTable)
|
||||
{
|
||||
BIT_addBitsFast(bitCPtr, CTable[symbol].val, CTable[symbol].nbBits);
|
||||
}
|
||||
|
||||
#define HUF_FLUSHBITS(s) BIT_flushBits(s)
|
||||
|
||||
#define HUF_FLUSHBITS_1(stream) \
|
||||
if (sizeof((stream)->bitContainer)*8 < HUF_TABLELOG_MAX*2+7) HUF_FLUSHBITS(stream)
|
||||
|
||||
#define HUF_FLUSHBITS_2(stream) \
|
||||
if (sizeof((stream)->bitContainer)*8 < HUF_TABLELOG_MAX*4+7) HUF_FLUSHBITS(stream)
|
||||
|
||||
FORCE_INLINE_TEMPLATE size_t
|
||||
HUF_compress1X_usingCTable_internal_body(void* dst, size_t dstSize,
|
||||
const void* src, size_t srcSize,
|
||||
const HUF_CElt* CTable)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*) src;
|
||||
BYTE* const ostart = (BYTE*)dst;
|
||||
BYTE* const oend = ostart + dstSize;
|
||||
BYTE* op = ostart;
|
||||
size_t n;
|
||||
BIT_CStream_t bitC;
|
||||
|
||||
/* init */
|
||||
if (dstSize < 8) return 0; /* not enough space to compress */
|
||||
{ size_t const initErr = BIT_initCStream(&bitC, op, oend-op);
|
||||
if (HUF_isError(initErr)) return 0; }
|
||||
|
||||
n = srcSize & ~3; /* join to mod 4 */
|
||||
switch (srcSize & 3)
|
||||
{
|
||||
case 3 : HUF_encodeSymbol(&bitC, ip[n+ 2], CTable);
|
||||
HUF_FLUSHBITS_2(&bitC);
|
||||
/* fall-through */
|
||||
case 2 : HUF_encodeSymbol(&bitC, ip[n+ 1], CTable);
|
||||
HUF_FLUSHBITS_1(&bitC);
|
||||
/* fall-through */
|
||||
case 1 : HUF_encodeSymbol(&bitC, ip[n+ 0], CTable);
|
||||
HUF_FLUSHBITS(&bitC);
|
||||
/* fall-through */
|
||||
case 0 : /* fall-through */
|
||||
default: break;
|
||||
}
|
||||
|
||||
for (; n>0; n-=4) { /* note : n&3==0 at this stage */
|
||||
HUF_encodeSymbol(&bitC, ip[n- 1], CTable);
|
||||
HUF_FLUSHBITS_1(&bitC);
|
||||
HUF_encodeSymbol(&bitC, ip[n- 2], CTable);
|
||||
HUF_FLUSHBITS_2(&bitC);
|
||||
HUF_encodeSymbol(&bitC, ip[n- 3], CTable);
|
||||
HUF_FLUSHBITS_1(&bitC);
|
||||
HUF_encodeSymbol(&bitC, ip[n- 4], CTable);
|
||||
HUF_FLUSHBITS(&bitC);
|
||||
}
|
||||
|
||||
return BIT_closeCStream(&bitC);
|
||||
}
|
||||
|
||||
#if DYNAMIC_BMI2
|
||||
|
||||
static TARGET_ATTRIBUTE("bmi2") size_t
|
||||
HUF_compress1X_usingCTable_internal_bmi2(void* dst, size_t dstSize,
|
||||
const void* src, size_t srcSize,
|
||||
const HUF_CElt* CTable)
|
||||
{
|
||||
return HUF_compress1X_usingCTable_internal_body(dst, dstSize, src, srcSize, CTable);
|
||||
}
|
||||
|
||||
static size_t
|
||||
HUF_compress1X_usingCTable_internal_default(void* dst, size_t dstSize,
|
||||
const void* src, size_t srcSize,
|
||||
const HUF_CElt* CTable)
|
||||
{
|
||||
return HUF_compress1X_usingCTable_internal_body(dst, dstSize, src, srcSize, CTable);
|
||||
}
|
||||
|
||||
static size_t
|
||||
HUF_compress1X_usingCTable_internal(void* dst, size_t dstSize,
|
||||
const void* src, size_t srcSize,
|
||||
const HUF_CElt* CTable, const int bmi2)
|
||||
{
|
||||
if (bmi2) {
|
||||
return HUF_compress1X_usingCTable_internal_bmi2(dst, dstSize, src, srcSize, CTable);
|
||||
}
|
||||
return HUF_compress1X_usingCTable_internal_default(dst, dstSize, src, srcSize, CTable);
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
static size_t
|
||||
HUF_compress1X_usingCTable_internal(void* dst, size_t dstSize,
|
||||
const void* src, size_t srcSize,
|
||||
const HUF_CElt* CTable, const int bmi2)
|
||||
{
|
||||
(void)bmi2;
|
||||
return HUF_compress1X_usingCTable_internal_body(dst, dstSize, src, srcSize, CTable);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable)
|
||||
{
|
||||
return HUF_compress1X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, /* bmi2 */ 0);
|
||||
}
|
||||
|
||||
|
||||
static size_t
|
||||
HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize,
|
||||
const void* src, size_t srcSize,
|
||||
const HUF_CElt* CTable, int bmi2)
|
||||
{
|
||||
size_t const segmentSize = (srcSize+3)/4; /* first 3 segments */
|
||||
const BYTE* ip = (const BYTE*) src;
|
||||
const BYTE* const iend = ip + srcSize;
|
||||
BYTE* const ostart = (BYTE*) dst;
|
||||
BYTE* const oend = ostart + dstSize;
|
||||
BYTE* op = ostart;
|
||||
|
||||
if (dstSize < 6 + 1 + 1 + 1 + 8) return 0; /* minimum space to compress successfully */
|
||||
if (srcSize < 12) return 0; /* no saving possible : too small input */
|
||||
op += 6; /* jumpTable */
|
||||
|
||||
{ CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, oend-op, ip, segmentSize, CTable, bmi2) );
|
||||
if (cSize==0) return 0;
|
||||
assert(cSize <= 65535);
|
||||
MEM_writeLE16(ostart, (U16)cSize);
|
||||
op += cSize;
|
||||
}
|
||||
|
||||
ip += segmentSize;
|
||||
{ CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, oend-op, ip, segmentSize, CTable, bmi2) );
|
||||
if (cSize==0) return 0;
|
||||
assert(cSize <= 65535);
|
||||
MEM_writeLE16(ostart+2, (U16)cSize);
|
||||
op += cSize;
|
||||
}
|
||||
|
||||
ip += segmentSize;
|
||||
{ CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, oend-op, ip, segmentSize, CTable, bmi2) );
|
||||
if (cSize==0) return 0;
|
||||
assert(cSize <= 65535);
|
||||
MEM_writeLE16(ostart+4, (U16)cSize);
|
||||
op += cSize;
|
||||
}
|
||||
|
||||
ip += segmentSize;
|
||||
{ CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, oend-op, ip, iend-ip, CTable, bmi2) );
|
||||
if (cSize==0) return 0;
|
||||
op += cSize;
|
||||
}
|
||||
|
||||
return op-ostart;
|
||||
}
|
||||
|
||||
size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable)
|
||||
{
|
||||
return HUF_compress4X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, /* bmi2 */ 0);
|
||||
}
|
||||
|
||||
|
||||
static size_t HUF_compressCTable_internal(
|
||||
BYTE* const ostart, BYTE* op, BYTE* const oend,
|
||||
const void* src, size_t srcSize,
|
||||
unsigned singleStream, const HUF_CElt* CTable, const int bmi2)
|
||||
{
|
||||
size_t const cSize = singleStream ?
|
||||
HUF_compress1X_usingCTable_internal(op, oend - op, src, srcSize, CTable, bmi2) :
|
||||
HUF_compress4X_usingCTable_internal(op, oend - op, src, srcSize, CTable, bmi2);
|
||||
if (HUF_isError(cSize)) { return cSize; }
|
||||
if (cSize==0) { return 0; } /* uncompressible */
|
||||
op += cSize;
|
||||
/* check compressibility */
|
||||
if ((size_t)(op-ostart) >= srcSize-1) { return 0; }
|
||||
return op-ostart;
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
U32 count[HUF_SYMBOLVALUE_MAX + 1];
|
||||
HUF_CElt CTable[HUF_SYMBOLVALUE_MAX + 1];
|
||||
huffNodeTable nodeTable;
|
||||
} HUF_compress_tables_t;
|
||||
|
||||
/* HUF_compress_internal() :
|
||||
* `workSpace` must a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */
|
||||
static size_t HUF_compress_internal (
|
||||
void* dst, size_t dstSize,
|
||||
const void* src, size_t srcSize,
|
||||
unsigned maxSymbolValue, unsigned huffLog,
|
||||
unsigned singleStream,
|
||||
void* workSpace, size_t wkspSize,
|
||||
HUF_CElt* oldHufTable, HUF_repeat* repeat, int preferRepeat,
|
||||
const int bmi2)
|
||||
{
|
||||
HUF_compress_tables_t* const table = (HUF_compress_tables_t*)workSpace;
|
||||
BYTE* const ostart = (BYTE*)dst;
|
||||
BYTE* const oend = ostart + dstSize;
|
||||
BYTE* op = ostart;
|
||||
|
||||
/* checks & inits */
|
||||
if (((size_t)workSpace & 3) != 0) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */
|
||||
if (wkspSize < sizeof(*table)) return ERROR(workSpace_tooSmall);
|
||||
if (!srcSize) return 0; /* Uncompressed */
|
||||
if (!dstSize) return 0; /* cannot fit anything within dst budget */
|
||||
if (srcSize > HUF_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); /* current block size limit */
|
||||
if (huffLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
|
||||
if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge);
|
||||
if (!maxSymbolValue) maxSymbolValue = HUF_SYMBOLVALUE_MAX;
|
||||
if (!huffLog) huffLog = HUF_TABLELOG_DEFAULT;
|
||||
|
||||
/* Heuristic : If old table is valid, use it for small inputs */
|
||||
if (preferRepeat && repeat && *repeat == HUF_repeat_valid) {
|
||||
return HUF_compressCTable_internal(ostart, op, oend,
|
||||
src, srcSize,
|
||||
singleStream, oldHufTable, bmi2);
|
||||
}
|
||||
|
||||
/* Scan input and build symbol stats */
|
||||
{ CHECK_V_F(largest, HIST_count_wksp (table->count, &maxSymbolValue, (const BYTE*)src, srcSize, table->count) );
|
||||
if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* single symbol, rle */
|
||||
if (largest <= (srcSize >> 7)+4) return 0; /* heuristic : probably not compressible enough */
|
||||
}
|
||||
|
||||
/* Check validity of previous table */
|
||||
if ( repeat
|
||||
&& *repeat == HUF_repeat_check
|
||||
&& !HUF_validateCTable(oldHufTable, table->count, maxSymbolValue)) {
|
||||
*repeat = HUF_repeat_none;
|
||||
}
|
||||
/* Heuristic : use existing table for small inputs */
|
||||
if (preferRepeat && repeat && *repeat != HUF_repeat_none) {
|
||||
return HUF_compressCTable_internal(ostart, op, oend,
|
||||
src, srcSize,
|
||||
singleStream, oldHufTable, bmi2);
|
||||
}
|
||||
|
||||
/* Build Huffman Tree */
|
||||
huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue);
|
||||
{ CHECK_V_F(maxBits, HUF_buildCTable_wksp(table->CTable, table->count,
|
||||
maxSymbolValue, huffLog,
|
||||
table->nodeTable, sizeof(table->nodeTable)) );
|
||||
huffLog = (U32)maxBits;
|
||||
/* Zero unused symbols in CTable, so we can check it for validity */
|
||||
memset(table->CTable + (maxSymbolValue + 1), 0,
|
||||
sizeof(table->CTable) - ((maxSymbolValue + 1) * sizeof(HUF_CElt)));
|
||||
}
|
||||
|
||||
/* Write table description header */
|
||||
{ CHECK_V_F(hSize, HUF_writeCTable (op, dstSize, table->CTable, maxSymbolValue, huffLog) );
|
||||
/* Check if using previous huffman table is beneficial */
|
||||
if (repeat && *repeat != HUF_repeat_none) {
|
||||
size_t const oldSize = HUF_estimateCompressedSize(oldHufTable, table->count, maxSymbolValue);
|
||||
size_t const newSize = HUF_estimateCompressedSize(table->CTable, table->count, maxSymbolValue);
|
||||
if (oldSize <= hSize + newSize || hSize + 12 >= srcSize) {
|
||||
return HUF_compressCTable_internal(ostart, op, oend,
|
||||
src, srcSize,
|
||||
singleStream, oldHufTable, bmi2);
|
||||
} }
|
||||
|
||||
/* Use the new huffman table */
|
||||
if (hSize + 12ul >= srcSize) { return 0; }
|
||||
op += hSize;
|
||||
if (repeat) { *repeat = HUF_repeat_none; }
|
||||
if (oldHufTable)
|
||||
memcpy(oldHufTable, table->CTable, sizeof(table->CTable)); /* Save new table */
|
||||
}
|
||||
return HUF_compressCTable_internal(ostart, op, oend,
|
||||
src, srcSize,
|
||||
singleStream, table->CTable, bmi2);
|
||||
}
|
||||
|
||||
|
||||
size_t HUF_compress1X_wksp (void* dst, size_t dstSize,
|
||||
const void* src, size_t srcSize,
|
||||
unsigned maxSymbolValue, unsigned huffLog,
|
||||
void* workSpace, size_t wkspSize)
|
||||
{
|
||||
return HUF_compress_internal(dst, dstSize, src, srcSize,
|
||||
maxSymbolValue, huffLog, 1 /*single stream*/,
|
||||
workSpace, wkspSize,
|
||||
NULL, NULL, 0, 0 /*bmi2*/);
|
||||
}
|
||||
|
||||
size_t HUF_compress1X_repeat (void* dst, size_t dstSize,
|
||||
const void* src, size_t srcSize,
|
||||
unsigned maxSymbolValue, unsigned huffLog,
|
||||
void* workSpace, size_t wkspSize,
|
||||
HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2)
|
||||
{
|
||||
return HUF_compress_internal(dst, dstSize, src, srcSize,
|
||||
maxSymbolValue, huffLog, 1 /*single stream*/,
|
||||
workSpace, wkspSize, hufTable,
|
||||
repeat, preferRepeat, bmi2);
|
||||
}
|
||||
|
||||
size_t HUF_compress1X (void* dst, size_t dstSize,
|
||||
const void* src, size_t srcSize,
|
||||
unsigned maxSymbolValue, unsigned huffLog)
|
||||
{
|
||||
unsigned workSpace[HUF_WORKSPACE_SIZE_U32];
|
||||
return HUF_compress1X_wksp(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, workSpace, sizeof(workSpace));
|
||||
}
|
||||
|
||||
/* HUF_compress4X_repeat():
|
||||
* compress input using 4 streams.
|
||||
* provide workspace to generate compression tables */
|
||||
size_t HUF_compress4X_wksp (void* dst, size_t dstSize,
|
||||
const void* src, size_t srcSize,
|
||||
unsigned maxSymbolValue, unsigned huffLog,
|
||||
void* workSpace, size_t wkspSize)
|
||||
{
|
||||
return HUF_compress_internal(dst, dstSize, src, srcSize,
|
||||
maxSymbolValue, huffLog, 0 /*4 streams*/,
|
||||
workSpace, wkspSize,
|
||||
NULL, NULL, 0, 0 /*bmi2*/);
|
||||
}
|
||||
|
||||
/* HUF_compress4X_repeat():
|
||||
* compress input using 4 streams.
|
||||
* re-use an existing huffman compression table */
|
||||
size_t HUF_compress4X_repeat (void* dst, size_t dstSize,
|
||||
const void* src, size_t srcSize,
|
||||
unsigned maxSymbolValue, unsigned huffLog,
|
||||
void* workSpace, size_t wkspSize,
|
||||
HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2)
|
||||
{
|
||||
return HUF_compress_internal(dst, dstSize, src, srcSize,
|
||||
maxSymbolValue, huffLog, 0 /* 4 streams */,
|
||||
workSpace, wkspSize,
|
||||
hufTable, repeat, preferRepeat, bmi2);
|
||||
}
|
||||
|
||||
size_t HUF_compress2 (void* dst, size_t dstSize,
|
||||
const void* src, size_t srcSize,
|
||||
unsigned maxSymbolValue, unsigned huffLog)
|
||||
{
|
||||
unsigned workSpace[HUF_WORKSPACE_SIZE_U32];
|
||||
return HUF_compress4X_wksp(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, workSpace, sizeof(workSpace));
|
||||
}
|
||||
|
||||
size_t HUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
||||
{
|
||||
return HUF_compress2(dst, maxDstSize, src, srcSize, 255, HUF_TABLELOG_DEFAULT);
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,795 @@
|
|||
/*
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
/* This header contains definitions
|
||||
* that shall **only** be used by modules within lib/compress.
|
||||
*/
|
||||
|
||||
#ifndef ZSTD_COMPRESS_H
|
||||
#define ZSTD_COMPRESS_H
|
||||
|
||||
/*-*************************************
|
||||
* Dependencies
|
||||
***************************************/
|
||||
#include "zstd_internal.h"
|
||||
#ifdef ZSTD_MULTITHREAD
|
||||
# include "zstdmt_compress.h"
|
||||
#endif
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Constants
|
||||
***************************************/
|
||||
#define kSearchStrength 8
|
||||
#define HASH_READ_SIZE 8
|
||||
#define ZSTD_DUBT_UNSORTED_MARK 1 /* For btlazy2 strategy, index 1 now means "unsorted".
|
||||
It could be confused for a real successor at index "1", if sorted as larger than its predecessor.
|
||||
It's not a big deal though : candidate will just be sorted again.
|
||||
Additionnally, candidate position 1 will be lost.
|
||||
But candidate 1 cannot hide a large tree of candidates, so it's a minimal loss.
|
||||
The benefit is that ZSTD_DUBT_UNSORTED_MARK cannot be misdhandled after table re-use with a different strategy
|
||||
Constant required by ZSTD_compressBlock_btlazy2() and ZSTD_reduceTable_internal() */
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Context memory management
|
||||
***************************************/
|
||||
typedef enum { ZSTDcs_created=0, ZSTDcs_init, ZSTDcs_ongoing, ZSTDcs_ending } ZSTD_compressionStage_e;
|
||||
typedef enum { zcss_init=0, zcss_load, zcss_flush } ZSTD_cStreamStage;
|
||||
|
||||
typedef enum {
|
||||
ZSTD_dictDefaultAttach = 0,
|
||||
ZSTD_dictForceAttach = 1,
|
||||
ZSTD_dictForceCopy = -1,
|
||||
} ZSTD_dictAttachPref_e;
|
||||
|
||||
typedef struct ZSTD_prefixDict_s {
|
||||
const void* dict;
|
||||
size_t dictSize;
|
||||
ZSTD_dictContentType_e dictContentType;
|
||||
} ZSTD_prefixDict;
|
||||
|
||||
typedef struct {
|
||||
U32 CTable[HUF_CTABLE_SIZE_U32(255)];
|
||||
HUF_repeat repeatMode;
|
||||
} ZSTD_hufCTables_t;
|
||||
|
||||
typedef struct {
|
||||
FSE_CTable offcodeCTable[FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)];
|
||||
FSE_CTable matchlengthCTable[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)];
|
||||
FSE_CTable litlengthCTable[FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)];
|
||||
FSE_repeat offcode_repeatMode;
|
||||
FSE_repeat matchlength_repeatMode;
|
||||
FSE_repeat litlength_repeatMode;
|
||||
} ZSTD_fseCTables_t;
|
||||
|
||||
typedef struct {
|
||||
ZSTD_hufCTables_t huf;
|
||||
ZSTD_fseCTables_t fse;
|
||||
} ZSTD_entropyCTables_t;
|
||||
|
||||
typedef struct {
|
||||
U32 off;
|
||||
U32 len;
|
||||
} ZSTD_match_t;
|
||||
|
||||
typedef struct {
|
||||
int price;
|
||||
U32 off;
|
||||
U32 mlen;
|
||||
U32 litlen;
|
||||
U32 rep[ZSTD_REP_NUM];
|
||||
} ZSTD_optimal_t;
|
||||
|
||||
typedef enum { zop_dynamic=0, zop_predef } ZSTD_OptPrice_e;
|
||||
|
||||
typedef struct {
|
||||
/* All tables are allocated inside cctx->workspace by ZSTD_resetCCtx_internal() */
|
||||
U32* litFreq; /* table of literals statistics, of size 256 */
|
||||
U32* litLengthFreq; /* table of litLength statistics, of size (MaxLL+1) */
|
||||
U32* matchLengthFreq; /* table of matchLength statistics, of size (MaxML+1) */
|
||||
U32* offCodeFreq; /* table of offCode statistics, of size (MaxOff+1) */
|
||||
ZSTD_match_t* matchTable; /* list of found matches, of size ZSTD_OPT_NUM+1 */
|
||||
ZSTD_optimal_t* priceTable; /* All positions tracked by optimal parser, of size ZSTD_OPT_NUM+1 */
|
||||
|
||||
U32 litSum; /* nb of literals */
|
||||
U32 litLengthSum; /* nb of litLength codes */
|
||||
U32 matchLengthSum; /* nb of matchLength codes */
|
||||
U32 offCodeSum; /* nb of offset codes */
|
||||
U32 litSumBasePrice; /* to compare to log2(litfreq) */
|
||||
U32 litLengthSumBasePrice; /* to compare to log2(llfreq) */
|
||||
U32 matchLengthSumBasePrice;/* to compare to log2(mlfreq) */
|
||||
U32 offCodeSumBasePrice; /* to compare to log2(offreq) */
|
||||
ZSTD_OptPrice_e priceType; /* prices can be determined dynamically, or follow a pre-defined cost structure */
|
||||
const ZSTD_entropyCTables_t* symbolCosts; /* pre-calculated dictionary statistics */
|
||||
} optState_t;
|
||||
|
||||
typedef struct {
|
||||
ZSTD_entropyCTables_t entropy;
|
||||
U32 rep[ZSTD_REP_NUM];
|
||||
} ZSTD_compressedBlockState_t;
|
||||
|
||||
typedef struct {
|
||||
BYTE const* nextSrc; /* next block here to continue on current prefix */
|
||||
BYTE const* base; /* All regular indexes relative to this position */
|
||||
BYTE const* dictBase; /* extDict indexes relative to this position */
|
||||
U32 dictLimit; /* below that point, need extDict */
|
||||
U32 lowLimit; /* below that point, no more data */
|
||||
} ZSTD_window_t;
|
||||
|
||||
typedef struct ZSTD_matchState_t ZSTD_matchState_t;
|
||||
struct ZSTD_matchState_t {
|
||||
ZSTD_window_t window; /* State for window round buffer management */
|
||||
U32 loadedDictEnd; /* index of end of dictionary */
|
||||
U32 nextToUpdate; /* index from which to continue table update */
|
||||
U32 nextToUpdate3; /* index from which to continue table update */
|
||||
U32 hashLog3; /* dispatch table : larger == faster, more memory */
|
||||
U32* hashTable;
|
||||
U32* hashTable3;
|
||||
U32* chainTable;
|
||||
optState_t opt; /* optimal parser state */
|
||||
const ZSTD_matchState_t *dictMatchState;
|
||||
};
|
||||
|
||||
typedef struct {
|
||||
ZSTD_compressedBlockState_t* prevCBlock;
|
||||
ZSTD_compressedBlockState_t* nextCBlock;
|
||||
ZSTD_matchState_t matchState;
|
||||
} ZSTD_blockState_t;
|
||||
|
||||
typedef struct {
|
||||
U32 offset;
|
||||
U32 checksum;
|
||||
} ldmEntry_t;
|
||||
|
||||
typedef struct {
|
||||
ZSTD_window_t window; /* State for the window round buffer management */
|
||||
ldmEntry_t* hashTable;
|
||||
BYTE* bucketOffsets; /* Next position in bucket to insert entry */
|
||||
U64 hashPower; /* Used to compute the rolling hash.
|
||||
* Depends on ldmParams.minMatchLength */
|
||||
} ldmState_t;
|
||||
|
||||
typedef struct {
|
||||
U32 enableLdm; /* 1 if enable long distance matching */
|
||||
U32 hashLog; /* Log size of hashTable */
|
||||
U32 bucketSizeLog; /* Log bucket size for collision resolution, at most 8 */
|
||||
U32 minMatchLength; /* Minimum match length */
|
||||
U32 hashEveryLog; /* Log number of entries to skip */
|
||||
U32 windowLog; /* Window log for the LDM */
|
||||
} ldmParams_t;
|
||||
|
||||
typedef struct {
|
||||
U32 offset;
|
||||
U32 litLength;
|
||||
U32 matchLength;
|
||||
} rawSeq;
|
||||
|
||||
typedef struct {
|
||||
rawSeq* seq; /* The start of the sequences */
|
||||
size_t pos; /* The position where reading stopped. <= size. */
|
||||
size_t size; /* The number of sequences. <= capacity. */
|
||||
size_t capacity; /* The capacity starting from `seq` pointer */
|
||||
} rawSeqStore_t;
|
||||
|
||||
struct ZSTD_CCtx_params_s {
|
||||
ZSTD_format_e format;
|
||||
ZSTD_compressionParameters cParams;
|
||||
ZSTD_frameParameters fParams;
|
||||
|
||||
int compressionLevel;
|
||||
int forceWindow; /* force back-references to respect limit of
|
||||
* 1<<wLog, even for dictionary */
|
||||
|
||||
ZSTD_dictAttachPref_e attachDictPref;
|
||||
|
||||
/* Multithreading: used to pass parameters to mtctx */
|
||||
unsigned nbWorkers;
|
||||
unsigned jobSize;
|
||||
unsigned overlapSizeLog;
|
||||
|
||||
/* Long distance matching parameters */
|
||||
ldmParams_t ldmParams;
|
||||
|
||||
/* Internal use, for createCCtxParams() and freeCCtxParams() only */
|
||||
ZSTD_customMem customMem;
|
||||
}; /* typedef'd to ZSTD_CCtx_params within "zstd.h" */
|
||||
|
||||
struct ZSTD_CCtx_s {
|
||||
ZSTD_compressionStage_e stage;
|
||||
int cParamsChanged; /* == 1 if cParams(except wlog) or compression level are changed in requestedParams. Triggers transmission of new params to ZSTDMT (if available) then reset to 0. */
|
||||
int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */
|
||||
ZSTD_CCtx_params requestedParams;
|
||||
ZSTD_CCtx_params appliedParams;
|
||||
U32 dictID;
|
||||
|
||||
int workSpaceOversizedDuration;
|
||||
void* workSpace;
|
||||
size_t workSpaceSize;
|
||||
size_t blockSize;
|
||||
unsigned long long pledgedSrcSizePlusOne; /* this way, 0 (default) == unknown */
|
||||
unsigned long long consumedSrcSize;
|
||||
unsigned long long producedCSize;
|
||||
XXH64_state_t xxhState;
|
||||
ZSTD_customMem customMem;
|
||||
size_t staticSize;
|
||||
|
||||
seqStore_t seqStore; /* sequences storage ptrs */
|
||||
ldmState_t ldmState; /* long distance matching state */
|
||||
rawSeq* ldmSequences; /* Storage for the ldm output sequences */
|
||||
size_t maxNbLdmSequences;
|
||||
rawSeqStore_t externSeqStore; /* Mutable reference to external sequences */
|
||||
ZSTD_blockState_t blockState;
|
||||
U32* entropyWorkspace; /* entropy workspace of HUF_WORKSPACE_SIZE bytes */
|
||||
|
||||
/* streaming */
|
||||
char* inBuff;
|
||||
size_t inBuffSize;
|
||||
size_t inToCompress;
|
||||
size_t inBuffPos;
|
||||
size_t inBuffTarget;
|
||||
char* outBuff;
|
||||
size_t outBuffSize;
|
||||
size_t outBuffContentSize;
|
||||
size_t outBuffFlushedSize;
|
||||
ZSTD_cStreamStage streamStage;
|
||||
U32 frameEnded;
|
||||
|
||||
/* Dictionary */
|
||||
ZSTD_CDict* cdictLocal;
|
||||
const ZSTD_CDict* cdict;
|
||||
ZSTD_prefixDict prefixDict; /* single-usage dictionary */
|
||||
|
||||
/* Multi-threading */
|
||||
#ifdef ZSTD_MULTITHREAD
|
||||
ZSTDMT_CCtx* mtctx;
|
||||
#endif
|
||||
};
|
||||
|
||||
typedef enum { ZSTD_dtlm_fast, ZSTD_dtlm_full } ZSTD_dictTableLoadMethod_e;
|
||||
|
||||
typedef enum { ZSTD_noDict = 0, ZSTD_extDict = 1, ZSTD_dictMatchState = 2 } ZSTD_dictMode_e;
|
||||
|
||||
|
||||
typedef size_t (*ZSTD_blockCompressor) (
|
||||
ZSTD_matchState_t* bs, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
|
||||
ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_dictMode_e dictMode);
|
||||
|
||||
|
||||
MEM_STATIC U32 ZSTD_LLcode(U32 litLength)
|
||||
{
|
||||
static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7,
|
||||
8, 9, 10, 11, 12, 13, 14, 15,
|
||||
16, 16, 17, 17, 18, 18, 19, 19,
|
||||
20, 20, 20, 20, 21, 21, 21, 21,
|
||||
22, 22, 22, 22, 22, 22, 22, 22,
|
||||
23, 23, 23, 23, 23, 23, 23, 23,
|
||||
24, 24, 24, 24, 24, 24, 24, 24,
|
||||
24, 24, 24, 24, 24, 24, 24, 24 };
|
||||
static const U32 LL_deltaCode = 19;
|
||||
return (litLength > 63) ? ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength];
|
||||
}
|
||||
|
||||
/* ZSTD_MLcode() :
|
||||
* note : mlBase = matchLength - MINMATCH;
|
||||
* because it's the format it's stored in seqStore->sequences */
|
||||
MEM_STATIC U32 ZSTD_MLcode(U32 mlBase)
|
||||
{
|
||||
static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
|
||||
16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
|
||||
32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37,
|
||||
38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39,
|
||||
40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
|
||||
41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41,
|
||||
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
|
||||
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 };
|
||||
static const U32 ML_deltaCode = 36;
|
||||
return (mlBase > 127) ? ZSTD_highbit32(mlBase) + ML_deltaCode : ML_Code[mlBase];
|
||||
}
|
||||
|
||||
/*! ZSTD_storeSeq() :
|
||||
* Store a sequence (literal length, literals, offset code and match length code) into seqStore_t.
|
||||
* `offsetCode` : distance to match + 3 (values 1-3 are repCodes).
|
||||
* `mlBase` : matchLength - MINMATCH
|
||||
*/
|
||||
MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const void* literals, U32 offsetCode, size_t mlBase)
|
||||
{
|
||||
#if defined(DEBUGLEVEL) && (DEBUGLEVEL >= 6)
|
||||
static const BYTE* g_start = NULL;
|
||||
if (g_start==NULL) g_start = (const BYTE*)literals; /* note : index only works for compression within a single segment */
|
||||
{ U32 const pos = (U32)((const BYTE*)literals - g_start);
|
||||
DEBUGLOG(6, "Cpos%7u :%3u literals, match%4u bytes at offCode%7u",
|
||||
pos, (U32)litLength, (U32)mlBase+MINMATCH, (U32)offsetCode);
|
||||
}
|
||||
#endif
|
||||
/* copy Literals */
|
||||
assert(seqStorePtr->lit + litLength <= seqStorePtr->litStart + 128 KB);
|
||||
ZSTD_wildcopy(seqStorePtr->lit, literals, litLength);
|
||||
seqStorePtr->lit += litLength;
|
||||
|
||||
/* literal Length */
|
||||
if (litLength>0xFFFF) {
|
||||
assert(seqStorePtr->longLengthID == 0); /* there can only be a single long length */
|
||||
seqStorePtr->longLengthID = 1;
|
||||
seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
|
||||
}
|
||||
seqStorePtr->sequences[0].litLength = (U16)litLength;
|
||||
|
||||
/* match offset */
|
||||
seqStorePtr->sequences[0].offset = offsetCode + 1;
|
||||
|
||||
/* match Length */
|
||||
if (mlBase>0xFFFF) {
|
||||
assert(seqStorePtr->longLengthID == 0); /* there can only be a single long length */
|
||||
seqStorePtr->longLengthID = 2;
|
||||
seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
|
||||
}
|
||||
seqStorePtr->sequences[0].matchLength = (U16)mlBase;
|
||||
|
||||
seqStorePtr->sequences++;
|
||||
}
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Match length counter
|
||||
***************************************/
|
||||
static unsigned ZSTD_NbCommonBytes (size_t val)
|
||||
{
|
||||
if (MEM_isLittleEndian()) {
|
||||
if (MEM_64bits()) {
|
||||
# if defined(_MSC_VER) && defined(_WIN64)
|
||||
unsigned long r = 0;
|
||||
_BitScanForward64( &r, (U64)val );
|
||||
return (unsigned)(r>>3);
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 4)
|
||||
return (__builtin_ctzll((U64)val) >> 3);
|
||||
# else
|
||||
static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2,
|
||||
0, 3, 1, 3, 1, 4, 2, 7,
|
||||
0, 2, 3, 6, 1, 5, 3, 5,
|
||||
1, 3, 4, 4, 2, 5, 6, 7,
|
||||
7, 0, 1, 2, 3, 3, 4, 6,
|
||||
2, 6, 5, 5, 3, 4, 5, 6,
|
||||
7, 1, 2, 4, 6, 4, 4, 5,
|
||||
7, 2, 6, 5, 7, 6, 7, 7 };
|
||||
return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
|
||||
# endif
|
||||
} else { /* 32 bits */
|
||||
# if defined(_MSC_VER)
|
||||
unsigned long r=0;
|
||||
_BitScanForward( &r, (U32)val );
|
||||
return (unsigned)(r>>3);
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3)
|
||||
return (__builtin_ctz((U32)val) >> 3);
|
||||
# else
|
||||
static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0,
|
||||
3, 2, 2, 1, 3, 2, 0, 1,
|
||||
3, 3, 1, 2, 2, 2, 2, 0,
|
||||
3, 1, 2, 0, 1, 0, 1, 1 };
|
||||
return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
|
||||
# endif
|
||||
}
|
||||
} else { /* Big Endian CPU */
|
||||
if (MEM_64bits()) {
|
||||
# if defined(_MSC_VER) && defined(_WIN64)
|
||||
unsigned long r = 0;
|
||||
_BitScanReverse64( &r, val );
|
||||
return (unsigned)(r>>3);
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 4)
|
||||
return (__builtin_clzll(val) >> 3);
|
||||
# else
|
||||
unsigned r;
|
||||
const unsigned n32 = sizeof(size_t)*4; /* calculate this way due to compiler complaining in 32-bits mode */
|
||||
if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; }
|
||||
if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
|
||||
r += (!val);
|
||||
return r;
|
||||
# endif
|
||||
} else { /* 32 bits */
|
||||
# if defined(_MSC_VER)
|
||||
unsigned long r = 0;
|
||||
_BitScanReverse( &r, (unsigned long)val );
|
||||
return (unsigned)(r>>3);
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3)
|
||||
return (__builtin_clz((U32)val) >> 3);
|
||||
# else
|
||||
unsigned r;
|
||||
if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
|
||||
r += (!val);
|
||||
return r;
|
||||
# endif
|
||||
} }
|
||||
}
|
||||
|
||||
|
||||
MEM_STATIC size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* const pInLimit)
|
||||
{
|
||||
const BYTE* const pStart = pIn;
|
||||
const BYTE* const pInLoopLimit = pInLimit - (sizeof(size_t)-1);
|
||||
|
||||
if (pIn < pInLoopLimit) {
|
||||
{ size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn);
|
||||
if (diff) return ZSTD_NbCommonBytes(diff); }
|
||||
pIn+=sizeof(size_t); pMatch+=sizeof(size_t);
|
||||
while (pIn < pInLoopLimit) {
|
||||
size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn);
|
||||
if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; }
|
||||
pIn += ZSTD_NbCommonBytes(diff);
|
||||
return (size_t)(pIn - pStart);
|
||||
} }
|
||||
if (MEM_64bits() && (pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; }
|
||||
if ((pIn<(pInLimit-1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn+=2; pMatch+=2; }
|
||||
if ((pIn<pInLimit) && (*pMatch == *pIn)) pIn++;
|
||||
return (size_t)(pIn - pStart);
|
||||
}
|
||||
|
||||
/** ZSTD_count_2segments() :
|
||||
* can count match length with `ip` & `match` in 2 different segments.
|
||||
* convention : on reaching mEnd, match count continue starting from iStart
|
||||
*/
|
||||
MEM_STATIC size_t
|
||||
ZSTD_count_2segments(const BYTE* ip, const BYTE* match,
|
||||
const BYTE* iEnd, const BYTE* mEnd, const BYTE* iStart)
|
||||
{
|
||||
const BYTE* const vEnd = MIN( ip + (mEnd - match), iEnd);
|
||||
size_t const matchLength = ZSTD_count(ip, match, vEnd);
|
||||
if (match + matchLength != mEnd) return matchLength;
|
||||
DEBUGLOG(7, "ZSTD_count_2segments: found a 2-parts match (current length==%zu)", matchLength);
|
||||
DEBUGLOG(7, "distance from match beginning to end dictionary = %zi", mEnd - match);
|
||||
DEBUGLOG(7, "distance from current pos to end buffer = %zi", iEnd - ip);
|
||||
DEBUGLOG(7, "next byte : ip==%02X, istart==%02X", ip[matchLength], *iStart);
|
||||
DEBUGLOG(7, "final match length = %zu", matchLength + ZSTD_count(ip+matchLength, iStart, iEnd));
|
||||
return matchLength + ZSTD_count(ip+matchLength, iStart, iEnd);
|
||||
}
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Hashes
|
||||
***************************************/
|
||||
static const U32 prime3bytes = 506832829U;
|
||||
static U32 ZSTD_hash3(U32 u, U32 h) { return ((u << (32-24)) * prime3bytes) >> (32-h) ; }
|
||||
MEM_STATIC size_t ZSTD_hash3Ptr(const void* ptr, U32 h) { return ZSTD_hash3(MEM_readLE32(ptr), h); } /* only in zstd_opt.h */
|
||||
|
||||
static const U32 prime4bytes = 2654435761U;
|
||||
static U32 ZSTD_hash4(U32 u, U32 h) { return (u * prime4bytes) >> (32-h) ; }
|
||||
static size_t ZSTD_hash4Ptr(const void* ptr, U32 h) { return ZSTD_hash4(MEM_read32(ptr), h); }
|
||||
|
||||
static const U64 prime5bytes = 889523592379ULL;
|
||||
static size_t ZSTD_hash5(U64 u, U32 h) { return (size_t)(((u << (64-40)) * prime5bytes) >> (64-h)) ; }
|
||||
static size_t ZSTD_hash5Ptr(const void* p, U32 h) { return ZSTD_hash5(MEM_readLE64(p), h); }
|
||||
|
||||
static const U64 prime6bytes = 227718039650203ULL;
|
||||
static size_t ZSTD_hash6(U64 u, U32 h) { return (size_t)(((u << (64-48)) * prime6bytes) >> (64-h)) ; }
|
||||
static size_t ZSTD_hash6Ptr(const void* p, U32 h) { return ZSTD_hash6(MEM_readLE64(p), h); }
|
||||
|
||||
static const U64 prime7bytes = 58295818150454627ULL;
|
||||
static size_t ZSTD_hash7(U64 u, U32 h) { return (size_t)(((u << (64-56)) * prime7bytes) >> (64-h)) ; }
|
||||
static size_t ZSTD_hash7Ptr(const void* p, U32 h) { return ZSTD_hash7(MEM_readLE64(p), h); }
|
||||
|
||||
static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL;
|
||||
static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u) * prime8bytes) >> (64-h)) ; }
|
||||
static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h); }
|
||||
|
||||
MEM_STATIC size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls)
|
||||
{
|
||||
switch(mls)
|
||||
{
|
||||
default:
|
||||
case 4: return ZSTD_hash4Ptr(p, hBits);
|
||||
case 5: return ZSTD_hash5Ptr(p, hBits);
|
||||
case 6: return ZSTD_hash6Ptr(p, hBits);
|
||||
case 7: return ZSTD_hash7Ptr(p, hBits);
|
||||
case 8: return ZSTD_hash8Ptr(p, hBits);
|
||||
}
|
||||
}
|
||||
|
||||
/*-*************************************
|
||||
* Round buffer management
|
||||
***************************************/
|
||||
/* Max current allowed */
|
||||
#define ZSTD_CURRENT_MAX ((3U << 29) + (1U << ZSTD_WINDOWLOG_MAX))
|
||||
/* Maximum chunk size before overflow correction needs to be called again */
|
||||
#define ZSTD_CHUNKSIZE_MAX \
|
||||
( ((U32)-1) /* Maximum ending current index */ \
|
||||
- ZSTD_CURRENT_MAX) /* Maximum beginning lowLimit */
|
||||
|
||||
/**
|
||||
* ZSTD_window_clear():
|
||||
* Clears the window containing the history by simply setting it to empty.
|
||||
*/
|
||||
MEM_STATIC void ZSTD_window_clear(ZSTD_window_t* window)
|
||||
{
|
||||
size_t const endT = (size_t)(window->nextSrc - window->base);
|
||||
U32 const end = (U32)endT;
|
||||
|
||||
window->lowLimit = end;
|
||||
window->dictLimit = end;
|
||||
}
|
||||
|
||||
/**
|
||||
* ZSTD_window_hasExtDict():
|
||||
* Returns non-zero if the window has a non-empty extDict.
|
||||
*/
|
||||
MEM_STATIC U32 ZSTD_window_hasExtDict(ZSTD_window_t const window)
|
||||
{
|
||||
return window.lowLimit < window.dictLimit;
|
||||
}
|
||||
|
||||
/**
|
||||
* ZSTD_matchState_dictMode():
|
||||
* Inspects the provided matchState and figures out what dictMode should be
|
||||
* passed to the compressor.
|
||||
*/
|
||||
MEM_STATIC ZSTD_dictMode_e ZSTD_matchState_dictMode(const ZSTD_matchState_t *ms)
|
||||
{
|
||||
return ZSTD_window_hasExtDict(ms->window) ?
|
||||
ZSTD_extDict :
|
||||
ms->dictMatchState != NULL ?
|
||||
ZSTD_dictMatchState :
|
||||
ZSTD_noDict;
|
||||
}
|
||||
|
||||
/**
|
||||
* ZSTD_window_needOverflowCorrection():
|
||||
* Returns non-zero if the indices are getting too large and need overflow
|
||||
* protection.
|
||||
*/
|
||||
MEM_STATIC U32 ZSTD_window_needOverflowCorrection(ZSTD_window_t const window,
|
||||
void const* srcEnd)
|
||||
{
|
||||
U32 const current = (U32)((BYTE const*)srcEnd - window.base);
|
||||
return current > ZSTD_CURRENT_MAX;
|
||||
}
|
||||
|
||||
/**
|
||||
* ZSTD_window_correctOverflow():
|
||||
* Reduces the indices to protect from index overflow.
|
||||
* Returns the correction made to the indices, which must be applied to every
|
||||
* stored index.
|
||||
*
|
||||
* The least significant cycleLog bits of the indices must remain the same,
|
||||
* which may be 0. Every index up to maxDist in the past must be valid.
|
||||
* NOTE: (maxDist & cycleMask) must be zero.
|
||||
*/
|
||||
MEM_STATIC U32 ZSTD_window_correctOverflow(ZSTD_window_t* window, U32 cycleLog,
|
||||
U32 maxDist, void const* src)
|
||||
{
|
||||
/* preemptive overflow correction:
|
||||
* 1. correction is large enough:
|
||||
* lowLimit > (3<<29) ==> current > 3<<29 + 1<<windowLog
|
||||
* 1<<windowLog <= newCurrent < 1<<chainLog + 1<<windowLog
|
||||
*
|
||||
* current - newCurrent
|
||||
* > (3<<29 + 1<<windowLog) - (1<<windowLog + 1<<chainLog)
|
||||
* > (3<<29) - (1<<chainLog)
|
||||
* > (3<<29) - (1<<30) (NOTE: chainLog <= 30)
|
||||
* > 1<<29
|
||||
*
|
||||
* 2. (ip+ZSTD_CHUNKSIZE_MAX - cctx->base) doesn't overflow:
|
||||
* After correction, current is less than (1<<chainLog + 1<<windowLog).
|
||||
* In 64-bit mode we are safe, because we have 64-bit ptrdiff_t.
|
||||
* In 32-bit mode we are safe, because (chainLog <= 29), so
|
||||
* ip+ZSTD_CHUNKSIZE_MAX - cctx->base < 1<<32.
|
||||
* 3. (cctx->lowLimit + 1<<windowLog) < 1<<32:
|
||||
* windowLog <= 31 ==> 3<<29 + 1<<windowLog < 7<<29 < 1<<32.
|
||||
*/
|
||||
U32 const cycleMask = (1U << cycleLog) - 1;
|
||||
U32 const current = (U32)((BYTE const*)src - window->base);
|
||||
U32 const newCurrent = (current & cycleMask) + maxDist;
|
||||
U32 const correction = current - newCurrent;
|
||||
assert((maxDist & cycleMask) == 0);
|
||||
assert(current > newCurrent);
|
||||
/* Loose bound, should be around 1<<29 (see above) */
|
||||
assert(correction > 1<<28);
|
||||
|
||||
window->base += correction;
|
||||
window->dictBase += correction;
|
||||
window->lowLimit -= correction;
|
||||
window->dictLimit -= correction;
|
||||
|
||||
DEBUGLOG(4, "Correction of 0x%x bytes to lowLimit=0x%x", correction,
|
||||
window->lowLimit);
|
||||
return correction;
|
||||
}
|
||||
|
||||
/**
|
||||
* ZSTD_window_enforceMaxDist():
|
||||
* Updates lowLimit so that:
|
||||
* (srcEnd - base) - lowLimit == maxDist + loadedDictEnd
|
||||
*
|
||||
* This allows a simple check that index >= lowLimit to see if index is valid.
|
||||
* This must be called before a block compression call, with srcEnd as the block
|
||||
* source end.
|
||||
*
|
||||
* If loadedDictEndPtr is not NULL, we set it to zero once we update lowLimit.
|
||||
* This is because dictionaries are allowed to be referenced as long as the last
|
||||
* byte of the dictionary is in the window, but once they are out of range,
|
||||
* they cannot be referenced. If loadedDictEndPtr is NULL, we use
|
||||
* loadedDictEnd == 0.
|
||||
*
|
||||
* In normal dict mode, the dict is between lowLimit and dictLimit. In
|
||||
* dictMatchState mode, lowLimit and dictLimit are the same, and the dictionary
|
||||
* is below them. forceWindow and dictMatchState are therefore incompatible.
|
||||
*/
|
||||
MEM_STATIC void ZSTD_window_enforceMaxDist(ZSTD_window_t* window,
|
||||
void const* srcEnd, U32 maxDist,
|
||||
U32* loadedDictEndPtr,
|
||||
const ZSTD_matchState_t** dictMatchStatePtr)
|
||||
{
|
||||
U32 const current = (U32)((BYTE const*)srcEnd - window->base);
|
||||
U32 loadedDictEnd = loadedDictEndPtr != NULL ? *loadedDictEndPtr : 0;
|
||||
DEBUGLOG(5, "ZSTD_window_enforceMaxDist: current=%u, maxDist=%u", current, maxDist);
|
||||
if (current > maxDist + loadedDictEnd) {
|
||||
U32 const newLowLimit = current - maxDist;
|
||||
if (window->lowLimit < newLowLimit) window->lowLimit = newLowLimit;
|
||||
if (window->dictLimit < window->lowLimit) {
|
||||
DEBUGLOG(5, "Update dictLimit to match lowLimit, from %u to %u",
|
||||
window->dictLimit, window->lowLimit);
|
||||
window->dictLimit = window->lowLimit;
|
||||
}
|
||||
if (loadedDictEndPtr)
|
||||
*loadedDictEndPtr = 0;
|
||||
if (dictMatchStatePtr)
|
||||
*dictMatchStatePtr = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* ZSTD_window_update():
|
||||
* Updates the window by appending [src, src + srcSize) to the window.
|
||||
* If it is not contiguous, the current prefix becomes the extDict, and we
|
||||
* forget about the extDict. Handles overlap of the prefix and extDict.
|
||||
* Returns non-zero if the segment is contiguous.
|
||||
*/
|
||||
MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window,
|
||||
void const* src, size_t srcSize)
|
||||
{
|
||||
BYTE const* const ip = (BYTE const*)src;
|
||||
U32 contiguous = 1;
|
||||
DEBUGLOG(5, "ZSTD_window_update");
|
||||
/* Check if blocks follow each other */
|
||||
if (src != window->nextSrc) {
|
||||
/* not contiguous */
|
||||
size_t const distanceFromBase = (size_t)(window->nextSrc - window->base);
|
||||
DEBUGLOG(5, "Non contiguous blocks, new segment starts at %u", window->dictLimit);
|
||||
window->lowLimit = window->dictLimit;
|
||||
assert(distanceFromBase == (size_t)(U32)distanceFromBase); /* should never overflow */
|
||||
window->dictLimit = (U32)distanceFromBase;
|
||||
window->dictBase = window->base;
|
||||
window->base = ip - distanceFromBase;
|
||||
// ms->nextToUpdate = window->dictLimit;
|
||||
if (window->dictLimit - window->lowLimit < HASH_READ_SIZE) window->lowLimit = window->dictLimit; /* too small extDict */
|
||||
contiguous = 0;
|
||||
}
|
||||
window->nextSrc = ip + srcSize;
|
||||
/* if input and dictionary overlap : reduce dictionary (area presumed modified by input) */
|
||||
if ( (ip+srcSize > window->dictBase + window->lowLimit)
|
||||
& (ip < window->dictBase + window->dictLimit)) {
|
||||
ptrdiff_t const highInputIdx = (ip + srcSize) - window->dictBase;
|
||||
U32 const lowLimitMax = (highInputIdx > (ptrdiff_t)window->dictLimit) ? window->dictLimit : (U32)highInputIdx;
|
||||
window->lowLimit = lowLimitMax;
|
||||
DEBUGLOG(5, "Overlapping extDict and input : new lowLimit = %u", window->lowLimit);
|
||||
}
|
||||
return contiguous;
|
||||
}
|
||||
|
||||
|
||||
/* debug functions */
|
||||
|
||||
MEM_STATIC double ZSTD_fWeight(U32 rawStat)
|
||||
{
|
||||
U32 const fp_accuracy = 8;
|
||||
U32 const fp_multiplier = (1 << fp_accuracy);
|
||||
U32 const stat = rawStat + 1;
|
||||
U32 const hb = ZSTD_highbit32(stat);
|
||||
U32 const BWeight = hb * fp_multiplier;
|
||||
U32 const FWeight = (stat << fp_accuracy) >> hb;
|
||||
U32 const weight = BWeight + FWeight;
|
||||
assert(hb + fp_accuracy < 31);
|
||||
return (double)weight / fp_multiplier;
|
||||
}
|
||||
|
||||
MEM_STATIC void ZSTD_debugTable(const U32* table, U32 max)
|
||||
{
|
||||
unsigned u, sum;
|
||||
for (u=0, sum=0; u<=max; u++) sum += table[u];
|
||||
DEBUGLOG(2, "total nb elts: %u", sum);
|
||||
for (u=0; u<=max; u++) {
|
||||
DEBUGLOG(2, "%2u: %5u (%.2f)",
|
||||
u, table[u], ZSTD_fWeight(sum) - ZSTD_fWeight(table[u]) );
|
||||
}
|
||||
}
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
/* ==============================================================
|
||||
* Private declarations
|
||||
* These prototypes shall only be called from within lib/compress
|
||||
* ============================================================== */
|
||||
|
||||
/* ZSTD_getCParamsFromCCtxParams() :
|
||||
* cParams are built depending on compressionLevel, src size hints,
|
||||
* LDM and manually set compression parameters.
|
||||
*/
|
||||
ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams(
|
||||
const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize);
|
||||
|
||||
/*! ZSTD_initCStream_internal() :
|
||||
* Private use only. Init streaming operation.
|
||||
* expects params to be valid.
|
||||
* must receive dict, or cdict, or none, but not both.
|
||||
* @return : 0, or an error code */
|
||||
size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
|
||||
const void* dict, size_t dictSize,
|
||||
const ZSTD_CDict* cdict,
|
||||
ZSTD_CCtx_params params, unsigned long long pledgedSrcSize);
|
||||
|
||||
void ZSTD_resetSeqStore(seqStore_t* ssPtr);
|
||||
|
||||
/*! ZSTD_compressStream_generic() :
|
||||
* Private use only. To be called from zstdmt_compress.c in single-thread mode. */
|
||||
size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
|
||||
ZSTD_outBuffer* output,
|
||||
ZSTD_inBuffer* input,
|
||||
ZSTD_EndDirective const flushMode);
|
||||
|
||||
/*! ZSTD_getCParamsFromCDict() :
|
||||
* as the name implies */
|
||||
ZSTD_compressionParameters ZSTD_getCParamsFromCDict(const ZSTD_CDict* cdict);
|
||||
|
||||
/* ZSTD_compressBegin_advanced_internal() :
|
||||
* Private use only. To be called from zstdmt_compress.c. */
|
||||
size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx,
|
||||
const void* dict, size_t dictSize,
|
||||
ZSTD_dictContentType_e dictContentType,
|
||||
ZSTD_dictTableLoadMethod_e dtlm,
|
||||
const ZSTD_CDict* cdict,
|
||||
ZSTD_CCtx_params params,
|
||||
unsigned long long pledgedSrcSize);
|
||||
|
||||
/* ZSTD_compress_advanced_internal() :
|
||||
* Private use only. To be called from zstdmt_compress.c. */
|
||||
size_t ZSTD_compress_advanced_internal(ZSTD_CCtx* cctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
const void* dict,size_t dictSize,
|
||||
ZSTD_CCtx_params params);
|
||||
|
||||
|
||||
/* ZSTD_writeLastEmptyBlock() :
|
||||
* output an empty Block with end-of-frame mark to complete a frame
|
||||
* @return : size of data written into `dst` (== ZSTD_blockHeaderSize (defined in zstd_internal.h))
|
||||
* or an error code if `dstCapcity` is too small (<ZSTD_blockHeaderSize)
|
||||
*/
|
||||
size_t ZSTD_writeLastEmptyBlock(void* dst, size_t dstCapacity);
|
||||
|
||||
|
||||
/* ZSTD_referenceExternalSequences() :
|
||||
* Must be called before starting a compression operation.
|
||||
* seqs must parse a prefix of the source.
|
||||
* This cannot be used when long range matching is enabled.
|
||||
* Zstd will use these sequences, and pass the literals to a secondary block
|
||||
* compressor.
|
||||
* @return : An error code on failure.
|
||||
* NOTE: seqs are not verified! Invalid sequences can cause out-of-bounds memory
|
||||
* access and data corruption.
|
||||
*/
|
||||
size_t ZSTD_referenceExternalSequences(ZSTD_CCtx* cctx, rawSeq* seq, size_t nbSeq);
|
||||
|
||||
|
||||
#endif /* ZSTD_COMPRESS_H */
|
||||
|
|
@ -0,0 +1,487 @@
|
|||
/*
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#include "zstd_compress_internal.h"
|
||||
#include "zstd_double_fast.h"
|
||||
|
||||
|
||||
void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms,
|
||||
ZSTD_compressionParameters const* cParams,
|
||||
void const* end, ZSTD_dictTableLoadMethod_e dtlm)
|
||||
{
|
||||
U32* const hashLarge = ms->hashTable;
|
||||
U32 const hBitsL = cParams->hashLog;
|
||||
U32 const mls = cParams->searchLength;
|
||||
U32* const hashSmall = ms->chainTable;
|
||||
U32 const hBitsS = cParams->chainLog;
|
||||
const BYTE* const base = ms->window.base;
|
||||
const BYTE* ip = base + ms->nextToUpdate;
|
||||
const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE;
|
||||
const U32 fastHashFillStep = 3;
|
||||
|
||||
/* Always insert every fastHashFillStep position into the hash tables.
|
||||
* Insert the other positions into the large hash table if their entry
|
||||
* is empty.
|
||||
*/
|
||||
for (; ip + fastHashFillStep - 1 <= iend; ip += fastHashFillStep) {
|
||||
U32 const current = (U32)(ip - base);
|
||||
U32 i;
|
||||
for (i = 0; i < fastHashFillStep; ++i) {
|
||||
size_t const smHash = ZSTD_hashPtr(ip + i, hBitsS, mls);
|
||||
size_t const lgHash = ZSTD_hashPtr(ip + i, hBitsL, 8);
|
||||
if (i == 0)
|
||||
hashSmall[smHash] = current + i;
|
||||
if (i == 0 || hashLarge[lgHash] == 0)
|
||||
hashLarge[lgHash] = current + i;
|
||||
/* Only load extra positions for ZSTD_dtlm_full */
|
||||
if (dtlm == ZSTD_dtlm_fast)
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
FORCE_INLINE_TEMPLATE
|
||||
size_t ZSTD_compressBlock_doubleFast_generic(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize,
|
||||
U32 const mls /* template */, ZSTD_dictMode_e const dictMode)
|
||||
{
|
||||
U32* const hashLong = ms->hashTable;
|
||||
const U32 hBitsL = cParams->hashLog;
|
||||
U32* const hashSmall = ms->chainTable;
|
||||
const U32 hBitsS = cParams->chainLog;
|
||||
const BYTE* const base = ms->window.base;
|
||||
const BYTE* const istart = (const BYTE*)src;
|
||||
const BYTE* ip = istart;
|
||||
const BYTE* anchor = istart;
|
||||
const U32 prefixLowestIndex = ms->window.dictLimit;
|
||||
const BYTE* const prefixLowest = base + prefixLowestIndex;
|
||||
const BYTE* const iend = istart + srcSize;
|
||||
const BYTE* const ilimit = iend - HASH_READ_SIZE;
|
||||
U32 offset_1=rep[0], offset_2=rep[1];
|
||||
U32 offsetSaved = 0;
|
||||
|
||||
const ZSTD_matchState_t* const dms = ms->dictMatchState;
|
||||
const U32* const dictHashLong = dictMode == ZSTD_dictMatchState ?
|
||||
dms->hashTable : NULL;
|
||||
const U32* const dictHashSmall = dictMode == ZSTD_dictMatchState ?
|
||||
dms->chainTable : NULL;
|
||||
const U32 dictStartIndex = dictMode == ZSTD_dictMatchState ?
|
||||
dms->window.dictLimit : 0;
|
||||
const BYTE* const dictBase = dictMode == ZSTD_dictMatchState ?
|
||||
dms->window.base : NULL;
|
||||
const BYTE* const dictStart = dictMode == ZSTD_dictMatchState ?
|
||||
dictBase + dictStartIndex : NULL;
|
||||
const BYTE* const dictEnd = dictMode == ZSTD_dictMatchState ?
|
||||
dms->window.nextSrc : NULL;
|
||||
const U32 dictIndexDelta = dictMode == ZSTD_dictMatchState ?
|
||||
prefixLowestIndex - (U32)(dictEnd - dictBase) :
|
||||
0;
|
||||
const U32 dictAndPrefixLength = (U32)(ip - prefixLowest + dictEnd - dictStart);
|
||||
|
||||
assert(dictMode == ZSTD_noDict || dictMode == ZSTD_dictMatchState);
|
||||
|
||||
/* init */
|
||||
ip += (dictAndPrefixLength == 0);
|
||||
if (dictMode == ZSTD_noDict) {
|
||||
U32 const maxRep = (U32)(ip - prefixLowest);
|
||||
if (offset_2 > maxRep) offsetSaved = offset_2, offset_2 = 0;
|
||||
if (offset_1 > maxRep) offsetSaved = offset_1, offset_1 = 0;
|
||||
}
|
||||
if (dictMode == ZSTD_dictMatchState) {
|
||||
/* dictMatchState repCode checks don't currently handle repCode == 0
|
||||
* disabling. */
|
||||
assert(offset_1 <= dictAndPrefixLength);
|
||||
assert(offset_2 <= dictAndPrefixLength);
|
||||
}
|
||||
|
||||
/* Main Search Loop */
|
||||
while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */
|
||||
size_t mLength;
|
||||
U32 offset;
|
||||
size_t const h2 = ZSTD_hashPtr(ip, hBitsL, 8);
|
||||
size_t const h = ZSTD_hashPtr(ip, hBitsS, mls);
|
||||
U32 const current = (U32)(ip-base);
|
||||
U32 const matchIndexL = hashLong[h2];
|
||||
U32 matchIndexS = hashSmall[h];
|
||||
const BYTE* matchLong = base + matchIndexL;
|
||||
const BYTE* match = base + matchIndexS;
|
||||
const U32 repIndex = current + 1 - offset_1;
|
||||
const BYTE* repMatch = (dictMode == ZSTD_dictMatchState
|
||||
&& repIndex < prefixLowestIndex) ?
|
||||
dictBase + (repIndex - dictIndexDelta) :
|
||||
base + repIndex;
|
||||
hashLong[h2] = hashSmall[h] = current; /* update hash tables */
|
||||
|
||||
/* check dictMatchState repcode */
|
||||
if (dictMode == ZSTD_dictMatchState
|
||||
&& ((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */)
|
||||
&& (MEM_read32(repMatch) == MEM_read32(ip+1)) ) {
|
||||
const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend;
|
||||
mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4;
|
||||
ip++;
|
||||
ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH);
|
||||
goto _match_stored;
|
||||
}
|
||||
|
||||
/* check noDict repcode */
|
||||
if ( dictMode == ZSTD_noDict
|
||||
&& ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1)))) {
|
||||
mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4;
|
||||
ip++;
|
||||
ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH);
|
||||
goto _match_stored;
|
||||
}
|
||||
|
||||
/* check prefix long match */
|
||||
if ( (matchIndexL > prefixLowestIndex) && (MEM_read64(matchLong) == MEM_read64(ip)) ) {
|
||||
mLength = ZSTD_count(ip+8, matchLong+8, iend) + 8;
|
||||
offset = (U32)(ip-matchLong);
|
||||
while (((ip>anchor) & (matchLong>prefixLowest)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */
|
||||
goto _match_found;
|
||||
}
|
||||
|
||||
/* check dictMatchState long match */
|
||||
if (dictMode == ZSTD_dictMatchState) {
|
||||
U32 const dictMatchIndexL = dictHashLong[h2];
|
||||
const BYTE* dictMatchL = dictBase + dictMatchIndexL;
|
||||
assert(dictMatchL < dictEnd);
|
||||
|
||||
if (dictMatchL > dictStart && MEM_read64(dictMatchL) == MEM_read64(ip)) {
|
||||
mLength = ZSTD_count_2segments(ip+8, dictMatchL+8, iend, dictEnd, prefixLowest) + 8;
|
||||
offset = (U32)(current - dictMatchIndexL - dictIndexDelta);
|
||||
while (((ip>anchor) & (dictMatchL>dictStart)) && (ip[-1] == dictMatchL[-1])) { ip--; dictMatchL--; mLength++; } /* catch up */
|
||||
goto _match_found;
|
||||
}
|
||||
}
|
||||
|
||||
/* check prefix short match */
|
||||
if ( (matchIndexS > prefixLowestIndex) && (MEM_read32(match) == MEM_read32(ip)) ) {
|
||||
goto _search_next_long;
|
||||
}
|
||||
|
||||
/* check dictMatchState short match */
|
||||
if (dictMode == ZSTD_dictMatchState) {
|
||||
U32 const dictMatchIndexS = dictHashSmall[h];
|
||||
match = dictBase + dictMatchIndexS;
|
||||
matchIndexS = dictMatchIndexS + dictIndexDelta;
|
||||
|
||||
if (match > dictStart && MEM_read32(match) == MEM_read32(ip)) {
|
||||
goto _search_next_long;
|
||||
}
|
||||
}
|
||||
|
||||
ip += ((ip-anchor) >> kSearchStrength) + 1;
|
||||
continue;
|
||||
|
||||
_search_next_long:
|
||||
|
||||
{
|
||||
size_t const hl3 = ZSTD_hashPtr(ip+1, hBitsL, 8);
|
||||
U32 const matchIndexL3 = hashLong[hl3];
|
||||
const BYTE* matchL3 = base + matchIndexL3;
|
||||
hashLong[hl3] = current + 1;
|
||||
|
||||
/* check prefix long +1 match */
|
||||
if ( (matchIndexL3 > prefixLowestIndex) && (MEM_read64(matchL3) == MEM_read64(ip+1)) ) {
|
||||
mLength = ZSTD_count(ip+9, matchL3+8, iend) + 8;
|
||||
ip++;
|
||||
offset = (U32)(ip-matchL3);
|
||||
while (((ip>anchor) & (matchL3>prefixLowest)) && (ip[-1] == matchL3[-1])) { ip--; matchL3--; mLength++; } /* catch up */
|
||||
goto _match_found;
|
||||
}
|
||||
|
||||
/* check dict long +1 match */
|
||||
if (dictMode == ZSTD_dictMatchState) {
|
||||
U32 const dictMatchIndexL3 = dictHashLong[hl3];
|
||||
const BYTE* dictMatchL3 = dictBase + dictMatchIndexL3;
|
||||
assert(dictMatchL3 < dictEnd);
|
||||
if (dictMatchL3 > dictStart && MEM_read64(dictMatchL3) == MEM_read64(ip+1)) {
|
||||
mLength = ZSTD_count_2segments(ip+1+8, dictMatchL3+8, iend, dictEnd, prefixLowest) + 8;
|
||||
ip++;
|
||||
offset = (U32)(current + 1 - dictMatchIndexL3 - dictIndexDelta);
|
||||
while (((ip>anchor) & (dictMatchL3>dictStart)) && (ip[-1] == dictMatchL3[-1])) { ip--; dictMatchL3--; mLength++; } /* catch up */
|
||||
goto _match_found;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* if no long +1 match, explore the short match we found */
|
||||
if (dictMode == ZSTD_dictMatchState && matchIndexS < prefixLowestIndex) {
|
||||
mLength = ZSTD_count_2segments(ip+4, match+4, iend, dictEnd, prefixLowest) + 4;
|
||||
offset = (U32)(current - matchIndexS);
|
||||
while (((ip>anchor) & (match>dictStart)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */
|
||||
} else {
|
||||
mLength = ZSTD_count(ip+4, match+4, iend) + 4;
|
||||
offset = (U32)(ip - match);
|
||||
while (((ip>anchor) & (match>prefixLowest)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */
|
||||
}
|
||||
|
||||
/* fall-through */
|
||||
|
||||
_match_found:
|
||||
offset_2 = offset_1;
|
||||
offset_1 = offset;
|
||||
|
||||
ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
|
||||
|
||||
_match_stored:
|
||||
/* match found */
|
||||
ip += mLength;
|
||||
anchor = ip;
|
||||
|
||||
if (ip <= ilimit) {
|
||||
/* Fill Table */
|
||||
hashLong[ZSTD_hashPtr(base+current+2, hBitsL, 8)] =
|
||||
hashSmall[ZSTD_hashPtr(base+current+2, hBitsS, mls)] = current+2; /* here because current+2 could be > iend-8 */
|
||||
hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] =
|
||||
hashSmall[ZSTD_hashPtr(ip-2, hBitsS, mls)] = (U32)(ip-2-base);
|
||||
|
||||
/* check immediate repcode */
|
||||
if (dictMode == ZSTD_dictMatchState) {
|
||||
while (ip <= ilimit) {
|
||||
U32 const current2 = (U32)(ip-base);
|
||||
U32 const repIndex2 = current2 - offset_2;
|
||||
const BYTE* repMatch2 = dictMode == ZSTD_dictMatchState
|
||||
&& repIndex2 < prefixLowestIndex ?
|
||||
dictBase - dictIndexDelta + repIndex2 :
|
||||
base + repIndex2;
|
||||
if ( ((U32)((prefixLowestIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */)
|
||||
&& (MEM_read32(repMatch2) == MEM_read32(ip)) ) {
|
||||
const BYTE* const repEnd2 = repIndex2 < prefixLowestIndex ? dictEnd : iend;
|
||||
size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixLowest) + 4;
|
||||
U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
|
||||
ZSTD_storeSeq(seqStore, 0, anchor, 0, repLength2-MINMATCH);
|
||||
hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2;
|
||||
hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2;
|
||||
ip += repLength2;
|
||||
anchor = ip;
|
||||
continue;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (dictMode == ZSTD_noDict) {
|
||||
while ( (ip <= ilimit)
|
||||
&& ( (offset_2>0)
|
||||
& (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) {
|
||||
/* store sequence */
|
||||
size_t const rLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4;
|
||||
U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; /* swap offset_2 <=> offset_1 */
|
||||
hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip-base);
|
||||
hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip-base);
|
||||
ZSTD_storeSeq(seqStore, 0, anchor, 0, rLength-MINMATCH);
|
||||
ip += rLength;
|
||||
anchor = ip;
|
||||
continue; /* faster when present ... (?) */
|
||||
} } } }
|
||||
|
||||
/* save reps for next block */
|
||||
rep[0] = offset_1 ? offset_1 : offsetSaved;
|
||||
rep[1] = offset_2 ? offset_2 : offsetSaved;
|
||||
|
||||
/* Return the last literals size */
|
||||
return iend - anchor;
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTD_compressBlock_doubleFast(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize)
|
||||
{
|
||||
const U32 mls = cParams->searchLength;
|
||||
switch(mls)
|
||||
{
|
||||
default: /* includes case 3 */
|
||||
case 4 :
|
||||
return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, cParams, src, srcSize, 4, ZSTD_noDict);
|
||||
case 5 :
|
||||
return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, cParams, src, srcSize, 5, ZSTD_noDict);
|
||||
case 6 :
|
||||
return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, cParams, src, srcSize, 6, ZSTD_noDict);
|
||||
case 7 :
|
||||
return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, cParams, src, srcSize, 7, ZSTD_noDict);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTD_compressBlock_doubleFast_dictMatchState(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize)
|
||||
{
|
||||
const U32 mls = cParams->searchLength;
|
||||
switch(mls)
|
||||
{
|
||||
default: /* includes case 3 */
|
||||
case 4 :
|
||||
return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, cParams, src, srcSize, 4, ZSTD_dictMatchState);
|
||||
case 5 :
|
||||
return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, cParams, src, srcSize, 5, ZSTD_dictMatchState);
|
||||
case 6 :
|
||||
return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, cParams, src, srcSize, 6, ZSTD_dictMatchState);
|
||||
case 7 :
|
||||
return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, cParams, src, srcSize, 7, ZSTD_dictMatchState);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static size_t ZSTD_compressBlock_doubleFast_extDict_generic(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize,
|
||||
U32 const mls /* template */)
|
||||
{
|
||||
U32* const hashLong = ms->hashTable;
|
||||
U32 const hBitsL = cParams->hashLog;
|
||||
U32* const hashSmall = ms->chainTable;
|
||||
U32 const hBitsS = cParams->chainLog;
|
||||
const BYTE* const istart = (const BYTE*)src;
|
||||
const BYTE* ip = istart;
|
||||
const BYTE* anchor = istart;
|
||||
const BYTE* const iend = istart + srcSize;
|
||||
const BYTE* const ilimit = iend - 8;
|
||||
const U32 prefixStartIndex = ms->window.dictLimit;
|
||||
const BYTE* const base = ms->window.base;
|
||||
const BYTE* const prefixStart = base + prefixStartIndex;
|
||||
const U32 dictStartIndex = ms->window.lowLimit;
|
||||
const BYTE* const dictBase = ms->window.dictBase;
|
||||
const BYTE* const dictStart = dictBase + dictStartIndex;
|
||||
const BYTE* const dictEnd = dictBase + prefixStartIndex;
|
||||
U32 offset_1=rep[0], offset_2=rep[1];
|
||||
|
||||
DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_extDict_generic (srcSize=%zu)", srcSize);
|
||||
|
||||
/* Search Loop */
|
||||
while (ip < ilimit) { /* < instead of <=, because (ip+1) */
|
||||
const size_t hSmall = ZSTD_hashPtr(ip, hBitsS, mls);
|
||||
const U32 matchIndex = hashSmall[hSmall];
|
||||
const BYTE* const matchBase = matchIndex < prefixStartIndex ? dictBase : base;
|
||||
const BYTE* match = matchBase + matchIndex;
|
||||
|
||||
const size_t hLong = ZSTD_hashPtr(ip, hBitsL, 8);
|
||||
const U32 matchLongIndex = hashLong[hLong];
|
||||
const BYTE* const matchLongBase = matchLongIndex < prefixStartIndex ? dictBase : base;
|
||||
const BYTE* matchLong = matchLongBase + matchLongIndex;
|
||||
|
||||
const U32 current = (U32)(ip-base);
|
||||
const U32 repIndex = current + 1 - offset_1; /* offset_1 expected <= current +1 */
|
||||
const BYTE* const repBase = repIndex < prefixStartIndex ? dictBase : base;
|
||||
const BYTE* const repMatch = repBase + repIndex;
|
||||
size_t mLength;
|
||||
hashSmall[hSmall] = hashLong[hLong] = current; /* update hash table */
|
||||
|
||||
if ((((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow : ensure repIndex doesn't overlap dict + prefix */
|
||||
& (repIndex > dictStartIndex))
|
||||
&& (MEM_read32(repMatch) == MEM_read32(ip+1)) ) {
|
||||
const BYTE* repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend;
|
||||
mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4;
|
||||
ip++;
|
||||
ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH);
|
||||
} else {
|
||||
if ((matchLongIndex > dictStartIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) {
|
||||
const BYTE* const matchEnd = matchLongIndex < prefixStartIndex ? dictEnd : iend;
|
||||
const BYTE* const lowMatchPtr = matchLongIndex < prefixStartIndex ? dictStart : prefixStart;
|
||||
U32 offset;
|
||||
mLength = ZSTD_count_2segments(ip+8, matchLong+8, iend, matchEnd, prefixStart) + 8;
|
||||
offset = current - matchLongIndex;
|
||||
while (((ip>anchor) & (matchLong>lowMatchPtr)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */
|
||||
offset_2 = offset_1;
|
||||
offset_1 = offset;
|
||||
ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
|
||||
|
||||
} else if ((matchIndex > dictStartIndex) && (MEM_read32(match) == MEM_read32(ip))) {
|
||||
size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8);
|
||||
U32 const matchIndex3 = hashLong[h3];
|
||||
const BYTE* const match3Base = matchIndex3 < prefixStartIndex ? dictBase : base;
|
||||
const BYTE* match3 = match3Base + matchIndex3;
|
||||
U32 offset;
|
||||
hashLong[h3] = current + 1;
|
||||
if ( (matchIndex3 > dictStartIndex) && (MEM_read64(match3) == MEM_read64(ip+1)) ) {
|
||||
const BYTE* const matchEnd = matchIndex3 < prefixStartIndex ? dictEnd : iend;
|
||||
const BYTE* const lowMatchPtr = matchIndex3 < prefixStartIndex ? dictStart : prefixStart;
|
||||
mLength = ZSTD_count_2segments(ip+9, match3+8, iend, matchEnd, prefixStart) + 8;
|
||||
ip++;
|
||||
offset = current+1 - matchIndex3;
|
||||
while (((ip>anchor) & (match3>lowMatchPtr)) && (ip[-1] == match3[-1])) { ip--; match3--; mLength++; } /* catch up */
|
||||
} else {
|
||||
const BYTE* const matchEnd = matchIndex < prefixStartIndex ? dictEnd : iend;
|
||||
const BYTE* const lowMatchPtr = matchIndex < prefixStartIndex ? dictStart : prefixStart;
|
||||
mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, prefixStart) + 4;
|
||||
offset = current - matchIndex;
|
||||
while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */
|
||||
}
|
||||
offset_2 = offset_1;
|
||||
offset_1 = offset;
|
||||
ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
|
||||
|
||||
} else {
|
||||
ip += ((ip-anchor) >> kSearchStrength) + 1;
|
||||
continue;
|
||||
} }
|
||||
|
||||
/* found a match : store it */
|
||||
ip += mLength;
|
||||
anchor = ip;
|
||||
|
||||
if (ip <= ilimit) {
|
||||
/* Fill Table */
|
||||
hashSmall[ZSTD_hashPtr(base+current+2, hBitsS, mls)] = current+2;
|
||||
hashLong[ZSTD_hashPtr(base+current+2, hBitsL, 8)] = current+2;
|
||||
hashSmall[ZSTD_hashPtr(ip-2, hBitsS, mls)] = (U32)(ip-2-base);
|
||||
hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base);
|
||||
/* check immediate repcode */
|
||||
while (ip <= ilimit) {
|
||||
U32 const current2 = (U32)(ip-base);
|
||||
U32 const repIndex2 = current2 - offset_2;
|
||||
const BYTE* repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2;
|
||||
if ( (((U32)((prefixStartIndex-1) - repIndex2) >= 3) /* intentional overflow : ensure repIndex2 doesn't overlap dict + prefix */
|
||||
& (repIndex2 > dictStartIndex))
|
||||
&& (MEM_read32(repMatch2) == MEM_read32(ip)) ) {
|
||||
const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend;
|
||||
size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4;
|
||||
U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
|
||||
ZSTD_storeSeq(seqStore, 0, anchor, 0, repLength2-MINMATCH);
|
||||
hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2;
|
||||
hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2;
|
||||
ip += repLength2;
|
||||
anchor = ip;
|
||||
continue;
|
||||
}
|
||||
break;
|
||||
} } }
|
||||
|
||||
/* save reps for next block */
|
||||
rep[0] = offset_1;
|
||||
rep[1] = offset_2;
|
||||
|
||||
/* Return the last literals size */
|
||||
return iend - anchor;
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTD_compressBlock_doubleFast_extDict(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize)
|
||||
{
|
||||
U32 const mls = cParams->searchLength;
|
||||
switch(mls)
|
||||
{
|
||||
default: /* includes case 3 */
|
||||
case 4 :
|
||||
return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, cParams, src, srcSize, 4);
|
||||
case 5 :
|
||||
return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, cParams, src, srcSize, 5);
|
||||
case 6 :
|
||||
return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, cParams, src, srcSize, 6);
|
||||
case 7 :
|
||||
return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, cParams, src, srcSize, 7);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
/*
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#ifndef ZSTD_DOUBLE_FAST_H
|
||||
#define ZSTD_DOUBLE_FAST_H
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include "mem.h" /* U32 */
|
||||
#include "zstd_compress_internal.h" /* ZSTD_CCtx, size_t */
|
||||
|
||||
void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms,
|
||||
ZSTD_compressionParameters const* cParams,
|
||||
void const* end, ZSTD_dictTableLoadMethod_e dtlm);
|
||||
size_t ZSTD_compressBlock_doubleFast(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_doubleFast_dictMatchState(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_doubleFast_extDict(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ZSTD_DOUBLE_FAST_H */
|
||||
|
|
@ -0,0 +1,380 @@
|
|||
/*
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#include "zstd_compress_internal.h"
|
||||
#include "zstd_fast.h"
|
||||
|
||||
|
||||
void ZSTD_fillHashTable(ZSTD_matchState_t* ms,
|
||||
ZSTD_compressionParameters const* cParams,
|
||||
void const* end, ZSTD_dictTableLoadMethod_e dtlm)
|
||||
{
|
||||
U32* const hashTable = ms->hashTable;
|
||||
U32 const hBits = cParams->hashLog;
|
||||
U32 const mls = cParams->searchLength;
|
||||
const BYTE* const base = ms->window.base;
|
||||
const BYTE* ip = base + ms->nextToUpdate;
|
||||
const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE;
|
||||
const U32 fastHashFillStep = 3;
|
||||
|
||||
/* Always insert every fastHashFillStep position into the hash table.
|
||||
* Insert the other positions if their hash entry is empty.
|
||||
*/
|
||||
for (; ip + fastHashFillStep - 1 <= iend; ip += fastHashFillStep) {
|
||||
U32 const current = (U32)(ip - base);
|
||||
U32 i;
|
||||
for (i = 0; i < fastHashFillStep; ++i) {
|
||||
size_t const hash = ZSTD_hashPtr(ip + i, hBits, mls);
|
||||
if (i == 0 || hashTable[hash] == 0)
|
||||
hashTable[hash] = current + i;
|
||||
/* Only load extra positions for ZSTD_dtlm_full */
|
||||
if (dtlm == ZSTD_dtlm_fast)
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
FORCE_INLINE_TEMPLATE
|
||||
size_t ZSTD_compressBlock_fast_generic(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize,
|
||||
U32 const hlog, U32 stepSize, U32 const mls,
|
||||
ZSTD_dictMode_e const dictMode)
|
||||
{
|
||||
U32* const hashTable = ms->hashTable;
|
||||
const BYTE* const base = ms->window.base;
|
||||
const BYTE* const istart = (const BYTE*)src;
|
||||
const BYTE* ip = istart;
|
||||
const BYTE* anchor = istart;
|
||||
const U32 prefixStartIndex = ms->window.dictLimit;
|
||||
const BYTE* const prefixStart = base + prefixStartIndex;
|
||||
const BYTE* const iend = istart + srcSize;
|
||||
const BYTE* const ilimit = iend - HASH_READ_SIZE;
|
||||
U32 offset_1=rep[0], offset_2=rep[1];
|
||||
U32 offsetSaved = 0;
|
||||
|
||||
const ZSTD_matchState_t* const dms = ms->dictMatchState;
|
||||
const U32* const dictHashTable = dictMode == ZSTD_dictMatchState ?
|
||||
dms->hashTable : NULL;
|
||||
const U32 dictStartIndex = dictMode == ZSTD_dictMatchState ?
|
||||
dms->window.dictLimit : 0;
|
||||
const BYTE* const dictBase = dictMode == ZSTD_dictMatchState ?
|
||||
dms->window.base : NULL;
|
||||
const BYTE* const dictStart = dictMode == ZSTD_dictMatchState ?
|
||||
dictBase + dictStartIndex : NULL;
|
||||
const BYTE* const dictEnd = dictMode == ZSTD_dictMatchState ?
|
||||
dms->window.nextSrc : NULL;
|
||||
const U32 dictIndexDelta = dictMode == ZSTD_dictMatchState ?
|
||||
prefixStartIndex - (U32)(dictEnd - dictBase) :
|
||||
0;
|
||||
const U32 dictAndPrefixLength = (U32)(ip - prefixStart + dictEnd - dictStart);
|
||||
|
||||
assert(dictMode == ZSTD_noDict || dictMode == ZSTD_dictMatchState);
|
||||
|
||||
/* otherwise, we would get index underflow when translating a dict index
|
||||
* into a local index */
|
||||
assert(dictMode != ZSTD_dictMatchState
|
||||
|| prefixStartIndex >= (U32)(dictEnd - dictBase));
|
||||
|
||||
/* init */
|
||||
stepSize += !stepSize; /* support stepSize of 0 */
|
||||
ip += (dictAndPrefixLength == 0);
|
||||
if (dictMode == ZSTD_noDict) {
|
||||
U32 const maxRep = (U32)(ip - prefixStart);
|
||||
if (offset_2 > maxRep) offsetSaved = offset_2, offset_2 = 0;
|
||||
if (offset_1 > maxRep) offsetSaved = offset_1, offset_1 = 0;
|
||||
}
|
||||
if (dictMode == ZSTD_dictMatchState) {
|
||||
/* dictMatchState repCode checks don't currently handle repCode == 0
|
||||
* disabling. */
|
||||
assert(offset_1 <= dictAndPrefixLength);
|
||||
assert(offset_2 <= dictAndPrefixLength);
|
||||
}
|
||||
|
||||
/* Main Search Loop */
|
||||
while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */
|
||||
size_t mLength;
|
||||
size_t const h = ZSTD_hashPtr(ip, hlog, mls);
|
||||
U32 const current = (U32)(ip-base);
|
||||
U32 const matchIndex = hashTable[h];
|
||||
const BYTE* match = base + matchIndex;
|
||||
const U32 repIndex = current + 1 - offset_1;
|
||||
const BYTE* repMatch = (dictMode == ZSTD_dictMatchState
|
||||
&& repIndex < prefixStartIndex) ?
|
||||
dictBase + (repIndex - dictIndexDelta) :
|
||||
base + repIndex;
|
||||
hashTable[h] = current; /* update hash table */
|
||||
|
||||
if ( (dictMode == ZSTD_dictMatchState)
|
||||
&& ((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow : ensure repIndex isn't overlapping dict + prefix */
|
||||
&& (MEM_read32(repMatch) == MEM_read32(ip+1)) ) {
|
||||
const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend;
|
||||
mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4;
|
||||
ip++;
|
||||
ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH);
|
||||
} else if ( dictMode == ZSTD_noDict
|
||||
&& ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1)))) {
|
||||
mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4;
|
||||
ip++;
|
||||
ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH);
|
||||
} else if ( (matchIndex <= prefixStartIndex)
|
||||
|| (MEM_read32(match) != MEM_read32(ip)) ) {
|
||||
if (dictMode == ZSTD_dictMatchState) {
|
||||
U32 const dictMatchIndex = dictHashTable[h];
|
||||
const BYTE* dictMatch = dictBase + dictMatchIndex;
|
||||
if (dictMatchIndex <= dictStartIndex ||
|
||||
MEM_read32(dictMatch) != MEM_read32(ip)) {
|
||||
assert(stepSize >= 1);
|
||||
ip += ((ip-anchor) >> kSearchStrength) + stepSize;
|
||||
continue;
|
||||
} else {
|
||||
/* found a dict match */
|
||||
U32 const offset = (U32)(current-dictMatchIndex-dictIndexDelta);
|
||||
mLength = ZSTD_count_2segments(ip+4, dictMatch+4, iend, dictEnd, prefixStart) + 4;
|
||||
while (((ip>anchor) & (dictMatch>dictStart))
|
||||
&& (ip[-1] == dictMatch[-1])) {
|
||||
ip--; dictMatch--; mLength++;
|
||||
} /* catch up */
|
||||
offset_2 = offset_1;
|
||||
offset_1 = offset;
|
||||
ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
|
||||
}
|
||||
} else {
|
||||
assert(stepSize >= 1);
|
||||
ip += ((ip-anchor) >> kSearchStrength) + stepSize;
|
||||
continue;
|
||||
}
|
||||
} else {
|
||||
/* found a regular match */
|
||||
U32 const offset = (U32)(ip-match);
|
||||
mLength = ZSTD_count(ip+4, match+4, iend) + 4;
|
||||
while (((ip>anchor) & (match>prefixStart))
|
||||
&& (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */
|
||||
offset_2 = offset_1;
|
||||
offset_1 = offset;
|
||||
ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
|
||||
}
|
||||
|
||||
/* match found */
|
||||
ip += mLength;
|
||||
anchor = ip;
|
||||
|
||||
if (ip <= ilimit) {
|
||||
/* Fill Table */
|
||||
hashTable[ZSTD_hashPtr(base+current+2, hlog, mls)] = current+2; /* here because current+2 could be > iend-8 */
|
||||
hashTable[ZSTD_hashPtr(ip-2, hlog, mls)] = (U32)(ip-2-base);
|
||||
|
||||
/* check immediate repcode */
|
||||
if (dictMode == ZSTD_dictMatchState) {
|
||||
while (ip <= ilimit) {
|
||||
U32 const current2 = (U32)(ip-base);
|
||||
U32 const repIndex2 = current2 - offset_2;
|
||||
const BYTE* repMatch2 = repIndex2 < prefixStartIndex ?
|
||||
dictBase - dictIndexDelta + repIndex2 :
|
||||
base + repIndex2;
|
||||
if ( ((U32)((prefixStartIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */)
|
||||
&& (MEM_read32(repMatch2) == MEM_read32(ip)) ) {
|
||||
const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend;
|
||||
size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4;
|
||||
U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
|
||||
ZSTD_storeSeq(seqStore, 0, anchor, 0, repLength2-MINMATCH);
|
||||
hashTable[ZSTD_hashPtr(ip, hlog, mls)] = current2;
|
||||
ip += repLength2;
|
||||
anchor = ip;
|
||||
continue;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (dictMode == ZSTD_noDict) {
|
||||
while ( (ip <= ilimit)
|
||||
&& ( (offset_2>0)
|
||||
& (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) {
|
||||
/* store sequence */
|
||||
size_t const rLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4;
|
||||
U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; /* swap offset_2 <=> offset_1 */
|
||||
hashTable[ZSTD_hashPtr(ip, hlog, mls)] = (U32)(ip-base);
|
||||
ZSTD_storeSeq(seqStore, 0, anchor, 0, rLength-MINMATCH);
|
||||
ip += rLength;
|
||||
anchor = ip;
|
||||
continue; /* faster when present ... (?) */
|
||||
} } } }
|
||||
|
||||
/* save reps for next block */
|
||||
rep[0] = offset_1 ? offset_1 : offsetSaved;
|
||||
rep[1] = offset_2 ? offset_2 : offsetSaved;
|
||||
|
||||
/* Return the last literals size */
|
||||
return iend - anchor;
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTD_compressBlock_fast(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize)
|
||||
{
|
||||
U32 const hlog = cParams->hashLog;
|
||||
U32 const mls = cParams->searchLength;
|
||||
U32 const stepSize = cParams->targetLength;
|
||||
assert(ms->dictMatchState == NULL);
|
||||
switch(mls)
|
||||
{
|
||||
default: /* includes case 3 */
|
||||
case 4 :
|
||||
return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, hlog, stepSize, 4, ZSTD_noDict);
|
||||
case 5 :
|
||||
return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, hlog, stepSize, 5, ZSTD_noDict);
|
||||
case 6 :
|
||||
return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, hlog, stepSize, 6, ZSTD_noDict);
|
||||
case 7 :
|
||||
return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, hlog, stepSize, 7, ZSTD_noDict);
|
||||
}
|
||||
}
|
||||
|
||||
size_t ZSTD_compressBlock_fast_dictMatchState(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize)
|
||||
{
|
||||
U32 const hlog = cParams->hashLog;
|
||||
U32 const mls = cParams->searchLength;
|
||||
U32 const stepSize = cParams->targetLength;
|
||||
assert(ms->dictMatchState != NULL);
|
||||
switch(mls)
|
||||
{
|
||||
default: /* includes case 3 */
|
||||
case 4 :
|
||||
return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, hlog, stepSize, 4, ZSTD_dictMatchState);
|
||||
case 5 :
|
||||
return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, hlog, stepSize, 5, ZSTD_dictMatchState);
|
||||
case 6 :
|
||||
return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, hlog, stepSize, 6, ZSTD_dictMatchState);
|
||||
case 7 :
|
||||
return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, hlog, stepSize, 7, ZSTD_dictMatchState);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static size_t ZSTD_compressBlock_fast_extDict_generic(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize,
|
||||
U32 const hlog, U32 stepSize, U32 const mls)
|
||||
{
|
||||
U32* hashTable = ms->hashTable;
|
||||
const BYTE* const base = ms->window.base;
|
||||
const BYTE* const dictBase = ms->window.dictBase;
|
||||
const BYTE* const istart = (const BYTE*)src;
|
||||
const BYTE* ip = istart;
|
||||
const BYTE* anchor = istart;
|
||||
const U32 dictStartIndex = ms->window.lowLimit;
|
||||
const BYTE* const dictStart = dictBase + dictStartIndex;
|
||||
const U32 prefixStartIndex = ms->window.dictLimit;
|
||||
const BYTE* const prefixStart = base + prefixStartIndex;
|
||||
const BYTE* const dictEnd = dictBase + prefixStartIndex;
|
||||
const BYTE* const iend = istart + srcSize;
|
||||
const BYTE* const ilimit = iend - 8;
|
||||
U32 offset_1=rep[0], offset_2=rep[1];
|
||||
|
||||
stepSize += !stepSize; /* support stepSize == 0 */
|
||||
|
||||
/* Search Loop */
|
||||
while (ip < ilimit) { /* < instead of <=, because (ip+1) */
|
||||
const size_t h = ZSTD_hashPtr(ip, hlog, mls);
|
||||
const U32 matchIndex = hashTable[h];
|
||||
const BYTE* const matchBase = matchIndex < prefixStartIndex ? dictBase : base;
|
||||
const BYTE* match = matchBase + matchIndex;
|
||||
const U32 current = (U32)(ip-base);
|
||||
const U32 repIndex = current + 1 - offset_1;
|
||||
const BYTE* const repBase = repIndex < prefixStartIndex ? dictBase : base;
|
||||
const BYTE* const repMatch = repBase + repIndex;
|
||||
size_t mLength;
|
||||
hashTable[h] = current; /* update hash table */
|
||||
assert(offset_1 <= current +1); /* check repIndex */
|
||||
|
||||
if ( (((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > dictStartIndex))
|
||||
&& (MEM_read32(repMatch) == MEM_read32(ip+1)) ) {
|
||||
const BYTE* repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend;
|
||||
mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4;
|
||||
ip++;
|
||||
ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH);
|
||||
} else {
|
||||
if ( (matchIndex < dictStartIndex) ||
|
||||
(MEM_read32(match) != MEM_read32(ip)) ) {
|
||||
assert(stepSize >= 1);
|
||||
ip += ((ip-anchor) >> kSearchStrength) + stepSize;
|
||||
continue;
|
||||
}
|
||||
{ const BYTE* matchEnd = matchIndex < prefixStartIndex ? dictEnd : iend;
|
||||
const BYTE* lowMatchPtr = matchIndex < prefixStartIndex ? dictStart : prefixStart;
|
||||
U32 offset;
|
||||
mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, prefixStart) + 4;
|
||||
while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */
|
||||
offset = current - matchIndex;
|
||||
offset_2 = offset_1;
|
||||
offset_1 = offset;
|
||||
ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
|
||||
} }
|
||||
|
||||
/* found a match : store it */
|
||||
ip += mLength;
|
||||
anchor = ip;
|
||||
|
||||
if (ip <= ilimit) {
|
||||
/* Fill Table */
|
||||
hashTable[ZSTD_hashPtr(base+current+2, hlog, mls)] = current+2;
|
||||
hashTable[ZSTD_hashPtr(ip-2, hlog, mls)] = (U32)(ip-2-base);
|
||||
/* check immediate repcode */
|
||||
while (ip <= ilimit) {
|
||||
U32 const current2 = (U32)(ip-base);
|
||||
U32 const repIndex2 = current2 - offset_2;
|
||||
const BYTE* repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2;
|
||||
if ( (((U32)((prefixStartIndex-1) - repIndex2) >= 3) & (repIndex2 > dictStartIndex)) /* intentional overflow */
|
||||
&& (MEM_read32(repMatch2) == MEM_read32(ip)) ) {
|
||||
const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend;
|
||||
size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4;
|
||||
U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
|
||||
ZSTD_storeSeq(seqStore, 0, anchor, 0, repLength2-MINMATCH);
|
||||
hashTable[ZSTD_hashPtr(ip, hlog, mls)] = current2;
|
||||
ip += repLength2;
|
||||
anchor = ip;
|
||||
continue;
|
||||
}
|
||||
break;
|
||||
} } }
|
||||
|
||||
/* save reps for next block */
|
||||
rep[0] = offset_1;
|
||||
rep[1] = offset_2;
|
||||
|
||||
/* Return the last literals size */
|
||||
return iend - anchor;
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTD_compressBlock_fast_extDict(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize)
|
||||
{
|
||||
U32 const hlog = cParams->hashLog;
|
||||
U32 const mls = cParams->searchLength;
|
||||
U32 const stepSize = cParams->targetLength;
|
||||
switch(mls)
|
||||
{
|
||||
default: /* includes case 3 */
|
||||
case 4 :
|
||||
return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, hlog, stepSize, 4);
|
||||
case 5 :
|
||||
return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, hlog, stepSize, 5);
|
||||
case 6 :
|
||||
return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, hlog, stepSize, 6);
|
||||
case 7 :
|
||||
return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, hlog, stepSize, 7);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
/*
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#ifndef ZSTD_FAST_H
|
||||
#define ZSTD_FAST_H
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include "mem.h" /* U32 */
|
||||
#include "zstd_compress_internal.h"
|
||||
|
||||
void ZSTD_fillHashTable(ZSTD_matchState_t* ms,
|
||||
ZSTD_compressionParameters const* cParams,
|
||||
void const* end, ZSTD_dictTableLoadMethod_e dtlm);
|
||||
size_t ZSTD_compressBlock_fast(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_fast_dictMatchState(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_fast_extDict(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ZSTD_FAST_H */
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,69 @@
|
|||
/*
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#ifndef ZSTD_LAZY_H
|
||||
#define ZSTD_LAZY_H
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include "zstd_compress_internal.h"
|
||||
|
||||
U32 ZSTD_insertAndFindFirstIndex(
|
||||
ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
|
||||
const BYTE* ip);
|
||||
|
||||
void ZSTD_preserveUnsortedMark (U32* const table, U32 const size, U32 const reducerValue); /*! used in ZSTD_reduceIndex(). pre-emptively increase value of ZSTD_DUBT_UNSORTED_MARK */
|
||||
|
||||
size_t ZSTD_compressBlock_btlazy2(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_lazy2(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_lazy(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_greedy(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
|
||||
|
||||
size_t ZSTD_compressBlock_btlazy2_dictMatchState(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_lazy2_dictMatchState(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_lazy_dictMatchState(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_greedy_dictMatchState(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
|
||||
|
||||
size_t ZSTD_compressBlock_greedy_extDict(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_lazy_extDict(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_lazy2_extDict(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_btlazy2_extDict(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ZSTD_LAZY_H */
|
||||
|
|
@ -0,0 +1,648 @@
|
|||
/*
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
*/
|
||||
|
||||
#include "zstd_ldm.h"
|
||||
|
||||
#include "debug.h"
|
||||
#include "zstd_fast.h" /* ZSTD_fillHashTable() */
|
||||
#include "zstd_double_fast.h" /* ZSTD_fillDoubleHashTable() */
|
||||
|
||||
#define LDM_BUCKET_SIZE_LOG 3
|
||||
#define LDM_MIN_MATCH_LENGTH 64
|
||||
#define LDM_HASH_RLOG 7
|
||||
#define LDM_HASH_CHAR_OFFSET 10
|
||||
|
||||
void ZSTD_ldm_adjustParameters(ldmParams_t* params,
|
||||
ZSTD_compressionParameters const* cParams)
|
||||
{
|
||||
params->windowLog = cParams->windowLog;
|
||||
ZSTD_STATIC_ASSERT(LDM_BUCKET_SIZE_LOG <= ZSTD_LDM_BUCKETSIZELOG_MAX);
|
||||
DEBUGLOG(4, "ZSTD_ldm_adjustParameters");
|
||||
if (!params->bucketSizeLog) params->bucketSizeLog = LDM_BUCKET_SIZE_LOG;
|
||||
if (!params->minMatchLength) params->minMatchLength = LDM_MIN_MATCH_LENGTH;
|
||||
if (cParams->strategy >= ZSTD_btopt) {
|
||||
/* Get out of the way of the optimal parser */
|
||||
U32 const minMatch = MAX(cParams->targetLength, params->minMatchLength);
|
||||
assert(minMatch >= ZSTD_LDM_MINMATCH_MIN);
|
||||
assert(minMatch <= ZSTD_LDM_MINMATCH_MAX);
|
||||
params->minMatchLength = minMatch;
|
||||
}
|
||||
if (params->hashLog == 0) {
|
||||
params->hashLog = MAX(ZSTD_HASHLOG_MIN, params->windowLog - LDM_HASH_RLOG);
|
||||
assert(params->hashLog <= ZSTD_HASHLOG_MAX);
|
||||
}
|
||||
if (params->hashEveryLog == 0) {
|
||||
params->hashEveryLog = params->windowLog < params->hashLog
|
||||
? 0
|
||||
: params->windowLog - params->hashLog;
|
||||
}
|
||||
params->bucketSizeLog = MIN(params->bucketSizeLog, params->hashLog);
|
||||
}
|
||||
|
||||
size_t ZSTD_ldm_getTableSize(ldmParams_t params)
|
||||
{
|
||||
size_t const ldmHSize = ((size_t)1) << params.hashLog;
|
||||
size_t const ldmBucketSizeLog = MIN(params.bucketSizeLog, params.hashLog);
|
||||
size_t const ldmBucketSize =
|
||||
((size_t)1) << (params.hashLog - ldmBucketSizeLog);
|
||||
size_t const totalSize = ldmBucketSize + ldmHSize * sizeof(ldmEntry_t);
|
||||
return params.enableLdm ? totalSize : 0;
|
||||
}
|
||||
|
||||
size_t ZSTD_ldm_getMaxNbSeq(ldmParams_t params, size_t maxChunkSize)
|
||||
{
|
||||
return params.enableLdm ? (maxChunkSize / params.minMatchLength) : 0;
|
||||
}
|
||||
|
||||
/** ZSTD_ldm_getSmallHash() :
|
||||
* numBits should be <= 32
|
||||
* If numBits==0, returns 0.
|
||||
* @return : the most significant numBits of value. */
|
||||
static U32 ZSTD_ldm_getSmallHash(U64 value, U32 numBits)
|
||||
{
|
||||
assert(numBits <= 32);
|
||||
return numBits == 0 ? 0 : (U32)(value >> (64 - numBits));
|
||||
}
|
||||
|
||||
/** ZSTD_ldm_getChecksum() :
|
||||
* numBitsToDiscard should be <= 32
|
||||
* @return : the next most significant 32 bits after numBitsToDiscard */
|
||||
static U32 ZSTD_ldm_getChecksum(U64 hash, U32 numBitsToDiscard)
|
||||
{
|
||||
assert(numBitsToDiscard <= 32);
|
||||
return (hash >> (64 - 32 - numBitsToDiscard)) & 0xFFFFFFFF;
|
||||
}
|
||||
|
||||
/** ZSTD_ldm_getTag() ;
|
||||
* Given the hash, returns the most significant numTagBits bits
|
||||
* after (32 + hbits) bits.
|
||||
*
|
||||
* If there are not enough bits remaining, return the last
|
||||
* numTagBits bits. */
|
||||
static U32 ZSTD_ldm_getTag(U64 hash, U32 hbits, U32 numTagBits)
|
||||
{
|
||||
assert(numTagBits < 32 && hbits <= 32);
|
||||
if (32 - hbits < numTagBits) {
|
||||
return hash & (((U32)1 << numTagBits) - 1);
|
||||
} else {
|
||||
return (hash >> (32 - hbits - numTagBits)) & (((U32)1 << numTagBits) - 1);
|
||||
}
|
||||
}
|
||||
|
||||
/** ZSTD_ldm_getBucket() :
|
||||
* Returns a pointer to the start of the bucket associated with hash. */
|
||||
static ldmEntry_t* ZSTD_ldm_getBucket(
|
||||
ldmState_t* ldmState, size_t hash, ldmParams_t const ldmParams)
|
||||
{
|
||||
return ldmState->hashTable + (hash << ldmParams.bucketSizeLog);
|
||||
}
|
||||
|
||||
/** ZSTD_ldm_insertEntry() :
|
||||
* Insert the entry with corresponding hash into the hash table */
|
||||
static void ZSTD_ldm_insertEntry(ldmState_t* ldmState,
|
||||
size_t const hash, const ldmEntry_t entry,
|
||||
ldmParams_t const ldmParams)
|
||||
{
|
||||
BYTE* const bucketOffsets = ldmState->bucketOffsets;
|
||||
*(ZSTD_ldm_getBucket(ldmState, hash, ldmParams) + bucketOffsets[hash]) = entry;
|
||||
bucketOffsets[hash]++;
|
||||
bucketOffsets[hash] &= ((U32)1 << ldmParams.bucketSizeLog) - 1;
|
||||
}
|
||||
|
||||
/** ZSTD_ldm_makeEntryAndInsertByTag() :
|
||||
*
|
||||
* Gets the small hash, checksum, and tag from the rollingHash.
|
||||
*
|
||||
* If the tag matches (1 << ldmParams.hashEveryLog)-1, then
|
||||
* creates an ldmEntry from the offset, and inserts it into the hash table.
|
||||
*
|
||||
* hBits is the length of the small hash, which is the most significant hBits
|
||||
* of rollingHash. The checksum is the next 32 most significant bits, followed
|
||||
* by ldmParams.hashEveryLog bits that make up the tag. */
|
||||
static void ZSTD_ldm_makeEntryAndInsertByTag(ldmState_t* ldmState,
|
||||
U64 const rollingHash,
|
||||
U32 const hBits,
|
||||
U32 const offset,
|
||||
ldmParams_t const ldmParams)
|
||||
{
|
||||
U32 const tag = ZSTD_ldm_getTag(rollingHash, hBits, ldmParams.hashEveryLog);
|
||||
U32 const tagMask = ((U32)1 << ldmParams.hashEveryLog) - 1;
|
||||
if (tag == tagMask) {
|
||||
U32 const hash = ZSTD_ldm_getSmallHash(rollingHash, hBits);
|
||||
U32 const checksum = ZSTD_ldm_getChecksum(rollingHash, hBits);
|
||||
ldmEntry_t entry;
|
||||
entry.offset = offset;
|
||||
entry.checksum = checksum;
|
||||
ZSTD_ldm_insertEntry(ldmState, hash, entry, ldmParams);
|
||||
}
|
||||
}
|
||||
|
||||
/** ZSTD_ldm_getRollingHash() :
|
||||
* Get a 64-bit hash using the first len bytes from buf.
|
||||
*
|
||||
* Giving bytes s = s_1, s_2, ... s_k, the hash is defined to be
|
||||
* H(s) = s_1*(a^(k-1)) + s_2*(a^(k-2)) + ... + s_k*(a^0)
|
||||
*
|
||||
* where the constant a is defined to be prime8bytes.
|
||||
*
|
||||
* The implementation adds an offset to each byte, so
|
||||
* H(s) = (s_1 + HASH_CHAR_OFFSET)*(a^(k-1)) + ... */
|
||||
static U64 ZSTD_ldm_getRollingHash(const BYTE* buf, U32 len)
|
||||
{
|
||||
U64 ret = 0;
|
||||
U32 i;
|
||||
for (i = 0; i < len; i++) {
|
||||
ret *= prime8bytes;
|
||||
ret += buf[i] + LDM_HASH_CHAR_OFFSET;
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
/** ZSTD_ldm_ipow() :
|
||||
* Return base^exp. */
|
||||
static U64 ZSTD_ldm_ipow(U64 base, U64 exp)
|
||||
{
|
||||
U64 ret = 1;
|
||||
while (exp) {
|
||||
if (exp & 1) { ret *= base; }
|
||||
exp >>= 1;
|
||||
base *= base;
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
U64 ZSTD_ldm_getHashPower(U32 minMatchLength) {
|
||||
DEBUGLOG(4, "ZSTD_ldm_getHashPower: mml=%u", minMatchLength);
|
||||
assert(minMatchLength >= ZSTD_LDM_MINMATCH_MIN);
|
||||
return ZSTD_ldm_ipow(prime8bytes, minMatchLength - 1);
|
||||
}
|
||||
|
||||
/** ZSTD_ldm_updateHash() :
|
||||
* Updates hash by removing toRemove and adding toAdd. */
|
||||
static U64 ZSTD_ldm_updateHash(U64 hash, BYTE toRemove, BYTE toAdd, U64 hashPower)
|
||||
{
|
||||
hash -= ((toRemove + LDM_HASH_CHAR_OFFSET) * hashPower);
|
||||
hash *= prime8bytes;
|
||||
hash += toAdd + LDM_HASH_CHAR_OFFSET;
|
||||
return hash;
|
||||
}
|
||||
|
||||
/** ZSTD_ldm_countBackwardsMatch() :
|
||||
* Returns the number of bytes that match backwards before pIn and pMatch.
|
||||
*
|
||||
* We count only bytes where pMatch >= pBase and pIn >= pAnchor. */
|
||||
static size_t ZSTD_ldm_countBackwardsMatch(
|
||||
const BYTE* pIn, const BYTE* pAnchor,
|
||||
const BYTE* pMatch, const BYTE* pBase)
|
||||
{
|
||||
size_t matchLength = 0;
|
||||
while (pIn > pAnchor && pMatch > pBase && pIn[-1] == pMatch[-1]) {
|
||||
pIn--;
|
||||
pMatch--;
|
||||
matchLength++;
|
||||
}
|
||||
return matchLength;
|
||||
}
|
||||
|
||||
/** ZSTD_ldm_fillFastTables() :
|
||||
*
|
||||
* Fills the relevant tables for the ZSTD_fast and ZSTD_dfast strategies.
|
||||
* This is similar to ZSTD_loadDictionaryContent.
|
||||
*
|
||||
* The tables for the other strategies are filled within their
|
||||
* block compressors. */
|
||||
static size_t ZSTD_ldm_fillFastTables(ZSTD_matchState_t* ms,
|
||||
ZSTD_compressionParameters const* cParams,
|
||||
void const* end)
|
||||
{
|
||||
const BYTE* const iend = (const BYTE*)end;
|
||||
|
||||
switch(cParams->strategy)
|
||||
{
|
||||
case ZSTD_fast:
|
||||
ZSTD_fillHashTable(ms, cParams, iend, ZSTD_dtlm_fast);
|
||||
break;
|
||||
|
||||
case ZSTD_dfast:
|
||||
ZSTD_fillDoubleHashTable(ms, cParams, iend, ZSTD_dtlm_fast);
|
||||
break;
|
||||
|
||||
case ZSTD_greedy:
|
||||
case ZSTD_lazy:
|
||||
case ZSTD_lazy2:
|
||||
case ZSTD_btlazy2:
|
||||
case ZSTD_btopt:
|
||||
case ZSTD_btultra:
|
||||
break;
|
||||
default:
|
||||
assert(0); /* not possible : not a valid strategy id */
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/** ZSTD_ldm_fillLdmHashTable() :
|
||||
*
|
||||
* Fills hashTable from (lastHashed + 1) to iend (non-inclusive).
|
||||
* lastHash is the rolling hash that corresponds to lastHashed.
|
||||
*
|
||||
* Returns the rolling hash corresponding to position iend-1. */
|
||||
static U64 ZSTD_ldm_fillLdmHashTable(ldmState_t* state,
|
||||
U64 lastHash, const BYTE* lastHashed,
|
||||
const BYTE* iend, const BYTE* base,
|
||||
U32 hBits, ldmParams_t const ldmParams)
|
||||
{
|
||||
U64 rollingHash = lastHash;
|
||||
const BYTE* cur = lastHashed + 1;
|
||||
|
||||
while (cur < iend) {
|
||||
rollingHash = ZSTD_ldm_updateHash(rollingHash, cur[-1],
|
||||
cur[ldmParams.minMatchLength-1],
|
||||
state->hashPower);
|
||||
ZSTD_ldm_makeEntryAndInsertByTag(state,
|
||||
rollingHash, hBits,
|
||||
(U32)(cur - base), ldmParams);
|
||||
++cur;
|
||||
}
|
||||
return rollingHash;
|
||||
}
|
||||
|
||||
|
||||
/** ZSTD_ldm_limitTableUpdate() :
|
||||
*
|
||||
* Sets cctx->nextToUpdate to a position corresponding closer to anchor
|
||||
* if it is far way
|
||||
* (after a long match, only update tables a limited amount). */
|
||||
static void ZSTD_ldm_limitTableUpdate(ZSTD_matchState_t* ms, const BYTE* anchor)
|
||||
{
|
||||
U32 const current = (U32)(anchor - ms->window.base);
|
||||
if (current > ms->nextToUpdate + 1024) {
|
||||
ms->nextToUpdate =
|
||||
current - MIN(512, current - ms->nextToUpdate - 1024);
|
||||
}
|
||||
}
|
||||
|
||||
static size_t ZSTD_ldm_generateSequences_internal(
|
||||
ldmState_t* ldmState, rawSeqStore_t* rawSeqStore,
|
||||
ldmParams_t const* params, void const* src, size_t srcSize)
|
||||
{
|
||||
/* LDM parameters */
|
||||
int const extDict = ZSTD_window_hasExtDict(ldmState->window);
|
||||
U32 const minMatchLength = params->minMatchLength;
|
||||
U64 const hashPower = ldmState->hashPower;
|
||||
U32 const hBits = params->hashLog - params->bucketSizeLog;
|
||||
U32 const ldmBucketSize = 1U << params->bucketSizeLog;
|
||||
U32 const hashEveryLog = params->hashEveryLog;
|
||||
U32 const ldmTagMask = (1U << params->hashEveryLog) - 1;
|
||||
/* Prefix and extDict parameters */
|
||||
U32 const dictLimit = ldmState->window.dictLimit;
|
||||
U32 const lowestIndex = extDict ? ldmState->window.lowLimit : dictLimit;
|
||||
BYTE const* const base = ldmState->window.base;
|
||||
BYTE const* const dictBase = extDict ? ldmState->window.dictBase : NULL;
|
||||
BYTE const* const dictStart = extDict ? dictBase + lowestIndex : NULL;
|
||||
BYTE const* const dictEnd = extDict ? dictBase + dictLimit : NULL;
|
||||
BYTE const* const lowPrefixPtr = base + dictLimit;
|
||||
/* Input bounds */
|
||||
BYTE const* const istart = (BYTE const*)src;
|
||||
BYTE const* const iend = istart + srcSize;
|
||||
BYTE const* const ilimit = iend - MAX(minMatchLength, HASH_READ_SIZE);
|
||||
/* Input positions */
|
||||
BYTE const* anchor = istart;
|
||||
BYTE const* ip = istart;
|
||||
/* Rolling hash */
|
||||
BYTE const* lastHashed = NULL;
|
||||
U64 rollingHash = 0;
|
||||
|
||||
while (ip <= ilimit) {
|
||||
size_t mLength;
|
||||
U32 const current = (U32)(ip - base);
|
||||
size_t forwardMatchLength = 0, backwardMatchLength = 0;
|
||||
ldmEntry_t* bestEntry = NULL;
|
||||
if (ip != istart) {
|
||||
rollingHash = ZSTD_ldm_updateHash(rollingHash, lastHashed[0],
|
||||
lastHashed[minMatchLength],
|
||||
hashPower);
|
||||
} else {
|
||||
rollingHash = ZSTD_ldm_getRollingHash(ip, minMatchLength);
|
||||
}
|
||||
lastHashed = ip;
|
||||
|
||||
/* Do not insert and do not look for a match */
|
||||
if (ZSTD_ldm_getTag(rollingHash, hBits, hashEveryLog) != ldmTagMask) {
|
||||
ip++;
|
||||
continue;
|
||||
}
|
||||
|
||||
/* Get the best entry and compute the match lengths */
|
||||
{
|
||||
ldmEntry_t* const bucket =
|
||||
ZSTD_ldm_getBucket(ldmState,
|
||||
ZSTD_ldm_getSmallHash(rollingHash, hBits),
|
||||
*params);
|
||||
ldmEntry_t* cur;
|
||||
size_t bestMatchLength = 0;
|
||||
U32 const checksum = ZSTD_ldm_getChecksum(rollingHash, hBits);
|
||||
|
||||
for (cur = bucket; cur < bucket + ldmBucketSize; ++cur) {
|
||||
size_t curForwardMatchLength, curBackwardMatchLength,
|
||||
curTotalMatchLength;
|
||||
if (cur->checksum != checksum || cur->offset <= lowestIndex) {
|
||||
continue;
|
||||
}
|
||||
if (extDict) {
|
||||
BYTE const* const curMatchBase =
|
||||
cur->offset < dictLimit ? dictBase : base;
|
||||
BYTE const* const pMatch = curMatchBase + cur->offset;
|
||||
BYTE const* const matchEnd =
|
||||
cur->offset < dictLimit ? dictEnd : iend;
|
||||
BYTE const* const lowMatchPtr =
|
||||
cur->offset < dictLimit ? dictStart : lowPrefixPtr;
|
||||
|
||||
curForwardMatchLength = ZSTD_count_2segments(
|
||||
ip, pMatch, iend,
|
||||
matchEnd, lowPrefixPtr);
|
||||
if (curForwardMatchLength < minMatchLength) {
|
||||
continue;
|
||||
}
|
||||
curBackwardMatchLength =
|
||||
ZSTD_ldm_countBackwardsMatch(ip, anchor, pMatch,
|
||||
lowMatchPtr);
|
||||
curTotalMatchLength = curForwardMatchLength +
|
||||
curBackwardMatchLength;
|
||||
} else { /* !extDict */
|
||||
BYTE const* const pMatch = base + cur->offset;
|
||||
curForwardMatchLength = ZSTD_count(ip, pMatch, iend);
|
||||
if (curForwardMatchLength < minMatchLength) {
|
||||
continue;
|
||||
}
|
||||
curBackwardMatchLength =
|
||||
ZSTD_ldm_countBackwardsMatch(ip, anchor, pMatch,
|
||||
lowPrefixPtr);
|
||||
curTotalMatchLength = curForwardMatchLength +
|
||||
curBackwardMatchLength;
|
||||
}
|
||||
|
||||
if (curTotalMatchLength > bestMatchLength) {
|
||||
bestMatchLength = curTotalMatchLength;
|
||||
forwardMatchLength = curForwardMatchLength;
|
||||
backwardMatchLength = curBackwardMatchLength;
|
||||
bestEntry = cur;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* No match found -- continue searching */
|
||||
if (bestEntry == NULL) {
|
||||
ZSTD_ldm_makeEntryAndInsertByTag(ldmState, rollingHash,
|
||||
hBits, current,
|
||||
*params);
|
||||
ip++;
|
||||
continue;
|
||||
}
|
||||
|
||||
/* Match found */
|
||||
mLength = forwardMatchLength + backwardMatchLength;
|
||||
ip -= backwardMatchLength;
|
||||
|
||||
{
|
||||
/* Store the sequence:
|
||||
* ip = current - backwardMatchLength
|
||||
* The match is at (bestEntry->offset - backwardMatchLength)
|
||||
*/
|
||||
U32 const matchIndex = bestEntry->offset;
|
||||
U32 const offset = current - matchIndex;
|
||||
rawSeq* const seq = rawSeqStore->seq + rawSeqStore->size;
|
||||
|
||||
/* Out of sequence storage */
|
||||
if (rawSeqStore->size == rawSeqStore->capacity)
|
||||
return ERROR(dstSize_tooSmall);
|
||||
seq->litLength = (U32)(ip - anchor);
|
||||
seq->matchLength = (U32)mLength;
|
||||
seq->offset = offset;
|
||||
rawSeqStore->size++;
|
||||
}
|
||||
|
||||
/* Insert the current entry into the hash table */
|
||||
ZSTD_ldm_makeEntryAndInsertByTag(ldmState, rollingHash, hBits,
|
||||
(U32)(lastHashed - base),
|
||||
*params);
|
||||
|
||||
assert(ip + backwardMatchLength == lastHashed);
|
||||
|
||||
/* Fill the hash table from lastHashed+1 to ip+mLength*/
|
||||
/* Heuristic: don't need to fill the entire table at end of block */
|
||||
if (ip + mLength <= ilimit) {
|
||||
rollingHash = ZSTD_ldm_fillLdmHashTable(
|
||||
ldmState, rollingHash, lastHashed,
|
||||
ip + mLength, base, hBits, *params);
|
||||
lastHashed = ip + mLength - 1;
|
||||
}
|
||||
ip += mLength;
|
||||
anchor = ip;
|
||||
}
|
||||
return iend - anchor;
|
||||
}
|
||||
|
||||
/*! ZSTD_ldm_reduceTable() :
|
||||
* reduce table indexes by `reducerValue` */
|
||||
static void ZSTD_ldm_reduceTable(ldmEntry_t* const table, U32 const size,
|
||||
U32 const reducerValue)
|
||||
{
|
||||
U32 u;
|
||||
for (u = 0; u < size; u++) {
|
||||
if (table[u].offset < reducerValue) table[u].offset = 0;
|
||||
else table[u].offset -= reducerValue;
|
||||
}
|
||||
}
|
||||
|
||||
size_t ZSTD_ldm_generateSequences(
|
||||
ldmState_t* ldmState, rawSeqStore_t* sequences,
|
||||
ldmParams_t const* params, void const* src, size_t srcSize)
|
||||
{
|
||||
U32 const maxDist = 1U << params->windowLog;
|
||||
BYTE const* const istart = (BYTE const*)src;
|
||||
BYTE const* const iend = istart + srcSize;
|
||||
size_t const kMaxChunkSize = 1 << 20;
|
||||
size_t const nbChunks = (srcSize / kMaxChunkSize) + ((srcSize % kMaxChunkSize) != 0);
|
||||
size_t chunk;
|
||||
size_t leftoverSize = 0;
|
||||
|
||||
assert(ZSTD_CHUNKSIZE_MAX >= kMaxChunkSize);
|
||||
/* Check that ZSTD_window_update() has been called for this chunk prior
|
||||
* to passing it to this function.
|
||||
*/
|
||||
assert(ldmState->window.nextSrc >= (BYTE const*)src + srcSize);
|
||||
/* The input could be very large (in zstdmt), so it must be broken up into
|
||||
* chunks to enforce the maximmum distance and handle overflow correction.
|
||||
*/
|
||||
assert(sequences->pos <= sequences->size);
|
||||
assert(sequences->size <= sequences->capacity);
|
||||
for (chunk = 0; chunk < nbChunks && sequences->size < sequences->capacity; ++chunk) {
|
||||
BYTE const* const chunkStart = istart + chunk * kMaxChunkSize;
|
||||
size_t const remaining = (size_t)(iend - chunkStart);
|
||||
BYTE const *const chunkEnd =
|
||||
(remaining < kMaxChunkSize) ? iend : chunkStart + kMaxChunkSize;
|
||||
size_t const chunkSize = chunkEnd - chunkStart;
|
||||
size_t newLeftoverSize;
|
||||
size_t const prevSize = sequences->size;
|
||||
|
||||
assert(chunkStart < iend);
|
||||
/* 1. Perform overflow correction if necessary. */
|
||||
if (ZSTD_window_needOverflowCorrection(ldmState->window, chunkEnd)) {
|
||||
U32 const ldmHSize = 1U << params->hashLog;
|
||||
U32 const correction = ZSTD_window_correctOverflow(
|
||||
&ldmState->window, /* cycleLog */ 0, maxDist, src);
|
||||
ZSTD_ldm_reduceTable(ldmState->hashTable, ldmHSize, correction);
|
||||
}
|
||||
/* 2. We enforce the maximum offset allowed.
|
||||
*
|
||||
* kMaxChunkSize should be small enough that we don't lose too much of
|
||||
* the window through early invalidation.
|
||||
* TODO: * Test the chunk size.
|
||||
* * Try invalidation after the sequence generation and test the
|
||||
* the offset against maxDist directly.
|
||||
*/
|
||||
ZSTD_window_enforceMaxDist(&ldmState->window, chunkEnd, maxDist, NULL, NULL);
|
||||
/* 3. Generate the sequences for the chunk, and get newLeftoverSize. */
|
||||
newLeftoverSize = ZSTD_ldm_generateSequences_internal(
|
||||
ldmState, sequences, params, chunkStart, chunkSize);
|
||||
if (ZSTD_isError(newLeftoverSize))
|
||||
return newLeftoverSize;
|
||||
/* 4. We add the leftover literals from previous iterations to the first
|
||||
* newly generated sequence, or add the `newLeftoverSize` if none are
|
||||
* generated.
|
||||
*/
|
||||
/* Prepend the leftover literals from the last call */
|
||||
if (prevSize < sequences->size) {
|
||||
sequences->seq[prevSize].litLength += (U32)leftoverSize;
|
||||
leftoverSize = newLeftoverSize;
|
||||
} else {
|
||||
assert(newLeftoverSize == chunkSize);
|
||||
leftoverSize += chunkSize;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
void ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, U32 const minMatch) {
|
||||
while (srcSize > 0 && rawSeqStore->pos < rawSeqStore->size) {
|
||||
rawSeq* seq = rawSeqStore->seq + rawSeqStore->pos;
|
||||
if (srcSize <= seq->litLength) {
|
||||
/* Skip past srcSize literals */
|
||||
seq->litLength -= (U32)srcSize;
|
||||
return;
|
||||
}
|
||||
srcSize -= seq->litLength;
|
||||
seq->litLength = 0;
|
||||
if (srcSize < seq->matchLength) {
|
||||
/* Skip past the first srcSize of the match */
|
||||
seq->matchLength -= (U32)srcSize;
|
||||
if (seq->matchLength < minMatch) {
|
||||
/* The match is too short, omit it */
|
||||
if (rawSeqStore->pos + 1 < rawSeqStore->size) {
|
||||
seq[1].litLength += seq[0].matchLength;
|
||||
}
|
||||
rawSeqStore->pos++;
|
||||
}
|
||||
return;
|
||||
}
|
||||
srcSize -= seq->matchLength;
|
||||
seq->matchLength = 0;
|
||||
rawSeqStore->pos++;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* If the sequence length is longer than remaining then the sequence is split
|
||||
* between this block and the next.
|
||||
*
|
||||
* Returns the current sequence to handle, or if the rest of the block should
|
||||
* be literals, it returns a sequence with offset == 0.
|
||||
*/
|
||||
static rawSeq maybeSplitSequence(rawSeqStore_t* rawSeqStore,
|
||||
U32 const remaining, U32 const minMatch)
|
||||
{
|
||||
rawSeq sequence = rawSeqStore->seq[rawSeqStore->pos];
|
||||
assert(sequence.offset > 0);
|
||||
/* Likely: No partial sequence */
|
||||
if (remaining >= sequence.litLength + sequence.matchLength) {
|
||||
rawSeqStore->pos++;
|
||||
return sequence;
|
||||
}
|
||||
/* Cut the sequence short (offset == 0 ==> rest is literals). */
|
||||
if (remaining <= sequence.litLength) {
|
||||
sequence.offset = 0;
|
||||
} else if (remaining < sequence.litLength + sequence.matchLength) {
|
||||
sequence.matchLength = remaining - sequence.litLength;
|
||||
if (sequence.matchLength < minMatch) {
|
||||
sequence.offset = 0;
|
||||
}
|
||||
}
|
||||
/* Skip past `remaining` bytes for the future sequences. */
|
||||
ZSTD_ldm_skipSequences(rawSeqStore, remaining, minMatch);
|
||||
return sequence;
|
||||
}
|
||||
|
||||
size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize)
|
||||
{
|
||||
unsigned const minMatch = cParams->searchLength;
|
||||
ZSTD_blockCompressor const blockCompressor =
|
||||
ZSTD_selectBlockCompressor(cParams->strategy, ZSTD_matchState_dictMode(ms));
|
||||
/* Input bounds */
|
||||
BYTE const* const istart = (BYTE const*)src;
|
||||
BYTE const* const iend = istart + srcSize;
|
||||
/* Input positions */
|
||||
BYTE const* ip = istart;
|
||||
|
||||
DEBUGLOG(5, "ZSTD_ldm_blockCompress: srcSize=%zu", srcSize);
|
||||
assert(rawSeqStore->pos <= rawSeqStore->size);
|
||||
assert(rawSeqStore->size <= rawSeqStore->capacity);
|
||||
/* Loop through each sequence and apply the block compressor to the lits */
|
||||
while (rawSeqStore->pos < rawSeqStore->size && ip < iend) {
|
||||
/* maybeSplitSequence updates rawSeqStore->pos */
|
||||
rawSeq const sequence = maybeSplitSequence(rawSeqStore,
|
||||
(U32)(iend - ip), minMatch);
|
||||
int i;
|
||||
/* End signal */
|
||||
if (sequence.offset == 0)
|
||||
break;
|
||||
|
||||
assert(sequence.offset <= (1U << cParams->windowLog));
|
||||
assert(ip + sequence.litLength + sequence.matchLength <= iend);
|
||||
|
||||
/* Fill tables for block compressor */
|
||||
ZSTD_ldm_limitTableUpdate(ms, ip);
|
||||
ZSTD_ldm_fillFastTables(ms, cParams, ip);
|
||||
/* Run the block compressor */
|
||||
DEBUGLOG(5, "calling block compressor on segment of size %u", sequence.litLength);
|
||||
{
|
||||
size_t const newLitLength =
|
||||
blockCompressor(ms, seqStore, rep, cParams, ip,
|
||||
sequence.litLength);
|
||||
ip += sequence.litLength;
|
||||
/* Update the repcodes */
|
||||
for (i = ZSTD_REP_NUM - 1; i > 0; i--)
|
||||
rep[i] = rep[i-1];
|
||||
rep[0] = sequence.offset;
|
||||
/* Store the sequence */
|
||||
ZSTD_storeSeq(seqStore, newLitLength, ip - newLitLength,
|
||||
sequence.offset + ZSTD_REP_MOVE,
|
||||
sequence.matchLength - MINMATCH);
|
||||
ip += sequence.matchLength;
|
||||
}
|
||||
}
|
||||
/* Fill the tables for the block compressor */
|
||||
ZSTD_ldm_limitTableUpdate(ms, ip);
|
||||
ZSTD_ldm_fillFastTables(ms, cParams, ip);
|
||||
/* Compress the last literals */
|
||||
return blockCompressor(ms, seqStore, rep, cParams,
|
||||
ip, iend - ip);
|
||||
}
|
||||
|
|
@ -0,0 +1,110 @@
|
|||
/*
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
*/
|
||||
|
||||
#ifndef ZSTD_LDM_H
|
||||
#define ZSTD_LDM_H
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include "zstd_compress_internal.h" /* ldmParams_t, U32 */
|
||||
#include "zstd.h" /* ZSTD_CCtx, size_t */
|
||||
|
||||
/*-*************************************
|
||||
* Long distance matching
|
||||
***************************************/
|
||||
|
||||
#define ZSTD_LDM_DEFAULT_WINDOW_LOG ZSTD_WINDOWLOG_DEFAULTMAX
|
||||
|
||||
/**
|
||||
* ZSTD_ldm_generateSequences():
|
||||
*
|
||||
* Generates the sequences using the long distance match finder.
|
||||
* Generates long range matching sequences in `sequences`, which parse a prefix
|
||||
* of the source. `sequences` must be large enough to store every sequence,
|
||||
* which can be checked with `ZSTD_ldm_getMaxNbSeq()`.
|
||||
* @returns 0 or an error code.
|
||||
*
|
||||
* NOTE: The user must have called ZSTD_window_update() for all of the input
|
||||
* they have, even if they pass it to ZSTD_ldm_generateSequences() in chunks.
|
||||
* NOTE: This function returns an error if it runs out of space to store
|
||||
* sequences.
|
||||
*/
|
||||
size_t ZSTD_ldm_generateSequences(
|
||||
ldmState_t* ldms, rawSeqStore_t* sequences,
|
||||
ldmParams_t const* params, void const* src, size_t srcSize);
|
||||
|
||||
/**
|
||||
* ZSTD_ldm_blockCompress():
|
||||
*
|
||||
* Compresses a block using the predefined sequences, along with a secondary
|
||||
* block compressor. The literals section of every sequence is passed to the
|
||||
* secondary block compressor, and those sequences are interspersed with the
|
||||
* predefined sequences. Returns the length of the last literals.
|
||||
* Updates `rawSeqStore.pos` to indicate how many sequences have been consumed.
|
||||
* `rawSeqStore.seq` may also be updated to split the last sequence between two
|
||||
* blocks.
|
||||
* @return The length of the last literals.
|
||||
*
|
||||
* NOTE: The source must be at most the maximum block size, but the predefined
|
||||
* sequences can be any size, and may be longer than the block. In the case that
|
||||
* they are longer than the block, the last sequences may need to be split into
|
||||
* two. We handle that case correctly, and update `rawSeqStore` appropriately.
|
||||
* NOTE: This function does not return any errors.
|
||||
*/
|
||||
size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_compressionParameters const* cParams,
|
||||
void const* src, size_t srcSize);
|
||||
|
||||
/**
|
||||
* ZSTD_ldm_skipSequences():
|
||||
*
|
||||
* Skip past `srcSize` bytes worth of sequences in `rawSeqStore`.
|
||||
* Avoids emitting matches less than `minMatch` bytes.
|
||||
* Must be called for data with is not passed to ZSTD_ldm_blockCompress().
|
||||
*/
|
||||
void ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize,
|
||||
U32 const minMatch);
|
||||
|
||||
|
||||
/** ZSTD_ldm_getTableSize() :
|
||||
* Estimate the space needed for long distance matching tables or 0 if LDM is
|
||||
* disabled.
|
||||
*/
|
||||
size_t ZSTD_ldm_getTableSize(ldmParams_t params);
|
||||
|
||||
/** ZSTD_ldm_getSeqSpace() :
|
||||
* Return an upper bound on the number of sequences that can be produced by
|
||||
* the long distance matcher, or 0 if LDM is disabled.
|
||||
*/
|
||||
size_t ZSTD_ldm_getMaxNbSeq(ldmParams_t params, size_t maxChunkSize);
|
||||
|
||||
/** ZSTD_ldm_getTableSize() :
|
||||
* Return prime8bytes^(minMatchLength-1) */
|
||||
U64 ZSTD_ldm_getHashPower(U32 minMatchLength);
|
||||
|
||||
/** ZSTD_ldm_adjustParameters() :
|
||||
* If the params->hashEveryLog is not set, set it to its default value based on
|
||||
* windowLog and params->hashLog.
|
||||
*
|
||||
* Ensures that params->bucketSizeLog is <= params->hashLog (setting it to
|
||||
* params->hashLog if it is not).
|
||||
*
|
||||
* Ensures that the minMatchLength >= targetLength during optimal parsing.
|
||||
*/
|
||||
void ZSTD_ldm_adjustParameters(ldmParams_t* params,
|
||||
ZSTD_compressionParameters const* cParams);
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ZSTD_FAST_H */
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,49 @@
|
|||
/*
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#ifndef ZSTD_OPT_H
|
||||
#define ZSTD_OPT_H
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include "zstd_compress_internal.h"
|
||||
|
||||
void ZSTD_updateTree(
|
||||
ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
|
||||
const BYTE* ip, const BYTE* iend); /* used in ZSTD_loadDictionaryContent() */
|
||||
|
||||
size_t ZSTD_compressBlock_btopt(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_btultra(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
|
||||
|
||||
size_t ZSTD_compressBlock_btopt_dictMatchState(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_btultra_dictMatchState(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
|
||||
|
||||
size_t ZSTD_compressBlock_btopt_extDict(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_btultra_extDict(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ZSTD_OPT_H */
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,156 @@
|
|||
/*
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#ifndef ZSTDMT_COMPRESS_H
|
||||
#define ZSTDMT_COMPRESS_H
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
/* Note : This is an internal API.
|
||||
* Some methods are still exposed (ZSTDLIB_API),
|
||||
* because it used to be the only way to invoke MT compression.
|
||||
* Now, it's recommended to use ZSTD_compress_generic() instead.
|
||||
* These methods will stop being exposed in a future version */
|
||||
|
||||
/* === Dependencies === */
|
||||
#include <stddef.h> /* size_t */
|
||||
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters */
|
||||
#include "zstd.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */
|
||||
|
||||
|
||||
/* === Memory management === */
|
||||
typedef struct ZSTDMT_CCtx_s ZSTDMT_CCtx;
|
||||
ZSTDLIB_API ZSTDMT_CCtx* ZSTDMT_createCCtx(unsigned nbWorkers);
|
||||
ZSTDLIB_API ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers,
|
||||
ZSTD_customMem cMem);
|
||||
ZSTDLIB_API size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx);
|
||||
|
||||
ZSTDLIB_API size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx);
|
||||
|
||||
|
||||
/* === Simple one-pass compression function === */
|
||||
|
||||
ZSTDLIB_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
int compressionLevel);
|
||||
|
||||
|
||||
|
||||
/* === Streaming functions === */
|
||||
|
||||
ZSTDLIB_API size_t ZSTDMT_initCStream(ZSTDMT_CCtx* mtctx, int compressionLevel);
|
||||
ZSTDLIB_API size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* mtctx, unsigned long long pledgedSrcSize); /**< if srcSize is not known at reset time, use ZSTD_CONTENTSIZE_UNKNOWN. Note: for compatibility with older programs, 0 means the same as ZSTD_CONTENTSIZE_UNKNOWN, but it will change in the future to mean "empty" */
|
||||
|
||||
ZSTDLIB_API size_t ZSTDMT_compressStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
|
||||
|
||||
ZSTDLIB_API size_t ZSTDMT_flushStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output); /**< @return : 0 == all flushed; >0 : still some data to be flushed; or an error code (ZSTD_isError()) */
|
||||
ZSTDLIB_API size_t ZSTDMT_endStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output); /**< @return : 0 == all flushed; >0 : still some data to be flushed; or an error code (ZSTD_isError()) */
|
||||
|
||||
|
||||
/* === Advanced functions and parameters === */
|
||||
|
||||
#ifndef ZSTDMT_JOBSIZE_MIN
|
||||
# define ZSTDMT_JOBSIZE_MIN (1U << 20) /* 1 MB - Minimum size of each compression job */
|
||||
#endif
|
||||
|
||||
ZSTDLIB_API size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
const ZSTD_CDict* cdict,
|
||||
ZSTD_parameters params,
|
||||
unsigned overlapLog);
|
||||
|
||||
ZSTDLIB_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx,
|
||||
const void* dict, size_t dictSize, /* dict can be released after init, a local copy is preserved within zcs */
|
||||
ZSTD_parameters params,
|
||||
unsigned long long pledgedSrcSize); /* pledgedSrcSize is optional and can be zero == unknown */
|
||||
|
||||
ZSTDLIB_API size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx,
|
||||
const ZSTD_CDict* cdict,
|
||||
ZSTD_frameParameters fparams,
|
||||
unsigned long long pledgedSrcSize); /* note : zero means empty */
|
||||
|
||||
/* ZSTDMT_parameter :
|
||||
* List of parameters that can be set using ZSTDMT_setMTCtxParameter() */
|
||||
typedef enum {
|
||||
ZSTDMT_p_jobSize, /* Each job is compressed in parallel. By default, this value is dynamically determined depending on compression parameters. Can be set explicitly here. */
|
||||
ZSTDMT_p_overlapSectionLog /* Each job may reload a part of previous job to enhance compressionr ratio; 0 == no overlap, 6(default) == use 1/8th of window, >=9 == use full window. This is a "sticky" parameter : its value will be re-used on next compression job */
|
||||
} ZSTDMT_parameter;
|
||||
|
||||
/* ZSTDMT_setMTCtxParameter() :
|
||||
* allow setting individual parameters, one at a time, among a list of enums defined in ZSTDMT_parameter.
|
||||
* The function must be called typically after ZSTD_createCCtx() but __before ZSTDMT_init*() !__
|
||||
* Parameters not explicitly reset by ZSTDMT_init*() remain the same in consecutive compression sessions.
|
||||
* @return : 0, or an error code (which can be tested using ZSTD_isError()) */
|
||||
ZSTDLIB_API size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, unsigned value);
|
||||
|
||||
/* ZSTDMT_getMTCtxParameter() :
|
||||
* Query the ZSTDMT_CCtx for a parameter value.
|
||||
* @return : 0, or an error code (which can be tested using ZSTD_isError()) */
|
||||
ZSTDLIB_API size_t ZSTDMT_getMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, unsigned* value);
|
||||
|
||||
|
||||
/*! ZSTDMT_compressStream_generic() :
|
||||
* Combines ZSTDMT_compressStream() with optional ZSTDMT_flushStream() or ZSTDMT_endStream()
|
||||
* depending on flush directive.
|
||||
* @return : minimum amount of data still to be flushed
|
||||
* 0 if fully flushed
|
||||
* or an error code
|
||||
* note : needs to be init using any ZSTD_initCStream*() variant */
|
||||
ZSTDLIB_API size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
|
||||
ZSTD_outBuffer* output,
|
||||
ZSTD_inBuffer* input,
|
||||
ZSTD_EndDirective endOp);
|
||||
|
||||
|
||||
/* ========================================================
|
||||
* === Private interface, for use by ZSTD_compress.c ===
|
||||
* === Not exposed in libzstd. Never invoke directly ===
|
||||
* ======================================================== */
|
||||
|
||||
size_t ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params, ZSTDMT_parameter parameter, unsigned value);
|
||||
|
||||
/* ZSTDMT_CCtxParam_setNbWorkers()
|
||||
* Set nbWorkers, and clamp it.
|
||||
* Also reset jobSize and overlapLog */
|
||||
size_t ZSTDMT_CCtxParam_setNbWorkers(ZSTD_CCtx_params* params, unsigned nbWorkers);
|
||||
|
||||
/*! ZSTDMT_updateCParams_whileCompressing() :
|
||||
* Updates only a selected set of compression parameters, to remain compatible with current frame.
|
||||
* New parameters will be applied to next compression job. */
|
||||
void ZSTDMT_updateCParams_whileCompressing(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_params* cctxParams);
|
||||
|
||||
/* ZSTDMT_getFrameProgression():
|
||||
* tells how much data has been consumed (input) and produced (output) for current frame.
|
||||
* able to count progression inside worker threads.
|
||||
*/
|
||||
ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx);
|
||||
|
||||
|
||||
/*! ZSTDMT_initCStream_internal() :
|
||||
* Private use only. Init streaming operation.
|
||||
* expects params to be valid.
|
||||
* must receive dict, or cdict, or none, but not both.
|
||||
* @return : 0, or an error code */
|
||||
size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
|
||||
const void* dict, size_t dictSize, ZSTD_dictContentType_e dictContentType,
|
||||
const ZSTD_CDict* cdict,
|
||||
ZSTD_CCtx_params params, unsigned long long pledgedSrcSize);
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ZSTDMT_COMPRESS_H */
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,213 @@
|
|||
/*
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
/* ***************************************************************
|
||||
* NOTES/WARNINGS
|
||||
******************************************************************/
|
||||
/* The streaming API defined here is deprecated.
|
||||
* Consider migrating towards ZSTD_compressStream() API in `zstd.h`
|
||||
* See 'lib/README.md'.
|
||||
*****************************************************************/
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#ifndef ZSTD_BUFFERED_H_23987
|
||||
#define ZSTD_BUFFERED_H_23987
|
||||
|
||||
/* *************************************
|
||||
* Dependencies
|
||||
***************************************/
|
||||
#include <stddef.h> /* size_t */
|
||||
#include "zstd.h" /* ZSTD_CStream, ZSTD_DStream, ZSTDLIB_API */
|
||||
|
||||
|
||||
/* ***************************************************************
|
||||
* Compiler specifics
|
||||
*****************************************************************/
|
||||
/* Deprecation warnings */
|
||||
/* Should these warnings be a problem,
|
||||
it is generally possible to disable them,
|
||||
typically with -Wno-deprecated-declarations for gcc
|
||||
or _CRT_SECURE_NO_WARNINGS in Visual.
|
||||
Otherwise, it's also possible to define ZBUFF_DISABLE_DEPRECATE_WARNINGS */
|
||||
#ifdef ZBUFF_DISABLE_DEPRECATE_WARNINGS
|
||||
# define ZBUFF_DEPRECATED(message) ZSTDLIB_API /* disable deprecation warnings */
|
||||
#else
|
||||
# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
|
||||
# define ZBUFF_DEPRECATED(message) [[deprecated(message)]] ZSTDLIB_API
|
||||
# elif (defined(__GNUC__) && (__GNUC__ >= 5)) || defined(__clang__)
|
||||
# define ZBUFF_DEPRECATED(message) ZSTDLIB_API __attribute__((deprecated(message)))
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3)
|
||||
# define ZBUFF_DEPRECATED(message) ZSTDLIB_API __attribute__((deprecated))
|
||||
# elif defined(_MSC_VER)
|
||||
# define ZBUFF_DEPRECATED(message) ZSTDLIB_API __declspec(deprecated(message))
|
||||
# else
|
||||
# pragma message("WARNING: You need to implement ZBUFF_DEPRECATED for this compiler")
|
||||
# define ZBUFF_DEPRECATED(message) ZSTDLIB_API
|
||||
# endif
|
||||
#endif /* ZBUFF_DISABLE_DEPRECATE_WARNINGS */
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Streaming functions
|
||||
***************************************/
|
||||
/* This is the easier "buffered" streaming API,
|
||||
* using an internal buffer to lift all restrictions on user-provided buffers
|
||||
* which can be any size, any place, for both input and output.
|
||||
* ZBUFF and ZSTD are 100% interoperable,
|
||||
* frames created by one can be decoded by the other one */
|
||||
|
||||
typedef ZSTD_CStream ZBUFF_CCtx;
|
||||
ZBUFF_DEPRECATED("use ZSTD_createCStream") ZBUFF_CCtx* ZBUFF_createCCtx(void);
|
||||
ZBUFF_DEPRECATED("use ZSTD_freeCStream") size_t ZBUFF_freeCCtx(ZBUFF_CCtx* cctx);
|
||||
|
||||
ZBUFF_DEPRECATED("use ZSTD_initCStream") size_t ZBUFF_compressInit(ZBUFF_CCtx* cctx, int compressionLevel);
|
||||
ZBUFF_DEPRECATED("use ZSTD_initCStream_usingDict") size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
|
||||
|
||||
ZBUFF_DEPRECATED("use ZSTD_compressStream") size_t ZBUFF_compressContinue(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr, const void* src, size_t* srcSizePtr);
|
||||
ZBUFF_DEPRECATED("use ZSTD_flushStream") size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr);
|
||||
ZBUFF_DEPRECATED("use ZSTD_endStream") size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr);
|
||||
|
||||
/*-*************************************************
|
||||
* Streaming compression - howto
|
||||
*
|
||||
* A ZBUFF_CCtx object is required to track streaming operation.
|
||||
* Use ZBUFF_createCCtx() and ZBUFF_freeCCtx() to create/release resources.
|
||||
* ZBUFF_CCtx objects can be reused multiple times.
|
||||
*
|
||||
* Start by initializing ZBUF_CCtx.
|
||||
* Use ZBUFF_compressInit() to start a new compression operation.
|
||||
* Use ZBUFF_compressInitDictionary() for a compression which requires a dictionary.
|
||||
*
|
||||
* Use ZBUFF_compressContinue() repetitively to consume input stream.
|
||||
* *srcSizePtr and *dstCapacityPtr can be any size.
|
||||
* The function will report how many bytes were read or written within *srcSizePtr and *dstCapacityPtr.
|
||||
* Note that it may not consume the entire input, in which case it's up to the caller to present again remaining data.
|
||||
* The content of `dst` will be overwritten (up to *dstCapacityPtr) at each call, so save its content if it matters or change @dst .
|
||||
* @return : a hint to preferred nb of bytes to use as input for next function call (it's just a hint, to improve latency)
|
||||
* or an error code, which can be tested using ZBUFF_isError().
|
||||
*
|
||||
* At any moment, it's possible to flush whatever data remains within buffer, using ZBUFF_compressFlush().
|
||||
* The nb of bytes written into `dst` will be reported into *dstCapacityPtr.
|
||||
* Note that the function cannot output more than *dstCapacityPtr,
|
||||
* therefore, some content might still be left into internal buffer if *dstCapacityPtr is too small.
|
||||
* @return : nb of bytes still present into internal buffer (0 if it's empty)
|
||||
* or an error code, which can be tested using ZBUFF_isError().
|
||||
*
|
||||
* ZBUFF_compressEnd() instructs to finish a frame.
|
||||
* It will perform a flush and write frame epilogue.
|
||||
* The epilogue is required for decoders to consider a frame completed.
|
||||
* Similar to ZBUFF_compressFlush(), it may not be able to output the entire internal buffer content if *dstCapacityPtr is too small.
|
||||
* In which case, call again ZBUFF_compressFlush() to complete the flush.
|
||||
* @return : nb of bytes still present into internal buffer (0 if it's empty)
|
||||
* or an error code, which can be tested using ZBUFF_isError().
|
||||
*
|
||||
* Hint : _recommended buffer_ sizes (not compulsory) : ZBUFF_recommendedCInSize() / ZBUFF_recommendedCOutSize()
|
||||
* input : ZBUFF_recommendedCInSize==128 KB block size is the internal unit, use this value to reduce intermediate stages (better latency)
|
||||
* output : ZBUFF_recommendedCOutSize==ZSTD_compressBound(128 KB) + 3 + 3 : ensures it's always possible to write/flush/end a full block. Skip some buffering.
|
||||
* By using both, it ensures that input will be entirely consumed, and output will always contain the result, reducing intermediate buffering.
|
||||
* **************************************************/
|
||||
|
||||
|
||||
typedef ZSTD_DStream ZBUFF_DCtx;
|
||||
ZBUFF_DEPRECATED("use ZSTD_createDStream") ZBUFF_DCtx* ZBUFF_createDCtx(void);
|
||||
ZBUFF_DEPRECATED("use ZSTD_freeDStream") size_t ZBUFF_freeDCtx(ZBUFF_DCtx* dctx);
|
||||
|
||||
ZBUFF_DEPRECATED("use ZSTD_initDStream") size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx);
|
||||
ZBUFF_DEPRECATED("use ZSTD_initDStream_usingDict") size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* dctx, const void* dict, size_t dictSize);
|
||||
|
||||
ZBUFF_DEPRECATED("use ZSTD_decompressStream") size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx,
|
||||
void* dst, size_t* dstCapacityPtr,
|
||||
const void* src, size_t* srcSizePtr);
|
||||
|
||||
/*-***************************************************************************
|
||||
* Streaming decompression howto
|
||||
*
|
||||
* A ZBUFF_DCtx object is required to track streaming operations.
|
||||
* Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources.
|
||||
* Use ZBUFF_decompressInit() to start a new decompression operation,
|
||||
* or ZBUFF_decompressInitDictionary() if decompression requires a dictionary.
|
||||
* Note that ZBUFF_DCtx objects can be re-init multiple times.
|
||||
*
|
||||
* Use ZBUFF_decompressContinue() repetitively to consume your input.
|
||||
* *srcSizePtr and *dstCapacityPtr can be any size.
|
||||
* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr.
|
||||
* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again.
|
||||
* The content of `dst` will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change `dst`.
|
||||
* @return : 0 when a frame is completely decoded and fully flushed,
|
||||
* 1 when there is still some data left within internal buffer to flush,
|
||||
* >1 when more data is expected, with value being a suggested next input size (it's just a hint, which helps latency),
|
||||
* or an error code, which can be tested using ZBUFF_isError().
|
||||
*
|
||||
* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize() and ZBUFF_recommendedDOutSize()
|
||||
* output : ZBUFF_recommendedDOutSize== 128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded.
|
||||
* input : ZBUFF_recommendedDInSize == 128KB + 3;
|
||||
* just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
|
||||
* *******************************************************************************/
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Tool functions
|
||||
***************************************/
|
||||
ZBUFF_DEPRECATED("use ZSTD_isError") unsigned ZBUFF_isError(size_t errorCode);
|
||||
ZBUFF_DEPRECATED("use ZSTD_getErrorName") const char* ZBUFF_getErrorName(size_t errorCode);
|
||||
|
||||
/** Functions below provide recommended buffer sizes for Compression or Decompression operations.
|
||||
* These sizes are just hints, they tend to offer better latency */
|
||||
ZBUFF_DEPRECATED("use ZSTD_CStreamInSize") size_t ZBUFF_recommendedCInSize(void);
|
||||
ZBUFF_DEPRECATED("use ZSTD_CStreamOutSize") size_t ZBUFF_recommendedCOutSize(void);
|
||||
ZBUFF_DEPRECATED("use ZSTD_DStreamInSize") size_t ZBUFF_recommendedDInSize(void);
|
||||
ZBUFF_DEPRECATED("use ZSTD_DStreamOutSize") size_t ZBUFF_recommendedDOutSize(void);
|
||||
|
||||
#endif /* ZSTD_BUFFERED_H_23987 */
|
||||
|
||||
|
||||
#ifdef ZBUFF_STATIC_LINKING_ONLY
|
||||
#ifndef ZBUFF_STATIC_H_30298098432
|
||||
#define ZBUFF_STATIC_H_30298098432
|
||||
|
||||
/* ====================================================================================
|
||||
* The definitions in this section are considered experimental.
|
||||
* They should never be used in association with a dynamic library, as they may change in the future.
|
||||
* They are provided for advanced usages.
|
||||
* Use them only in association with static linking.
|
||||
* ==================================================================================== */
|
||||
|
||||
/*--- Dependency ---*/
|
||||
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters, ZSTD_customMem */
|
||||
#include "zstd.h"
|
||||
|
||||
|
||||
/*--- Custom memory allocator ---*/
|
||||
/*! ZBUFF_createCCtx_advanced() :
|
||||
* Create a ZBUFF compression context using external alloc and free functions */
|
||||
ZBUFF_DEPRECATED("use ZSTD_createCStream_advanced") ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem);
|
||||
|
||||
/*! ZBUFF_createDCtx_advanced() :
|
||||
* Create a ZBUFF decompression context using external alloc and free functions */
|
||||
ZBUFF_DEPRECATED("use ZSTD_createDStream_advanced") ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem);
|
||||
|
||||
|
||||
/*--- Advanced Streaming Initialization ---*/
|
||||
ZBUFF_DEPRECATED("use ZSTD_initDStream_usingDict") size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
|
||||
const void* dict, size_t dictSize,
|
||||
ZSTD_parameters params, unsigned long long pledgedSrcSize);
|
||||
|
||||
|
||||
#endif /* ZBUFF_STATIC_H_30298098432 */
|
||||
#endif /* ZBUFF_STATIC_LINKING_ONLY */
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
/*
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
/*-*************************************
|
||||
* Dependencies
|
||||
***************************************/
|
||||
#include "error_private.h"
|
||||
#include "zbuff.h"
|
||||
|
||||
/*-****************************************
|
||||
* ZBUFF Error Management (deprecated)
|
||||
******************************************/
|
||||
|
||||
/*! ZBUFF_isError() :
|
||||
* tells if a return value is an error code */
|
||||
unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); }
|
||||
/*! ZBUFF_getErrorName() :
|
||||
* provides error code string from function result (useful for debugging) */
|
||||
const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue