xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' of https://github.com/taosdata/TDengine into feat/TD-26174

This commit is contained in:
liuyao 2023-09-26 10:41:31 +08:00
parent 82e18e32b0 92f5803303
commit ca2325f886
179 changed files with 65421 additions and 161 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
cmake/cmake.define
View File

@ -83,6 +83,16 @@ ELSE ()
SET(TAOS_LIB taos)
ENDIF ()
# build TSZ by default
IF ("${TSZ_ENABLED}" MATCHES "false")
set(VAR_TSZ "" CACHE INTERNAL "global variant empty" )
ELSE()
# define add
MESSAGE(STATUS "build with TSZ enabled")
ADD_DEFINITIONS(-DTD_TSZ)
set(VAR_TSZ "TSZ" CACHE INTERNAL "global variant tsz" )
ENDIF()
IF (TD_WINDOWS)
MESSAGE("${Yellow} set compiler flag for Windows! ${ColourReset}")
SET(COMMON_FLAGS "/w /D_WIN32 /DWIN32 /Zi /MTd")

4
include/common/tglobal.h
View File

@ -145,6 +145,7 @@ extern bool tsUseAdapter;
extern int32_t tsMetaCacheMaxSize;
extern int32_t tsSlowLogThreshold;
extern int32_t tsSlowLogScope;
extern int32_t tsTimeSeriesThreshold;
// client
extern int32_t tsMinSlidingTime;
@ -159,10 +160,11 @@ extern char buildinfo[];
// lossy
extern char tsLossyColumns[];
extern double tsFPrecision;
extern float tsFPrecision;
extern double tsDPrecision;
extern uint32_t tsMaxRange;
extern uint32_t tsCurRange;
extern bool tsIfAdtFse;
extern char tsCompressor[];
// tfs

2
include/common/tgrant.h
View File

@ -30,6 +30,8 @@ extern "C" {
#define GRANTS_COL_MAX_LEN 196
#endif
#define GRANT_HEART_BEAT_MIN 2
typedef enum {
TSDB_GRANT_ALL,
TSDB_GRANT_TIME,

15
include/common/tmsg.h
View File

@ -1464,6 +1464,11 @@ typedef struct {
int32_t learnerProgress; // use one reservered
} SVnodeLoad;
typedef struct {
int32_t vgId;
int64_t nTimeSeries;
} SVnodeLoadLite;
typedef struct {
int8_t syncState;
int64_t syncTerm;
@ -1511,6 +1516,16 @@ int32_t tSerializeSStatusReq(void* buf, int32_t bufLen, SStatusReq* pReq);
int32_t tDeserializeSStatusReq(void* buf, int32_t bufLen, SStatusReq* pReq);
void tFreeSStatusReq(SStatusReq* pReq);
typedef struct {
int32_t dnodeId;
int64_t clusterId;
SArray* pVloads;
} SNotifyReq;
int32_t tSerializeSNotifyReq(void* buf, int32_t bufLen, SNotifyReq* pReq);
int32_t tDeserializeSNotifyReq(void* buf, int32_t bufLen, SNotifyReq* pReq);
void tFreeSNotifyReq(SNotifyReq* pReq);
typedef struct {
int32_t dnodeId;
int64_t clusterId;

5
include/common/tmsgdef.h
View File

@ -179,8 +179,7 @@ enum { // WARN: new msg should be appended to segment tail
TD_DEF_MSG_TYPE(TDMT_MND_STREAM_HEARTBEAT, "stream-heartbeat", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_RETRIEVE_IP_WHITE, "retrieve-ip-white", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_GET_USER_WHITELIST, "get-user-whitelist", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_NOTIFY, "notify", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_BALANCE_VGROUP_LEADER, "balance-vgroup-leader", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_RESTORE_DNODE, "restore-dnode", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_PAUSE_STREAM, "pause-stream", NULL, NULL)
@ -189,6 +188,8 @@ enum { // WARN: new msg should be appended to segment tail
TD_DEF_MSG_TYPE(TDMT_MND_STREAM_BEGIN_CHECKPOINT, "stream-begin-checkpoint", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_STREAM_NODECHANGE_CHECK, "stream-nodechange-check", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_TRIM_DB_TIMER, "trim-db-tmr", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_GRANT_NOTIFY, "grant-notify", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL)
TD_NEW_MSG_SEG(TDMT_VND_MSG)
TD_DEF_MSG_TYPE(TDMT_VND_SUBMIT, "submit", SSubmitReq, SSubmitRsp)

5
include/libs/executor/storageapi.h
View File

@ -146,6 +146,7 @@ typedef struct SSnapContext {
typedef struct {
int64_t uid;
int64_t ctbNum;
int32_t colNum;
} SMetaStbStats;
// void tqReaderSetColIdList(STqReader *pReader, SArray *pColIdList);
@ -289,8 +290,8 @@ typedef struct SStoreMeta {
// db name, vgId, numOfTables, numOfSTables
int32_t (*getNumOfChildTables)(
void* pVnode, int64_t uid,
int64_t* numOfTables); // int32_t metaGetStbStats(SMeta *pMeta, int64_t uid, SMetaStbStats *pInfo);
void* pVnode, int64_t uid, int64_t* numOfTables,
int32_t* numOfCols); // int32_t metaGetStbStats(SMeta *pMeta, int64_t uid, SMetaStbStats *pInfo);
void (*getBasicInfo)(void* pVnode, const char** dbname, int32_t* vgId, int64_t* numOfTables,
int64_t* numOfNormalTables); // vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId) &
// metaGetTbNum(SMeta *pMeta) & metaGetNtbNum(SMeta *pMeta);

2
include/libs/monitor/monitor.h
View File

@ -191,7 +191,7 @@ typedef struct {
} SMonBmInfo;
typedef struct {
SArray *pVloads; // SVnodeLoad
SArray *pVloads; // SVnodeLoad/SVnodeLoadLite
} SMonVloadInfo;
typedef struct {

1
include/libs/nodes/nodes.h
View File

@ -124,6 +124,7 @@ int32_t nodesListStrictAppendList(SNodeList* pTarget, SNodeList* pSrc);
int32_t nodesListPushFront(SNodeList* pList, SNode* pNode);
SListCell* nodesListErase(SNodeList* pList, SListCell* pCell);
void nodesListInsertList(SNodeList* pTarget, SListCell* pPos, SNodeList* pSrc);
void nodesListInsertListAfterPos(SNodeList* pTarget, SListCell* pPos, SNodeList* pSrc);
SNode* nodesListGetNode(SNodeList* pList, int32_t index);
SListCell* nodesListGetCell(SNodeList* pList, int32_t index);
void nodesDestroyList(SNodeList* pList);

3
include/libs/nodes/querynodes.h
View File

@ -536,6 +536,9 @@ int32_t nodesMergeConds(SNode** pDst, SNodeList** pSrc);
const char* operatorTypeStr(EOperatorType type);
const char* logicConditionTypeStr(ELogicConditionType type);
bool nodesIsStar(SNode* pNode);
bool nodesIsTableStar(SNode* pNode);
#ifdef __cplusplus
}
#endif

2
include/libs/parser/parser.h
View File

@ -64,7 +64,7 @@ typedef struct SParseContext {
SArray* pTableMetaPos; // sql table pos => catalog data pos
SArray* pTableVgroupPos; // sql table pos => catalog data pos
int64_t allocatorId;
int32_t biMode;
int8_t biMode;
} SParseContext;
int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery);

2
include/os/osThread.h
View File

@ -46,7 +46,7 @@ typedef HANDLE TdThreadMutexAttr; // windows api
typedef struct {
SRWLOCK lock;
int8_t excl;
} TdThreadRwlock; // pthread api
} TdThreadRwlock; // windows api
typedef pthread_attr_t TdThreadAttr; // pthread api
typedef pthread_once_t TdThreadOnce; // pthread api
typedef HANDLE TdThreadRwlockAttr; // windows api

51
include/td_sz.h Normal file
View File

@ -0,0 +1,51 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_SZ_H
#define _TD_SZ_H
#include "defines.h"
#ifdef __cplusplus
extern "C" {
#endif
void cost_start();
double cost_end(const char* tag);
//
// Init success return 1 else 0
//
void tdszInit(float fPrecision, double dPrecision, unsigned int maxIntervals, unsigned int intervals, int ifAdtFse, const char* compressor);
//
// compress interface to tdengine return value is count of output with bytes
//
int tdszCompress(int type, const char * input, const int nelements, const char * output);
//
// decompress interface to tdengine return value is count of output with bytes
//
int tdszDecompress(int type, const char * input, int compressedSize, const int nelements, const char * output);
//
// Exit call
//
void tdszExit();
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _SZ_H ----- */

9
include/util/tcompression.h
View File

@ -54,9 +54,16 @@ extern "C" {
#ifdef TD_TSZ
extern bool lossyFloat;
extern bool lossyDouble;
int32_t tsCompressInit();
int32_t tsCompressInit(char* lossyColumns, float fPrecision, double dPrecision, uint32_t maxIntervals, uint32_t intervals,
int32_t ifAdtFse, const char* compressor);
void tsCompressExit();
int32_t tsCompressFloatLossyImp(const char *const input, const int32_t nelements, char *const output);
int32_t tsDecompressFloatLossyImp(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output);
int32_t tsCompressDoubleLossyImp(const char *const input, const int32_t nelements, char *const output);
int32_t tsDecompressDoubleLossyImp(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output);
static FORCE_INLINE int32_t tsCompressFloatLossy(const char *const input, int32_t inputSize, const int32_t nelements,
char *const output, int32_t outputSize, char algorithm,
char *const buffer, int32_t bufferSize) {

5
include/util/tconfig.h
View File

@ -43,6 +43,7 @@ typedef enum {
CFG_DTYPE_INT32,
CFG_DTYPE_INT64,
CFG_DTYPE_FLOAT,
CFG_DTYPE_DOUBLE,
CFG_DTYPE_STRING,
CFG_DTYPE_DIR,
CFG_DTYPE_LOCALE,
@ -64,6 +65,7 @@ typedef struct SConfigItem {
union {
bool bval;
float fval;
double dval;
int32_t i32;
int64_t i64;
char *str;
@ -101,7 +103,8 @@ int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfg
int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal, int8_t scope);
int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, int8_t scope);
int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval, int8_t scope);
int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, double minval, double maxval, int8_t scope);
int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, float minval, float maxval, int8_t scope);
int32_t cfgAddDouble(SConfig *pCfg, const char *name, double defaultVal, double minval, double maxval, int8_t scope);
int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope);
int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope);
int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope);

35
packaging/tools/install.sh
View File

@ -33,14 +33,17 @@ adapterName="taosadapter"
benchmarkName="taosBenchmark"
dumpName="taosdump"
demoName="taosdemo"
xname="taosx"
clientName2="taos"
serverName2="${clientName2}d"
configFile2="${clientName2}.cfg"
productName2="TDengine"
emailName2="taosdata.com"
xname2="${clientName2}x"
adapterName2="${clientName2}adapter"
explorerName="${clientName2}-explorer"
benchmarkName2="${clientName2}Benchmark"
demoName2="${clientName2}demo"
dumpName2="${clientName2}dump"
@ -235,6 +238,12 @@ function install_bin() {
[ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || :
if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then
${csudo}rm -f ${bin_link_dir}/${xname2} || :
${csudo}rm -f ${bin_link_dir}/${explorerName} || :
#Make link
[ -x ${install_main_dir}/bin/${xname2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${xname2} ${bin_link_dir}/${xname2} || :
[ -x ${install_main_dir}/bin/${explorerName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${explorerName} ${bin_link_dir}/${explorerName} || :
[ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript2} || :
fi
}
@ -693,9 +702,29 @@ function clean_service_on_systemd() {
fi
${csudo}systemctl disable tarbitratord &>/dev/null || echo &>/dev/null
${csudo}rm -f ${tarbitratord_service_config}
# if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then
# ${csudo}rm -f ${service_config_dir}/${serverName2}.service
# fi
if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then
x_service_config="${service_config_dir}/${xName2}.service"
if [ -e "$x_service_config" ]; then
if systemctl is-active --quiet ${xName2}; then
echo "${productName2} ${xName2} is running, stopping it..."
${csudo}systemctl stop ${xName2} &>/dev/null || echo &>/dev/null
fi
${csudo}systemctl disable ${xName2} &>/dev/null || echo &>/dev/null
${csudo}rm -f ${x_service_config}
fi
explorer_service_config="${service_config_dir}/${explorerName2}.service"
if [ -e "$explorer_service_config" ]; then
if systemctl is-active --quiet ${explorerName2}; then
echo "${productName2} ${explorerName2} is running, stopping it..."
${csudo}systemctl stop ${explorerName2} &>/dev/null || echo &>/dev/null
fi
${csudo}systemctl disable ${explorerName2} &>/dev/null || echo &>/dev/null
${csudo}rm -f ${explorer_service_config}
${csudo}rm -f /etc/${clientName2}/explorer.toml
fi
fi
}
function install_service_on_systemd() {

45
packaging/tools/remove.sh
View File

@ -123,10 +123,11 @@ function clean_bin() {
${csudo}rm -f ${bin_link_dir}/set_core || :
${csudo}rm -f ${bin_link_dir}/TDinsight.sh || :
${csudo}rm -f ${bin_link_dir}/${keeperName2} || :
# ${csudo}rm -f ${bin_link_dir}/${xName2} || :
# ${csudo}rm -f ${bin_link_dir}/${explorerName2} || :
if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then
${csudo}rm -f ${bin_link_dir}/${xName2} || :
${csudo}rm -f ${bin_link_dir}/${explorerName2} || :
${csudo}rm -f ${bin_link_dir}/${clientName2} || :
${csudo}rm -f ${bin_link_dir}/${benchmarkName2} || :
${csudo}rm -f ${bin_link_dir}/${dumpName2} || :
@ -195,26 +196,28 @@ function clean_service_on_systemd() {
fi
${csudo}systemctl disable ${tarbitrator_service_name} &>/dev/null || echo &>/dev/null
# x_service_config="${service_config_dir}/${xName2}.service"
# if [ -e "$x_service_config" ]; then
# if systemctl is-active --quiet ${xName2}; then
# echo "${productName2} ${xName2} is running, stopping it..."
# ${csudo}systemctl stop ${xName2} &>/dev/null || echo &>/dev/null
# fi
# ${csudo}systemctl disable ${xName2} &>/dev/null || echo &>/dev/null
# ${csudo}rm -f ${x_service_config}
# fi
if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then
x_service_config="${service_config_dir}/${xName2}.service"
if [ -e "$x_service_config" ]; then
if systemctl is-active --quiet ${xName2}; then
echo "${productName2} ${xName2} is running, stopping it..."
${csudo}systemctl stop ${xName2} &>/dev/null || echo &>/dev/null
fi
${csudo}systemctl disable ${xName2} &>/dev/null || echo &>/dev/null
${csudo}rm -f ${x_service_config}
fi
# explorer_service_config="${service_config_dir}/${explorerName2}.service"
# if [ -e "$explorer_service_config" ]; then
# if systemctl is-active --quiet ${explorerName2}; then
# echo "${productName2} ${explorerName2} is running, stopping it..."
# ${csudo}systemctl stop ${explorerName2} &>/dev/null || echo &>/dev/null
# fi
# ${csudo}systemctl disable ${explorerName2} &>/dev/null || echo &>/dev/null
# ${csudo}rm -f ${explorer_service_config}
# ${csudo}rm -f /etc/${clientName2}/explorer.toml
# fi
explorer_service_config="${service_config_dir}/${explorerName2}.service"
if [ -e "$explorer_service_config" ]; then
if systemctl is-active --quiet ${explorerName2}; then
echo "${productName2} ${explorerName2} is running, stopping it..."
${csudo}systemctl stop ${explorerName2} &>/dev/null || echo &>/dev/null
fi
${csudo}systemctl disable ${explorerName2} &>/dev/null || echo &>/dev/null
${csudo}rm -f ${explorer_service_config}
${csudo}rm -f /etc/${clientName2}/explorer.toml
fi
fi
}
function clean_service_on_sysvinit() {

32
source/common/src/tglobal.c
View File

@ -20,6 +20,7 @@
#include "tgrant.h"
#include "tlog.h"
#include "tmisce.h"
#include "defines.h"
#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL)
#include "cus_name.h"
@ -144,6 +145,7 @@ bool tsUseAdapter = false;
int32_t tsMetaCacheMaxSize = -1; // MB
int32_t tsSlowLogThreshold = 3; // seconds
int32_t tsSlowLogScope = SLOW_LOG_TYPE_ALL;
int32_t tsTimeSeriesThreshold = 50;
/*
* denote if the server needs to compress response message at the application layer to client, including query rsp,
@ -211,14 +213,15 @@ int64_t tsTickPerMin[] = {60000L, 60000000L, 60000000000L};
*/
int64_t tsTickPerHour[] = {3600000L, 3600000000L, 3600000000000L};
// lossy compress 6
// lossy compress 7
char tsLossyColumns[32] = ""; // "float|double" means all float and double columns can be lossy compressed. set empty
// can close lossy compress.
// below option can take effect when tsLossyColumns not empty
double tsFPrecision = 1E-8; // float column precision
float tsFPrecision = 1E-8; // float column precision
double tsDPrecision = 1E-16; // double column precision
uint32_t tsMaxRange = 500; // max range
uint32_t tsCurRange = 100; // range
uint32_t tsMaxRange = 500; // max quantization intervals
uint32_t tsCurRange = 100; // current quantization intervals
bool tsIfAdtFse = false; // ADT-FSE algorithom or original huffman algorithom
char tsCompressor[32] = "ZSTD_COMPRESSOR"; // ZSTD_COMPRESSOR or GZIP_COMPRESSOR
// udf
@ -635,6 +638,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "trimVDbIntervalSec", tsTrimVDbIntervalSec, 1, 100000, CFG_SCOPE_SERVER) != 0) return -1;
if (cfgAddInt32(pCfg, "uptimeInterval", tsUptimeInterval, 1, 100000, CFG_SCOPE_SERVER) != 0) return -1;
if (cfgAddInt32(pCfg, "queryRsmaTolerance", tsQueryRsmaTolerance, 0, 900000, CFG_SCOPE_SERVER) != 0) return -1;
if (cfgAddInt32(pCfg, "timeseriesThreshold", tsTimeSeriesThreshold, 0, 2000, CFG_SCOPE_SERVER) != 0) return -1;
if (cfgAddInt64(pCfg, "walFsyncDataSizeLimit", tsWalFsyncDataSizeLimit, 100 * 1024 * 1024, INT64_MAX,
CFG_SCOPE_SERVER) != 0)
@ -651,6 +655,14 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "cacheLazyLoadThreshold", tsCacheLazyLoadThreshold, 0, 100000, CFG_SCOPE_SERVER) != 0)
return -1;
if (cfgAddString(pCfg, "LossyColumns", tsLossyColumns, CFG_SCOPE_SERVER) != 0) return -1;
if (cfgAddFloat(pCfg, "FPrecision", tsFPrecision, 0.0f, 100000.0f, CFG_SCOPE_SERVER) != 0) return -1;
if (cfgAddDouble(pCfg, "DPrecision", tsDPrecision, 0.0f, 1000000.0f, CFG_SCOPE_SERVER) != 0) return -1;
if (cfgAddInt32(pCfg, "MaxRange", tsMaxRange, 0, 65536, CFG_SCOPE_SERVER) != 0) return -1;
if (cfgAddInt32(pCfg, "CurRange", tsCurRange, 0, 65536, CFG_SCOPE_SERVER) != 0) return -1;
if (cfgAddBool(pCfg, "IfAdtFse", tsIfAdtFse, CFG_SCOPE_SERVER) != 0) return -1;
if (cfgAddString(pCfg, "Compressor", tsCompressor, CFG_SCOPE_SERVER) != 0) return -1;
if (cfgAddBool(pCfg, "filterScalarMode", tsFilterScalarMode, CFG_SCOPE_SERVER) != 0) return -1;
if (cfgAddInt32(pCfg, "keepTimeOffset", tsKeepTimeOffset, 0, 23, CFG_SCOPE_SERVER) != 0) return -1;
if (cfgAddInt32(pCfg, "maxStreamBackendCache", tsMaxStreamBackendCache, 16, 1024, CFG_SCOPE_SERVER) != 0) return -1;
@ -1043,6 +1055,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsTrimVDbIntervalSec = cfgGetItem(pCfg, "trimVDbIntervalSec")->i32;
tsUptimeInterval = cfgGetItem(pCfg, "uptimeInterval")->i32;
tsQueryRsmaTolerance = cfgGetItem(pCfg, "queryRsmaTolerance")->i32;
tsTimeSeriesThreshold = cfgGetItem(pCfg, "timeseriesThreshold")->i32;
tsWalFsyncDataSizeLimit = cfgGetItem(pCfg, "walFsyncDataSizeLimit")->i64;
@ -1067,6 +1080,15 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsCacheLazyLoadThreshold = cfgGetItem(pCfg, "cacheLazyLoadThreshold")->i32;
tstrncpy(tsLossyColumns, cfgGetItem(pCfg, "LossyColumns")->str, sizeof(tsLossyColumns));
tsFPrecision = cfgGetItem(pCfg, "FPrecision")->fval;
tsDPrecision = cfgGetItem(pCfg, "DPrecision")->dval;
tsMaxRange = cfgGetItem(pCfg, "MaxRange")->i32;
tsCurRange = cfgGetItem(pCfg, "CurRange")->i32;
tsIfAdtFse = cfgGetItem(pCfg, "IfAdtFse")->bval;
tstrncpy(tsCompressor, cfgGetItem(pCfg, "Compressor")->str, sizeof(tsCompressor));
tsDisableStream = cfgGetItem(pCfg, "disableStream")->bval;
tsStreamBufferSize = cfgGetItem(pCfg, "streamBufferSize")->i64;
@ -1458,6 +1480,8 @@ int32_t taosApplyLocalCfg(SConfig *pCfg, char *name) {
tqDebugFlag = cfgGetItem(pCfg, "tqDebugFlag")->i32;
} else if (strcasecmp("ttlFlushThreshold", name) == 0) {
tsTtlFlushThreshold = cfgGetItem(pCfg, "ttlFlushThreshold")->i32;
} else if (strcasecmp("timeseriesThreshold", name) == 0) {
tsTimeSeriesThreshold = cfgGetItem(pCfg, "timeseriesThreshold")->i32;
}
break;
}

62
source/common/src/tmsg.c
View File

@ -1035,6 +1035,68 @@ int32_t tDeserializeSMDropFullTextReq(void *buf, int32_t bufLen, SMDropFullTextR
return 0;
}
int32_t tSerializeSNotifyReq(void *buf, int32_t bufLen, SNotifyReq *pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI32(&encoder, pReq->dnodeId) < 0) return -1;
if (tEncodeI64(&encoder, pReq->clusterId) < 0) return -1;
int32_t nVgroup = taosArrayGetSize(pReq->pVloads);
if (tEncodeI32(&encoder, nVgroup) < 0) return -1;
for (int32_t i = 0; i < nVgroup; ++i) {
SVnodeLoadLite *vload = TARRAY_GET_ELEM(pReq->pVloads, i);
if (tEncodeI32(&encoder, vload->vgId) < 0) return -1;
if (tEncodeI64(&encoder, vload->nTimeSeries) < 0) return -1;
}
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSNotifyReq(void *buf, int32_t bufLen, SNotifyReq *pReq) {
int32_t code = TSDB_CODE_INVALID_MSG;
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) goto _exit;
if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) goto _exit;
if (tDecodeI64(&decoder, &pReq->clusterId) < 0) goto _exit;
int32_t nVgroup = 0;
if (tDecodeI32(&decoder, &nVgroup) < 0) goto _exit;
if (nVgroup > 0) {
pReq->pVloads = taosArrayInit(nVgroup, sizeof(SVnodeLoadLite));
if (!pReq->pVloads) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
for (int32_t i = 0; i < nVgroup; ++i) {
SVnodeLoadLite vload;
if (tDecodeI32(&decoder, &(vload.vgId)) < 0) goto _exit;
if (tDecodeI64(&decoder, &(vload.nTimeSeries)) < 0) goto _exit;
taosArrayPush(pReq->pVloads, &vload);
}
}
code = 0;
_exit:
tEndDecode(&decoder);
tDecoderClear(&decoder);
return code;
}
void tFreeSNotifyReq(SNotifyReq *pReq) {
if (pReq) {
taosArrayDestroy(pReq->pVloads);
}
}
int32_t tSerializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);

6
source/dnode/mgmt/mgmt_dnode/inc/dmInt.h
View File

@ -28,6 +28,7 @@ typedef struct SDnodeMgmt {
const char *path;
const char *name;
TdThread statusThread;
TdThread notifyThread;
TdThread monitorThread;
TdThread crashReportThread;
SSingleWorker mgmtWorker;
@ -36,6 +37,7 @@ typedef struct SDnodeMgmt {
ProcessDropNodeFp processDropNodeFp;
SendMonitorReportFp sendMonitorReportFp;
GetVnodeLoadsFp getVnodeLoadsFp;
GetVnodeLoadsFp getVnodeLoadsLiteFp;
GetMnodeLoadsFp getMnodeLoadsFp;
GetQnodeLoadsFp getQnodeLoadsFp;
int32_t statusSeq;
@ -44,17 +46,21 @@ typedef struct SDnodeMgmt {
// dmHandle.c
SArray *dmGetMsgHandles();
void dmSendStatusReq(SDnodeMgmt *pMgmt);
void dmSendNotifyReq(SDnodeMgmt *pMgmt);
int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t dmProcessAuthRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t dmProcessServerRunStatus(SDnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t dmProcessGrantReq(void *pInfo, SRpcMsg *pMsg);
int32_t dmProcessGrantNotify(void *pInfo, SRpcMsg *pMsg);
// dmWorker.c
int32_t dmPutNodeMsgToMgmtQueue(SDnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t dmStartStatusThread(SDnodeMgmt *pMgmt);
void dmStopStatusThread(SDnodeMgmt *pMgmt);
int32_t dmStartNotifyThread(SDnodeMgmt *pMgmt);
void dmStopNotifyThread(SDnodeMgmt *pMgmt);
int32_t dmStartMonitorThread(SDnodeMgmt *pMgmt);
void dmStopMonitorThread(SDnodeMgmt *pMgmt);
int32_t dmStartCrashReportThread(SDnodeMgmt *pMgmt);

31
source/dnode/mgmt/mgmt_dnode/src/dmHandle.c
View File

@ -170,6 +170,36 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) {
dmProcessStatusRsp(pMgmt, &rpcRsp);
}
void dmSendNotifyReq(SDnodeMgmt *pMgmt) {
SNotifyReq req = {0};
taosThreadRwlockRdlock(&pMgmt->pData->lock);
req.dnodeId = pMgmt->pData->dnodeId;
taosThreadRwlockUnlock(&pMgmt->pData->lock);
req.clusterId = pMgmt->pData->clusterId;
SMonVloadInfo vinfo = {0};
(*pMgmt->getVnodeLoadsLiteFp)(&vinfo);
req.pVloads = vinfo.pVloads;
int32_t contLen = tSerializeSNotifyReq(NULL, 0, &req);
void *pHead = rpcMallocCont(contLen);
tSerializeSNotifyReq(pHead, contLen, &req);
tFreeSNotifyReq(&req);
SRpcMsg rpcMsg = {.pCont = pHead,
.contLen = contLen,
.msgType = TDMT_MND_NOTIFY,
.info.ahandle = (void *)0x9527,
.info.refId = 0,
.info.noResp = 1};
SEpSet epSet = {0};
dmGetMnodeEpSet(pMgmt->pData, &epSet);
rpcSendRequest(pMgmt->msgCb.clientRpc, &epSet, &rpcMsg, NULL);
}
int32_t dmProcessAuthRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
dError("auth rsp is received, but not supported yet");
return 0;
@ -395,6 +425,7 @@ SArray *dmGetMsgHandles() {
// Requests handled by MNODE
if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT_NOTIFY, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_AUTH_RSP, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;
code = 0;

7
source/dnode/mgmt/mgmt_dnode/src/dmInt.c
View File

@ -20,6 +20,11 @@ static int32_t dmStartMgmt(SDnodeMgmt *pMgmt) {
if (dmStartStatusThread(pMgmt) != 0) {
return -1;
}
#if defined(TD_ENTERPRISE) && !defined(_TD_DARWIN_64)
if (dmStartNotifyThread(pMgmt) != 0) {
return -1;
}
#endif
if (dmStartMonitorThread(pMgmt) != 0) {
return -1;
}
@ -33,6 +38,7 @@ static void dmStopMgmt(SDnodeMgmt *pMgmt) {
pMgmt->pData->stopped = true;
dmStopMonitorThread(pMgmt);
dmStopStatusThread(pMgmt);
dmStopNotifyThread(pMgmt);
dmStopCrashReportThread(pMgmt);
}
@ -52,6 +58,7 @@ static int32_t dmOpenMgmt(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
pMgmt->processDropNodeFp = pInput->processDropNodeFp;
pMgmt->sendMonitorReportFp = pInput->sendMonitorReportFp;
pMgmt->getVnodeLoadsFp = pInput->getVnodeLoadsFp;
pMgmt->getVnodeLoadsLiteFp = pInput->getVnodeLoadsLiteFp;
pMgmt->getMnodeLoadsFp = pInput->getMnodeLoadsFp;
pMgmt->getQnodeLoadsFp = pInput->getQnodeLoadsFp;

48
source/dnode/mgmt/mgmt_dnode/src/dmWorker.c
View File

@ -53,6 +53,26 @@ static void *dmStatusThreadFp(void *param) {
return NULL;
}
tsem_t dmNotifySem;
static void *dmNotifyThreadFp(void *param) {
SDnodeMgmt *pMgmt = param;
int64_t lastTime = taosGetTimestampMs();
setThreadName("dnode-notify");
if (tsem_init(&dmNotifySem, 0, 0) != 0) {
return NULL;
}
while (1) {
if (pMgmt->pData->dropped || pMgmt->pData->stopped) break;
tsem_wait(&dmNotifySem);
dmSendNotifyReq(pMgmt);
}
return NULL;
}
static void *dmMonitorThreadFp(void *param) {
SDnodeMgmt *pMgmt = param;
int64_t lastTime = taosGetTimestampMs();
@ -132,7 +152,6 @@ static void *dmCrashReportThreadFp(void *param) {
return NULL;
}
int32_t dmStartStatusThread(SDnodeMgmt *pMgmt) {
TdThreadAttr thAttr;
taosThreadAttrInit(&thAttr);
@ -154,6 +173,29 @@ void dmStopStatusThread(SDnodeMgmt *pMgmt) {
}
}
int32_t dmStartNotifyThread(SDnodeMgmt *pMgmt) {
TdThreadAttr thAttr;
taosThreadAttrInit(&thAttr);
taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
if (taosThreadCreate(&pMgmt->notifyThread, &thAttr, dmNotifyThreadFp, pMgmt) != 0) {
dError("failed to create notify thread since %s", strerror(errno));
return -1;
}
taosThreadAttrDestroy(&thAttr);
tmsgReportStartup("dnode-notify", "initialized");
return 0;
}
void dmStopNotifyThread(SDnodeMgmt *pMgmt) {
if (taosCheckPthreadValid(pMgmt->notifyThread)) {
tsem_post(&dmNotifySem);
taosThreadJoin(pMgmt->notifyThread, NULL);
taosThreadClear(&pMgmt->notifyThread);
}
tsem_destroy(&dmNotifySem);
}
int32_t dmStartMonitorThread(SDnodeMgmt *pMgmt) {
TdThreadAttr thAttr;
taosThreadAttrInit(&thAttr);
@ -204,7 +246,6 @@ void dmStopCrashReportThread(SDnodeMgmt *pMgmt) {
}
}
static void dmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
SDnodeMgmt *pMgmt = pInfo->ahandle;
int32_t code = -1;
@ -251,6 +292,9 @@ static void dmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
case TDMT_MND_GRANT:
code = dmProcessGrantReq(&pMgmt->pData->clusterId, pMsg);
break;
case TDMT_MND_GRANT_NOTIFY:
code = dmProcessGrantNotify(NULL, pMsg);
break;
default:
terrno = TSDB_CODE_MSG_NOT_PROCESSED;
dGError("msg:%p, not processed in mgmt queue", pMsg);

1
source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
View File

@ -178,6 +178,7 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_CONN, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_HEARTBEAT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STATUS, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_NOTIFY, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_SYSTABLE_RETRIEVE, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_AUTH, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_SHOW_VARIABLES, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;

22
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -40,6 +40,28 @@ void vmGetVnodeLoads(SVnodeMgmt *pMgmt, SMonVloadInfo *pInfo, bool isReset) {
taosThreadRwlockUnlock(&pMgmt->lock);
}
void vmGetVnodeLoadsLite(SVnodeMgmt *pMgmt, SMonVloadInfo *pInfo) {
pInfo->pVloads = taosArrayInit(pMgmt->state.totalVnodes, sizeof(SVnodeLoadLite));
if (!pInfo->pVloads) return;
taosThreadRwlockRdlock(&pMgmt->lock);
void *pIter = taosHashIterate(pMgmt->hash, NULL);
while (pIter) {
SVnodeObj **ppVnode = pIter;
if (ppVnode == NULL || *ppVnode == NULL) continue;
SVnodeObj *pVnode = *ppVnode;
SVnodeLoadLite vload = {0};
if (vnodeGetLoadLite(pVnode->pImpl, &vload) == 0) {
taosArrayPush(pInfo->pVloads, &vload);
}
pIter = taosHashIterate(pMgmt->hash, pIter);
}
taosThreadRwlockUnlock(&pMgmt->lock);
}
void vmGetMonitorInfo(SVnodeMgmt *pMgmt, SMonVmInfo *pInfo) {
SMonVloadInfo vloads = {0};
vmGetVnodeLoads(pMgmt, &vloads, true);

1
source/dnode/mgmt/node_mgmt/inc/dmMgmt.h
View File

@ -119,6 +119,7 @@ int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
// dmMonitor.c
void dmSendMonitorReport();
void dmGetVnodeLoads(SMonVloadInfo *pInfo);
void dmGetVnodeLoadsLite(SMonVloadInfo *pInfo);
void dmGetMnodeLoads(SMonMloadInfo *pInfo);
void dmGetQnodeLoads(SQnodeLoad *pInfo);

1
source/dnode/mgmt/node_mgmt/inc/dmNodes.h
View File

@ -35,6 +35,7 @@ void smGetMonitorInfo(void *pMgmt, SMonSmInfo *pInfo);
void bmGetMonitorInfo(void *pMgmt, SMonBmInfo *pInfo);
void vmGetVnodeLoads(void *pMgmt, SMonVloadInfo *pInfo, bool isReset);
void vmGetVnodeLoadsLite(void *pMgmt, SMonVloadInfo *pInfo);
void mmGetMnodeLoads(void *pMgmt, SMonMloadInfo *pInfo);
void qmGetQnodeLoads(void *pMgmt, SQnodeLoad *pInfo);

1
source/dnode/mgmt/node_mgmt/src/dmEnv.c
View File

@ -419,6 +419,7 @@ SMgmtInputOpt dmBuildMgmtInputOpt(SMgmtWrapper *pWrapper) {
.processDropNodeFp = dmProcessDropNodeReq,
.sendMonitorReportFp = dmSendMonitorReport,
.getVnodeLoadsFp = dmGetVnodeLoads,
.getVnodeLoadsLiteFp = dmGetVnodeLoadsLite,
.getMnodeLoadsFp = dmGetMnodeLoads,
.getQnodeLoadsFp = dmGetQnodeLoads,
};

15
source/dnode/mgmt/node_mgmt/src/dmMgmt.c
View File

@ -19,6 +19,10 @@
#include "index.h"
#include "qworker.h"
#include "tstream.h"
#ifdef TD_TSZ
#include "tglobal.h"
#include "tcompression.h"
#endif
static bool dmRequireNode(SDnode *pDnode, SMgmtWrapper *pWrapper) {
SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper);
@ -111,6 +115,11 @@ int32_t dmInitDnode(SDnode *pDnode) {
goto _OVER;
}
#ifdef TD_TSZ
// compress module init
tsCompressInit(tsLossyColumns, tsFPrecision, tsDPrecision, tsMaxRange, tsCurRange, (int)tsIfAdtFse, tsCompressor);
#endif
pDnode->wrappers[DNODE].func = dmGetMgmtFunc();
pDnode->wrappers[MNODE].func = mmGetMgmtFunc();
pDnode->wrappers[VNODE].func = vmGetMgmtFunc();
@ -180,6 +189,12 @@ void dmCleanupDnode(SDnode *pDnode) {
streamMetaCleanup();
indexCleanup();
taosConvDestroy();
#ifdef TD_TSZ
// compress destroy
tsCompressExit();
#endif
dDebug("dnode is closed, ptr:%p", pDnode);
}

11
source/dnode/mgmt/node_mgmt/src/dmMonitor.c
View File

@ -119,6 +119,17 @@ void dmGetVnodeLoads(SMonVloadInfo *pInfo) {
}
}
void dmGetVnodeLoadsLite(SMonVloadInfo *pInfo) {
SDnode *pDnode = dmInstance();
SMgmtWrapper *pWrapper = &pDnode->wrappers[VNODE];
if (dmMarkWrapper(pWrapper) == 0) {
if (pWrapper->pMgmt != NULL) {
vmGetVnodeLoadsLite(pWrapper->pMgmt, pInfo);
}
dmReleaseWrapper(pWrapper);
}
}
void dmGetMnodeLoads(SMonMloadInfo *pInfo) {
SDnode *pDnode = dmInstance();
SMgmtWrapper *pWrapper = &pDnode->wrappers[MNODE];

1
source/dnode/mgmt/node_util/inc/dmUtil.h
View File

@ -121,6 +121,7 @@ typedef struct {
ProcessDropNodeFp processDropNodeFp;
SendMonitorReportFp sendMonitorReportFp;
GetVnodeLoadsFp getVnodeLoadsFp;
GetVnodeLoadsFp getVnodeLoadsLiteFp;
GetMnodeLoadsFp getMnodeLoadsFp;
GetQnodeLoadsFp getQnodeLoadsFp;
} SMgmtInputOpt;

47
source/dnode/mnode/impl/src/mndDnode.c
View File

@ -71,6 +71,7 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq);
static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq);
static int32_t mndProcessConfigDnodeRsp(SRpcMsg *pRsp);
static int32_t mndProcessStatusReq(SRpcMsg *pReq);
static int32_t mndProcessNotifyReq(SRpcMsg *pReq);
static int32_t mndProcessRestoreDnodeReq(SRpcMsg *pReq);
static int32_t mndRetrieveConfigs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
@ -80,6 +81,12 @@ static void mndCancelGetNextDnode(SMnode *pMnode, void *pIter);
static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pInMCfgReq, int32_t opLen, int32_t *pOutValue);
#ifdef _GRANT
int32_t mndUpdClusterInfo(SRpcMsg *pReq);
#else
static int32_t mndUpdClusterInfo(SRpcMsg *pReq) { return 0; }
#endif
int32_t mndInitDnode(SMnode *pMnode) {
SSdbTable table = {
.sdbType = SDB_DNODE,
@ -97,6 +104,7 @@ int32_t mndInitDnode(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_MND_CONFIG_DNODE, mndProcessConfigDnodeReq);
mndSetMsgHandle(pMnode, TDMT_DND_CONFIG_DNODE_RSP, mndProcessConfigDnodeRsp);
mndSetMsgHandle(pMnode, TDMT_MND_STATUS, mndProcessStatusReq);
mndSetMsgHandle(pMnode, TDMT_MND_NOTIFY, mndProcessNotifyReq);
mndSetMsgHandle(pMnode, TDMT_MND_DNODE_LIST, mndProcessDnodeListReq);
mndSetMsgHandle(pMnode, TDMT_MND_SHOW_VARIABLES, mndProcessShowVariablesReq);
mndSetMsgHandle(pMnode, TDMT_MND_RESTORE_DNODE, mndProcessRestoreDnodeReq);
@ -543,6 +551,10 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
mGTrace("dnode:%d, status received, accessTimes:%d check:%d online:%d reboot:%d changed:%d statusSeq:%d", pDnode->id,
pDnode->accessTimes, needCheck, online, reboot, dnodeChanged, statusReq.statusSeq);
if (reboot) {
tsGrantHBInterval = GRANT_HEART_BEAT_MIN;
}
for (int32_t v = 0; v < taosArrayGetSize(statusReq.pVloads); ++v) {
SVnodeLoad *pVload = taosArrayGet(statusReq.pVloads, v);
@ -676,6 +688,41 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
_OVER:
mndReleaseDnode(pMnode, pDnode);
taosArrayDestroy(statusReq.pVloads);
mndUpdClusterInfo(pReq);
return code;
}
static int32_t mndProcessNotifyReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
SNotifyReq notifyReq = {0};
int32_t code = 0;
if ((code = tDeserializeSNotifyReq(pReq->pCont, pReq->contLen, &notifyReq)) != 0) {
terrno = code;
goto _OVER;
}
int64_t clusterid = mndGetClusterId(pMnode);
if (notifyReq.clusterId != 0 && notifyReq.clusterId != clusterid) {
code = TSDB_CODE_MND_DNODE_DIFF_CLUSTER;
mWarn("dnode:%d, its clusterid:%" PRId64 " differ from current cluster:%" PRId64 " since %s", notifyReq.dnodeId,
notifyReq.clusterId, clusterid, tstrerror(code));
goto _OVER;
}
int32_t nVgroup = taosArrayGetSize(notifyReq.pVloads);
for (int32_t v = 0; v < nVgroup; ++v) {
SVnodeLoadLite *pVload = taosArrayGet(notifyReq.pVloads, v);
SVgObj *pVgroup = mndAcquireVgroup(pMnode, pVload->vgId);
if (pVgroup != NULL) {
pVgroup->numOfTimeSeries = pVload->nTimeSeries;
mndReleaseVgroup(pMnode, pVgroup);
}
}
_OVER:
mndUpdClusterInfo(pReq);
tFreeSNotifyReq(&notifyReq);
return code;
}

1
source/dnode/mnode/impl/src/mndGrant.c
View File

@ -130,6 +130,7 @@ void grantReset(SMnode *pMnode, EGrantType grant, uint64_t value) {}
void grantAdd(EGrantType grant, uint64_t value) {}
void grantRestore(EGrantType grant, uint64_t value) {}
int32_t dmProcessGrantReq(void *pInfo, SRpcMsg *pMsg) { return TSDB_CODE_SUCCESS; }
int32_t dmProcessGrantNotify(void *pInfo, SRpcMsg *pMsg) { return TSDB_CODE_SUCCESS; }
#endif

4
source/dnode/mnode/impl/src/mndStb.c
View File

@ -1678,6 +1678,10 @@ static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray
return -1;
}
if ((terrno = grantCheck(TSDB_GRANT_TIMESERIES)) != 0) {
return -1;
}
pNew->numOfColumns = pNew->numOfColumns + ncols;
if (mndAllocStbSchemas(pOld, pNew) != 0) {
return -1;

8
source/dnode/vnode/inc/vnode.h
View File

@ -86,11 +86,13 @@ void *vnodeGetIdx(void *pVnode);
void *vnodeGetIvtIdx(void *pVnode);
int32_t vnodeGetCtbNum(SVnode *pVnode, int64_t suid, int64_t *num);
int32_t vnodeGetStbColumnNum(SVnode *pVnode, tb_uid_t suid, int *num);
int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num);
int32_t vnodeGetAllCtbNum(SVnode *pVnode, int64_t *num);
void vnodeResetLoad(SVnode *pVnode, SVnodeLoad *pLoad);
int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad);
int32_t vnodeGetLoadLite(SVnode *pVnode, SVnodeLoadLite *pLoad);
int32_t vnodeValidateTableHash(SVnode *pVnode, char *tableFName);
int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg);
@ -134,7 +136,7 @@ bool metaTbInFilterCache(void *pVnode, tb_uid_t suid, int8_t type);
int32_t metaPutTbToFilterCache(void *pVnode, tb_uid_t suid, int8_t type);
int32_t metaSizeOfTbFilterCache(void *pVnode, int8_t type);
int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables);
int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables, int32_t *numOfCols);
// tsdb
typedef struct STsdbReader STsdbReader;
@ -288,10 +290,10 @@ typedef struct {
int64_t numOfSTables;
int64_t numOfCTables;
int64_t numOfNTables;
int64_t numOfCmprTables;
int64_t numOfReportedTimeSeries;
int64_t numOfNTimeSeries;
int64_t numOfTimeSeries;
int64_t itvTimeSeries;
// int64_t itvTimeSeries;
int64_t pointsWritten;
int64_t totalStorage;
int64_t compStorage;

2
source/dnode/vnode/src/inc/meta.h
View File

@ -71,7 +71,7 @@ int32_t metaCacheDrop(SMeta* pMeta, int64_t uid);
int32_t metaStatsCacheUpsert(SMeta* pMeta, SMetaStbStats* pInfo);
int32_t metaStatsCacheDrop(SMeta* pMeta, int64_t uid);
int32_t metaStatsCacheGet(SMeta* pMeta, int64_t uid, SMetaStbStats* pInfo);
void metaUpdateStbStats(SMeta* pMeta, int64_t uid, int64_t delta);
void metaUpdateStbStats(SMeta* pMeta, int64_t uid, int64_t deltaCtb, int32_t deltaCol);
int32_t metaUidFilterCacheGet(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, LRUHandle** pHandle);
struct SMeta {

3
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -170,7 +170,8 @@ int32_t metaTbGroupCacheClear(SMeta* pMeta, uint64_t suid);
int metaAddIndexToSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq);
int metaDropIndexFromSTable(SMeta* pMeta, int64_t version, SDropIndexReq* pReq);
int64_t metaGetTimeSeriesNum(SMeta* pMeta);
int64_t metaGetTimeSeriesNum(SMeta* pMeta, int type);
void metaUpdTimeSeriesNum(SMeta* pMeta);
SMCtbCursor* metaOpenCtbCursor(void* pVnode, tb_uid_t uid, int lock);
int32_t metaResumeCtbCursor(SMCtbCursor* pCtbCur, int8_t first);
void metaPauseCtbCursor(SMCtbCursor* pCtbCur);

60
source/dnode/vnode/src/meta/metaQuery.c
View File

@ -696,22 +696,30 @@ int64_t metaGetTbNum(SMeta *pMeta) {
return pMeta->pVnode->config.vndStats.numOfCTables + pMeta->pVnode->config.vndStats.numOfNTables;
}
// N.B. Called by statusReq per second
int64_t metaGetTimeSeriesNum(SMeta *pMeta) {
// sum of (number of columns of stable - 1) * number of ctables (excluding timestamp column)
int64_t nTables = metaGetTbNum(pMeta);
if (nTables - pMeta->pVnode->config.vndStats.numOfCmprTables > 100 ||
pMeta->pVnode->config.vndStats.numOfTimeSeries <= 0 ||
++pMeta->pVnode->config.vndStats.itvTimeSeries % (60 * 5) == 0) {
int64_t num = 0;
vnodeGetTimeSeriesNum(pMeta->pVnode, &num);
pMeta->pVnode->config.vndStats.numOfTimeSeries = num;
pMeta->pVnode->config.vndStats.itvTimeSeries = (TD_VID(pMeta->pVnode) % 100) * 2;
pMeta->pVnode->config.vndStats.numOfCmprTables = nTables;
void metaUpdTimeSeriesNum(SMeta *pMeta) {
int64_t nCtbTimeSeries = 0;
if (vnodeGetTimeSeriesNum(pMeta->pVnode, &nCtbTimeSeries) == 0) {
atomic_store_64(&pMeta->pVnode->config.vndStats.numOfTimeSeries, nCtbTimeSeries);
}
}
return pMeta->pVnode->config.vndStats.numOfTimeSeries + pMeta->pVnode->config.vndStats.numOfNTimeSeries;
static FORCE_INLINE int64_t metaGetTimeSeriesNumImpl(SMeta *pMeta, bool forceUpd) {
// sum of (number of columns of stable - 1) * number of ctables (excluding timestamp column)
SVnodeStats *pStats = &pMeta->pVnode->config.vndStats;
if (forceUpd || pStats->numOfTimeSeries <= 0) {
metaUpdTimeSeriesNum(pMeta);
}
return pStats->numOfTimeSeries + pStats->numOfNTimeSeries;
}
// type: 1 reported timeseries
int64_t metaGetTimeSeriesNum(SMeta *pMeta, int type) {
int64_t nTimeSeries = metaGetTimeSeriesNumImpl(pMeta, false);
if (type == 1) {
atomic_store_64(&pMeta->pVnode->config.vndStats.numOfReportedTimeSeries, nTimeSeries);
}
return nTimeSeries;
}
typedef struct {
@ -1509,9 +1517,10 @@ _exit:
return code;
}
int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables) {
int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables, int32_t *numOfCols) {
int32_t code = 0;
*numOfTables = 0;
if (!numOfTables && !numOfCols) goto _exit;
SVnode *pVnodeObj = pVnode;
metaRLock(pVnodeObj->pMeta);
@ -1520,19 +1529,26 @@ int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables) {
SMetaStbStats state = {0};
if (metaStatsCacheGet(pVnodeObj->pMeta, uid, &state) == TSDB_CODE_SUCCESS) {
metaULock(pVnodeObj->pMeta);
*numOfTables = state.ctbNum;
if (numOfTables) *numOfTables = state.ctbNum;
if (numOfCols) *numOfCols = state.colNum;
ASSERTS(state.colNum > 0, "vgId:%d, suid:%" PRIi64 " nCols:%d <= 0 in metaCache", TD_VID(pVnodeObj), uid,
state.colNum);
goto _exit;
}
// slow path: search TDB
int64_t ctbNum = 0;
int32_t colNum = 0;
vnodeGetCtbNum(pVnode, uid, &ctbNum);
vnodeGetStbColumnNum(pVnode, uid, &colNum);
metaULock(pVnodeObj->pMeta);
*numOfTables = ctbNum;
if (numOfTables) *numOfTables = ctbNum;
if (numOfCols) *numOfCols = colNum;
state.uid = uid;
state.ctbNum = ctbNum;
state.colNum = colNum;
// upsert the cache
metaWLock(pVnodeObj->pMeta);
@ -1543,12 +1559,12 @@ _exit:
return code;
}
void metaUpdateStbStats(SMeta *pMeta, int64_t uid, int64_t delta) {
void metaUpdateStbStats(SMeta *pMeta, int64_t uid, int64_t deltaCtb, int32_t deltaCol) {
SMetaStbStats stats = {0};
if (metaStatsCacheGet(pMeta, uid, &stats) == TSDB_CODE_SUCCESS) {
stats.ctbNum += delta;
stats.ctbNum += deltaCtb;
stats.colNum += deltaCol;
metaStatsCacheUpsert(pMeta, &stats);
}
}

103
source/dnode/vnode/src/meta/metaTable.c
View File

@ -15,6 +15,8 @@
#include "meta.h"
extern tsem_t dmNotifySem;
static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema);
static int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema);
static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME);
@ -26,7 +28,7 @@ static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME);
static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME);
static int metaUpdateSuidIdx(SMeta *pMeta, const SMetaEntry *pME);
static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry);
static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type);
static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid);
static void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey);
// opt ins_tables query
static int metaUpdateBtimeIdx(SMeta *pMeta, const SMetaEntry *pME);
@ -34,6 +36,7 @@ static int metaDeleteBtimeIdx(SMeta *pMeta, const SMetaEntry *pME);
static int metaUpdateNcolIdx(SMeta *pMeta, const SMetaEntry *pME);
static int metaDeleteNcolIdx(SMeta *pMeta, const SMetaEntry *pME);
static void metaGetEntryInfo(const SMetaEntry *pEntry, SMetaInfo *pInfo) {
pInfo->uid = pEntry->uid;
pInfo->version = pEntry->version;
@ -191,6 +194,14 @@ int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSche
return 0;
}
static inline void metaTimeSeriesNotifyCheck(SMeta *pMeta) {
#if defined(TD_ENTERPRISE) && !defined(_TD_DARWIN_64)
int64_t nTimeSeries = metaGetTimeSeriesNum(pMeta, 0);
int64_t deltaTS = nTimeSeries - pMeta->pVnode->config.vndStats.numOfReportedTimeSeries;
if (deltaTS > tsTimeSeriesThreshold) tsem_post(&dmNotifySem);
#endif
}
int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
SMetaEntry me = {0};
int kLen = 0;
@ -292,7 +303,7 @@ int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq, SArray *tb
for (int32_t iChild = 0; iChild < taosArrayGetSize(tbUidList); iChild++) {
tb_uid_t uid = *(tb_uid_t *)taosArrayGet(tbUidList, iChild);
metaDropTableByUid(pMeta, uid, NULL);
metaDropTableByUid(pMeta, uid, NULL, NULL);
}
// drop super table
@ -304,8 +315,12 @@ _drop_super_table:
tdbTbDelete(pMeta->pUidIdx, &pReq->suid, sizeof(tb_uid_t), pMeta->txn);
tdbTbDelete(pMeta->pSuidIdx, &pReq->suid, sizeof(tb_uid_t), pMeta->txn);
metaStatsCacheDrop(pMeta, pReq->suid);
metaULock(pMeta);
metaUpdTimeSeriesNum(pMeta);
_exit:
tdbFree(pKey);
tdbFree(pData);
@ -376,6 +391,8 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
nStbEntry.stbEntry.schemaRow = pReq->schemaRow;
nStbEntry.stbEntry.schemaTag = pReq->schemaTag;
int32_t deltaCol = pReq->schemaRow.nCols - oStbEntry.stbEntry.schemaRow.nCols;
metaWLock(pMeta);
// compare two entry
if (oStbEntry.stbEntry.schemaRow.version != pReq->schemaRow.version) {
@ -390,8 +407,18 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
// metaStatsCacheDrop(pMeta, nStbEntry.uid);
if (deltaCol != 0) {
metaUpdateStbStats(pMeta, pReq->suid, 0, deltaCol);
}
metaULock(pMeta);
if (deltaCol != 0) {
int64_t ctbNum;
metaGetStbStats(pMeta->pVnode, pReq->suid, &ctbNum, NULL);
pMeta->pVnode->config.vndStats.numOfTimeSeries += (ctbNum * deltaCol);
}
_exit:
if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf);
tDecoderClear(&dc);
tdbTbcClose(pTbDbc);
@ -734,6 +761,7 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs
metaReaderClear(&mr);
// build SMetaEntry
SVnodeStats *pStats = &pMeta->pVnode->config.vndStats;
me.version = ver;
me.type = pReq->type;
me.uid = pReq->uid;
@ -767,10 +795,13 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs
}
#endif
++pMeta->pVnode->config.vndStats.numOfCTables;
++pStats->numOfCTables;
int32_t nCols = 0;
metaGetStbStats(pMeta->pVnode, me.ctbEntry.suid, 0, &nCols);
pStats->numOfTimeSeries += nCols - 1;
metaWLock(pMeta);
metaUpdateStbStats(pMeta, me.ctbEntry.suid, 1);
metaUpdateStbStats(pMeta, me.ctbEntry.suid, 1, 0);
metaUidCacheClear(pMeta, me.ctbEntry.suid);
metaTbGroupCacheClear(pMeta, me.ctbEntry.suid);
metaULock(pMeta);
@ -782,12 +813,14 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs
me.ntbEntry.schemaRow = pReq->ntb.schemaRow;
me.ntbEntry.ncid = me.ntbEntry.schemaRow.pSchema[me.ntbEntry.schemaRow.nCols - 1].colId + 1;
++pMeta->pVnode->config.vndStats.numOfNTables;
pMeta->pVnode->config.vndStats.numOfNTimeSeries += me.ntbEntry.schemaRow.nCols - 1;
++pStats->numOfNTables;
pStats->numOfNTimeSeries += me.ntbEntry.schemaRow.nCols - 1;
}
if (metaHandleEntry(pMeta, &me) < 0) goto _err;
metaTimeSeriesNotifyCheck(pMeta);
if (pMetaRsp) {
*pMetaRsp = taosMemoryCalloc(1, sizeof(STableMetaRsp));
@ -817,7 +850,8 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUi
void *pData = NULL;
int nData = 0;
int rc = 0;
tb_uid_t uid;
tb_uid_t uid = 0;
tb_uid_t suid = 0;
int type;
rc = tdbTbGet(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, &pData, &nData);
@ -828,9 +862,19 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUi
uid = *(tb_uid_t *)pData;
metaWLock(pMeta);
metaDropTableByUid(pMeta, uid, &type);
rc = metaDropTableByUid(pMeta, uid, &type, &suid);
metaULock(pMeta);
if (rc < 0) goto _exit;
if (type == TSDB_CHILD_TABLE) {
int32_t nCols = 0;
SVnodeStats *pStats = &pMeta->pVnode->config.vndStats;
if (metaGetStbStats(pMeta->pVnode, suid, NULL, &nCols) == 0) {
pStats->numOfTimeSeries -= nCols - 1;
}
}
if ((type == TSDB_CHILD_TABLE || type == TSDB_NORMAL_TABLE) && tbUids) {
taosArrayPush(tbUids, &uid);
}
@ -839,20 +883,48 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUi
*tbUid = uid;
}
_exit:
tdbFree(pData);
return 0;
return rc;
}
void metaDropTables(SMeta *pMeta, SArray *tbUids) {
if (taosArrayGetSize(tbUids) == 0) return;
int64_t nCtbDropped = 0;
SSHashObj *suidHash = tSimpleHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT));
metaWLock(pMeta);
for (int i = 0; i < taosArrayGetSize(tbUids); ++i) {
tb_uid_t uid = *(tb_uid_t *)taosArrayGet(tbUids, i);
metaDropTableByUid(pMeta, uid, NULL);
tb_uid_t suid = 0;
int type;
metaDropTableByUid(pMeta, uid, &type, &suid);
if (type == TSDB_CHILD_TABLE && suid != 0 && suidHash) {
int64_t *pVal = tSimpleHashGet(suidHash, &suid, sizeof(tb_uid_t));
if (pVal) {
nCtbDropped = *pVal + 1;
} else {
nCtbDropped = 1;
}
tSimpleHashPut(suidHash, &suid, sizeof(tb_uid_t), &nCtbDropped, sizeof(int64_t));
}
metaDebug("batch drop table:%" PRId64, uid);
}
metaULock(pMeta);
// update timeseries
void *pCtbDropped = NULL;
int32_t iter = 0;
while ((pCtbDropped = tSimpleHashIterate(suidHash, pCtbDropped, &iter))) {
tb_uid_t *pSuid = tSimpleHashGetKey(pCtbDropped, NULL);
int32_t nCols = 0;
SVnodeStats *pStats = &pMeta->pVnode->config.vndStats;
if (metaGetStbStats(pMeta->pVnode, *pSuid, NULL, &nCols) == 0) {
pStats->numOfTimeSeries -= *(int64_t *)pCtbDropped * (nCols - 1);
}
}
tSimpleHashCleanup(suidHash);
}
static int32_t metaFilterTableByHash(SMeta *pMeta, SArray *uidList) {
@ -987,7 +1059,7 @@ static int metaDeleteTtl(SMeta *pMeta, const SMetaEntry *pME) {
return ttlMgrDeleteTtl(pMeta->pTtlMgr, &ctx);
}
static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) {
static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid) {
void *pData = NULL;
int nData = 0;
int rc = 0;
@ -1012,9 +1084,11 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) {
if (type) *type = e.type;
if (e.type == TSDB_CHILD_TABLE) {
if (pSuid) *pSuid = e.ctbEntry.suid;
void *tData = NULL;
int tLen = 0;
if (tdbTbGet(pMeta->pUidIdx, &e.ctbEntry.suid, sizeof(tb_uid_t), &tData, &tLen) == 0) {
STbDbKey tbDbKey = {.uid = e.ctbEntry.suid, .version = ((SUidIdxVal *)tData)[0].version};
if (tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &tData, &tLen) == 0) {
@ -1075,8 +1149,7 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) {
tdbTbDelete(pMeta->pCtbIdx, &(SCtbIdxKey){.suid = e.ctbEntry.suid, .uid = uid}, sizeof(SCtbIdxKey), pMeta->txn);
--pMeta->pVnode->config.vndStats.numOfCTables;
metaUpdateStbStats(pMeta, e.ctbEntry.suid, -1);
metaUpdateStbStats(pMeta, e.ctbEntry.suid, -1, 0);
metaUidCacheClear(pMeta, e.ctbEntry.suid);
metaTbGroupCacheClear(pMeta, e.ctbEntry.suid);
} else if (e.type == TSDB_NORMAL_TABLE) {
@ -1243,6 +1316,9 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
terrno = TSDB_CODE_VND_COL_ALREADY_EXISTS;
goto _err;
}
if ((terrno = grantCheck(TSDB_GRANT_TIMESERIES)) < 0) {
goto _err;
}
pSchema->version++;
pSchema->nCols++;
pNewSchema = taosMemoryMalloc(sizeof(SSchema) * pSchema->nCols);
@ -1255,6 +1331,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
strcpy(pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].name, pAlterTbReq->colName);
++pMeta->pVnode->config.vndStats.numOfNTimeSeries;
metaTimeSeriesNotifyCheck(pMeta);
break;
case TSDB_ALTER_TABLE_DROP_COLUMN:
if (pColumn == NULL) {

7
source/dnode/vnode/src/tsdb/tsdbCache.c
View File

@ -3061,7 +3061,7 @@ static int32_t tsdbCacheLoadBlockS3(STsdbFD *pFD, uint8_t **ppBlock) {
code = s3GetObjectBlock(pFD->objName, block_offset, tsS3BlockSize * pFD->szPage, ppBlock);
if (code != TSDB_CODE_SUCCESS) {
// taosMemoryFree(pBlock);
code = TSDB_CODE_OUT_OF_MEMORY;
// code = TSDB_CODE_OUT_OF_MEMORY;
return code;
}
@ -3100,7 +3100,10 @@ int32_t tsdbCacheGetBlockS3(SLRUCache *pCache, STsdbFD *pFD, LRUHandle **handle)
taosThreadMutexUnlock(&pTsdb->bMutex);
*handle = NULL;
return 0;
if (!pBlock) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
return code;
}
size_t charge = tsS3BlockSize * pFD->szPage;

3
source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
View File

@ -175,6 +175,9 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno) {
int32_t code = tsdbCacheGetBlockS3(pFD->pTsdb->bCache, pFD, &handle);
if (code != TSDB_CODE_SUCCESS || handle == NULL) {
tsdbBCacheRelease(pFD->pTsdb->bCache, handle);
if (!handle) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
goto _exit;
}

4
source/dnode/vnode/src/vnd/vnodeCos.c
View File

@ -301,14 +301,14 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t block_
apr_table_add(headers, COS_RANGE, range_buf);
s = cos_get_object_to_buffer(options, &bucket, &object, headers, NULL, &download_buffer, &resp_headers);
log_status(s);
if (!cos_status_is_ok(s)) {
vError("s3: %s", s->error_msg);
vError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno));
code = terrno;
code = TAOS_SYSTEM_ERROR(EIO);
return code;
}
log_status(s);
// print_headers(resp_headers);
int64_t len = 0;
int64_t size = 0;

21
source/dnode/vnode/src/vnd/vnodeQuery.c
View File

@ -388,7 +388,7 @@ int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) {
pLoad->cacheUsage = tsdbCacheGetUsage(pVnode);
pLoad->numOfCachedTables = tsdbCacheGetElems(pVnode);
pLoad->numOfTables = metaGetTbNum(pVnode->pMeta);
pLoad->numOfTimeSeries = metaGetTimeSeriesNum(pVnode->pMeta);
pLoad->numOfTimeSeries = metaGetTimeSeriesNum(pVnode->pMeta, 1);
pLoad->totalStorage = (int64_t)3 * 1073741824;
pLoad->compStorage = (int64_t)2 * 1073741824;
pLoad->pointsWritten = 100;
@ -400,6 +400,15 @@ int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) {
return 0;
}
int32_t vnodeGetLoadLite(SVnode *pVnode, SVnodeLoadLite *pLoad) {
SSyncState syncState = syncGetState(pVnode->sync);
if (syncState.state == TAOS_SYNC_STATE_LEADER) {
pLoad->vgId = TD_VID(pVnode);
pLoad->nTimeSeries = metaGetTimeSeriesNum(pVnode->pMeta, 1);
return 0;
}
return -1;
}
/**
* @brief Reset the statistics value by monitor interval
*
@ -544,8 +553,8 @@ int32_t vnodeGetCtbNum(SVnode *pVnode, int64_t suid, int64_t *num) {
return TSDB_CODE_SUCCESS;
}
static int32_t vnodeGetStbColumnNum(SVnode *pVnode, tb_uid_t suid, int *num) {
SSchemaWrapper *pSW = metaGetTableSchema(pVnode->pMeta, suid, -1, 1);
int32_t vnodeGetStbColumnNum(SVnode *pVnode, tb_uid_t suid, int *num) {
SSchemaWrapper *pSW = metaGetTableSchema(pVnode->pMeta, suid, -1, 0);
if (pSW) {
*num = pSW->nCols;
tDeleteSchemaWrapper(pSW);
@ -634,10 +643,8 @@ int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num) {
tb_uid_t suid = *(tb_uid_t *)taosArrayGet(suidList, i);
int64_t ctbNum = 0;
metaGetStbStats(pVnode, suid, &ctbNum);
int numOfCols = 0;
vnodeGetStbColumnNum(pVnode, suid, &numOfCols);
int32_t numOfCols = 0;
metaGetStbStats(pVnode, suid, &ctbNum, &numOfCols);
*num += ctbNum * (numOfCols - 1);
}

4
source/libs/executor/src/scanoperator.c
View File

@ -3928,7 +3928,7 @@ static void buildVnodeFilteredTbCount(SOperatorInfo* pOperator, STableCountScanO
pAPI->metaFn.getTableUidByName(pInfo->readHandle.vnode, pSupp->stbNameFilter, &uid);
int64_t numOfChildTables = 0;
pAPI->metaFn.getNumOfChildTables(pInfo->readHandle.vnode, uid, &numOfChildTables);
pAPI->metaFn.getNumOfChildTables(pInfo->readHandle.vnode, uid, &numOfChildTables, NULL);
fillTableCountScanDataBlock(pSupp, dbName, pSupp->stbNameFilter, numOfChildTables, pRes);
} else {
@ -3979,7 +3979,7 @@ static void buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, S
pRes->info.id.groupId = groupId;
int64_t ctbNum = 0;
int32_t code = pAPI->metaFn.getNumOfChildTables(pInfo->readHandle.vnode, stbUid, &ctbNum);
int32_t code = pAPI->metaFn.getNumOfChildTables(pInfo->readHandle.vnode, stbUid, &ctbNum, NULL);
fillTableCountScanDataBlock(pSupp, dbName, varDataVal(stbName), ctbNum, pRes);
}

30
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -1595,6 +1595,26 @@ void nodesListInsertList(SNodeList* pTarget, SListCell* pPos, SNodeList* pSrc) {
nodesFree(pSrc);
}
void nodesListInsertListAfterPos(SNodeList* pTarget, SListCell* pPos, SNodeList* pSrc) {
if (NULL == pTarget || NULL == pPos || NULL == pSrc || NULL == pSrc->pHead) {
return;
}
if (NULL == pPos->pNext) {
pTarget->pTail = pSrc->pHead;
} else {
pPos->pNext->pPrev = pSrc->pHead;
}
pSrc->pHead->pPrev = pPos;
pSrc->pTail->pNext = pPos->pNext;
pPos->pNext = pSrc->pHead;
pTarget->length += pSrc->length;
nodesFree(pSrc);
}
SNode* nodesListGetNode(SNodeList* pList, int32_t index) {
SNode* node;
FOREACH(node, pList) {
@ -2318,3 +2338,13 @@ SValueNode* nodesMakeValueNodeFromBool(bool b) {
}
return pValNode;
}
bool nodesIsStar(SNode* pNode) {
return (QUERY_NODE_COLUMN == nodeType(pNode)) && ('\0' == ((SColumnNode*)pNode)->tableAlias[0]) &&
(0 == strcmp(((SColumnNode*)pNode)->colName, "*"));
}
bool nodesIsTableStar(SNode* pNode) {
return (QUERY_NODE_COLUMN == nodeType(pNode)) && ('\0' != ((SColumnNode*)pNode)->tableAlias[0]) &&
(0 == strcmp(((SColumnNode*)pNode)->colName, "*"));
}

3
source/libs/parser/CMakeLists.txt
View File

@ -1,4 +1,7 @@
aux_source_directory(src PARSER_SRC)
IF (TD_BI_SUPPORT)
LIST(APPEND PARSER_SRC ${TD_ENTERPRISE_DIR}/src/plugins/bi/src/biRewriteQuery.c)
ENDIF ()
add_library(parser STATIC ${PARSER_SRC})
target_include_directories(
parser

1
source/libs/parser/inc/parInt.h
View File

@ -36,7 +36,6 @@ int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pS
int32_t calculateConstant(SParseContext* pParseCxt, SQuery* pQuery);
int32_t translatePostCreateStream(SParseContext* pParseCxt, SQuery* pQuery, void** pResRow);
int32_t translatePostCreateSmaIndex(SParseContext* pParseCxt, SQuery* pQuery, void** pResRow);
#ifdef __cplusplus
}
#endif

55
source/libs/parser/inc/parTranslater.h Normal file
View File

@ -0,0 +1,55 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_PARSER_TRANS_H_
#define _TD_PARSER_TRANS_H_
#ifdef __cplusplus
extern "C" {
#endif
#include "parToken.h"
#include "parUtil.h"
#include "parser.h"
typedef struct STranslateContext {
SParseContext* pParseCxt;
int32_t errCode;
SMsgBuf msgBuf;
SArray* pNsLevel; // element is SArray*, the element of this subarray is STableNode*
int32_t currLevel;
int32_t levelNo;
ESqlClause currClause;
SNode* pCurrStmt;
SCmdMsgInfo* pCmdMsg;
SHashObj* pDbs;
SHashObj* pTables;
SHashObj* pTargetTables;
SExplainOptions* pExplainOpt;
SParseMetaCache* pMetaCache;
bool createStream;
bool stableQuery;
bool showRewrite;
SNode* pPrevRoot;
SNode* pPostRoot;
} STranslateContext;
int32_t biRewriteSelectStar(STranslateContext* pCxt, SSelectStmt* pSelect);
int32_t findTable(STranslateContext* pCxt, const char* pTableAlias, STableNode** pOutput);
#ifdef __cplusplus
}
#endif
#endif /*_TD_PARSER_TRANS_H_*/

63
source/libs/parser/src/parTranslater.c
View File

@ -14,6 +14,7 @@
*/
#include "parInt.h"
#include "parTranslater.h"
#include "catalog.h"
#include "cmdnodes.h"
@ -35,28 +36,6 @@ typedef struct SRewriteTbNameContext {
char* pTbName;
} SRewriteTbNameContext;
typedef struct STranslateContext {
SParseContext* pParseCxt;
int32_t errCode;
SMsgBuf msgBuf;
SArray* pNsLevel; // element is SArray*, the element of this subarray is STableNode*
int32_t currLevel;
int32_t levelNo;
ESqlClause currClause;
SNode* pCurrStmt;
SCmdMsgInfo* pCmdMsg;
SHashObj* pDbs;
SHashObj* pTables;
SHashObj* pTargetTables;
SExplainOptions* pExplainOpt;
SParseMetaCache* pMetaCache;
bool createStream;
bool stableQuery;
bool showRewrite;
SNode* pPrevRoot;
SNode* pPostRoot;
} STranslateContext;
typedef struct SBuildTopicContext {
bool colExists;
bool colNotFound;
@ -1419,7 +1398,7 @@ static EDealRes haveVectorFunction(SNode* pNode, void* pContext) {
return DEAL_RES_CONTINUE;
}
static int32_t findTable(STranslateContext* pCxt, const char* pTableAlias, STableNode** pOutput) {
int32_t findTable(STranslateContext* pCxt, const char* pTableAlias, STableNode** pOutput) {
SArray* pTables = taosArrayGetP(pCxt->pNsLevel, pCxt->currLevel);
size_t nums = taosArrayGetSize(pTables);
for (size_t i = 0; i < nums; ++i) {
@ -1810,17 +1789,8 @@ static int32_t translateBlockDistFunc(STranslateContext* pCtx, SFunctionNode* pF
return TSDB_CODE_SUCCESS;
}
static bool isStar(SNode* pNode) {
return (QUERY_NODE_COLUMN == nodeType(pNode)) && ('\0' == ((SColumnNode*)pNode)->tableAlias[0]) &&
(0 == strcmp(((SColumnNode*)pNode)->colName, "*"));
}
static bool isTableStar(SNode* pNode) {
return (QUERY_NODE_COLUMN == nodeType(pNode)) && ('\0' != ((SColumnNode*)pNode)->tableAlias[0]) &&
(0 == strcmp(((SColumnNode*)pNode)->colName, "*"));
}
static bool isStarParam(SNode* pNode) { return isStar(pNode) || isTableStar(pNode); }
static bool isStarParam(SNode* pNode) { return nodesIsStar(pNode) || nodesIsTableStar(pNode); }
static int32_t translateMultiResFunc(STranslateContext* pCxt, SFunctionNode* pFunc) {
if (!fmIsMultiResFunc(pFunc->funcId)) {
@ -2938,9 +2908,9 @@ static int32_t createMultiResFuncsParas(STranslateContext* pCxt, SNodeList* pSrc
SNodeList* pExprs = NULL;
SNode* pPara = NULL;
FOREACH(pPara, pSrcParas) {
if (isStar(pPara)) {
if (nodesIsStar(pPara)) {
code = createAllColumns(pCxt, true, &pExprs);
} else if (isTableStar(pPara)) {
} else if (nodesIsTableStar(pPara)) {
code = createTableAllCols(pCxt, (SColumnNode*)pPara, true, &pExprs);
} else {
code = nodesListMakeStrictAppend(&pExprs, nodesCloneNode(pPara));
@ -3022,11 +2992,18 @@ static int32_t createTags(STranslateContext* pCxt, SNodeList** pOutput) {
return TSDB_CODE_SUCCESS;
}
#ifndef TD_ENTERPRISE
int32_t biRewriteSelectStar(STranslateContext* pCxt, SSelectStmt* pSelect) {
return TSDB_CODE_SUCCESS;
}
#endif
static int32_t translateStar(STranslateContext* pCxt, SSelectStmt* pSelect) {
SNode* pNode = NULL;
WHERE_EACH(pNode, pSelect->pProjectionList) {
int32_t code = TSDB_CODE_SUCCESS;
if (isStar(pNode)) {
if (nodesIsStar(pNode)) {
SNodeList* pCols = NULL;
code = createAllColumns(pCxt, false, &pCols);
if (TSDB_CODE_SUCCESS == code) {
@ -3046,7 +3023,7 @@ static int32_t translateStar(STranslateContext* pCxt, SSelectStmt* pSelect) {
ERASE_NODE(pSelect->pProjectionList);
continue;
}
} else if (isTableStar(pNode)) {
} else if (nodesIsTableStar(pNode)) {
SNodeList* pCols = NULL;
code = createTableAllCols(pCxt, (SColumnNode*)pNode, false, &pCols);
if (TSDB_CODE_SUCCESS == code) {
@ -3935,6 +3912,9 @@ static int32_t translateSelectFrom(STranslateContext* pCxt, SSelectStmt* pSelect
if (TSDB_CODE_SUCCESS == code) {
code = translateHaving(pCxt, pSelect);
}
if (TSDB_CODE_SUCCESS == code && pCxt->pParseCxt->biMode != 0) {
code = biRewriteSelectStar(pCxt, pSelect);
}
if (TSDB_CODE_SUCCESS == code) {
code = translateSelectList(pCxt, pSelect);
}
@ -9537,11 +9517,6 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) {
return TSDB_CODE_SUCCESS;
}
static int32_t rewriteQueryForBI(STranslateContext* pParseCxt, SQuery* pQuery) {
return TSDB_CODE_SUCCESS;
}
int32_t translate(SParseContext* pParseCxt, SQuery* pQuery, SParseMetaCache* pMetaCache) {
STranslateContext cxt = {0};
@ -9549,13 +9524,9 @@ int32_t translate(SParseContext* pParseCxt, SQuery* pQuery, SParseMetaCache* pMe
if (TSDB_CODE_SUCCESS == code) {
code = rewriteQuery(&cxt, pQuery);
}
if (TSDB_CODE_SUCCESS == code && pParseCxt->biMode != 0) {
code = rewriteQueryForBI(&cxt, pQuery);
}
if (TSDB_CODE_SUCCESS == code) {
code = translateQuery(&cxt, pQuery->pRoot);
}
if (TSDB_CODE_SUCCESS == code && (cxt.pPrevRoot || cxt.pPostRoot)) {
pQuery->pPrevRoot = cxt.pPrevRoot;
pQuery->pPostRoot = cxt.pPostRoot;

3
source/util/CMakeLists.txt
View File

@ -18,12 +18,13 @@ target_include_directories(
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
PRIVATE "${TD_SOURCE_DIR}/include/common"
PRIVATE "${GRANT_CFG_INCLUDE_DIR}"
PRIVATE "${TD_SOURCE_DIR}/utils/TSZ/sz/inc"
)
target_link_libraries(
util
PUBLIC os
PUBLIC lz4_static
PUBLIC api cjson geos_c
PUBLIC api cjson geos_c TSZ
)
if(${BUILD_TEST})

17
source/util/src/tcompression.c
View File

@ -70,20 +70,13 @@ bool lossyFloat = false;
bool lossyDouble = false;
// init call
int32_t tsCompressInit() {
int32_t tsCompressInit(char* lossyColumns, float fPrecision, double dPrecision, uint32_t maxIntervals, uint32_t intervals,
int32_t ifAdtFse, const char* compressor) {
// config
if (lossyColumns[0] == 0) {
lossyFloat = false;
lossyDouble = false;
return 0;
}
lossyFloat = strstr(lossyColumns, "float") != NULL;
lossyDouble = strstr(lossyColumns, "double") != NULL;
if (lossyFloat == false && lossyDouble == false) return 0;
tdszInit(fPrecision, dPrecision, maxRange, curRange, Compressor);
tdszInit(fPrecision, dPrecision, maxIntervals, intervals, ifAdtFse, compressor);
if (lossyFloat) uTrace("lossy compression float is opened. ");
if (lossyDouble) uTrace("lossy compression double is opened. ");
return 1;
@ -2377,7 +2370,7 @@ int32_t tsCompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_
int32_t tsDecompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t nBuf) {
#ifdef TD_TSZ
if (HEAD_ALGO(pIn[0]) == ALGO_SZ_LOSSY) {
if (HEAD_ALGO(((uint8_t *)pIn)[0]) == ALGO_SZ_LOSSY) {
// decompress lossy
return tsDecompressFloatLossyImp(pIn, nIn, nEle, pOut);
} else {
@ -2424,7 +2417,7 @@ int32_t tsCompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32
int32_t tsDecompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t nBuf) {
#ifdef TD_TSZ
if (HEAD_ALGO(input[0]) == ALGO_SZ_LOSSY) {
if (HEAD_ALGO(((uint8_t *)pIn)[0]) == ALGO_SZ_LOSSY) {
// decompress lossy
return tsDecompressDoubleLossyImp(pIn, nIn, nEle, pOut);
} else {

17
source/util/src/tconfig.c
View File

@ -321,6 +321,7 @@ int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcTy
case CFG_DTYPE_INT64:
return cfgSetInt64(pItem, value, stype);
case CFG_DTYPE_FLOAT:
case CFG_DTYPE_DOUBLE:
return cfgSetFloat(pItem, value, stype);
case CFG_DTYPE_STRING:
return cfgSetString(pItem, value, stype);
@ -405,7 +406,7 @@ int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t
return cfgAddItem(pCfg, &item, name);
}
int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, double minval, double maxval, int8_t scope) {
int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, float minval, float maxval, int8_t scope) {
if (defaultVal < minval || defaultVal > maxval) {
terrno = TSDB_CODE_OUT_OF_RANGE;
return -1;
@ -415,6 +416,16 @@ int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, double mi
return cfgAddItem(pCfg, &item, name);
}
int32_t cfgAddDouble(SConfig *pCfg, const char *name, double defaultVal, double minval, double maxval, int8_t scope) {
if (defaultVal < minval || defaultVal > maxval) {
terrno = TSDB_CODE_OUT_OF_RANGE;
return -1;
}
SConfigItem item = {.dtype = CFG_DTYPE_DOUBLE, .dval = defaultVal, .fmin = minval, .fmax = maxval, .scope = scope};
return cfgAddItem(pCfg, &item, name);
}
int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope) {
SConfigItem item = {.dtype = CFG_DTYPE_STRING, .scope = scope};
item.str = taosStrdup(defaultVal);
@ -496,6 +507,8 @@ const char *cfgDtypeStr(ECfgDataType type) {
return "int64";
case CFG_DTYPE_FLOAT:
return "float";
case CFG_DTYPE_DOUBLE:
return "double";
case CFG_DTYPE_STRING:
return "string";
case CFG_DTYPE_DIR:
@ -524,6 +537,7 @@ void cfgDumpItemValue(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t *p
len = snprintf(buf, bufSize, "%" PRId64, pItem->i64);
break;
case CFG_DTYPE_FLOAT:
case CFG_DTYPE_DOUBLE:
len = snprintf(buf, bufSize, "%f", pItem->fval);
break;
case CFG_DTYPE_STRING:
@ -621,6 +635,7 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) {
}
break;
case CFG_DTYPE_FLOAT:
case CFG_DTYPE_DOUBLE:
if (dump) {
printf("%s %s %.2f", src, name, pItem->fval);
printf("\n");

3
tests/parallel_test/cases.task
View File

@ -201,6 +201,8 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/multilevel.py
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/ttl.py
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/ttlChangeOnWrite.py
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/compress_tsz1.py
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/compress_tsz2.py
,,n,system-test,python3 ./test.py -f 0-others/compatibility.py
,,n,system-test,python3 ./test.py -f 0-others/tag_index_basic.py
,,n,system-test,python3 ./test.py -f 0-others/udfpy_main.py
@ -1020,6 +1022,7 @@
,,y,script,./test.sh -f tsim/query/partitionby.sim
,,y,script,./test.sh -f tsim/query/tableCount.sim
,,y,script,./test.sh -f tsim/query/show_db_table_kind.sim
,,y,script,./test.sh -f tsim/query/bi_star_table.sim
,,y,script,./test.sh -f tsim/query/tag_scan.sim
,,y,script,./test.sh -f tsim/query/nullColSma.sim
,,y,script,./test.sh -f tsim/query/bug3398.sim

5
tests/script/tsim/parser/limit.sim
View File

@ -40,16 +40,19 @@ while $i < $halfNum
$c = $x / 10
$c = $c * 10
$c = $x - $c
$c2 = $c
$binary = 'binary . $c
$binary = $binary . '
$nchar = 'nchar . $c
$nchar = $nchar . '
$ts = $ts + $i
print insert into $tb values ( $ts , $c , $c , $c , $c , $c , $c , true, $binary , $nchar )
sql insert into $tb values ( $ts , $c , $c , $c , $c , $c , $c , true, $binary , $nchar )
$ts = $ts + $halfNum
sql insert into $tb1 values ( $ts , $c , NULL , $c , NULL , $c , $c , true, $binary , $nchar )
print insert into $tb1 values ( $ts , $c2 , NULL , $c2 , NULL , $c2 , $c2 , true, $binary , $nchar )
sql insert into $tb1 values ( $ts , $c2 , NULL , $c2 , NULL , $c2 , $c2 , true, $binary , $nchar )
$x = $x + 1
endw

8
tests/script/tsim/parser/limit_stb.sim
View File

@ -32,11 +32,13 @@ sql select * from $stb limit 5
if $rows != 5 then
return -1
endi
sql select * from $stb limit 5 offset 1
print select * from $stb order by ts limit 5 offset 1
sql select * from $stb order by ts limit 5 offset 1
if $rows != 5 then
return -1
endi
if $data01 != 1 then
print $data01 $data41
if $data01 != 0 then
return -1
endi
#if $data41 != 5 then
@ -48,10 +50,12 @@ if $rows != 5 then
return -1
endi
print select * from $stb order by ts desc limit 5 offset 1
sql select * from $stb order by ts desc limit 5 offset 1
if $rows != 5 then
return -1
endi
print $data01 $data11 $data41
if $data01 != 9 then
return -1
endi

111
tests/script/tsim/query/bi_star_table.sim Normal file
View File

@ -0,0 +1,111 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
sql connect
sql drop database if exists db1;
sql create database db1 vgroups 3;
sql create database db1;
sql use db1;
sql create stable sta (ts timestamp, f1 int, f2 binary(200)) tags(t1 int, t2 int, t3 int);
sql create stable stb (ts timestamp, f1 int, f2 binary(200)) tags(t1 int, t2 int, t3 int);
sql create table tba1 using sta tags(1, 1, 1);
sql create table tba2 using sta tags(2, 2, 2);
sql insert into tba1 values(now, 1, "1");
sql insert into tba2 values(now + 1s, 2, "2");
sql create table tbn1 (ts timestamp, f1 int);
sql create database db2 vgroups 3;
sql create database db2;
sql use db2;
sql create stable sta (ts timestamp, f1 int, f2 binary(200)) tags(t1 int, t2 int, t3 int);
sql create stable stb (ts timestamp, f1 int, f2 binary(200)) tags(t1 int, t2 int, t3 int);
sql create table tba1 using sta tags(1, 1, 1);
sql create table tba2 using sta tags(2, 2, 2);
set_bi_mode 1
sql select * from db1.sta order by ts;
if $cols != 7 then
return -1
endi
if $data06 != tba1 then
return -1
endi
sql select last(*) from db1.sta;
if $cols != 4 then
return -1
endi
if $data03 != tba2 then
return -1
endi
sql select last_row(*) from db1.sta;
if $cols != 4 then
return -1
endi
if $data03 != tba2 then
return -1
endi
sql select first(*) from db1.sta;
if $cols != 4 then
return -1
endi
if $data03 != tba1 then
return -1
endi
print "=====table star ====================="
sql select b.* from db1.sta b order by ts;
if $cols != 7 then
return -1
endi
if $data06 != tba1 then
return -1
endi
sql select last(b.*) from db1.sta b;
if $cols != 4 then
return -1
endi
if $data03 != tba2 then
return -1
endi
sql select last_row(b.*) from db1.sta b;
if $cols != 4 then
return -1
endi
if $data03 != tba2 then
return -1
endi
sql select first(b.*) from db1.sta b;
if $cols != 4 then
return -1
endi
if $data03 != tba1 then
return -1
endi
sql select * from (select f1 from db1.sta);
if $cols != 1 then
return -1
endi
set_bi_mode 0
sql select * from db1.sta order by ts;
if $cols != 6 then
return -1
endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT

208
tests/system-test/0-others/compress_tsz1.py Normal file
View File

@ -0,0 +1,208 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import random
import time
import math
import taos
from util.log import *
from util.cases import *
from util.sql import *
class TDTestCase:
updatecfgDict = {'lossyColumns':'float-double','IfAdtFse':1}
# init
def init(self, conn, logSql, replicaVar=1):
seed = time.time() % 10000
random.seed(seed)
self.replicaVar = int(replicaVar)
tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor(), True)
# get col value and total max min ...
def getColsValues(self, i):
# c1 bigint c2 bigint c3 float c4 double c5 int
c5 = 0 # none flag 0: not have none 1: c3 is none 2: c4 is none 3: c3 and c4 is none
c1 = random.randint(-100, 100)
c2 = random.randint(-10000, 100000)
# c3 value
c3 = c1 + c2 * 0.00001 # 6 bit
c4 = c1 + c2 * 0.00000000001 # 12 bit
# set null
if random.randint(1, 10) == 5:
c3 = None
c5 = 1
if random.randint(1, 10) == 7:
c4 = None
c5 = 2
# set none flag
if c3 is None and c4 is None:
c5 = 3
# combine values
values = f"({self.ts},{c1},{c2},"
# c3
if c3 is None:
values += "null,"
else:
values += "%.8f,"%c3
# c4
if c4 is None:
values += "null,"
else:
values += "%.15f,"%c4
# c5
values += f"{c5}) "
# move next
self.ts += 1
return values
# insert data
def insertData(self):
tdLog.info("insert data ....")
sqls = ""
for i in range(self.childCnt):
# insert child table
values = ""
pre_insert = f"insert into t{i} values "
for j in range(self.childRow):
if values == "":
values = self.getColsValues(i)
else:
values += self.getColsValues(i)
# batch insert
if j % self.batchSize == 0 and values != "":
sql = pre_insert + values
tdSql.execute(sql)
values = ""
# append last
if values != "":
sql = pre_insert + values
tdSql.execute(sql)
values = ""
# random flush
if random.randint(1, 3) == 2:
sql = "flush database db;"
tdLog.info(sql)
tdSql.execute(sql)
# insert finished
tdLog.info(f"insert data successfully.\n"
f" inserted child table = {self.childCnt}\n"
f" inserted child rows = {self.childRow}\n"
f" total inserted rows = {self.childCnt*self.childRow}\n")
return
# prepareEnv
def prepareEnv(self):
# init
self.ts = 1680000000000*1000
self.childCnt = 10
self.childRow = 100000
self.batchSize = 5000
# create database db
sql = f"create database db vgroups 2 precision 'us' "
tdLog.info(sql)
tdSql.execute(sql)
sql = f"use db"
tdSql.execute(sql)
# create super talbe st
sql = f"create table st(ts timestamp, c1 bigint, c2 bigint, c3 float, c4 double, c5 int) tags(area int)"
tdLog.info(sql)
tdSql.execute(sql)
# create child table
for i in range(self.childCnt):
sql = f"create table t{i} using st tags({i}) "
tdSql.execute(sql)
# insert data
self.insertData()
# where
def checkCorrect(self):
sql = f"select * from st"
tdSql.query(sql)
rowCnt = tdSql.getRows()
for i in range(rowCnt):
# get cols val
c1 = tdSql.queryResult[i][1]
c2 = tdSql.queryResult[i][2]
c3 = tdSql.queryResult[i][3]
c4 = tdSql.queryResult[i][4]
c5 = tdSql.queryResult[i][5]
# calc expect value
ec3 = c1 + c2 * 0.00001 # 6 bit
ec4 = c1 + c2 * 0.00000000001 # 12 bit
# none flag c5
if c5 == 3:
if c3 is not None or c4 is not None:
tdLog.exit(f"Not expect . both c3 and c4 must none. c3 = {c3} c4={c4} c5={c5}")
elif c5 == 2:
if c4 is not None:
tdLog.exit(f"Not expect . c4 must be none. c4={c4} c5={c5}")
if math.isclose(c3, ec3, rel_tol=1e-06) == False:
tdLog.exit(f"Not expect . c3 value invalid. c3={c3} expect={ec3} fabs={math.fabs(c3-ec3)}")
elif c5 == 1:
if c3 is not None:
tdLog.exit(f"Not expect . c3 must be none. c3={c3} c5={c5}")
if math.isclose(c4, ec4, rel_tol=1e-12) == False:
tdLog.exit(f"Not expect . c4 value invalid. c4={c4} expect={ec4} fabs={math.fabs(c4-ec4)}")
else:
if math.isclose(c3, ec3, rel_tol=1e-06) == False:
tdLog.exit(f"Not expect . c3 invalid. c3={c3} expect={ec3} fabs={math.fabs(c3-ec3)}")
if math.isclose(c4, ec4, rel_tol=1e-12) == False:
tdLog.exit(f"Not expect . c4 invalid. c4={c4} expect={ec4} fabs={math.fabs(c4-ec4)}")
# successful
tdLog.info(f"check data correct ok. sql={sql}")
# run
def run(self):
# prepare env insert data
self.prepareEnv()
# check where
self.checkCorrect()
# stop
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())

208
tests/system-test/0-others/compress_tsz2.py Normal file
View File

@ -0,0 +1,208 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import random
import time
import math
import taos
from util.log import *
from util.cases import *
from util.sql import *
class TDTestCase:
updatecfgDict = {'lossyColumns':'float-double','IfAdtFse':0}
# init
def init(self, conn, logSql, replicaVar=1):
seed = time.time() % 10000
random.seed(seed)
self.replicaVar = int(replicaVar)
tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor(), True)
# get col value and total max min ...
def getColsValues(self, i):
# c1 bigint c2 bigint c3 float c4 double c5 int
c5 = 0 # none flag 0: not have none 1: c3 is none 2: c4 is none 3: c3 and c4 is none
c1 = random.randint(-100, 100)
c2 = random.randint(-10000, 100000)
# c3 value
c3 = c1 + c2 * 0.00001 # 6 bit
c4 = c1 + c2 * 0.00000000001 # 12 bit
# set null
if random.randint(1, 10) == 5:
c3 = None
c5 = 1
if random.randint(1, 10) == 7:
c4 = None
c5 = 2
# set none flag
if c3 is None and c4 is None:
c5 = 3
# combine values
values = f"({self.ts},{c1},{c2},"
# c3
if c3 is None:
values += "null,"
else:
values += "%.8f,"%c3
# c4
if c4 is None:
values += "null,"
else:
values += "%.15f,"%c4
# c5
values += f"{c5}) "
# move next
self.ts += 1
return values
# insert data
def insertData(self):
tdLog.info("insert data ....")
sqls = ""
for i in range(self.childCnt):
# insert child table
values = ""
pre_insert = f"insert into t{i} values "
for j in range(self.childRow):
if values == "":
values = self.getColsValues(i)
else:
values += self.getColsValues(i)
# batch insert
if j % self.batchSize == 0 and values != "":
sql = pre_insert + values
tdSql.execute(sql)
values = ""
# append last
if values != "":
sql = pre_insert + values
tdSql.execute(sql)
values = ""
# random flush
if random.randint(1, 3) == 2:
sql = "flush database db;"
tdLog.info(sql)
tdSql.execute(sql)
# insert finished
tdLog.info(f"insert data successfully.\n"
f" inserted child table = {self.childCnt}\n"
f" inserted child rows = {self.childRow}\n"
f" total inserted rows = {self.childCnt*self.childRow}\n")
return
# prepareEnv
def prepareEnv(self):
# init
self.ts = 1680000000000*1000
self.childCnt = 10
self.childRow = 100000
self.batchSize = 5000
# create database db
sql = f"create database db vgroups 2 precision 'us' "
tdLog.info(sql)
tdSql.execute(sql)
sql = f"use db"
tdSql.execute(sql)
# create super talbe st
sql = f"create table st(ts timestamp, c1 bigint, c2 bigint, c3 float, c4 double, c5 int) tags(area int)"
tdLog.info(sql)
tdSql.execute(sql)
# create child table
for i in range(self.childCnt):
sql = f"create table t{i} using st tags({i}) "
tdSql.execute(sql)
# insert data
self.insertData()
# where
def checkCorrect(self):
sql = f"select * from st"
tdSql.query(sql)
rowCnt = tdSql.getRows()
for i in range(rowCnt):
# get cols val
c1 = tdSql.queryResult[i][1]
c2 = tdSql.queryResult[i][2]
c3 = tdSql.queryResult[i][3]
c4 = tdSql.queryResult[i][4]
c5 = tdSql.queryResult[i][5]
# calc expect value
ec3 = c1 + c2 * 0.00001 # 6 bit
ec4 = c1 + c2 * 0.00000000001 # 12 bit
# none flag c5
if c5 == 3:
if c3 is not None or c4 is not None:
tdLog.exit(f"Not expect . both c3 and c4 must none. c3 = {c3} c4={c4} c5={c5}")
elif c5 == 2:
if c4 is not None:
tdLog.exit(f"Not expect . c4 must be none. c4={c4} c5={c5}")
if math.isclose(c3, ec3, rel_tol=1e-06) == False:
tdLog.exit(f"Not expect . c3 value invalid. c3={c3} expect={ec3} fabs={math.fabs(c3-ec3)}")
elif c5 == 1:
if c3 is not None:
tdLog.exit(f"Not expect . c3 must be none. c3={c3} c5={c5}")
if math.isclose(c4, ec4, rel_tol=1e-12) == False:
tdLog.exit(f"Not expect . c4 value invalid. c4={c4} expect={ec4} fabs={math.fabs(c4-ec4)}")
else:
if math.isclose(c3, ec3, rel_tol=1e-06) == False:
tdLog.exit(f"Not expect . c3 invalid. c3={c3} expect={ec3} fabs={math.fabs(c3-ec3)}")
if math.isclose(c4, ec4, rel_tol=1e-12) == False:
tdLog.exit(f"Not expect . c4 invalid. c4={c4} expect={ec4} fabs={math.fabs(c4-ec4)}")
# successful
tdLog.info(f"check data correct ok. sql={sql}")
# run
def run(self):
# prepare env insert data
self.prepareEnv()
# check where
self.checkCorrect()
# stop
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())

70
tests/system-test/1-insert/row64kInsertByCsv.py Executable file
View File

@ -0,0 +1,70 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import random
import os
import time
import taos
import subprocess
import string
from faker import Faker
from util.log import tdLog
from util.cases import tdCases
from util.sql import tdSql
from util.dnodes import tdDnodes
from util.dnodes import *
class TDTestCase:
updatecfgDict = {'maxSQLLength':1048576,'debugFlag': 143 ,"querySmaOptimize":1}
def init(self, conn, logSql, replicaVar):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
self.testcasePath = os.path.split(__file__)[0]
self.testcasePath = self.testcasePath.replace('\\', '//')
self.testcaseFilename = os.path.split(__file__)[-1]
os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename))
now = time.time()
self.ts = int(round(now * 1000))
self.num = 100
def insertFromCsvOfLength65500(self):
tdLog.info('test insert from csv of length 65500')
os.system(f"taosBenchmark -f {self.testcasePath}//tableColumn4096.json")
tdSql.execute(f"insert into db4096.ctb00 file '{self.testcasePath}//tableColumn4096csvLength64k.csv'")
tdSql.query("select count(*) from db4096.ctb00")
tdSql.checkData(0, 0, 1)
tdSql.query("select length(c4092) from db4096.ctb00")
tdSql.checkData(0, 0, 16375)
def run(self):
tdSql.prepare()
startTime_all = time.time()
self.insertFromCsvOfLength65500()
endTime_all = time.time()
print("total time %ds" % (endTime_all - startTime_all))
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

2
tests/system-test/1-insert/rowlength64k.py
View File

@ -32,6 +32,7 @@ class TDTestCase:
tdSql.init(conn.cursor(), logSql)
self.testcasePath = os.path.split(__file__)[0]
self.testcasePath = self.testcasePath.replace('\\', '//')
self.testcaseFilename = os.path.split(__file__)[-1]
os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename))
@ -1263,7 +1264,6 @@ class TDTestCase:
self.ins_query()
def run(self):
tdSql.prepare()

66
tests/system-test/1-insert/tableColumn4096.json Normal file
View File

@ -0,0 +1,66 @@
{
"filetype": "insert",
"cfgdir": "/etc/taos",
"host": "localhost",
"port": 6030,
"rest_port": 6041,
"user": "root",
"password": "taosdata",
"thread_count": 100,
"create_table_thread_count": 24,
"result_file": "taosBenchmark_result.log",
"confirm_parameter_prompt": "no",
"insert_interval": 0,
"num_of_records_per_req": 1000000,
"max_sql_len": 1024000,
"databases": [
{
"dbinfo": {
"name": "db4096",
"drop": "yes",
"replica": 1,
"duration": 10,
"precision": "ms",
"keep": 3650,
"comp": 2,
"vgroups": 2,
"buffer": 1000
},
"super_tables": [
{
"name": "stb0",
"child_table_exists": "no",
"childtable_count": 1,
"childtable_prefix": "ctb0",
"escape_character": "no",
"auto_create_table": "no",
"batch_create_tbl_num": 500,
"data_source": "rand",
"insert_mode": "taosc",
"rollup": null,
"interlace_rows": 0,
"line_protocol": null,
"tcp_transfer": "no",
"insert_rows": 0,
"childtable_limit": 0,
"childtable_offset": 0,
"rows_per_tbl": 0,
"max_sql_len": 1048576,
"disorder_ratio": 0,
"disorder_range": 1000,
"timestamp_step": 1000,
"start_timestamp": "2022-10-22 17:20:36",
"sample_format": "csv",
"sample_file": "./sample.csv",
"tags_file": "",
"columns": [{ "type": "INT","count": 4090},{"type": "BINARY","len":16374,"count": 2},{"type": "BINARY","len":16375,"count": 1}],
"tags": [{"type": "TINYINT", "count": 1},{"type": "NCHAR","count": 1}]
}
]
}
],
"prepare_rand": 10000,
"chinese": "no",
"streams": false,
"test_log": "/root/testlog/"
}

1
tests/system-test/1-insert/tableColumn4096csvLength64k.csv Normal file
View File

File diff suppressed because one or more lines are too long

4
utils/CMakeLists.txt
View File

@ -2,3 +2,7 @@
ADD_SUBDIRECTORY(tsim)
ADD_SUBDIRECTORY(test/c)
#ADD_SUBDIRECTORY(comparisonTest/tdengine)
IF (NOT "${TSZ_ENABLED}" MATCHES "false")
ADD_SUBDIRECTORY(TSZ)
ENDIF()

27
utils/TSZ/CMakeLists.txt Normal file
View File

@ -0,0 +1,27 @@
CMAKE_MINIMUM_REQUIRED(VERSION 3.0)
PROJECT(TDengine)
# include
INCLUDE_DIRECTORIES(sz/inc)
INCLUDE_DIRECTORIES(zstd/)
INCLUDE_DIRECTORIES(zstd/common/)
# source
AUX_SOURCE_DIRECTORY(sz/src SRC1)
AUX_SOURCE_DIRECTORY(zstd/dictBuilder SRC2)
AUX_SOURCE_DIRECTORY(zstd/common SRC3)
AUX_SOURCE_DIRECTORY(zstd/compress SRC4)
AUX_SOURCE_DIRECTORY(zstd/decompress SRC5)
AUX_SOURCE_DIRECTORY(zstd/deprecated SRC6)
AUX_SOURCE_DIRECTORY(zstd/legacy SRC7)
# archive
ADD_LIBRARY(TSZ STATIC ${SRC1} ${SRC2} ${SRC3} ${SRC4} ${SRC5} ${SRC6} ${SRC7})
TARGET_INCLUDE_DIRECTORIES(TSZ PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/sz/inc ${TD_SOURCE_DIR}/include)
# windows ignore warning
IF (TD_WINDOWS)
SET_TARGET_PROPERTIES(TSZ PROPERTIES COMPILE_FLAGS -w)
ENDIF ()

29
utils/TSZ/LICENSE Normal file
View File

@ -0,0 +1,29 @@
BSD 3-Clause License
Copyright (c) 2021, taosdata
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
3. Neither the name of the copyright holder nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

39
utils/TSZ/README.md Normal file
View File

@ -0,0 +1,39 @@
# TSZ
Error-bounded Lossy Data Compressor For Float/Double
TSZ algorithm is come from SZ algorithm, Github url is https://github.com/szcompressor.
Bellow is aspect of improvement :
1) Better speed and size
SZ head size about 24 bytes, we are reduced to 2 bytes.
we delete some no use code and some unnecessary function could be droped.
2) Support multi-threads, interface is thread-safety.
3) Remove 2D 3D 4D 5D function, only 1D be remained.
4) Remove int8 int16 int32 and other datatype, only float double be remained.
5) Optimize code speed
6) Other optimize...
After modify, TSZ become faster、smaller and independent. TSZ more suitable for small block data compression.
## ADT-FSE
ADT-FSE is an optimization algorithm of TSZ based on timeseries database scenario, which can provide higher compression rate and faster compression and decompression speed for small files.
Usage:
By default, ADT-FSE algorithm is enabled. Control it by `ifAdtFse` configuration.
Citation:
```
@inproceedings{
adtfse_sc2023,
author = {Tao Lu, Yu Zhong, Zibin Sun, Xiang Chen, You Zhou, Fei Wu, Ying Yang, Yunxin Huang, and Yafei Yang},
title = {ADT-FSE: A New Encoder for SZ},
year = {2023},
publisher = {IEEE Press},
booktitle = {Proceedings of the International Conference on High Performance Computing, Networking, Storage and Analysis},
location = {Denver, Colorado},
series = {SC'23}
}
```

47
utils/TSZ/sz/inc/ByteToolkit.h Normal file
View File

@ -0,0 +1,47 @@
/**
* @file ByteToolkit.h
* @author Sheng Di
* @date July, 2017
* @brief Header file for the ByteToolkit.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _ByteToolkit_H
#define _ByteToolkit_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdio.h>
int bytesToInt_bigEndian(unsigned char* bytes);
void intToBytes_bigEndian(unsigned char *b, unsigned int num);
long bytesToLong_bigEndian(unsigned char* b);
void longToBytes_bigEndian(unsigned char *b, long num);
short getExponent_float(float value);
short getPrecisionReqLength_float(float precision);
short getExponent_double(double value);
short getPrecisionReqLength_double(double precision);
float bytesToFloat(unsigned char* bytes);
void floatToBytes(unsigned char *b, float num);
double bytesToDouble(unsigned char* bytes);
void doubleToBytes(unsigned char *b, double num);
int getMaskRightCode(int m);
int getLeftMovingCode(int kMod8);
int getRightMovingSteps(int kMod8, int resiBitLength);
int getRightMovingCode(int kMod8, int resiBitLength);
size_t bytesToSize(unsigned char* bytes, int size_type);
void sizeToBytes(unsigned char* outBytes, size_t size, int size_type);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _ByteToolkit_H ----- */

75
utils/TSZ/sz/inc/CompressElement.h Normal file
View File

@ -0,0 +1,75 @@
/**
* @file CompressElement.h
* @author Sheng Di
* @date April, 2016
* @brief Header file for Compress Elements such as DoubleCompressELement.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#include <stdint.h>
#ifndef _CompressElement_H
#define _CompressElement_H
#ifdef __cplusplus
extern "C" {
#endif
typedef struct DoubleValueCompressElement
{
double data;
long curValue;
unsigned char curBytes[8]; //big_endian
int reqBytesLength;
int resiBitsLength;
} DoubleValueCompressElement;
typedef struct FloatValueCompressElement
{
float data; // diffValue + medianValue
int curValue; // diff int value
unsigned char curBytes[4]; // dif bytes value diffValue->iValue big_endian
int reqBytesLength;
int resiBitsLength;
} FloatValueCompressElement;
typedef struct LossyCompressionElement
{
int leadingZeroBytes; //0,1,2,or 3
unsigned char integerMidBytes[8];
int integerMidBytes_Length; //they are mid_bits actually
//char curBytes[8];
//int curBytes_Length; //4 for single_precision or 8 for double_precision
int resMidBitsLength;
int residualMidBits;
} LossyCompressionElement;
short computeGroupNum_float(float value);
short computeGroupNum_double(double value);
void listAdd_double(double last3CmprsData[3], double value);
void listAdd_float(float last3CmprsData[3], float value);
void listAdd_int(int64_t last3CmprsData[3], int64_t value);
void listAdd_int32(int32_t last3CmprsData[3], int32_t value);
void listAdd_float_group(float *groups, int *flags, char groupNum, float oriValue, float decValue, char* curGroupID);
void listAdd_double_group(double *groups, int *flags, char groupNum, double oriValue, double decValue, char* curGroupID);
int validPrediction_double(double minErr, double precision);
int validPrediction_float(float minErr, float precision);
double* generateGroupErrBounds(int errorBoundMode, double realPrecision, double pwrErrBound);
int generateGroupMaxIntervalCount(double* groupErrBounds);
void new_LossyCompressionElement(LossyCompressionElement *lce, int leadingNum, unsigned char* intMidBytes,
int intMidBytes_Length, int resiMidBitsLength, int resiBits);
void updateLossyCompElement_Double(unsigned char* curBytes, unsigned char* preBytes,
int reqBytesLength, int resiBitsLength, LossyCompressionElement *lce);
void updateLossyCompElement_Float(unsigned char* curBytes, unsigned char* preBytes,
int reqBytesLength, int resiBitsLength, LossyCompressionElement *lce);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _CompressElement_H ----- */

36
utils/TSZ/sz/inc/DynamicByteArray.h Normal file
View File

@ -0,0 +1,36 @@
/**
* @file DynamicByteArray.h
* @author Sheng Di
* @date April, 2016
* @brief Header file for Dynamic Byte Array.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _DynamicByteArray_H
#define _DynamicByteArray_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdio.h>
typedef struct DynamicByteArray
{
unsigned char* array;
size_t size;
size_t capacity;
} DynamicByteArray;
void new_DBA(DynamicByteArray **dba, size_t cap);
void convertDBAtoBytes(DynamicByteArray *dba, unsigned char** bytes);
void free_DBA(DynamicByteArray *dba);
unsigned char getDBA_Data(DynamicByteArray *dba, size_t pos);
void addDBA_Data(DynamicByteArray *dba, unsigned char value);
void memcpyDBA_Data(DynamicByteArray *dba, unsigned char* data, size_t length);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _DynamicByteArray_H ----- */

35
utils/TSZ/sz/inc/DynamicIntArray.h Normal file
View File

@ -0,0 +1,35 @@
/**
* @file DynamicIntArray.h
* @author Sheng Di
* @date April, 2016
* @brief Header file for Dynamic Int Array.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _DynamicIntArray_H
#define _DynamicIntArray_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdio.h>
typedef struct DynamicIntArray
{
unsigned char* array; //char* (one byte) is enough, don't have to be int*
size_t size;
size_t capacity;
} DynamicIntArray;
void new_DIA(DynamicIntArray **dia, size_t cap);
void convertDIAtoInts(DynamicIntArray *dia, unsigned char **data);
void free_DIA(DynamicIntArray *dia);
int getDIA_Data(DynamicIntArray *dia, size_t pos);
void addDIA_Data(DynamicIntArray *dia, int value);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _DynamicIntArray_H ----- */

71
utils/TSZ/sz/inc/Huffman.h Normal file
View File

@ -0,0 +1,71 @@
/**
* @file Huffman.h
* @author Sheng Di
* @date Aug., 2016
* @brief Header file for the exponential segment constructor.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _Huffman_H
#define _Huffman_H
#ifdef __cplusplus
extern "C" {
#endif
//Note: when changing the following settings, intvCapacity in sz.h should be changed as well.
//#define allNodes 131072
//#define stateNum 65536
typedef struct node_t {
struct node_t *left, *right;
size_t freq;
char t; //in_node:0; otherwise:1
unsigned int c;
} *node;
typedef struct HuffmanTree {
unsigned int stateNum;
unsigned int allNodes;
struct node_t* pool;
node *qqq, *qq; //the root node of the HuffmanTree is qq[1]
int n_nodes; //n_nodes is for compression
int qend;
unsigned long **code;
unsigned char *cout;
int n_inode; //n_inode is for decompression
int maxBitCount;
} HuffmanTree;
HuffmanTree* createHuffmanTree(int stateNum);
HuffmanTree* createDefaultHuffmanTree();
node new_node(HuffmanTree *huffmanTree, size_t freq, unsigned int c, node a, node b);
node new_node2(HuffmanTree *huffmanTree, unsigned int c, unsigned char t);
void qinsert(HuffmanTree *huffmanTree, node n);
node qremove(HuffmanTree *huffmanTree);
void build_code(HuffmanTree *huffmanTree, node n, int len, unsigned long out1, unsigned long out2);
void init(HuffmanTree *huffmanTree, int *s, size_t length);
void init_static(HuffmanTree *huffmanTree, int *s, size_t length);
void encode(HuffmanTree *huffmanTree, int *s, size_t length, unsigned char *out, size_t *outSize);
void decode(unsigned char *s, size_t targetLength, node t, int *out);
void pad_tree_uchar(HuffmanTree* huffmanTree, unsigned char* L, unsigned char* R, unsigned int* C, unsigned char* t, unsigned int i, node root);
void pad_tree_ushort(HuffmanTree* huffmanTree, unsigned short* L, unsigned short* R, unsigned int* C, unsigned char* t, unsigned int i, node root);
void pad_tree_uint(HuffmanTree* huffmanTree, unsigned int* L, unsigned int* R, unsigned int* C, unsigned char* t, unsigned int i, node root);
unsigned int convert_HuffTree_to_bytes_anyStates(HuffmanTree* huffmanTree, int nodeCount, unsigned char** out);
void unpad_tree_uchar(HuffmanTree* huffmanTree, unsigned char* L, unsigned char* R, unsigned int* C, unsigned char *t, unsigned int i, node root);
void unpad_tree_ushort(HuffmanTree* huffmanTree, unsigned short* L, unsigned short* R, unsigned int* C, unsigned char* t, unsigned int i, node root);
void unpad_tree_uint(HuffmanTree* huffmanTree, unsigned int* L, unsigned int* R, unsigned int* C, unsigned char* t, unsigned int i, node root);
node reconstruct_HuffTree_from_bytes_anyStates(HuffmanTree *huffmanTree, unsigned char* bytes, int nodeCount);
void encode_withTree(HuffmanTree* huffmanTree, int *s, size_t length, unsigned char **out, size_t *outSize);
void decode_withTree(HuffmanTree* huffmanTree, unsigned char *s, size_t targetLength, int *out);
void SZ_ReleaseHuffman(HuffmanTree* huffmanTree);
#ifdef __cplusplus
}
#endif
#endif

92
utils/TSZ/sz/inc/TightDataPointStorageD.h Normal file
View File

@ -0,0 +1,92 @@
/**
* @file TightDataPointStorageD.h
* @author Sheng Di
* @date April, 2016
* @brief Header file for the tight data point storage (TDPS).
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _TightDataPointStorageD_H
#define _TightDataPointStorageD_H
#include <stdbool.h>
#include "pub.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct TightDataPointStorageD
{
int ifAdtFse;
size_t dataSeriesLength;
int allSameData;
double realPrecision;
double medianValue;
char reqLength;
char radExpo; //used to compute reqLength based on segmented precisions in "pw_rel_compression"
double minLogValue;
int stateNum;
int allNodes;
size_t exactDataNum;
double reservedValue;
unsigned char* FseCode; // fse code of tp_code
size_t FseCode_size;
unsigned char* transCodeBits; // extra bitstream of transcoding
size_t transCodeBits_size;
unsigned char* typeArray; //its size is dataSeriesLength/4 (or xxx/4+1)
size_t typeArray_size;
unsigned char* leadNumArray; //its size is exactDataNum/4 (or exactDataNum/4+1)
size_t leadNumArray_size;
unsigned char* exactMidBytes;
size_t exactMidBytes_size;
unsigned char* residualMidBits;
size_t residualMidBits_size;
unsigned int intervals;
unsigned char isLossless; //a mark to denote whether it's lossless compression (1 is yes, 0 is no)
size_t segment_size;
unsigned char* raBytes;
size_t raBytes_size;
unsigned char plus_bits;
unsigned char max_bits;
} TightDataPointStorageD;
void new_TightDataPointStorageD_Empty(TightDataPointStorageD **self);
int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **self, unsigned char* flatBytes, size_t flatBytesLength, sz_exedata* pde_exe, sz_params* pde_params);
void new_TightDataPointStorageD(TightDataPointStorageD **self,
size_t dataSeriesLength, size_t exactDataNum,
int* type, unsigned char* exactMidBytes, size_t exactMidBytes_size,
unsigned char* leadNumIntArray, //leadNumIntArray contains readable numbers....
unsigned char* resiMidBits, size_t resiMidBits_size,
unsigned char resiBitLength,
double realPrecision, double medianValue, char reqLength,
unsigned int intervals, unsigned char radExpo);
void convertTDPStoBytes_double(TightDataPointStorageD* tdps, unsigned char* bytes, unsigned char* dsLengthBytes, unsigned char sameByte);
bool convertTDPStoFlatBytes_double(TightDataPointStorageD *tdps, unsigned char* bytes, size_t *size);
void free_TightDataPointStorageD(TightDataPointStorageD *tdps);
void free_TightDataPointStorageD2(TightDataPointStorageD *tdps);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _TightDataPointStorageD_H ----- */

97
utils/TSZ/sz/inc/TightDataPointStorageF.h Normal file
View File

@ -0,0 +1,97 @@
/**
* @file TightDataPointStorageF.h
* @author Sheng Di and Dingwen Tao
* @date Aug, 2016
* @brief Header file for the tight data point storage (TDPS).
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _TightDataPointStorageF_H
#define _TightDataPointStorageF_H
#include <stdio.h>
#include <stdbool.h>
#include "pub.h"
#include "bitstream.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct TightDataPointStorageF
{
int ifAdtFse;
size_t dataSeriesLength;
int allSameData;
double realPrecision; //it's used as the pwrErrBoundRatio when errBoundMode==PW_REL
float medianValue;
char reqLength;
char radExpo; //used to compute reqLength based on segmented precisions in "pw_rel_compression"
int stateNum;
int allNodes;
size_t exactDataNum;
float reservedValue;
float minLogValue;
unsigned char* FseCode; // fse code of tp_code
size_t FseCode_size;
unsigned char* transCodeBits; // extra bitstream of transcoding
size_t transCodeBits_size;
unsigned char* typeArray; //its size is dataSeriesLength/4 (or xxx/4+1)
size_t typeArray_size;
unsigned char* leadNumArray; //its size is exactDataNum/4 (or exactDataNum/4+1)
size_t leadNumArray_size;
unsigned char* exactMidBytes;
size_t exactMidBytes_size;
unsigned char* residualMidBits;
size_t residualMidBits_size;
unsigned int intervals; //quantization_intervals
unsigned char isLossless; //a mark to denote whether it's lossless compression (1 is yes, 0 is no)
size_t segment_size;
unsigned char* raBytes;
size_t raBytes_size;
unsigned char plus_bits;
unsigned char max_bits;
} TightDataPointStorageF;
void new_TightDataPointStorageF_Empty(TightDataPointStorageF **self);
int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **self, unsigned char* flatBytes, size_t flatBytesLength, sz_exedata* pde_exe, sz_params* pde_params);
void new_TightDataPointStorageF(TightDataPointStorageF **self,
size_t dataSeriesLength, size_t exactDataNum,
int* type, unsigned char* exactMidBytes, size_t exactMidBytes_size,
unsigned char* leadNumIntArray, //leadNumIntArray contains readable numbers....
unsigned char* resiMidBits, size_t resiMidBits_size,
unsigned char resiBitLength,
double realPrecision, float medianValue, char reqLength, unsigned int intervals,
unsigned char radExpo);
void convertTDPStoBytes_float(TightDataPointStorageF* tdps, unsigned char* bytes, unsigned char* dsLengthBytes, unsigned char sameByte);
bool convertTDPStoFlatBytes_float(TightDataPointStorageF *tdps, unsigned char* bytes, size_t *size);
void free_TightDataPointStorageF(TightDataPointStorageF *tdps);
void free_TightDataPointStorageF2(TightDataPointStorageF *tdps);
void printTDPS(TightDataPointStorageF *tdps);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _TightDataPointStorageF_H ----- */

33
utils/TSZ/sz/inc/TypeManager.h Normal file
View File

@ -0,0 +1,33 @@
/**
* @file TypeManager.h
* @author Sheng Di
* @date July, 2017
* @brief Header file for the TypeManager.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _TypeManager_H
#define _TypeManager_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdio.h>
#include <stdint.h>
//TypeManager.c
void convertByteArray2IntArray_fast_2b(size_t stepLength, unsigned char* byteArray, size_t byteArrayLength, unsigned char **intArray);
size_t convertIntArray2ByteArray_fast_2b(unsigned char* timeStepType, size_t timeStepTypeLength, unsigned char **result);
int getLeftMovingSteps(size_t k, unsigned char resiBitLength);
size_t convertIntArray2ByteArray_fast_dynamic(unsigned char* timeStepType, unsigned char resiBitLength, size_t nbEle, unsigned char **bytes);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _TypeManager_H ----- */

39
utils/TSZ/sz/inc/conf.h Normal file
View File

@ -0,0 +1,39 @@
/**
* @file conf.h
* @author Sheng Di
* @date July, 2017
* @brief Header file for the conf.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _Conf_H
#define _Conf_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdio.h>
//
// set default value
//
void setDefaulParams(sz_exedata* exedata, sz_params* params);
//conf.c
void updateQuantizationInfo(int quant_intervals);
int SZ_ReadConf(const char* sz_cfgFile);
int SZ_LoadConf(const char* sz_cfgFile);
unsigned int roundUpToPowerOf2(unsigned int base);
double computeABSErrBoundFromPSNR(double psnr, double threshold, double value_range);
double computeABSErrBoundFromNORM_ERR(double normErr, size_t nbEle);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _Conf_H ----- */

79
utils/TSZ/sz/inc/dataCompression.h Normal file
View File

@ -0,0 +1,79 @@
/**
* @file dataCompression.h
* @author Sheng Di
* @date July, 2017
* @brief Header file for the dataCompression.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _DataCompression_H
#define _DataCompression_H
#ifdef __cplusplus
extern "C" {
#endif
#include "sz.h"
#include <stdio.h>
#include <stdbool.h>
#define computeMinMax(data) \
for(i=1;i<size;i++)\
{\
data_ = data[i];\
if(min>data_)\
min = data_;\
else if(max<data_)\
max = data_;\
}\
//dataCompression.c
double computeRangeSize_double(double* oriData, size_t size, double* valueRangeSize, double* medianValue);
float computeRangeSize_float(float* oriData, size_t size, float* valueRangeSize, float* medianValue);
double min_d(double a, double b);
double max_d(double a, double b);
float min_f(float a, float b);
float max_f(float a, float b);
double getRealPrecision_double(double valueRangeSize, int errBoundMode, double absErrBound, double relBoundRatio, int *status);
double getRealPrecision_float(float valueRangeSize, int errBoundMode, double absErrBound, double relBoundRatio, int *status);
double getRealPrecision_int(long valueRangeSize, int errBoundMode, double absErrBound, double relBoundRatio, int *status);
void symTransform_8bytes(unsigned char data[8]);
void symTransform_2bytes(unsigned char data[2]);
void symTransform_4bytes(unsigned char data[4]);
void compressSingleFloatValue(FloatValueCompressElement *vce, float tgtValue, float precision, float medianValue,
int reqLength, int reqBytesLength, int resiBitsLength);
void compressSingleDoubleValue(DoubleValueCompressElement *vce, double tgtValue, double precision, double medianValue,
int reqLength, int reqBytesLength, int resiBitsLength);
int compIdenticalLeadingBytesCount_double(unsigned char* preBytes, unsigned char* curBytes);
int compIdenticalLeadingBytesCount_float(unsigned char* preBytes, unsigned char* curBytes);
void addExactData(DynamicByteArray *exactMidByteArray, DynamicIntArray *exactLeadNumArray,
DynamicIntArray *resiBitArray, LossyCompressionElement *lce);
int getPredictionCoefficients(int layers, int dimension, int **coeff_array, int *status);
int computeBlockEdgeSize_3D(int segmentSize);
int computeBlockEdgeSize_2D(int segmentSize);
int generateLossyCoefficients_float(float* oriData, double precision, size_t nbEle, int* reqBytesLength, int* resiBitsLength, float* medianValue, float* decData);
int compressExactDataArray_float(float* oriData, double precision, size_t nbEle, unsigned char** leadArray, unsigned char** midArray, unsigned char** resiArray,
int reqLength, int reqBytesLength, int resiBitsLength, float medianValue);
void decompressExactDataArray_float(unsigned char* leadNum, unsigned char* exactMidBytes, unsigned char* residualMidBits, size_t nbEle, int reqLength, float medianValue, float** decData);
int generateLossyCoefficients_double(double* oriData, double precision, size_t nbEle, int* reqBytesLength, int* resiBitsLength, double* medianValue, double* decData);
int compressExactDataArray_double(double* oriData, double precision, size_t nbEle, unsigned char** leadArray, unsigned char** midArray, unsigned char** resiArray,
int reqLength, int reqBytesLength, int resiBitsLength, double medianValue);
void decompressExactDataArray_double(unsigned char* leadNum, unsigned char* exactMidBytes, unsigned char* residualMidBits, size_t nbEle, int reqLength, double medianValue, double** decData);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _DataCompression_H ----- */

108
utils/TSZ/sz/inc/defines.h Normal file
View File

@ -0,0 +1,108 @@
/**
* @file defines.h
* @author Sheng Di
* @date July, 2019
* @brief Header file for the dataCompression.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _SZ_DEFINES_H
#define _SZ_DEFINES_H
// define data format version
#define DATA_FROMAT_VER1 1 // curretn version
#define PASTRI 103
// #define HZ 102 //deprecated
#define SZ 101
#define SZ_Transpose 104
//prediction mode of temporal dimension based compression
#define SZ_PREVIOUS_VALUE_ESTIMATE 0
#define MIN_NUM_OF_ELEMENTS 10 //if the # elements <= 20, skip the compression
#define SZ_ABS 0
#define REL 1
#define VR_REL 1 //alternative name to REL
#define ABS_AND_REL 2
#define ABS_OR_REL 3
#define PSNR 4
#define NORM 5
#define PW_REL 10
#define ABS_AND_PW_REL 11
#define ABS_OR_PW_REL 12
#define REL_AND_PW_REL 13
#define REL_OR_PW_REL 14
#define SZ_FLOAT 0
#define SZ_DOUBLE 1
#define SZ_UINT8 2
#define SZ_INT8 3
#define SZ_UINT16 4
#define SZ_INT16 5
#define SZ_UINT32 6
#define SZ_INT32 7
#define SZ_UINT64 8
#define SZ_INT64 9
#define LITTLE_ENDIAN_DATA 0 //refers to the endian type of the data read from the disk
#define BIG_ENDIAN_DATA 1 //big_endian (ppc, max, etc.) ; little_endian (x86, x64, etc.)
#define LITTLE_ENDIAN_SYSTEM 0 //refers to the endian type of the system
#define BIG_ENDIAN_SYSTEM 1
#define DynArrayInitLen 1024
#define MIN_ZLIB_DEC_ALLOMEM_BYTES 1000000
//#define maxRangeRadius 32768
//#define maxRangeRadius 1048576//131072
#define SZ_BEST_SPEED 0
#define SZ_BEST_COMPRESSION 1
#define SZ_DEFAULT_COMPRESSION 2
#define SZ_TEMPORAL_COMPRESSION 3
#define SZ_NO_REGRESSION 0
#define SZ_WITH_LINEAR_REGRESSION 1
#define SZ_PWR_MIN_TYPE 0
#define SZ_PWR_AVG_TYPE 1
#define SZ_PWR_MAX_TYPE 2
#define SZ_FORCE_SNAPSHOT_COMPRESSION 0
#define SZ_FORCE_TEMPORAL_COMPRESSION 1
#define SZ_PERIO_TEMPORAL_COMPRESSION 2
//SUCCESS returning status
#define SZ_SUCCESS 0 //successful
#define SZ_FAILED -1 //Not successful
#define SZ_FERR -2 //Failed to open input file
#define SZ_TERR -3 //wrong data type (should be only float or double)
#define SZ_DERR -4 //dimension error
#define SZ_MERR -5 //sz_mode error
#define SZ_BERR -6 //bound-mode error (should be only SZ_ABS, REL, ABS_AND_REL, ABS_OR_REL, or PW_REL)
#define SZ_LITTER_ELEMENT -7
#define SZ_ALGORITHM_ERR -8
#define SZ_FORMAT_ERR -9
#define SZ_MAINTAIN_VAR_DATA 0
#define SZ_DESTROY_WHOLE_VARSET 1
#define GROUP_COUNT 16 //2^{16}=65536
// metaData remove some by tickduan
#define MetaDataByteLength 2 // original is 28 bytes
#define MetaDataByteLength_double 2 // original is 36 bytes
#define numOfBufferedSteps 1 //the number of time steps in the buffer
#define GZIP_COMPRESSOR 0 //i.e., ZLIB_COMPRSSOR
#define ZSTD_COMPRESSOR 1
#endif /* _SZ_DEFINES_H */

171
utils/TSZ/sz/inc/dictionary.h Normal file
View File

@ -0,0 +1,171 @@
/*-------------------------------------------------------------------------*/
/**
@file dictionary.h
@author N. Devillard
@brief Implements a dictionary for string variables.
This module implements a simple dictionary object, i.e. a list
of string/string associations. This object is useful to store e.g.
informations retrieved from a configuration file (ini files).
*/
/*--------------------------------------------------------------------------*/
#ifndef _DICTIONARY_H_
#define _DICTIONARY_H_
/*---------------------------------------------------------------------------
Includes
---------------------------------------------------------------------------*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
/*---------------------------------------------------------------------------
New types
---------------------------------------------------------------------------*/
#ifdef __cplusplus
extern "C" {
#endif
/*-------------------------------------------------------------------------*/
/**
@brief Dictionary object
This object contains a list of string/string associations. Each
association is identified by a unique string key. Looking up values
in the dictionary is speeded up by the use of a (hopefully collision-free)
hash function.
*/
/*-------------------------------------------------------------------------*/
typedef struct _dictionary_ {
int n ; /** Number of entries in dictionary */
int size ; /** Storage size */
char ** val ; /** List of string values */
char ** key ; /** List of string keys */
unsigned * hash ; /** List of hash values for keys */
} dictionary ;
/*---------------------------------------------------------------------------
Function prototypes
---------------------------------------------------------------------------*/
/*-------------------------------------------------------------------------*/
/**
@brief Compute the hash key for a string.
@param key Character string to use for key.
@return 1 unsigned int on at least 32 bits.
This hash function has been taken from an Article in Dr Dobbs Journal.
This is normally a collision-free function, distributing keys evenly.
The key is stored anyway in the struct so that collision can be avoided
by comparing the key itself in last resort.
*/
/*--------------------------------------------------------------------------*/
unsigned dictionary_hash(const char * key);
/*-------------------------------------------------------------------------*/
/**
@brief Create a new dictionary object.
@param size Optional initial size of the dictionary.
@return 1 newly allocated dictionary objet.
This function allocates a new dictionary object of given size and returns
it. If you do not know in advance (roughly) the number of entries in the
dictionary, give size=0.
*/
/*--------------------------------------------------------------------------*/
dictionary * dictionary_new(int size);
/*-------------------------------------------------------------------------*/
/**
@brief Delete a dictionary object
@param d dictionary object to deallocate.
@return void
Deallocate a dictionary object and all memory associated to it.
*/
/*--------------------------------------------------------------------------*/
void dictionary_del(dictionary * vd);
/*-------------------------------------------------------------------------*/
/**
@brief Get a value from a dictionary.
@param d dictionary object to search.
@param key Key to look for in the dictionary.
@param def Default value to return if key not found.
@return 1 pointer to internally allocated character string.
This function locates a key in a dictionary and returns a pointer to its
value, or the passed 'def' pointer if no such key can be found in
dictionary. The returned character pointer points to data internal to the
dictionary object, you should not try to free it or modify it.
*/
/*--------------------------------------------------------------------------*/
char * dictionary_get(dictionary * d, const char * key, char * def);
/*-------------------------------------------------------------------------*/
/**
@brief Set a value in a dictionary.
@param d dictionary object to modify.
@param key Key to modify or add.
@param val Value to add.
@return int 0 if Ok, anything else otherwise
If the given key is found in the dictionary, the associated value is
replaced by the provided one. If the key cannot be found in the
dictionary, it is added to it.
It is Ok to provide a NULL value for val, but NULL values for the dictionary
or the key are considered as errors: the function will return immediately
in such a case.
Notice that if you dictionary_set a variable to NULL, a call to
dictionary_get will return a NULL value: the variable will be found, and
its value (NULL) is returned. In other words, setting the variable
content to NULL is equivalent to deleting the variable from the
dictionary. It is not possible (in this implementation) to have a key in
the dictionary without value.
This function returns non-zero in case of failure.
*/
/*--------------------------------------------------------------------------*/
int dictionary_set(dictionary * vd, const char * key, const char * val);
/*-------------------------------------------------------------------------*/
/**
@brief Delete a key in a dictionary
@param d dictionary object to modify.
@param key Key to remove.
@return void
This function deletes a key in a dictionary. Nothing is done if the
key cannot be found.
*/
/*--------------------------------------------------------------------------*/
void dictionary_unset(dictionary * d, const char * key);
/*-------------------------------------------------------------------------*/
/**
@brief Dump a dictionary to an opened file pointer.
@param d Dictionary to dump
@param f Opened file pointer.
@return void
Dumps a dictionary onto an opened file pointer. Key pairs are printed out
as @c [Key]=[Value], one per line. It is Ok to provide stdout or stderr as
output file pointers.
*/
/*--------------------------------------------------------------------------*/
void dictionary_dump(dictionary * d, FILE * out);
#ifdef __cplusplus
}
#endif
#endif

321
utils/TSZ/sz/inc/iniparser.h Normal file
View File

@ -0,0 +1,321 @@
/*-------------------------------------------------------------------------*/
/**
@file iniparser.h
@author N. Devillard
@brief Parser for ini files.
*/
/*--------------------------------------------------------------------------*/
#ifndef _INIPARSER_H_
#define _INIPARSER_H_
/*---------------------------------------------------------------------------
Includes
---------------------------------------------------------------------------*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
/*
* The following #include is necessary on many Unixes but not Linux.
* It is not needed for Windows platforms.
* Uncomment it if needed.
*/
/* #include <unistd.h> */
#include "dictionary.h"
/*-------------------------------------------------------------------------*/
/**
@brief Get number of sections in a dictionary
@param d Dictionary to examine
@return int Number of sections found in dictionary
This function returns the number of sections found in a dictionary.
The test to recognize sections is done on the string stored in the
dictionary: a section name is given as "section" whereas a key is
stored as "section:key", thus the test looks for entries that do not
contain a colon.
This clearly fails in the case a section name contains a colon, but
this should simply be avoided.
This function returns -1 in case of error.
*/
/*--------------------------------------------------------------------------*/
int iniparser_getnsec(dictionary * d);
/*-------------------------------------------------------------------------*/
/**
@brief Get name for section n in a dictionary.
@param d Dictionary to examine
@param n Section number (from 0 to nsec-1).
@return Pointer to char string
This function locates the n-th section in a dictionary and returns
its name as a pointer to a string statically allocated inside the
dictionary. Do not free or modify the returned string!
This function returns NULL in case of error.
*/
/*--------------------------------------------------------------------------*/
char * iniparser_getsecname(dictionary * d, int n);
/*-------------------------------------------------------------------------*/
/**
@brief Save a dictionary to a loadable ini file
@param d Dictionary to dump
@param f Opened file pointer to dump to
@return void
This function dumps a given dictionary into a loadable ini file.
It is Ok to specify @c stderr or @c stdout as output files.
*/
/*--------------------------------------------------------------------------*/
void iniparser_dump_ini(dictionary * d, FILE * f);
/*-------------------------------------------------------------------------*/
/**
@brief Save a dictionary section to a loadable ini file
@param d Dictionary to dump
@param s Section name of dictionary to dump
@param f Opened file pointer to dump to
@return void
This function dumps a given section of a given dictionary into a loadable ini
file. It is Ok to specify @c stderr or @c stdout as output files.
*/
/*--------------------------------------------------------------------------*/
void iniparser_dumpsection_ini(dictionary * d, char * s, FILE * f);
/*-------------------------------------------------------------------------*/
/**
@brief Dump a dictionary to an opened file pointer.
@param d Dictionary to dump.
@param f Opened file pointer to dump to.
@return void
This function prints out the contents of a dictionary, one element by
line, onto the provided file pointer. It is OK to specify @c stderr
or @c stdout as output files. This function is meant for debugging
purposes mostly.
*/
/*--------------------------------------------------------------------------*/
void iniparser_dump(dictionary * d, FILE * f);
/*-------------------------------------------------------------------------*/
/**
@brief Get the number of keys in a section of a dictionary.
@param d Dictionary to examine
@param s Section name of dictionary to examine
@return Number of keys in section
*/
/*--------------------------------------------------------------------------*/
int iniparser_getsecnkeys(dictionary * d, char * s);
/*-------------------------------------------------------------------------*/
/**
@brief Get the number of keys in a section of a dictionary.
@param d Dictionary to examine
@param s Section name of dictionary to examine
@return pointer to statically allocated character strings
This function queries a dictionary and finds all keys in a given section.
Each pointer in the returned char pointer-to-pointer is pointing to
a string allocated in the dictionary; do not free or modify them.
This function returns NULL in case of error.
*/
/*--------------------------------------------------------------------------*/
char ** iniparser_getseckeys(dictionary * d, char * s);
/*-------------------------------------------------------------------------*/
/**
@brief Get the string associated to a key
@param d Dictionary to search
@param key Key string to look for
@param def Default value to return if key not found.
@return pointer to statically allocated character string
This function queries a dictionary for a key. A key as read from an
ini file is given as "section:key". If the key cannot be found,
the pointer passed as 'def' is returned.
The returned char pointer is pointing to a string allocated in
the dictionary, do not free or modify it.
*/
/*--------------------------------------------------------------------------*/
char * iniparser_getstring(dictionary * d, const char * key, char * def);
/*-------------------------------------------------------------------------*/
/**
@brief Get the string associated to a key, convert to an int
@param d Dictionary to search
@param key Key string to look for
@param notfound Value to return in case of error
@return integer
This function queries a dictionary for a key. A key as read from an
ini file is given as "section:key". If the key cannot be found,
the notfound value is returned.
Supported values for integers include the usual C notation
so decimal, octal (starting with 0) and hexadecimal (starting with 0x)
are supported. Examples:
- "42" -> 42
- "042" -> 34 (octal -> decimal)
- "0x42" -> 66 (hexa -> decimal)
Warning: the conversion may overflow in various ways. Conversion is
totally outsourced to strtol(), see the associated man page for overflow
handling.
Credits: Thanks to A. Becker for suggesting strtol()
*/
/*--------------------------------------------------------------------------*/
int iniparser_getint(dictionary * d, const char * key, int notfound);
/*-------------------------------------------------------------------------*/
/**
@brief Get the string associated to a key, convert to a long
@param d Dictionary to search
@param key Key string to look for
@param notfound Value to return in case of error
@return long
Credits: This function bases completely on int iniparser_getint and was
slightly modified to return long instead of int.
*/
/*--------------------------------------------------------------------------*/
long iniparser_getlint(dictionary * d, const char * key, int notfound);
/*-------------------------------------------------------------------------*/
/**
@brief Get the string associated to a key, convert to a double
@param d Dictionary to search
@param key Key string to look for
@param notfound Value to return in case of error
@return double
This function queries a dictionary for a key. A key as read from an
ini file is given as "section:key". If the key cannot be found,
the notfound value is returned.
*/
/*--------------------------------------------------------------------------*/
double iniparser_getdouble(dictionary * d, const char * key, double notfound);
/*-------------------------------------------------------------------------*/
/**
@brief Get the string associated to a key, convert to a boolean
@param d Dictionary to search
@param key Key string to look for
@param notfound Value to return in case of error
@return integer
This function queries a dictionary for a key. A key as read from an
ini file is given as "section:key". If the key cannot be found,
the notfound value is returned.
A true boolean is found if one of the following is matched:
- A string starting with 'y'
- A string starting with 'Y'
- A string starting with 't'
- A string starting with 'T'
- A string starting with '1'
A false boolean is found if one of the following is matched:
- A string starting with 'n'
- A string starting with 'N'
- A string starting with 'f'
- A string starting with 'F'
- A string starting with '0'
The notfound value returned if no boolean is identified, does not
necessarily have to be 0 or 1.
*/
/*--------------------------------------------------------------------------*/
int iniparser_getboolean(dictionary * d, const char * key, int notfound);
/*-------------------------------------------------------------------------*/
/**
@brief Set an entry in a dictionary.
@param ini Dictionary to modify.
@param entry Entry to modify (entry name)
@param val New value to associate to the entry.
@return int 0 if Ok, -1 otherwise.
If the given entry can be found in the dictionary, it is modified to
contain the provided value. If it cannot be found, -1 is returned.
It is Ok to set val to NULL.
*/
/*--------------------------------------------------------------------------*/
int iniparser_set(dictionary * ini, const char * entry, const char * val);
/*-------------------------------------------------------------------------*/
/**
@brief Delete an entry in a dictionary
@param ini Dictionary to modify
@param entry Entry to delete (entry name)
@return void
If the given entry can be found, it is deleted from the dictionary.
*/
/*--------------------------------------------------------------------------*/
void iniparser_unset(dictionary * ini, const char * entry);
/*-------------------------------------------------------------------------*/
/**
@brief Finds out if a given entry exists in a dictionary
@param ini Dictionary to search
@param entry Name of the entry to look for
@return integer 1 if entry exists, 0 otherwise
Finds out if a given entry exists in the dictionary. Since sections
are stored as keys with NULL associated values, this is the only way
of querying for the presence of sections in a dictionary.
*/
/*--------------------------------------------------------------------------*/
int iniparser_find_entry(dictionary * ini, const char * entry) ;
/*-------------------------------------------------------------------------*/
/**
@brief Parse an ini file and return an allocated dictionary object
@param ininame Name of the ini file to read.
@return Pointer to newly allocated dictionary
This is the parser for ini files. This function is called, providing
the name of the file to be read. It returns a dictionary object that
should not be accessed directly, but through accessor functions
instead.
The returned dictionary must be freed using iniparser_freedict().
*/
/*--------------------------------------------------------------------------*/
dictionary * iniparser_load(const char * ininame);
/*-------------------------------------------------------------------------*/
/**
@brief Free all memory associated to an ini dictionary
@param d Dictionary to free
@return void
Free all memory associated to an ini dictionary.
It is mandatory to call this function before the dictionary object
gets out of the current context.
*/
/*--------------------------------------------------------------------------*/
void iniparser_freedict(dictionary * d);
#endif

72
utils/TSZ/sz/inc/pub.h Normal file
View File

@ -0,0 +1,72 @@
/**
* @file sz.h
* @author Sheng Di
* @date April, 2015
* @brief Header file for the whole compressor.
* (C) 2015 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _PUB_H
#define _PUB_H
#include <stdio.h>
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
#endif
/* array meta data and compression parameters for SZ_Init_Params() */
typedef struct sz_params
{
int dataType;
int ifAdtFse; // 0 (ADT-FSE algorithm) or 1 (original Huffman encoding)
unsigned int max_quant_intervals; //max number of quantization intervals for quantization
unsigned int quantization_intervals;
unsigned int maxRangeRadius;
int sol_ID;// it's SZ or SZ_Transpose, unless the setting is PASTRI compression mode (./configure --enable-pastri)
int losslessCompressor;
int sampleDistance; //2 bytes
float predThreshold; // 2 bytes
int szMode; //* 0 (best speed) or 1 (better compression with Zstd/Gzip) or 3 temporal-dimension based compression
int errorBoundMode; //4bits (0.5byte), //SZ_ABS, REL, ABS_AND_REL, or ABS_OR_REL, PSNR, or PW_REL, PSNR
double absErrBound; //absolute error bound for float
double absErrBoundDouble; // for double
double relBoundRatio; //value range based relative error bound ratio
double psnr; //PSNR
double normErr;
double pw_relBoundRatio; //point-wise relative error bound
int segment_size; //only used for 2D/3D data compression with pw_relBoundRatio (deprecated)
int pwr_type; //only used for 2D/3D data compression with pw_relBoundRatio
int protectValueRange; //0 or 1
float fmin, fmax;
double dmin, dmax;
int snapshotCmprStep; //perform single-snapshot-based compression if time_step == snapshotCmprStep
int predictionMode;
int accelerate_pw_rel_compression;
int plus_bits;
int randomAccess;
int withRegression;
} sz_params;
typedef struct sz_exedata
{
char optQuantMode; //opt Quantization (0: fixed ; 1: optimized)
int intvCapacity; // the number of intervals for the linear-scaling quantization
int intvRadius; // the number of intervals for the radius of the quantization range (intvRadius=intvCapacity/2)
unsigned int SZ_SIZE_TYPE; //the length (# bytes) of the size_t in the system at runtime //4 or 8: sizeof(size_t)
} sz_exedata;
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _PUB_H ----- */

196
utils/TSZ/sz/inc/sz.h Normal file
View File

@ -0,0 +1,196 @@
/**
* @file sz.h
* @author Sheng Di
* @date April, 2015
* @brief Header file for the whole compressor.
* (C) 2015 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _SZ_H
#define _SZ_H
#include <stdio.h>
#include <stdint.h>
#include <stdbool.h>
#include <time.h> /* For time(), in seconds */
#include "pub.h"
#include "CompressElement.h"
#include "DynamicByteArray.h"
#include "DynamicIntArray.h"
#include "TightDataPointStorageD.h"
#include "TightDataPointStorageF.h"
#include "conf.h"
#include "dataCompression.h"
#include "ByteToolkit.h"
#include "TypeManager.h"
#include "sz_float.h"
#include "sz_double.h"
#include "szd_float.h"
#include "szd_double.h"
#include "utility.h"
#ifdef _WIN32
#define PATH_SEPARATOR ';'
#define INLINE
#else
#define PATH_SEPARATOR ':'
#define INLINE inline
#endif
#ifdef __cplusplus
extern "C" {
#endif
void cost_start();
double cost_end(const char* tag);
void show_rate( int in_len, int out_len);
//typedef char int8_t;
//typedef unsigned char uint8_t;
//typedef short int16_t;
//typedef unsigned short uint16_t;
//typedef int int32_t;
//typedef unsigned int uint32_t;
//typedef long int64_t;
//typedef unsigned long uint64_t;
#include "defines.h"
//Note: the following setting should be consistent with stateNum in Huffman.h
//#define intvCapacity 65536
//#define intvRadius 32768
//#define intvCapacity 131072
//#define intvRadius 65536
#define SZ_COMPUTE_1D_NUMBER_OF_BLOCKS( COUNT, NUM_BLOCKS, BLOCK_SIZE ) \
if (COUNT <= BLOCK_SIZE){ \
NUM_BLOCKS = 1; \
} \
else{ \
NUM_BLOCKS = COUNT / BLOCK_SIZE; \
} \
#define SZ_COMPUTE_2D_NUMBER_OF_BLOCKS( COUNT, NUM_BLOCKS, BLOCK_SIZE ) \
if (COUNT <= BLOCK_SIZE){ \
NUM_BLOCKS = 1; \
} \
else{ \
NUM_BLOCKS = COUNT / BLOCK_SIZE; \
} \
#define SZ_COMPUTE_3D_NUMBER_OF_BLOCKS( COUNT, NUM_BLOCKS, BLOCK_SIZE ) \
if (COUNT <= BLOCK_SIZE){ \
NUM_BLOCKS = 1; \
} \
else{ \
NUM_BLOCKS = COUNT / BLOCK_SIZE; \
} \
#define SZ_COMPUTE_BLOCKCOUNT( COUNT, NUM_BLOCKS, SPLIT_INDEX, \
EARLY_BLOCK_COUNT, LATE_BLOCK_COUNT ) \
EARLY_BLOCK_COUNT = LATE_BLOCK_COUNT = COUNT / NUM_BLOCKS; \
SPLIT_INDEX = COUNT % NUM_BLOCKS; \
if (0 != SPLIT_INDEX) { \
EARLY_BLOCK_COUNT = EARLY_BLOCK_COUNT + 1; \
} \
//typedef unsigned long unsigned long;
//typedef unsigned int uint;
typedef union lint16
{
unsigned short usvalue;
short svalue;
unsigned char byte[2];
} lint16;
typedef union lint32
{
int ivalue;
unsigned int uivalue;
unsigned char byte[4];
} lint32;
typedef union lint64
{
long lvalue;
unsigned long ulvalue;
unsigned char byte[8];
} lint64;
typedef union ldouble
{
double value;
unsigned long lvalue;
unsigned char byte[8];
} ldouble;
typedef union lfloat
{
float value;
unsigned int ivalue;
unsigned char byte[4];
} lfloat;
typedef struct sz_metadata
{
unsigned char ver; //only used for checking the version by calling SZ_GetMetaData()
int isConstant; //only used for checking if the data are constant values by calling SZ_GetMetaData()
int isLossless; //only used for checking if the data compression was lossless, used only by calling SZ_GetMetaData()
int sizeType; //only used for checking whether the size type is "int" or "long" in the compression, used only by calling SZ_GetMetaData()
size_t dataSeriesLength; //# number of data points in the dataset
int defactoNBBins; //real number of quantization bins
struct sz_params* conf_params; //configuration parameters
} sz_metadata;
/*We use a linked list to maintain time-step meta info for time-step based compression*/
typedef struct sz_tsc_metainfo
{
int totalNumOfSteps;
int currentStep;
char metadata_filename[256];
FILE *metadata_file;
unsigned char* bit_array; //sihuan added
size_t intersect_size; //sihuan added
int64_t* hist_index; //sihuan added: prestep index
} sz_tsc_metadata;
extern unsigned char versionNumber;
//-------------------key global variables--------------
extern int dataEndianType; //*endian type of the data read from disk
extern int sysEndianType; //*sysEndianType is actually set automatically.
extern sz_params *confparams_cpr;
extern sz_exedata *exe_params;
void SZ_Finalize();
int SZ_Init(const char *configFilePath);
int SZ_Init_Params(sz_params *params);
//
// compress output data to outData and return outSize
//
size_t SZ_compress_args(int dataType, void *data, size_t r1, unsigned char* outData, sz_params* params);
//
// decompress output data to outData and return outSize
//
size_t SZ_decompress(int dataType, unsigned char *bytes, size_t byteLength, size_t r1, unsigned char* outData);
void convertSZParamsToBytes(sz_params* params, unsigned char* result, char optQuantMode);
void convertBytesToSZParams(unsigned char* bytes, sz_params* params, sz_exedata* pde_exe);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _SZ_H ----- */

47
utils/TSZ/sz/inc/sz_double.h Normal file
View File

@ -0,0 +1,47 @@
/**
* @file sz_double.h
* @author Sheng Di
* @date July, 2017
* @brief Header file for the sz_double.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _SZ_Double_H
#define _SZ_Double_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdio.h>
#include <stdbool.h>
unsigned char* SZ_skip_compress_double(double* data, size_t dataLength, size_t* outSize);
void computeReqLength_double(double realPrecision, short radExpo, int* reqLength, double* medianValue);
unsigned int optimize_intervals_double_1D(double *oriData, size_t dataLength, double realPrecision);
unsigned int optimize_intervals_double_1D_opt(double *oriData, size_t dataLength, double realPrecision);
TightDataPointStorageD* SZ_compress_double_1D_MDQ(double *oriData,
size_t dataLength, double realPrecision, double valueRangeSize, double medianValue_d);
void SZ_compress_args_double_StoreOriData(double* oriData, size_t dataLength, unsigned char* newByteData, size_t *outSize);
bool SZ_compress_args_double_NoCkRngeNoGzip_1D( unsigned char* newByteData, double *oriData, size_t dataLength, double realPrecision, size_t *outSize, double valueRangeSize, double medianValue_d);
void SZ_compress_args_double_withinRange(unsigned char* newByteData, double *oriData, size_t dataLength, size_t *outSize);
int SZ_compress_args_double(double *oriData, size_t r1, unsigned char* newByteData, size_t *outSize, sz_params* params);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _SZ_Double_H ----- */

40
utils/TSZ/sz/inc/sz_float.h Normal file
View File

@ -0,0 +1,40 @@
/**
* @file sz_float.h
* @author Sheng Di
* @date July, 2017
* @brief Header file for the sz_float.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _SZ_Float_H
#define _SZ_Float_H
#ifdef __cplusplus
extern "C" {
#endif
void computeReqLength_float(double realPrecision, short radExpo, int* reqLength, float* medianValue);
unsigned int optimize_intervals_float_1D(float *oriData, size_t dataLength, double realPrecision);
unsigned int optimize_intervals_float_1D_opt(float *oriData, size_t dataLength, double realPrecision);
TightDataPointStorageF* SZ_compress_float_1D_MDQ(float *oriData,
size_t dataLength, float realPrecision, float valueRangeSize, float medianValue_f);
bool SZ_compress_args_float_NoCkRngeNoGzip_1D( unsigned char* newByteData, float *oriData,
size_t dataLength, double realPrecision, size_t *outSize, float valueRangeSize, float medianValue_f);
void SZ_compress_args_float_withinRange(unsigned char* newByteData, float *oriData, size_t dataLength, size_t *outSize);
int SZ_compress_args_float(float *oriData, size_t r1, unsigned char* newByteData, size_t *outSize, sz_params* params);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _SZ_Float_H ----- */

29
utils/TSZ/sz/inc/szd_double.h Normal file
View File

@ -0,0 +1,29 @@
/**
* @file szd_double.h
* @author Sheng Di
* @date July, 2017
* @brief Header file for the szd_double.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _SZD_Double_H
#define _SZD_Double_H
#ifdef __cplusplus
extern "C" {
#endif
#include "TightDataPointStorageD.h"
void decompressDataSeries_double_1D(double* data, size_t dataSeriesLength, double* hist_data, TightDataPointStorageD* tdps);
void getSnapshotData_double_1D(double* data, size_t dataSeriesLength, TightDataPointStorageD* tdps, int errBoundMode, int compressionType, double* hist_data, sz_params* pde_params);
int SZ_decompress_args_double(double* newData, size_t r1, unsigned char* cmpBytes, size_t cmpSize, int compressionType, double* hist_data, sz_exedata* pde_exe, sz_params* pde_params);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _SZD_Double_H ----- */

32
utils/TSZ/sz/inc/szd_float.h Normal file
View File

@ -0,0 +1,32 @@
/**
* @file szd_float.h
* @author Sheng Di
* @date July, 2017
* @brief Header file for the szd_float.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _SZD_Float_H
#define _SZD_Float_H
#ifdef __cplusplus
extern "C" {
#endif
#include "TightDataPointStorageF.h"
void decompressDataSeries_float_1D(float* data, size_t dataSeriesLength, float* hist_data, TightDataPointStorageF* tdps);
void getSnapshotData_float_1D(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps, int errBoundMode, int compressionType, float* hist_data, sz_params* pde_params);
int SZ_decompress_args_float(float* newData, size_t r1, unsigned char* cmpBytes, size_t cmpSize, int compressionType, float* hist_data, sz_exedata* pde_exe, sz_params* pde_params);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _SZD_Float_H ----- */

124
utils/TSZ/sz/inc/transcode.h Normal file
View File

@ -0,0 +1,124 @@
/**
* @file transcode.h
* @author Yu Zhong (lx1zhong@qq.com)
* @brief Header file for ADT-FSE algorithm
* @date 2022-08-25
*
* @copyright Copyright (c) 2022
*
*/
#ifndef _Transcode_H
#define _Transcode_H
#include "sz.h"
#include "mem.h"
#ifdef __cplusplus
extern "C" {
#endif
#ifdef WINDOWS
int32_t BUILDIN_CLZ_EX(uint32_t val);
#else
#define BUILDIN_CLZ_EX(val) __builtin_clz(val)
#endif
/**
* dict = {
0: [0,0], 1:[1,0], 2:[2,0], 3:[3,0],
4: [4,0], 5:[5,0], 6:[6,0], 7:[7,0],
8: [8,0], 9:[9,0], 10:[10,0], 11:[11,0],
12: [12,0], 13:[13,0], 14:[14,0], 15:[15,0],
16: [16,1], 18:[17,1], 20:[18,1], 22:[19,1],
24: [20,2], 28:[21,2], 32:[22,3], 40:[23,3],
48: [24,4], 64:[25,6], 128:[26,7], 256:[27,8],
512: [28,9], 1024:[29,10], 2048:[30,11], 4096:[31,12],
8192: [32,13], 16384:[33,14],
-8192: [35,13], -16384:[34,14],
-512: [39,9], -1024:[38,10], -2048:[37,11], -4096:[36,12],
-48: [43,4], -64:[42,6], -128:[41,7], -256:[40,8],
-24: [47,2], -28:[46,2], -32:[45,3], -40:[44,3],
-16: [51,1], -18:[50,1], -20:[49,1], -22:[48,1],
-12: [55,0], -13:[54,0], -14:[53,0], -15:[52,0],
-8: [59,0], -9:[58,0], -10:[57,0], -11:[56,0],
-4: [63,0], -5:[62,0], -6:[61,0], -7:[60,0],
-1:[66,0], -2:[65,0], -3:[64,0],
#-16384:[67,0]
}
*
*/
MEM_STATIC int Int2code(int factor)
{
static const uint8_t Ft_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7,
8, 9, 10, 11, 12, 13, 14, 15,
16, 16, 17, 17, 18, 18, 19, 19,
20, 20, 20, 20, 21, 21, 21, 21,
22, 22, 22, 22, 22, 22, 22, 22,
23, 23, 23, 23, 23, 23, 23, 23,
24, 24, 24, 24, 24, 24, 24, 24,
24, 24, 24, 24, 24, 24, 24, 24 };
if (factor >= 0)
return (factor > 63) ? 50 - BUILDIN_CLZ_EX(factor) : Ft_Code[factor];
else {
factor = -factor;
return (factor > 63) ? 17 + BUILDIN_CLZ_EX(factor) : 67 - Ft_Code[factor];
}
}
/**
* reverse_dict = {
0: [0,0], 1:[1,0], 2:[2,0], 3:[3,0],
4: [4,0], 5:[5,0], 6:[6,0], 7:[7,0],
8: [8,0], 9:[9,0], 10:[10,0], 11:[11,0],
12: [12,0], 13:[13,0], 14:[14,0], 15:[15,0],
16: [16,1], 17:[18,1], 18:[20,1], 19:[22,1],
20: [24,2], 21:[28,2], 22:[32,3], 23:[40,3],
24: [48,4], 25:[64,6], 26:[128,7], 27:[256,8],
28: [512,9], 29:[1024,10], 30:[2048,11], 31:[4096,12],
32: [8192,13], 33:[16384,14],
35:[-8192,13], 34:[-16384,14],
39:[-512,9], 38:[-1024,10], 37:[-2048,11], 36:[-4096,12],
43:[-48,4], 42:[-64,6], 41:[-128,7], 40:[-256,8],
47:[-24,2], 46:[-28,2], 45:[-32,3], 44:[-40,3],
51:[-16,1], 50:[-18,1], 49:[-20,1], 48:[-22,1],
55:[-12,0], 54:[-13,0], 53:[-14,0], 52:[-15,0],
59:[-8,0], 58:[-9,0], 57:[-10,0], 56:[-11,0],
63:[-4,0], 62:[-5,0], 61:[-6,0], 60:[-7,0],
66:[-1,0], 65:[-2,0], 64:[-3,0],
#67:[-16384,0]
}
*
*/
static const int code2int[67][2] = { {0,0},{1,0},{2,0},{3,0},
{4,0},{5,0},{6,0},{7,0},
{8,0},{9,0},{10,0},{11,0},
{12,0},{13,0},{14,0},{15,0},
{16,1},{18,1},{20,1},{22,1},
{24,2},{28,2},{32,3},{40,3},
{48,4},{64,6},{128,7},{256,8},
{512,9},{1024,10},{2048,11},{4096,12},
{8192,13},{16384,14},
{-16384,14}, {-8192,13},
{-4096,12},{-2048,11},{-1024,10},{-512,9},
{-256,8},{-128,7},{-64,6},{-48,4},
{-40,3},{-32,3},{-28,2},{-24,2},
{-22,1},{-20,1},{-18,1},{-16,1},
{-15,0},{-14,0},{-13,0},{-12,0},
{-11,0},{-10,0},{-9,0},{-8,0},
{-7,0},{-6,0},{-5,0},{-4,0},
{-3,0},{-2,0},{-1,0} };
void encode_with_fse(int *type, size_t dataSeriesLength, unsigned int intervals,
unsigned char **FseCode, size_t *FseCode_size,
unsigned char **transCodeBits, size_t *transCodeBits_size);
void decode_with_fse(int *type, size_t dataSeriesLength, unsigned int intervals,
unsigned char *FseCode, size_t FseCode_size,
unsigned char *transCodeBits, size_t transCodeBits_size);
#ifdef __cplusplus
}
#endif
#endif

29
utils/TSZ/sz/inc/utility.h Normal file
View File

@ -0,0 +1,29 @@
/**
* @file utility.h
* @author Sheng Di, Sihuan Li
* @date July, 2018
* @brief Header file for the utility.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _UTILITY_H
#define _UTILITY_H
#include "sz.h"
#ifdef __cplusplus
extern "C" {
#endif
int is_lossless_compressed_data(unsigned char* compressedBytes, size_t cmpSize);
unsigned long sz_lossless_compress(int losslessCompressor, unsigned char* data, unsigned long dataLength, unsigned char* compressBytes);
unsigned long sz_lossless_decompress(int losslessCompressor, unsigned char* compressBytes, unsigned long cmpSize, unsigned char** oriData, unsigned long targetOriSize);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _UTILITY_H ----- */

298
utils/TSZ/sz/src/ByteToolkit.c Normal file
View File

@ -0,0 +1,298 @@
/**
* @file ByteToolkit.c
* @author Sheng Di
* @date April, 2016
* @brief Byte Toolkit
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#include <stdlib.h>
#include <string.h>
#include "sz.h"
INLINE int bytesToInt_bigEndian(unsigned char* bytes)
{
int res;
unsigned char* des = (unsigned char*)&res;
des[0] = bytes[3];
des[1] = bytes[2];
des[2] = bytes[1];
des[3] = bytes[0];
return res;
}
/**
* @unsigned char *b the variable to store the converted bytes (length=4)
* @unsigned int num
* */
INLINE void intToBytes_bigEndian(unsigned char *b, unsigned int num)
{
unsigned char* sou =(unsigned char*)&num;
b[0] = sou[3];
b[1] = sou[2];
b[2] = sou[1];
b[3] = sou[0];
}
/**
* @endianType: refers to the endian_type of unsigned char* b.
* */
INLINE long bytesToLong_bigEndian(unsigned char* b) {
long temp = 0;
long res = 0;
res <<= 8;
temp = b[0] & 0xff;
res |= temp;
res <<= 8;
temp = b[1] & 0xff;
res |= temp;
res <<= 8;
temp = b[2] & 0xff;
res |= temp;
res <<= 8;
temp = b[3] & 0xff;
res |= temp;
res <<= 8;
temp = b[4] & 0xff;
res |= temp;
res <<= 8;
temp = b[5] & 0xff;
res |= temp;
res <<= 8;
temp = b[6] & 0xff;
res |= temp;
res <<= 8;
temp = b[7] & 0xff;
res |= temp;
return res;
}
INLINE void longToBytes_bigEndian(unsigned char *b, long num)
{
unsigned char* sou = (unsigned char*)&num;
if(sizeof(num) == 8) {
// 8 bytes
b[7] = sou[0];
b[6] = sou[1];
b[5] = sou[2];
b[4] = sou[3];
b[3] = sou[4];
b[2] = sou[5];
b[1] = sou[6];
b[0] = sou[7];
} else {
memset(b, 0, 4);
b[7] = sou[0];
b[6] = sou[1];
b[5] = sou[2];
b[4] = sou[3];
}
}
//TODO: debug: lfBuf.lvalue could be actually little_endian....
INLINE short getExponent_float(float value)
{
//int ivalue = floatToBigEndianInt(value);
lfloat lbuf;
lbuf.value = value;
int ivalue = lbuf.ivalue;
int expValue = (ivalue & 0x7F800000) >> 23;
expValue -= 127;
return (short)expValue;
}
INLINE short getPrecisionReqLength_float(float precision)
{
lfloat lbuf;
lbuf.value = precision;
int ivalue = lbuf.ivalue;
int expValue = (ivalue & 0x7F800000) >> 23;
expValue -= 127;
return (short)expValue;
}
INLINE short getExponent_double(double value)
{
//long lvalue = doubleToBigEndianLong(value);
ldouble lbuf;
lbuf.value = value;
long lvalue = lbuf.lvalue;
int expValue = (int)((lvalue & 0x7FF0000000000000) >> 52);
expValue -= 1023;
return (short)expValue;
}
INLINE short getPrecisionReqLength_double(double precision)
{
ldouble lbuf;
lbuf.value = precision;
long lvalue = lbuf.lvalue;
int expValue = (int)((lvalue & 0x7FF0000000000000) >> 52);
expValue -= 1023;
// unsigned char the1stManBit = (unsigned char)((lvalue & 0x0008000000000000) >> 51);
// if(the1stManBit==1)
// expValue--;
return (short)expValue;
}
//the byte to input is in the big-endian format
INLINE float bytesToFloat(unsigned char* bytes)
{
lfloat buf;
memcpy(buf.byte, bytes, 4);
if(sysEndianType==LITTLE_ENDIAN_SYSTEM)
symTransform_4bytes(buf.byte);
return buf.value;
}
INLINE void floatToBytes(unsigned char *b, float num)
{
lfloat buf;
buf.value = num;
memcpy(b, buf.byte, 4);
if(sysEndianType==LITTLE_ENDIAN_SYSTEM)
symTransform_4bytes(b);
}
//the byte to input is in the big-endian format
INLINE double bytesToDouble(unsigned char* bytes)
{
ldouble buf;
memcpy(buf.byte, bytes, 8);
if(sysEndianType==LITTLE_ENDIAN_SYSTEM)
symTransform_8bytes(buf.byte);
return buf.value;
}
INLINE void doubleToBytes(unsigned char *b, double num)
{
ldouble buf;
buf.value = num;
memcpy(b, buf.byte, 8);
if(sysEndianType==LITTLE_ENDIAN_SYSTEM)
symTransform_8bytes(b);
}
INLINE int getMaskRightCode(int m) {
switch (m) {
case 1:
return 0x01;
case 2:
return 0x03;
case 3:
return 0x07;
case 4:
return 0x0F;
case 5:
return 0x1F;
case 6:
return 0x3F;
case 7:
return 0X7F;
case 8:
return 0XFF;
default:
return 0;
}
}
INLINE int getLeftMovingCode(int kMod8)
{
return getMaskRightCode(8 - kMod8);
}
INLINE int getRightMovingSteps(int kMod8, int resiBitLength) {
return 8 - kMod8 - resiBitLength;
}
INLINE int getRightMovingCode(int kMod8, int resiBitLength)
{
int rightMovingSteps = 8 - kMod8 - resiBitLength;
if(rightMovingSteps < 0)
{
switch(-rightMovingSteps)
{
case 1:
return 0x80;
case 2:
return 0xC0;
case 3:
return 0xE0;
case 4:
return 0xF0;
case 5:
return 0xF8;
case 6:
return 0xFC;
case 7:
return 0XFE;
default:
return 0;
}
}
else //if(rightMovingSteps >= 0)
{
int a = getMaskRightCode(8 - kMod8);
int b = getMaskRightCode(8 - kMod8 - resiBitLength);
int c = a - b;
return c;
}
}
INLINE size_t bytesToSize(unsigned char* bytes, int size_type)
{
size_t result = 0;
if(size_type == 4)
result = bytesToInt_bigEndian(bytes);//4
else
result = bytesToLong_bigEndian(bytes);//8
return result;
}
INLINE void sizeToBytes(unsigned char* outBytes, size_t size, int size_type)
{
if(size_type == 4)
intToBytes_bigEndian(outBytes, (unsigned int)size);//4
else
longToBytes_bigEndian(outBytes, (unsigned long)size);//8
}
void convertSZParamsToBytes(sz_params* params, unsigned char* result, char optQuantMode)
{
//unsigned char* result = (unsigned char*)malloc(16);
unsigned char buf = 0;
buf = optQuantMode;
buf = (buf << 1) | dataEndianType;
buf = (buf << 1) | sysEndianType;
buf = (buf << 2) | params->szMode;
result[0] = buf;
}
void convertBytesToSZParams(unsigned char* bytes, sz_params* params, sz_exedata* pde_exe)
{
unsigned char flag1 = bytes[0];
pde_exe->optQuantMode = (flag1 & 0x40) >> 6;
dataEndianType = (flag1 & 0x20) >> 5;
params->szMode = (flag1 & 0x0c) >> 2;
}

237
utils/TSZ/sz/src/CompressElement.c Normal file
View File

@ -0,0 +1,237 @@
/**
* @file CompressElement.c
* @author Sheng Di
* @date May, 2016
* @brief Functions of CompressElement
* (C) 2015 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef WINDOWS
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wchar-subscripts"
#endif
#include <stdlib.h>
#include <stdio.h>
#include <math.h>
#include <string.h>
#include "sz.h"
#include "CompressElement.h"
INLINE short computeGroupNum_float(float value)
{
short expo = getExponent_float(value);
if(expo < 0)
expo = -1;
return expo;
}
INLINE short computeGroupNum_double(double value)
{
short expo = getExponent_double(value);
if(expo < 0)
expo = -1;
return expo;
}
/**
* Add preceding neighbor values to a buffer.
* @param last3CmprsData buffer
* @param value the value to be added to the buffer
* */
INLINE void listAdd_double(double last3CmprsData[3], double value)
{
last3CmprsData[2] = last3CmprsData[1];
last3CmprsData[1] = last3CmprsData[0];
last3CmprsData[0] = value;
}
INLINE void listAdd_float(float last3CmprsData[3], float value)
{
last3CmprsData[2] = last3CmprsData[1];
last3CmprsData[1] = last3CmprsData[0];
last3CmprsData[0] = value;
}
INLINE void listAdd_int(int64_t last3CmprsData[3], int64_t value)
{
last3CmprsData[2] = last3CmprsData[1];
last3CmprsData[1] = last3CmprsData[0];
last3CmprsData[0] = value;
}
INLINE void listAdd_int32(int32_t last3CmprsData[3], int32_t value)
{
last3CmprsData[2] = last3CmprsData[1];
last3CmprsData[1] = last3CmprsData[0];
last3CmprsData[0] = value;
}
INLINE void listAdd_float_group(float *groups, int *flags, char groupNum, float oriValue, float decValue, char* curGroupID)
{
if(groupNum>=0)
{
if(flags[groupNum]==0)
flags[groupNum] = 1;
groups[groupNum] = decValue;
}
else
{
groups[0] = decValue;
flags[0] = 1;
}
if(oriValue>=0)
*curGroupID = groupNum+2; //+[-1,0,1,2,3,....,16] is mapped to [1,2,....,18]
else
*curGroupID = -(groupNum+2); //-[-1,0,1,2,3,....,16] is mapped to [-1,-2,....,-18]
}
INLINE void listAdd_double_group(double *groups, int *flags, char groupNum, double oriValue, double decValue, char* curGroupID)
{
if(groupNum>=0)
{
if(flags[groupNum]==0)
flags[groupNum] = 1;
groups[groupNum] = decValue;
}
else
{
groups[0] = decValue;
flags[0] = 1;
}
if(oriValue>=0)
*curGroupID = groupNum+2; //+[-1,0,1,2,3,....,16] is mapped to [1,2,....,18]
else
*curGroupID = -(groupNum+2); //-[-1,0,1,2,3,....,16] is mapped to [-1,-2,....,-18]
}
/**
* Determine whether the prediction value minErr is valid.
*
* */
INLINE int validPrediction_double(double minErr, double precision)
{
if(minErr<=precision)
return 1;
else
return 0;
}
INLINE int validPrediction_float(float minErr, float precision)
{
if(minErr<=precision)
return 1;
else
return 0;
}
double* generateGroupErrBounds(int errorBoundMode, double realPrecision, double pwrErrBound)
{
double pwrError;
double* result = (double*)malloc(GROUP_COUNT*sizeof(double));
int i = 0;
for(i=0;i<GROUP_COUNT;i++)
{
pwrError = ((double)pow(2, i))*pwrErrBound;
switch(errorBoundMode)
{
case ABS_AND_PW_REL:
case REL_AND_PW_REL:
result[i] = pwrError<realPrecision?pwrError:realPrecision;
break;
case ABS_OR_PW_REL:
case REL_OR_PW_REL:
result[i] = pwrError<realPrecision?realPrecision:pwrError;
break;
case PW_REL:
result[i] = pwrError;
break;
}
}
return result;
}
int generateGroupMaxIntervalCount(double* groupErrBounds)
{
int i = 0;
int maxCount = 0, count = 0;
for(i=0;i<GROUP_COUNT;i++)
{
count = (int)(pow(2, i)/groupErrBounds[i] + 0.5);
if(maxCount<count)
maxCount = count;
}
return maxCount;
}
void new_LossyCompressionElement(LossyCompressionElement *lce, int leadingNum, unsigned char* intMidBytes,
int intMidBytes_Length, int resiMidBitsLength, int resiBits)
{
lce->leadingZeroBytes = leadingNum; //0,1,2,or 3
memcpy(lce->integerMidBytes,intMidBytes,intMidBytes_Length);
lce->integerMidBytes_Length = intMidBytes_Length; //they are mid_bits actually
lce->resMidBitsLength = resiMidBitsLength;
lce->residualMidBits = resiBits;
}
void updateLossyCompElement_Double(unsigned char* curBytes, unsigned char* preBytes,
int reqBytesLength, int resiBitsLength, LossyCompressionElement *lce)
{
int resiIndex, intMidBytes_Length = 0;
int leadingNum = compIdenticalLeadingBytesCount_double(preBytes, curBytes); //in fact, float is enough for both single-precision and double-precisiond ata.
int fromByteIndex = leadingNum;
int toByteIndex = reqBytesLength; //later on: should use "< toByteIndex" to tarverse....
if(fromByteIndex < toByteIndex)
{
intMidBytes_Length = reqBytesLength - leadingNum;
memcpy(lce->integerMidBytes, &(curBytes[fromByteIndex]), intMidBytes_Length);
}
int resiBits = 0;
if(resiBitsLength!=0)
{
resiIndex = reqBytesLength;
if(resiIndex < 8)
resiBits = (curBytes[resiIndex] & 0xFF) >> (8-resiBitsLength);
}
lce->leadingZeroBytes = leadingNum;
lce->integerMidBytes_Length = intMidBytes_Length;
lce->resMidBitsLength = resiBitsLength;
lce->residualMidBits = resiBits;
}
INLINE void updateLossyCompElement_Float(unsigned char* diffBytes, unsigned char* preDiffBytes,
int reqBytesLength, int resiBitsLength, LossyCompressionElement *lce)
{
int resiIndex, intMidBytes_Length = 0;
int leadingNum = compIdenticalLeadingBytesCount_float(preDiffBytes, diffBytes); //in fact, float is enough for both single-precision and double-precisiond ata.
int fromByteIndex = leadingNum;
int toByteIndex = reqBytesLength; //later on: should use "< toByteIndex" to tarverse....
if(fromByteIndex < toByteIndex)
{
intMidBytes_Length = reqBytesLength - leadingNum;
// set lce mid data
memcpy(lce->integerMidBytes, &(diffBytes[fromByteIndex]), intMidBytes_Length);
}
int resiBits = 0;
if(resiBitsLength!=0)
{
resiIndex = reqBytesLength;
if(resiIndex < 8)
resiBits = (diffBytes[resiIndex] & 0xFF) >> (8-resiBitsLength);
}
// set lce
lce->leadingZeroBytes = leadingNum;
lce->integerMidBytes_Length = intMidBytes_Length;
lce->resMidBitsLength = resiBitsLength;
lce->residualMidBits = resiBits;
}
#ifndef WINDOWS
#pragma GCC diagnostic pop
#endif

72
utils/TSZ/sz/src/DynamicByteArray.c Normal file
View File

@ -0,0 +1,72 @@
/**
* @file DynamicByteArray.c
* @author Sheng Di
* @date May, 2016
* @brief Dynamic Byte Array
* (C) 2015 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include "DynamicByteArray.h"
#include "sz.h"
void new_DBA(DynamicByteArray **dba, size_t cap) {
*dba = (DynamicByteArray *)malloc(sizeof(DynamicByteArray));
(*dba)->size = 0;
(*dba)->capacity = cap;
(*dba)->array = (unsigned char*)malloc(sizeof(unsigned char)*cap);
}
void convertDBAtoBytes(DynamicByteArray *dba, unsigned char** bytes)
{
size_t size = dba->size;
if(size>0)
*bytes = (unsigned char*)malloc(size * sizeof(unsigned char));
else
{
*bytes = NULL;
return ;
}
memcpy(*bytes, dba->array, size*sizeof(unsigned char));
}
void free_DBA(DynamicByteArray *dba)
{
free(dba->array);
free(dba);
}
INLINE unsigned char getDBA_Data(DynamicByteArray *dba, size_t pos)
{
if(pos>=dba->size)
{
printf("Error: wrong position of DBA (impossible case unless bugs elsewhere in the code?).\n");
exit(0);
}
return dba->array[pos];
}
INLINE void addDBA_Data(DynamicByteArray *dba, unsigned char value)
{
if(dba->size==dba->capacity)
{
dba->capacity = dba->capacity << 1;
dba->array = (unsigned char *)realloc(dba->array, dba->capacity*sizeof(unsigned char));
}
dba->array[dba->size] = value;
dba->size ++;
}
INLINE void memcpyDBA_Data(DynamicByteArray *dba, unsigned char* data, size_t length)
{
if(dba->size + length > dba->capacity)
{
dba->capacity = dba->size + length;
dba->array = (unsigned char *)realloc(dba->array, dba->capacity*sizeof(unsigned char));
}
memcpy(&(dba->array[dba->size]), data, length);
dba->size += length;
}

58
utils/TSZ/sz/src/DynamicIntArray.c Normal file
View File

@ -0,0 +1,58 @@
/**
* @file DynamicIntArray.c
* @author Sheng Di
* @date May, 2016
* @brief Dynamic Int Array
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include "DynamicIntArray.h"
#include "sz.h"
void new_DIA(DynamicIntArray **dia, size_t cap) {
*dia = (DynamicIntArray *)malloc(sizeof(DynamicIntArray));
(*dia)->size = 0;
(*dia)->capacity = cap;
(*dia)->array = (unsigned char*)malloc(sizeof(unsigned char)*cap);
}
void convertDIAtoInts(DynamicIntArray *dia, unsigned char **data)
{
size_t size = dia->size;
if(size>0)
*data = (unsigned char*)malloc(size * sizeof(char));
else
*data = NULL;
memcpy(*data, dia->array, size*sizeof(unsigned char));
}
void free_DIA(DynamicIntArray *dia)
{
free(dia->array);
free(dia);
}
int getDIA_Data(DynamicIntArray *dia, size_t pos)
{
if(pos>=dia->size)
{
printf("Error: wrong position of DIA.\n");
exit(0);
}
return dia->array[pos];
}
INLINE void addDIA_Data(DynamicIntArray *dia, int value)
{
if(dia->size==dia->capacity)
{
dia->capacity = dia->capacity << 1;
dia->array = (unsigned char *)realloc(dia->array, dia->capacity*sizeof(unsigned char));
}
dia->array[dia->size] = (unsigned char)value;
dia->size ++;
}

765
utils/TSZ/sz/src/Huffman.c Normal file
View File

@ -0,0 +1,765 @@
/**
* @file Huffman.c
* @author Sheng Di
* @date Aug., 2016
* @brief Customized Huffman Encoding, Compression and Decompression functions
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "Huffman.h"
#include "sz.h"
HuffmanTree* createHuffmanTree(int stateNum)
{
HuffmanTree *huffmanTree = (HuffmanTree*)malloc(sizeof(HuffmanTree));
memset(huffmanTree, 0, sizeof(HuffmanTree));
huffmanTree->stateNum = stateNum;
huffmanTree->allNodes = 2*stateNum;
huffmanTree->pool = (struct node_t*)malloc(huffmanTree->allNodes*2*sizeof(struct node_t));
huffmanTree->qqq = (node*)malloc(huffmanTree->allNodes*2*sizeof(node));
huffmanTree->code = (unsigned long**)malloc(huffmanTree->stateNum*sizeof(unsigned long*));
huffmanTree->cout = (unsigned char *)malloc(huffmanTree->stateNum*sizeof(unsigned char));
memset(huffmanTree->pool, 0, huffmanTree->allNodes*2*sizeof(struct node_t));
memset(huffmanTree->qqq, 0, huffmanTree->allNodes*2*sizeof(node));
memset(huffmanTree->code, 0, huffmanTree->stateNum*sizeof(unsigned long*));
memset(huffmanTree->cout, 0, huffmanTree->stateNum*sizeof(unsigned char));
huffmanTree->qq = huffmanTree->qqq - 1;
huffmanTree->n_nodes = 0;
huffmanTree->n_inode = 0;
huffmanTree->qend = 1;
return huffmanTree;
}
HuffmanTree* createDefaultHuffmanTree()
{
int maxRangeRadius = 32768;
int stateNum = maxRangeRadius << 1; //*2
return createHuffmanTree(stateNum);
}
node new_node(HuffmanTree* huffmanTree, size_t freq, unsigned int c, node a, node b)
{
node n = huffmanTree->pool + huffmanTree->n_nodes++;
if (freq)
{
n->c = c;
n->freq = freq;
n->t = 1;
}
else {
n->left = a;
n->right = b;
n->freq = a->freq + b->freq;
n->t = 0;
//n->c = 0;
}
return n;
}
node new_node2(HuffmanTree *huffmanTree, unsigned int c, unsigned char t)
{
huffmanTree->pool[huffmanTree->n_nodes].c = c;
huffmanTree->pool[huffmanTree->n_nodes].t = t;
return huffmanTree->pool + huffmanTree->n_nodes++;
}
/* priority queue */
void qinsert(HuffmanTree *huffmanTree, node n)
{
int j, i = huffmanTree->qend++;
while ((j = (i>>1))) //j=i/2
{
if (huffmanTree->qq[j]->freq <= n->freq) break;
huffmanTree->qq[i] = huffmanTree->qq[j], i = j;
}
huffmanTree->qq[i] = n;
}
node qremove(HuffmanTree* huffmanTree)
{
int i, l;
node n = huffmanTree->qq[i = 1];
node p;
if (huffmanTree->qend < 2) return 0;
huffmanTree->qend --;
huffmanTree->qq[i] = huffmanTree->qq[huffmanTree->qend];
while ((l = (i<<1)) < huffmanTree->qend) //l=(i*2)
{
if (l + 1 < huffmanTree->qend && huffmanTree->qq[l + 1]->freq < huffmanTree->qq[l]->freq) l++;
if(huffmanTree->qq[i]->freq > huffmanTree->qq[l]->freq)
{
p = huffmanTree->qq[i];
huffmanTree->qq[i] = huffmanTree->qq[l];
huffmanTree->qq[l] = p;
i = l;
}
else
{
break;
}
}
return n;
}
/* walk the tree and put 0s and 1s */
/**
* @out1 should be set to 0.
* @out2 should be 0 as well.
* @index: the index of the byte
* */
void build_code(HuffmanTree *huffmanTree, node n, int len, unsigned long out1, unsigned long out2)
{
if (n->t) {
huffmanTree->code[n->c] = (unsigned long*)malloc(2*sizeof(unsigned long));
if(len<=64)
{
(huffmanTree->code[n->c])[0] = out1 << (64 - len);
(huffmanTree->code[n->c])[1] = out2;
}
else
{
(huffmanTree->code[n->c])[0] = out1;
(huffmanTree->code[n->c])[1] = out2 << (128 - len);
}
huffmanTree->cout[n->c] = (unsigned char)len;
return;
}
int index = len >> 6; //=len/64
if(index == 0)
{
out1 = out1 << 1;
out1 = out1 | 0;
build_code(huffmanTree, n->left, len + 1, out1, 0);
out1 = out1 | 1;
build_code(huffmanTree, n->right, len + 1, out1, 0);
}
else
{
if(len%64!=0)
out2 = out2 << 1;
out2 = out2 | 0;
build_code(huffmanTree, n->left, len + 1, out1, out2);
out2 = out2 | 1;
build_code(huffmanTree, n->right, len + 1, out1, out2);
}
}
/**
* Compute the frequency of the data and build the Huffman tree
* @param HuffmanTree* huffmanTree (output)
* @param int *s (input)
* @param size_t length (input)
* */
void init(HuffmanTree* huffmanTree, int *s, size_t length)
{
size_t i, index;
size_t *freq = (size_t *)malloc(huffmanTree->allNodes*sizeof(size_t));
memset(freq, 0, huffmanTree->allNodes*sizeof(size_t));
for(i = 0;i < length;i++)
{
index = s[i];
freq[index]++;
}
for (i = 0; i < huffmanTree->allNodes; i++)
if (freq[i])
qinsert(huffmanTree, new_node(huffmanTree, freq[i], i, 0, 0));
while (huffmanTree->qend > 2)
qinsert(huffmanTree, new_node(huffmanTree, 0, 0, qremove(huffmanTree), qremove(huffmanTree)));
build_code(huffmanTree, huffmanTree->qq[1], 0, 0, 0);
free(freq);
}
void init_static(HuffmanTree* huffmanTree, int *s, size_t length)
{
size_t i;
size_t *freq = (size_t *)malloc(huffmanTree->allNodes*sizeof(size_t));
memset(freq, 0, huffmanTree->allNodes*sizeof(size_t));
for (i = 0; i < huffmanTree->allNodes; i++)
if (freq[i])
qinsert(huffmanTree, new_node(huffmanTree, freq[i], i, 0, 0));
while (huffmanTree->qend > 2)
qinsert(huffmanTree, new_node(huffmanTree, 0, 0, qremove(huffmanTree), qremove(huffmanTree)));
build_code(huffmanTree, huffmanTree->qq[1], 0, 0, 0);
free(freq);
}
void encode(HuffmanTree *huffmanTree, int *s, size_t length, unsigned char *out, size_t *outSize)
{
size_t i = 0;
unsigned char bitSize = 0, byteSize, byteSizep;
int state;
unsigned char *p = out;
int lackBits = 0;
//long totalBitSize = 0, maxBitSize = 0, bitSize21 = 0, bitSize32 = 0;
for (i = 0;i<length;i++)
{
state = s[i];
bitSize = huffmanTree->cout[state];
//printf("%d %d : %d %u\n",i, state, bitSize, (code[state])[0] >> (64-cout[state]));
//debug: compute the average bitSize and the count that is over 32...
/*if(bitSize>=21)
bitSize21++;
if(bitSize>=32)
bitSize32++;
if(maxBitSize<bitSize)
maxBitSize = bitSize;
totalBitSize+=bitSize;*/
if(lackBits==0)
{
byteSize = bitSize%8==0 ? bitSize/8 : bitSize/8+1; //it's equal to the number of bytes involved (for *outSize)
byteSizep = bitSize/8; //it's used to move the pointer p for next data
if(byteSize<=8)
{
longToBytes_bigEndian(p, (huffmanTree->code[state])[0]);
p += byteSizep;
}
else //byteSize>8
{
longToBytes_bigEndian(p, (huffmanTree->code[state])[0]);
p += 8;
longToBytes_bigEndian(p, (huffmanTree->code[state])[1]);
p += (byteSizep - 8);
}
*outSize += byteSize;
lackBits = bitSize%8==0 ? 0 : 8 - bitSize%8;
}
else
{
*p = (*p) | (unsigned char)((huffmanTree->code[state])[0] >> (64 - lackBits));
if(lackBits < bitSize)
{
p++;
//(*outSize)++;
long newCode = (huffmanTree->code[state])[0] << lackBits;
longToBytes_bigEndian(p, newCode);
if(bitSize<=64)
{
bitSize -= lackBits;
byteSize = bitSize%8==0 ? bitSize/8 : bitSize/8+1;
byteSizep = bitSize/8;
p += byteSizep;
(*outSize)+=byteSize;
lackBits = bitSize%8==0 ? 0 : 8 - bitSize%8;
}
else //bitSize > 64
{
byteSizep = 7; //must be 7 bytes, because lackBits!=0
p+=byteSizep;
(*outSize)+=byteSize;
bitSize -= 64;
if(lackBits < bitSize)
{
*p = (*p) | (unsigned char)((huffmanTree->code[state])[0] >> (64 - lackBits));
p++;
//(*outSize)++;
newCode = (huffmanTree->code[state])[1] << lackBits;
longToBytes_bigEndian(p, newCode);
bitSize -= lackBits;
byteSize = bitSize%8==0 ? bitSize/8 : bitSize/8+1;
byteSizep = bitSize/8;
p += byteSizep;
(*outSize)+=byteSize;
lackBits = bitSize%8==0 ? 0 : 8 - bitSize%8;
}
else //lackBits >= bitSize
{
*p = (*p) | (unsigned char)((huffmanTree->code[state])[0] >> (64 - bitSize));
lackBits -= bitSize;
}
}
}
else //lackBits >= bitSize
{
lackBits -= bitSize;
if(lackBits==0)
p++;
}
}
}
// for(i=0;i<stateNum;i++)
// if(code[i]!=NULL) free(code[i]);
/*printf("max bitsize = %d\n", maxBitSize);
printf("bitSize21 ratio = %f\n", ((float)bitSize21)/length);
printf("bitSize32 ratio = %f\n", ((float)bitSize32)/length);
printf("avg bit size = %f\n", ((float)totalBitSize)/length);*/
}
void decode(unsigned char *s, size_t targetLength, node t, int *out)
{
size_t i = 0, byteIndex = 0, count = 0;
int r;
node n = t;
if(n->t) //root->t==1 means that all state values are the same (constant)
{
for(count=0;count<targetLength;count++)
out[count] = n->c;
return;
}
for(i=0;count<targetLength;i++)
{
byteIndex = i>>3; //i/8
r = i%8;
if(((s[byteIndex] >> (7-r)) & 0x01) == 0)
n = n->left;
else
n = n->right;
if (n->t) {
//putchar(n->c);
out[count] = n->c;
n = t;
count++;
}
}
// putchar('\n');
if (t != n) printf("garbage input\n");
return;
}
void pad_tree_uchar(HuffmanTree* huffmanTree, unsigned char* L, unsigned char* R, unsigned int* C, unsigned char* t, unsigned int i, node root)
{
C[i] = root->c;
t[i] = root->t;
node lroot = root->left;
if(lroot!=0)
{
huffmanTree->n_inode++;
L[i] = huffmanTree->n_inode;
pad_tree_uchar(huffmanTree, L,R,C,t, huffmanTree->n_inode, lroot);
}
node rroot = root->right;
if(rroot!=0)
{
huffmanTree->n_inode++;
R[i] = huffmanTree->n_inode;
pad_tree_uchar(huffmanTree, L,R,C,t, huffmanTree->n_inode, rroot);
}
}
void pad_tree_ushort(HuffmanTree* huffmanTree, unsigned short* L, unsigned short* R, unsigned int* C, unsigned char* t, unsigned int i, node root)
{
C[i] = root->c;
t[i] = root->t;
node lroot = root->left;
if(lroot!=0)
{
huffmanTree->n_inode++;
L[i] = huffmanTree->n_inode;
pad_tree_ushort(huffmanTree,L,R,C,t,huffmanTree->n_inode, lroot);
}
node rroot = root->right;
if(rroot!=0)
{
huffmanTree->n_inode++;
R[i] = huffmanTree->n_inode;
pad_tree_ushort(huffmanTree,L,R,C,t,huffmanTree->n_inode, rroot);
}
}
void pad_tree_uint(HuffmanTree* huffmanTree, unsigned int* L, unsigned int* R, unsigned int* C, unsigned char* t, unsigned int i, node root)
{
C[i] = root->c;
t[i] = root->t;
node lroot = root->left;
if(lroot!=0)
{
huffmanTree->n_inode++;
L[i] = huffmanTree->n_inode;
pad_tree_uint(huffmanTree,L,R,C,t,huffmanTree->n_inode, lroot);
}
node rroot = root->right;
if(rroot!=0)
{
huffmanTree->n_inode++;
R[i] = huffmanTree->n_inode;
pad_tree_uint(huffmanTree,L,R,C,t,huffmanTree->n_inode, rroot);
}
}
unsigned int convert_HuffTree_to_bytes_anyStates(HuffmanTree* huffmanTree, int nodeCount, unsigned char** out)
{
if(nodeCount<=256)
{
unsigned char* L = (unsigned char*)malloc(nodeCount*sizeof(unsigned char));
memset(L, 0, nodeCount*sizeof(unsigned char));
unsigned char* R = (unsigned char*)malloc(nodeCount*sizeof(unsigned char));
memset(R, 0, nodeCount*sizeof(unsigned char));
unsigned int* C = (unsigned int*)malloc(nodeCount*sizeof(unsigned int));
memset(C, 0, nodeCount*sizeof(unsigned int));
unsigned char* t = (unsigned char*)malloc(nodeCount*sizeof(unsigned char));
memset(t, 0, nodeCount*sizeof(unsigned char));
pad_tree_uchar(huffmanTree,L,R,C,t,0,huffmanTree->qq[1]);
unsigned int totalSize = 1+3*nodeCount*sizeof(unsigned char)+nodeCount*sizeof(unsigned int);
*out = (unsigned char*)malloc(totalSize*sizeof(unsigned char));
(*out)[0] = (unsigned char)sysEndianType;
memcpy(*out+1, L, nodeCount*sizeof(unsigned char));
memcpy((*out)+1+nodeCount*sizeof(unsigned char),R,nodeCount*sizeof(unsigned char));
memcpy((*out)+1+2*nodeCount*sizeof(unsigned char),C,nodeCount*sizeof(unsigned int));
memcpy((*out)+1+2*nodeCount*sizeof(unsigned char)+nodeCount*sizeof(unsigned int), t, nodeCount*sizeof(unsigned char));
free(L);
free(R);
free(C);
free(t);
return totalSize;
}
else if(nodeCount<=65536)
{
unsigned short* L = (unsigned short*)malloc(nodeCount*sizeof(unsigned short));
memset(L, 0, nodeCount*sizeof(unsigned short));
unsigned short* R = (unsigned short*)malloc(nodeCount*sizeof(unsigned short));
memset(R, 0, nodeCount*sizeof(unsigned short));
unsigned int* C = (unsigned int*)malloc(nodeCount*sizeof(unsigned int));
memset(C, 0, nodeCount*sizeof(unsigned int));
unsigned char* t = (unsigned char*)malloc(nodeCount*sizeof(unsigned char));
memset(t, 0, nodeCount*sizeof(unsigned char));
pad_tree_ushort(huffmanTree,L,R,C,t,0,huffmanTree->qq[1]);
unsigned int totalSize = 1+2*nodeCount*sizeof(unsigned short)+nodeCount*sizeof(unsigned char) + nodeCount*sizeof(unsigned int);
*out = (unsigned char*)malloc(totalSize);
(*out)[0] = (unsigned char)sysEndianType;
memcpy(*out+1, L, nodeCount*sizeof(unsigned short));
memcpy((*out)+1+nodeCount*sizeof(unsigned short),R,nodeCount*sizeof(unsigned short));
memcpy((*out)+1+2*nodeCount*sizeof(unsigned short),C,nodeCount*sizeof(unsigned int));
memcpy((*out)+1+2*nodeCount*sizeof(unsigned short)+nodeCount*sizeof(unsigned int),t,nodeCount*sizeof(unsigned char));
free(L);
free(R);
free(C);
free(t);
return totalSize;
}
else //nodeCount>65536
{
unsigned int* L = (unsigned int*)malloc(nodeCount*sizeof(unsigned int));
memset(L, 0, nodeCount*sizeof(unsigned int));
unsigned int* R = (unsigned int*)malloc(nodeCount*sizeof(unsigned int));
memset(R, 0, nodeCount*sizeof(unsigned int));
unsigned int* C = (unsigned int*)malloc(nodeCount*sizeof(unsigned int));
memset(C, 0, nodeCount*sizeof(unsigned int));
unsigned char* t = (unsigned char*)malloc(nodeCount*sizeof(unsigned char));
memset(t, 0, nodeCount*sizeof(unsigned char));
pad_tree_uint(huffmanTree, L,R,C,t,0,huffmanTree->qq[1]);
//debug
//node root = new_node2(0,0);
//unpad_tree_uint(L,R,C,t,0,root);
unsigned int totalSize = 1+3*nodeCount*sizeof(unsigned int)+nodeCount*sizeof(unsigned char);
*out = (unsigned char*)malloc(totalSize);
(*out)[0] = (unsigned char)sysEndianType;
memcpy(*out+1, L, nodeCount*sizeof(unsigned int));
memcpy((*out)+1+nodeCount*sizeof(unsigned int),R,nodeCount*sizeof(unsigned int));
memcpy((*out)+1+2*nodeCount*sizeof(unsigned int),C,nodeCount*sizeof(unsigned int));
memcpy((*out)+1+3*nodeCount*sizeof(unsigned int),t,nodeCount*sizeof(unsigned char));
free(L);
free(R);
free(C);
free(t);
return totalSize;
}
}
void unpad_tree_uchar(HuffmanTree* huffmanTree, unsigned char* L, unsigned char* R, unsigned int* C, unsigned char *t, unsigned int i, node root)
{
//root->c = C[i];
if(root->t==0)
{
unsigned char l, r;
l = L[i];
if(l!=0)
{
node lroot = new_node2(huffmanTree,C[l],t[l]);
root->left = lroot;
unpad_tree_uchar(huffmanTree,L,R,C,t,l,lroot);
}
r = R[i];
if(r!=0)
{
node rroot = new_node2(huffmanTree,C[r],t[r]);
root->right = rroot;
unpad_tree_uchar(huffmanTree,L,R,C,t,r,rroot);
}
}
}
void unpad_tree_ushort(HuffmanTree* huffmanTree, unsigned short* L, unsigned short* R, unsigned int* C, unsigned char* t, unsigned int i, node root)
{
//root->c = C[i];
if(root->t==0)
{
unsigned short l, r;
l = L[i];
if(l!=0)
{
node lroot = new_node2(huffmanTree,C[l],t[l]);
root->left = lroot;
unpad_tree_ushort(huffmanTree,L,R,C,t,l,lroot);
}
r = R[i];
if(r!=0)
{
node rroot = new_node2(huffmanTree,C[r],t[r]);
root->right = rroot;
unpad_tree_ushort(huffmanTree,L,R,C,t,r,rroot);
}
}
}
void unpad_tree_uint(HuffmanTree* huffmanTree, unsigned int* L, unsigned int* R, unsigned int* C, unsigned char* t, unsigned int i, node root)
{
//root->c = C[i];
if(root->t==0)
{
unsigned int l, r;
l = L[i];
if(l!=0)
{
node lroot = new_node2(huffmanTree,C[l],t[l]);
root->left = lroot;
unpad_tree_uint(huffmanTree,L,R,C,t,l,lroot);
}
r = R[i];
if(r!=0)
{
node rroot = new_node2(huffmanTree,C[r],t[r]);
root->right = rroot;
unpad_tree_uint(huffmanTree,L,R,C,t,r,rroot);
}
}
}
node reconstruct_HuffTree_from_bytes_anyStates(HuffmanTree *huffmanTree, unsigned char* bytes, int nodeCount)
{
if(nodeCount<=256)
{
unsigned char* L = (unsigned char*)malloc(nodeCount*sizeof(unsigned char));
memset(L, 0, nodeCount*sizeof(unsigned char));
unsigned char* R = (unsigned char*)malloc(nodeCount*sizeof(unsigned char));
memset(R, 0, nodeCount*sizeof(unsigned char));
unsigned int* C = (unsigned int*)malloc(nodeCount*sizeof(unsigned int));
memset(C, 0, nodeCount*sizeof(unsigned int));
unsigned char* t = (unsigned char*)malloc(nodeCount*sizeof(unsigned char));
memset(t, 0, nodeCount*sizeof(unsigned char));
unsigned char cmpSysEndianType = bytes[0];
if(cmpSysEndianType!=(unsigned char)sysEndianType)
{
unsigned char* p = (unsigned char*)(bytes+1+2*nodeCount*sizeof(unsigned char));
size_t i = 0, size = nodeCount*sizeof(unsigned int);
while(1)
{
symTransform_4bytes(p);
i+=sizeof(unsigned int);
if(i<size)
p+=sizeof(unsigned int);
else
break;
}
}
memcpy(L, bytes+1, nodeCount*sizeof(unsigned char));
memcpy(R, bytes+1+nodeCount*sizeof(unsigned char), nodeCount*sizeof(unsigned char));
memcpy(C, bytes+1+2*nodeCount*sizeof(unsigned char), nodeCount*sizeof(unsigned int));
memcpy(t, bytes+1+2*nodeCount*sizeof(unsigned char)+nodeCount*sizeof(unsigned int), nodeCount*sizeof(unsigned char));
node root = new_node2(huffmanTree, C[0],t[0]);
unpad_tree_uchar(huffmanTree,L,R,C,t,0,root);
free(L);
free(R);
free(C);
free(t);
return root;
}
else if(nodeCount<=65536)
{
unsigned short* L = (unsigned short*)malloc(nodeCount*sizeof(unsigned short));
memset(L, 0, nodeCount*sizeof(unsigned short));
unsigned short* R = (unsigned short*)malloc(nodeCount*sizeof(unsigned short));
memset(R, 0, nodeCount*sizeof(unsigned short));
unsigned int* C = (unsigned int*)malloc(nodeCount*sizeof(unsigned int));
memset(C, 0, nodeCount*sizeof(unsigned int));
unsigned char* t = (unsigned char*)malloc(nodeCount*sizeof(unsigned char));
memset(t, 0, nodeCount*sizeof(unsigned char));
unsigned char cmpSysEndianType = bytes[0];
if(cmpSysEndianType!=(unsigned char)sysEndianType)
{
unsigned char* p = (unsigned char*)(bytes+1);
size_t i = 0, size = 2*nodeCount*sizeof(unsigned short);
while(1)
{
symTransform_2bytes(p);
i+=sizeof(unsigned short);
if(i<size)
p+=sizeof(unsigned short);
else
break;
}
size = nodeCount*sizeof(unsigned int);
while(1)
{
symTransform_4bytes(p);
i+=sizeof(unsigned int);
if(i<size)
p+=sizeof(unsigned int);
else
break;
}
}
memcpy(L, bytes+1, nodeCount*sizeof(unsigned short));
memcpy(R, bytes+1+nodeCount*sizeof(unsigned short), nodeCount*sizeof(unsigned short));
memcpy(C, bytes+1+2*nodeCount*sizeof(unsigned short), nodeCount*sizeof(unsigned int));
memcpy(t, bytes+1+2*nodeCount*sizeof(unsigned short)+nodeCount*sizeof(unsigned int), nodeCount*sizeof(unsigned char));
node root = new_node2(huffmanTree,0,0);
unpad_tree_ushort(huffmanTree,L,R,C,t,0,root);
free(L);
free(R);
free(C);
free(t);
return root;
}
else //nodeCount>65536
{
unsigned int* L = (unsigned int*)malloc(nodeCount*sizeof(unsigned int));
memset(L, 0, nodeCount*sizeof(unsigned int));
unsigned int* R = (unsigned int*)malloc(nodeCount*sizeof(unsigned int));
memset(R, 0, nodeCount*sizeof(unsigned int));
unsigned int* C = (unsigned int*)malloc(nodeCount*sizeof(unsigned int));
memset(C, 0, nodeCount*sizeof(unsigned int));
unsigned char* t = (unsigned char*)malloc(nodeCount*sizeof(unsigned char));
memset(t, 0, nodeCount*sizeof(unsigned char));
unsigned char cmpSysEndianType = bytes[0];
if(cmpSysEndianType!=(unsigned char)sysEndianType)
{
unsigned char* p = (unsigned char*)(bytes+1);
size_t i = 0, size = 3*nodeCount*sizeof(unsigned int);
while(1)
{
symTransform_4bytes(p);
i+=sizeof(unsigned int);
if(i<size)
p+=sizeof(unsigned int);
else
break;
}
}
memcpy(L, bytes+1, nodeCount*sizeof(unsigned int));
memcpy(R, bytes+1+nodeCount*sizeof(unsigned int), nodeCount*sizeof(unsigned int));
memcpy(C, bytes+1+2*nodeCount*sizeof(unsigned int), nodeCount*sizeof(unsigned int));
memcpy(t, bytes+1+3*nodeCount*sizeof(unsigned int), nodeCount*sizeof(unsigned char));
node root = new_node2(huffmanTree,0,0);
unpad_tree_uint(huffmanTree,L,R,C,t,0,root);
free(L);
free(R);
free(C);
free(t);
return root;
}
}
void encode_withTree(HuffmanTree* huffmanTree, int *s, size_t length, unsigned char **out, size_t *outSize)
{
size_t i;
int nodeCount = 0;
unsigned char *treeBytes, buffer[4];
init(huffmanTree, s, length);
for (i = 0; i < huffmanTree->stateNum; i++)
if (huffmanTree->code[i]) nodeCount++;
nodeCount = nodeCount*2-1;
unsigned int treeByteSize = convert_HuffTree_to_bytes_anyStates(huffmanTree,nodeCount, &treeBytes);
//printf("treeByteSize = %d\n", treeByteSize);
*out = (unsigned char*)malloc(length*sizeof(int)+treeByteSize);
intToBytes_bigEndian(buffer, nodeCount);
memcpy(*out, buffer, 4);
intToBytes_bigEndian(buffer, huffmanTree->stateNum/2); //real number of intervals
memcpy(*out+4, buffer, 4);
memcpy(*out+8, treeBytes, treeByteSize);
free(treeBytes);
size_t enCodeSize = 0;
encode(huffmanTree, s, length, *out+8+treeByteSize, &enCodeSize);
*outSize = 8+treeByteSize+enCodeSize;
}
/**
* @par *out rememmber to allocate targetLength short_type data for it beforehand.
*
* */
void decode_withTree(HuffmanTree* huffmanTree, unsigned char *s, size_t targetLength, int *out)
{
size_t encodeStartIndex;
size_t nodeCount = bytesToInt_bigEndian(s);
node root = reconstruct_HuffTree_from_bytes_anyStates(huffmanTree,s+8, nodeCount);
//sdi: Debug
/* build_code(root, 0, 0, 0);
int i;
unsigned long code_1, code_2;
for (i = 0; i < stateNum; i++)
if (code[i])
{
printf("%d: %lu,%lu ; %u\n", i, (code[i])[0],(code[i])[1], cout[i]);
//code_1 = (code[i])[0];
}*/
if(nodeCount<=256)
encodeStartIndex = 1+3*nodeCount*sizeof(unsigned char)+nodeCount*sizeof(unsigned int);
else if(nodeCount<=65536)
encodeStartIndex = 1+2*nodeCount*sizeof(unsigned short)+nodeCount*sizeof(unsigned char)+nodeCount*sizeof(unsigned int);
else
encodeStartIndex = 1+3*nodeCount*sizeof(unsigned int)+nodeCount*sizeof(unsigned char);
decode(s+8+encodeStartIndex, targetLength, root, out);
}
void SZ_ReleaseHuffman(HuffmanTree* huffmanTree)
{
size_t i;
free(huffmanTree->pool);
huffmanTree->pool = NULL;
free(huffmanTree->qqq);
huffmanTree->qqq = NULL;
for(i=0;i<huffmanTree->stateNum;i++)
{
if(huffmanTree->code[i]!=NULL)
free(huffmanTree->code[i]);
}
free(huffmanTree->code);
huffmanTree->code = NULL;
free(huffmanTree->cout);
huffmanTree->cout = NULL;
free(huffmanTree);
huffmanTree = NULL;
}

420
utils/TSZ/sz/src/TightDataPointStorageD.c Normal file
View File

@ -0,0 +1,420 @@
/**
* @file TightPointDataStorageD.c
* @author Sheng Di and Dingwen Tao
* @date Aug, 2016
* @brief The functions used to construct the tightPointDataStorage element for storing compressed bytes.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include "TightDataPointStorageD.h"
#include "sz.h"
#include "Huffman.h"
#include "transcode.h"
#include "fse.h"
void new_TightDataPointStorageD_Empty(TightDataPointStorageD **this)
{
TightDataPointStorageD* tdps = (TightDataPointStorageD*)malloc(sizeof(TightDataPointStorageD));
memset(tdps, 0, sizeof(TightDataPointStorageD));
*this = tdps;
}
int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsigned char* flatBytes, size_t flatBytesLength, sz_exedata* pde_exe, sz_params* pde_params)
{
new_TightDataPointStorageD_Empty(this);
size_t i, index = 0;
unsigned char version = flatBytes[index++]; //3
unsigned char sameRByte = flatBytes[index++]; //1
// parse data format
switch (version)
{
case DATA_FROMAT_VER1:
break;
default:
printf(" error, compressed data format can not be recognised. ver=%d\n ", version);
return SZ_ABS;
}
int same = sameRByte & 0x01;
(*this)->ifAdtFse = (sameRByte & 0x02)>>1; //0000,0010
(*this)->isLossless = (sameRByte & 0x10)>>4;
pde_exe->SZ_SIZE_TYPE = ((sameRByte & 0x40)>>6)==1?8:4;
//pde_params->protectValueRange = (sameRByte & 0x04)>>2;
pde_params->accelerate_pw_rel_compression = (sameRByte & 0x08) >> 3;
int errorBoundMode = SZ_ABS;
convertBytesToSZParams(&(flatBytes[index]), pde_params, pde_exe);
index += MetaDataByteLength_double;
int isRegression = (sameRByte >> 7) & 0x01;
unsigned char dsLengthBytes[8];
for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++)
dsLengthBytes[i] = flatBytes[index++];
(*this)->dataSeriesLength = bytesToSize(dsLengthBytes, pde_exe->SZ_SIZE_TYPE);
if((*this)->isLossless==1)
{
//(*this)->exactMidBytes = flatBytes+8;
return errorBoundMode;
}
else if(same==1)
{
(*this)->allSameData = 1;
(*this)->exactMidBytes = &(flatBytes[index]);
return errorBoundMode;
}
else
(*this)->allSameData = 0;
if(isRegression == 1)
{
(*this)->raBytes_size = flatBytesLength - 3 - 1 - MetaDataByteLength_double - pde_exe->SZ_SIZE_TYPE;
(*this)->raBytes = &(flatBytes[index]);
return errorBoundMode;
}
unsigned char byteBuf[8];
for (i = 0; i < 4; i++)
byteBuf[i] = flatBytes[index++];
int max_quant_intervals = bytesToInt_bigEndian(byteBuf);// 4
pde_params->maxRangeRadius = max_quant_intervals/2;
for (i = 0; i < 4; i++)
byteBuf[i] = flatBytes[index++];
(*this)->intervals = bytesToInt_bigEndian(byteBuf);// 4
for (i = 0; i < 8; i++)
byteBuf[i] = flatBytes[index++];
(*this)->medianValue = bytesToDouble(byteBuf);//8
(*this)->reqLength = flatBytes[index++]; //1
for (i = 0; i < 8; i++)
byteBuf[i] = flatBytes[index++];
(*this)->realPrecision = bytesToDouble(byteBuf);//8
if ((*this)->ifAdtFse == 0) {
for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++)
byteBuf[i] = flatBytes[index++];
(*this)->typeArray_size = bytesToSize(byteBuf, pde_exe->SZ_SIZE_TYPE);
} else {
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
byteBuf[i] = flatBytes[index++];
(*this)->FseCode_size = bytesToSize(byteBuf, pde_exe->SZ_SIZE_TYPE);// 4
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
byteBuf[i] = flatBytes[index++];
(*this)->transCodeBits_size = bytesToSize(byteBuf, pde_exe->SZ_SIZE_TYPE);// 4
}
for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++)
byteBuf[i] = flatBytes[index++];
(*this)->exactDataNum = bytesToSize(byteBuf, pde_exe->SZ_SIZE_TYPE);// ST
for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++)
byteBuf[i] = flatBytes[index++];
(*this)->exactMidBytes_size = bytesToSize(byteBuf, pde_exe->SZ_SIZE_TYPE);// ST
size_t logicLeadNumBitsNum = (*this)->exactDataNum * 2;
if (logicLeadNumBitsNum % 8 == 0)
{
(*this)->leadNumArray_size = logicLeadNumBitsNum >> 3;
}
else
{
(*this)->leadNumArray_size = (logicLeadNumBitsNum >> 3) + 1;
}
if ((*this)->ifAdtFse == 0) {
(*this)->typeArray = &flatBytes[index];
//retrieve the number of states (i.e., stateNum)
(*this)->allNodes = bytesToInt_bigEndian((*this)->typeArray); //the first 4 bytes store the stateNum
(*this)->stateNum = ((*this)->allNodes+1)/2;
index+=(*this)->typeArray_size;
} else {
(*this)->FseCode = &flatBytes[index];
index+=(*this)->FseCode_size;
(*this)->transCodeBits = &flatBytes[index];
index+=(*this)->transCodeBits_size;
}
// todo need check length
(*this)->residualMidBits_size = flatBytesLength - 1 - 1 - MetaDataByteLength - pde_exe->SZ_SIZE_TYPE - 4 - 4 - 4 - 1 - 8
- pde_exe->SZ_SIZE_TYPE - pde_exe->SZ_SIZE_TYPE
- (*this)->leadNumArray_size - (*this)->exactMidBytes_size;
if ((*this)->ifAdtFse == 0) {
(*this)->residualMidBits_size = (*this)->residualMidBits_size - (*this)->typeArray_size - pde_exe->SZ_SIZE_TYPE ;
} else {
(*this)->residualMidBits_size = (*this)->residualMidBits_size - (*this)->FseCode_size - (*this)->transCodeBits_size - pde_exe->SZ_SIZE_TYPE - pde_exe->SZ_SIZE_TYPE;
}
(*this)->leadNumArray = &flatBytes[index];
index+=(*this)->leadNumArray_size;
(*this)->exactMidBytes = &flatBytes[index];
index+=(*this)->exactMidBytes_size;
(*this)->residualMidBits = &flatBytes[index];
return errorBoundMode;
}
/**
*
* type's length == dataSeriesLength
* exactMidBytes's length == exactMidBytes_size
* leadNumIntArray's length == exactDataNum
* escBytes's length == escBytes_size
* resiBitLength's length == resiBitLengthSize
* */
void new_TightDataPointStorageD(TightDataPointStorageD **this,
size_t dataSeriesLength, size_t exactDataNum,
int* type, unsigned char* exactMidBytes, size_t exactMidBytes_size,
unsigned char* leadNumIntArray, //leadNumIntArray contains readable numbers....
unsigned char* resiMidBits, size_t resiMidBits_size,
unsigned char resiBitLength,
double realPrecision, double medianValue, char reqLength, unsigned int intervals,
unsigned char radExpo)
{
//int i = 0;
*this = (TightDataPointStorageD *)malloc(sizeof(TightDataPointStorageD));
memset(*this, 0, sizeof(TightDataPointStorageD));
(*this)->ifAdtFse = confparams_cpr->ifAdtFse;
(*this)->allSameData = 0;
(*this)->realPrecision = realPrecision;
(*this)->medianValue = medianValue;
(*this)->reqLength = reqLength;
(*this)->dataSeriesLength = dataSeriesLength;
(*this)->exactDataNum = exactDataNum;
if ((*this)->ifAdtFse == 0) {
// huffman
int stateNum = 2 * intervals;
HuffmanTree* huffmanTree = createHuffmanTree(stateNum);
encode_withTree(huffmanTree, type, dataSeriesLength, &(*this)->typeArray, &(*this)->typeArray_size);
SZ_ReleaseHuffman(huffmanTree);
}
else {
// fse
encode_with_fse(type, dataSeriesLength, intervals, &((*this)->FseCode), &((*this)->FseCode_size),
&((*this)->transCodeBits), &((*this)->transCodeBits_size));
}
(*this)->exactMidBytes = exactMidBytes;
(*this)->exactMidBytes_size = exactMidBytes_size;
(*this)->leadNumArray_size = convertIntArray2ByteArray_fast_2b(leadNumIntArray, exactDataNum, &((*this)->leadNumArray));
(*this)->residualMidBits_size = convertIntArray2ByteArray_fast_dynamic(resiMidBits, resiBitLength, exactDataNum, &((*this)->residualMidBits));
(*this)->intervals = intervals;
(*this)->isLossless = 0;
(*this)->radExpo = radExpo;
}
void convertTDPStoBytes_double(TightDataPointStorageD* tdps, unsigned char* bytes, unsigned char* dsLengthBytes, unsigned char sameByte)
{
size_t i, k = 0;
unsigned char intervalsBytes[4];
unsigned char typeArrayLengthBytes[8];
unsigned char exactLengthBytes[8];
unsigned char exactMidBytesLength[8];
unsigned char realPrecisionBytes[8];
unsigned char fsecodeLengthBytes[8];
unsigned char transcodeLengthBytes[8];
unsigned char medianValueBytes[8];
unsigned char max_quant_intervals_Bytes[4];
bytes[k++] = versionNumber;
bytes[k++] = sameByte; //1 byte
convertSZParamsToBytes(confparams_cpr, &(bytes[k]), exe_params->optQuantMode);
k = k + MetaDataByteLength_double;
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST: 4 or 8 bytes
bytes[k++] = dsLengthBytes[i];
intToBytes_bigEndian(max_quant_intervals_Bytes, confparams_cpr->max_quant_intervals);
for(i = 0;i<4;i++)//4
bytes[k++] = max_quant_intervals_Bytes[i];
intToBytes_bigEndian(intervalsBytes, tdps->intervals);
for(i = 0;i<4;i++)//4
bytes[k++] = intervalsBytes[i];
doubleToBytes(medianValueBytes, tdps->medianValue);
for (i = 0; i < 8; i++)// 8
bytes[k++] = medianValueBytes[i];
bytes[k++] = tdps->reqLength; //1 byte
doubleToBytes(realPrecisionBytes, tdps->realPrecision);
for (i = 0; i < 8; i++)// 8
bytes[k++] = realPrecisionBytes[i];
if (tdps->ifAdtFse == 0) {
sizeToBytes(typeArrayLengthBytes, tdps->typeArray_size, exe_params->SZ_SIZE_TYPE);
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
bytes[k++] = typeArrayLengthBytes[i];
} else {
sizeToBytes(fsecodeLengthBytes, tdps->FseCode_size, exe_params->SZ_SIZE_TYPE);
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
bytes[k++] = fsecodeLengthBytes[i];
sizeToBytes(transcodeLengthBytes, tdps->transCodeBits_size, exe_params->SZ_SIZE_TYPE);
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
bytes[k++] = transcodeLengthBytes[i];
}
sizeToBytes(exactLengthBytes, tdps->exactDataNum, exe_params->SZ_SIZE_TYPE);
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
bytes[k++] = exactLengthBytes[i];
sizeToBytes(exactMidBytesLength, tdps->exactMidBytes_size, exe_params->SZ_SIZE_TYPE);
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
bytes[k++] = exactMidBytesLength[i];
if(confparams_cpr->errorBoundMode>=PW_REL)
{
doubleToBytes(exactMidBytesLength, tdps->minLogValue);
for(i = 0;i < 8; i++)
bytes[k++] = exactMidBytesLength[i];
}
// copy data
if (tdps->ifAdtFse == 0) {
memcpy(&(bytes[k]), tdps->typeArray, tdps->typeArray_size);
k += tdps->typeArray_size;
} else {
memcpy(&(bytes[k]), tdps->FseCode, tdps->FseCode_size);
k += tdps->FseCode_size;
memcpy(&(bytes[k]), tdps->transCodeBits, tdps->transCodeBits_size);
k += tdps->transCodeBits_size;
}
memcpy(&(bytes[k]), tdps->leadNumArray, tdps->leadNumArray_size);
k += tdps->leadNumArray_size;
memcpy(&(bytes[k]), tdps->exactMidBytes, tdps->exactMidBytes_size);
k += tdps->exactMidBytes_size;
if(tdps->residualMidBits!=NULL)
{
memcpy(&(bytes[k]), tdps->residualMidBits, tdps->residualMidBits_size);
k += tdps->residualMidBits_size;
}
}
//Convert TightDataPointStorageD to bytes...
bool convertTDPStoFlatBytes_double(TightDataPointStorageD *tdps, unsigned char* bytes, size_t *size)
{
size_t i, k = 0;
unsigned char dsLengthBytes[8];
if(exe_params->SZ_SIZE_TYPE==4)
intToBytes_bigEndian(dsLengthBytes, tdps->dataSeriesLength);//4
else
longToBytes_bigEndian(dsLengthBytes, tdps->dataSeriesLength);//8
unsigned char sameByte = tdps->allSameData==1?(unsigned char)1:(unsigned char)0;
//sameByte = sameByte | (confparams_cpr->szMode << 1);
sameByte = sameByte | (tdps->ifAdtFse << 1); //0000,0010
if(tdps->isLossless)
sameByte = (unsigned char) (sameByte | 0x10);
if(confparams_cpr->errorBoundMode>=PW_REL)
sameByte = (unsigned char) (sameByte | 0x20); // 00100000, the 5th bit
if(exe_params->SZ_SIZE_TYPE==8)
sameByte = (unsigned char) (sameByte | 0x40); // 01000000, the 6th bit
if(confparams_cpr->errorBoundMode == PW_REL && confparams_cpr->accelerate_pw_rel_compression)
sameByte = (unsigned char) (sameByte | 0x08);
if(tdps->allSameData==1)
{
size_t totalByteLength = 1 + 1 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + tdps->exactMidBytes_size;
//bytes = (unsigned char *)malloc(sizeof(unsigned char)*totalByteLength); // comment by tickduan
if(totalByteLength >= tdps->dataSeriesLength * sizeof(double))
{
return false;
}
bytes[k++] = versionNumber;
bytes[k++] = sameByte;
convertSZParamsToBytes(confparams_cpr, &(bytes[k]), exe_params->optQuantMode);
k = k + MetaDataByteLength_double;
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
bytes[k++] = dsLengthBytes[i];
for (i = 0; i < tdps->exactMidBytes_size; i++)
bytes[k++] = tdps->exactMidBytes[i];
*size = totalByteLength;
}
else
{
size_t residualMidBitsLength = tdps->residualMidBits == NULL ? 0 : tdps->residualMidBits_size;
size_t totalByteLength = 1 + 1 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + 4 + 4 + 8 + 1 + 8
+ exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE
+ tdps->leadNumArray_size
+ tdps->exactMidBytes_size
+ residualMidBitsLength;
if (tdps->ifAdtFse == 0) {
totalByteLength += tdps->typeArray_size + exe_params->SZ_SIZE_TYPE;
} else {
totalByteLength += tdps->FseCode_size + tdps->transCodeBits_size + exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE;
}
//*bytes = (unsigned char *)malloc(sizeof(unsigned char)*totalByteLength); comment by tickduan
if(totalByteLength >= tdps->dataSeriesLength * sizeof(double))
{
return false;
}
convertTDPStoBytes_double(tdps, bytes, dsLengthBytes, sameByte);
*size = totalByteLength;
}
return true;
}
void free_TightDataPointStorageD(TightDataPointStorageD *tdps)
{
if(tdps->leadNumArray!=NULL)
free(tdps->leadNumArray);
if(tdps->FseCode!=NULL)
free(tdps->FseCode);
if(tdps->transCodeBits!=NULL)
free(tdps->transCodeBits);
if(tdps->exactMidBytes!=NULL)
free(tdps->exactMidBytes);
if(tdps->residualMidBits!=NULL)
free(tdps->residualMidBits);
if(tdps->typeArray)
free(tdps->typeArray);
free(tdps);
}
/**
* to free the memory used in the decompression
* */
void free_TightDataPointStorageD2(TightDataPointStorageD *tdps)
{
free(tdps);
}

434
utils/TSZ/sz/src/TightDataPointStorageF.c Normal file
View File

@ -0,0 +1,434 @@
/**
* @file TightPointDataStorageF.c
* @author Sheng Di and Dingwen Tao
* @date Aug, 2016
* @brief The functions used to construct the tightPointDataStorage element for storing compressed bytes.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include "TightDataPointStorageF.h"
#include "sz.h"
#include "Huffman.h"
#include "transcode.h"
#include "fse.h"
void new_TightDataPointStorageF_Empty(TightDataPointStorageF **this)
{
TightDataPointStorageF* tdpf = (TightDataPointStorageF*)malloc(sizeof(TightDataPointStorageF));
memset(tdpf, 0, sizeof(TightDataPointStorageF));
*this = tdpf;
}
int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsigned char* flatBytes, size_t flatBytesLength, sz_exedata* pde_exe, sz_params* pde_params)
{
new_TightDataPointStorageF_Empty(this);
size_t i, index = 0;
//
// parse tdps
//
// 1 version(1)
unsigned char version = flatBytes[index++]; //1
unsigned char sameRByte = flatBytes[index++]; //1
// parse data format
switch (version)
{
case DATA_FROMAT_VER1:
break;
default:
printf(" error, float compressed data format can not be recognised. ver=%d\n ", version);
return SZ_ABS;
}
// 2 same(1) //note that 1000,0000 is reserved for regression tag.
int same = sameRByte & 0x01; //0000,0001
(*this)->ifAdtFse = (sameRByte & 0x02)>>1; //0000,0110
(*this)->isLossless = (sameRByte & 0x10)>>4; //0001,0000 //0010,0000
pde_exe->SZ_SIZE_TYPE = ((sameRByte & 0x40)>>6)==1?8:4; //0100,0000
int errorBoundMode = SZ_ABS;
// 3 meta(2)
convertBytesToSZParams(&(flatBytes[index]), pde_params, pde_exe);
index += MetaDataByteLength;
// 4 element count(4)
unsigned char dsLengthBytes[8];
for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++)
dsLengthBytes[i] = flatBytes[index++];
(*this)->dataSeriesLength = bytesToSize(dsLengthBytes, pde_exe->SZ_SIZE_TYPE);// 4 or 8
if((*this)->isLossless==1)
{
//(*this)->exactMidBytes = flatBytes+8;
return errorBoundMode;
}
else if(same==1)
{
(*this)->allSameData = 1;
(*this)->exactMidBytes = &(flatBytes[index]);
return errorBoundMode;
}
else
(*this)->allSameData = 0;
// regression
int isRegression = (sameRByte >> 7) & 0x01;
if(isRegression == 1)
{
(*this)->raBytes_size = flatBytesLength - 1 - 1 - MetaDataByteLength - pde_exe->SZ_SIZE_TYPE;
(*this)->raBytes = &(flatBytes[index]);
return errorBoundMode;
}
// 5 quant intervals(4)
unsigned char byteBuf[8];
for (i = 0; i < 4; i++)
byteBuf[i] = flatBytes[index++];
int max_quant_intervals = bytesToInt_bigEndian(byteBuf);// 4
pde_params->maxRangeRadius = max_quant_intervals/2;
// 6 intervals
for (i = 0; i < 4; i++)
byteBuf[i] = flatBytes[index++];
(*this)->intervals = bytesToInt_bigEndian(byteBuf);// 4
// 7 median
for (i = 0; i < 4; i++)
byteBuf[i] = flatBytes[index++];
(*this)->medianValue = bytesToFloat(byteBuf); //4
// 8 reqLength
(*this)->reqLength = flatBytes[index++]; //1
// 9 realPrecision(8)
for (i = 0; i < 8; i++)
byteBuf[i] = flatBytes[index++];
(*this)->realPrecision = bytesToDouble(byteBuf);//8
// 10 typeArray_size
if ((*this)->ifAdtFse == 0) {
for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++)
byteBuf[i] = flatBytes[index++];
(*this)->typeArray_size = bytesToSize(byteBuf, pde_exe->SZ_SIZE_TYPE);// 4
} else {
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
byteBuf[i] = flatBytes[index++];
(*this)->FseCode_size = bytesToSize(byteBuf, pde_exe->SZ_SIZE_TYPE);// 4
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
byteBuf[i] = flatBytes[index++];
(*this)->transCodeBits_size = bytesToSize(byteBuf, pde_exe->SZ_SIZE_TYPE);// 4
}
// 11 exactNum
for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++)
byteBuf[i] = flatBytes[index++];
(*this)->exactDataNum = bytesToSize(byteBuf, pde_exe->SZ_SIZE_TYPE);// ST
// 12 mid size
for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++)
byteBuf[i] = flatBytes[index++];
(*this)->exactMidBytes_size = bytesToSize(byteBuf, pde_exe->SZ_SIZE_TYPE);// STqq
// calc leadNumArray_size
size_t logicLeadNumBitsNum = (*this)->exactDataNum * 2;
if (logicLeadNumBitsNum % 8 == 0)
{
(*this)->leadNumArray_size = logicLeadNumBitsNum >> 3;
}
else
{
(*this)->leadNumArray_size = (logicLeadNumBitsNum >> 3) + 1;
}
// 13 typeArray
if ((*this)->ifAdtFse == 0) {
(*this)->typeArray = &flatBytes[index];
//retrieve the number of states (i.e., stateNum)
(*this)->allNodes = bytesToInt_bigEndian((*this)->typeArray); //the first 4 bytes store the stateNum
(*this)->stateNum = ((*this)->allNodes+1)/2;
index+=(*this)->typeArray_size;
} else {
(*this)->FseCode = &flatBytes[index];
index+=(*this)->FseCode_size;
(*this)->transCodeBits = &flatBytes[index];
index+=(*this)->transCodeBits_size;
}
// 14 leadNumArray
(*this)->leadNumArray = &flatBytes[index];
index += (*this)->leadNumArray_size;
// 15 exactMidBytes
(*this)->exactMidBytes = &flatBytes[index];
index+=(*this)->exactMidBytes_size;
// 16 residualMidBits
(*this)->residualMidBits = &flatBytes[index];
// calc residualMidBits_size
(*this)->residualMidBits_size = flatBytesLength - 1 - 1 - MetaDataByteLength - pde_exe->SZ_SIZE_TYPE - 4 - 4 - 4 - 1 - 8
- pde_exe->SZ_SIZE_TYPE - pde_exe->SZ_SIZE_TYPE
- (*this)->leadNumArray_size - (*this)->exactMidBytes_size;
if ((*this)->ifAdtFse == 0) {
(*this)->residualMidBits_size = (*this)->residualMidBits_size - (*this)->typeArray_size - pde_exe->SZ_SIZE_TYPE ;
} else {
(*this)->residualMidBits_size = (*this)->residualMidBits_size - (*this)->FseCode_size - (*this)->transCodeBits_size - pde_exe->SZ_SIZE_TYPE - pde_exe->SZ_SIZE_TYPE;
}
// printTDPS(*this);
return errorBoundMode;
}
/**
*
* type's length == dataSeriesLength
* exactMidBytes's length == exactMidBytes_size
* leadNumIntArray's length == exactDataNum
* escBytes's length == escBytes_size
* resiBitLength's length == resiBitLengthSize
* */
void new_TightDataPointStorageF(TightDataPointStorageF **this,
size_t dataSeriesLength, size_t exactDataNum,
int* type, unsigned char* exactMidBytes, size_t exactMidBytes_size,
unsigned char* leadNumIntArray, //leadNumIntArray contains readable numbers....
unsigned char* resiMidBits, size_t resiMidBits_size,
unsigned char resiBitLength,
double realPrecision, float medianValue, char reqLength, unsigned int intervals,
unsigned char radExpo) {
*this = (TightDataPointStorageF *)malloc(sizeof(TightDataPointStorageF));
memset(*this, 0, sizeof(TightDataPointStorageF));
(*this)->ifAdtFse = confparams_cpr->ifAdtFse;
(*this)->allSameData = 0;
(*this)->realPrecision = realPrecision;
(*this)->medianValue = medianValue;
(*this)->reqLength = reqLength;
(*this)->dataSeriesLength = dataSeriesLength;
(*this)->exactDataNum = exactDataNum;
if ((*this)->ifAdtFse == 0) {
// huffman
int stateNum = 2 * intervals;
HuffmanTree* huffmanTree = createHuffmanTree(stateNum);
encode_withTree(huffmanTree, type, dataSeriesLength, &(*this)->typeArray, &(*this)->typeArray_size);
SZ_ReleaseHuffman(huffmanTree);
}
else {
// fse
encode_with_fse(type, dataSeriesLength, intervals, &((*this)->FseCode), &((*this)->FseCode_size),
&((*this)->transCodeBits), &((*this)->transCodeBits_size));
}
(*this)->exactMidBytes = exactMidBytes;
(*this)->exactMidBytes_size = exactMidBytes_size;
(*this)->leadNumArray_size = convertIntArray2ByteArray_fast_2b(leadNumIntArray, exactDataNum, &((*this)->leadNumArray));
(*this)->residualMidBits_size = convertIntArray2ByteArray_fast_dynamic(resiMidBits, resiBitLength, exactDataNum, &((*this)->residualMidBits));
(*this)->intervals = intervals;
(*this)->isLossless = 0;
(*this)->radExpo = radExpo;
}
void convertTDPStoBytes_float(TightDataPointStorageF* tdps, unsigned char* bytes, unsigned char* dsLengthBytes, unsigned char sameByte)
{
size_t i, k = 0;
unsigned char intervalsBytes[4];
unsigned char typeArrayLengthBytes[8];
unsigned char exactLengthBytes[8];
unsigned char exactMidBytesLength[8];
unsigned char realPrecisionBytes[8];
unsigned char fsecodeLengthBytes[8];
unsigned char transcodeLengthBytes[8];
unsigned char medianValueBytes[4];
unsigned char max_quant_intervals_Bytes[4];
// 1 version
bytes[k++] = versionNumber;
// 2 same
bytes[k++] = sameByte; //1 byte
// 3 meta
convertSZParamsToBytes(confparams_cpr, &(bytes[k]), exe_params->optQuantMode);
k = k + MetaDataByteLength;
// 4 element count
for(i = 0; i < exe_params->SZ_SIZE_TYPE; i++)//ST: 4 or 8 bytes
bytes[k++] = dsLengthBytes[i];
intToBytes_bigEndian(max_quant_intervals_Bytes, confparams_cpr->max_quant_intervals);
// 5 max_quant_intervals length
for(i = 0;i<4;i++)//4
bytes[k++] = max_quant_intervals_Bytes[i];
// 6 intervals
intToBytes_bigEndian(intervalsBytes, tdps->intervals);
for(i = 0;i<4;i++)//4
bytes[k++] = intervalsBytes[i];
// 7 median
floatToBytes(medianValueBytes, tdps->medianValue);
for (i = 0; i < 4; i++)// 4
bytes[k++] = medianValueBytes[i];
// 8 reqLength
bytes[k++] = tdps->reqLength; //1 byte
// 9 realPrecision
doubleToBytes(realPrecisionBytes, tdps->realPrecision);
for (i = 0; i < 8; i++)// 8
bytes[k++] = realPrecisionBytes[i];
// 10 typeArray size
if (tdps->ifAdtFse == 0) {
sizeToBytes(typeArrayLengthBytes, tdps->typeArray_size, exe_params->SZ_SIZE_TYPE);
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
bytes[k++] = typeArrayLengthBytes[i];
} else {
sizeToBytes(fsecodeLengthBytes, tdps->FseCode_size, exe_params->SZ_SIZE_TYPE);
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
bytes[k++] = fsecodeLengthBytes[i];
sizeToBytes(transcodeLengthBytes, tdps->transCodeBits_size, exe_params->SZ_SIZE_TYPE);
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
bytes[k++] = transcodeLengthBytes[i];
}
// 11 exactDataNum leadNum calc by this , so not save leadNum
sizeToBytes(exactLengthBytes, tdps->exactDataNum, exe_params->SZ_SIZE_TYPE);
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
bytes[k++] = exactLengthBytes[i];
// 12 Mid size
sizeToBytes(exactMidBytesLength, tdps->exactMidBytes_size, exe_params->SZ_SIZE_TYPE);
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
bytes[k++] = exactMidBytesLength[i];
// 13 typeArray
if (tdps->ifAdtFse == 0) {
memcpy(&(bytes[k]), tdps->typeArray, tdps->typeArray_size);
k += tdps->typeArray_size;
} else {
memcpy(&(bytes[k]), tdps->FseCode, tdps->FseCode_size);
k += tdps->FseCode_size;
memcpy(&(bytes[k]), tdps->transCodeBits, tdps->transCodeBits_size);
k += tdps->transCodeBits_size;
}
// 14 leadNumArray_size
memcpy(&(bytes[k]), tdps->leadNumArray, tdps->leadNumArray_size);
k += tdps->leadNumArray_size;
// 15 mid data
memcpy(&(bytes[k]), tdps->exactMidBytes, tdps->exactMidBytes_size);
k += tdps->exactMidBytes_size;
// 16 residualMidBits
if(tdps->residualMidBits!=NULL)
{
memcpy(&(bytes[k]), tdps->residualMidBits, tdps->residualMidBits_size);
k += tdps->residualMidBits_size;
}
}
//convert TightDataPointStorageD to bytes...
bool convertTDPStoFlatBytes_float(TightDataPointStorageF *tdps, unsigned char* bytes, size_t *size)
{
// printTDPS(tdps);
size_t i, k = 0;
unsigned char dsLengthBytes[8];
if(exe_params->SZ_SIZE_TYPE==4)
intToBytes_bigEndian(dsLengthBytes, tdps->dataSeriesLength);//4
else
longToBytes_bigEndian(dsLengthBytes, tdps->dataSeriesLength);//8
unsigned char sameByte = tdps->allSameData==1?(unsigned char)1:(unsigned char)0; //0000,0001
//sameByte = sameByte | (confparams_cpr->szMode << 1); //0000,0110 (no need because of convertSZParamsToBytes
sameByte = sameByte | (tdps->ifAdtFse << 1); // 0000,0010
if(tdps->isLossless)
sameByte = (unsigned char) (sameByte | 0x10); // 0001,0000
if(confparams_cpr->errorBoundMode>=PW_REL)
sameByte = (unsigned char) (sameByte | 0x20); // 0010,0000, the 5th bit
if(exe_params->SZ_SIZE_TYPE==8)
sameByte = (unsigned char) (sameByte | 0x40); // 0100,0000, the 6th bit
if(confparams_cpr->errorBoundMode == PW_REL && confparams_cpr->accelerate_pw_rel_compression)
sameByte = (unsigned char) (sameByte | 0x08); //0000,1000
//if(confparams_cpr->protectValueRange)
// sameByte = (unsigned char) (sameByte | 0x04); //0000,0100
if(tdps->allSameData == 1 )
{
//
// same format
//
size_t totalByteLength = 1 + 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + tdps->exactMidBytes_size;
//*bytes = (unsigned char *)malloc(sizeof(unsigned char)*totalByteLength); // not need malloc comment by tickduan
// check output buffer enough
if(totalByteLength >= tdps->dataSeriesLength * sizeof(float) )
{
*size = 0;
return false;
}
// 1 version 1 byte
bytes[k++] = versionNumber;
// 2 same flag 1 bytes
bytes[k++] = sameByte;
// 3 metaData 26 bytes
convertSZParamsToBytes(confparams_cpr, &(bytes[k]), exe_params->optQuantMode);
k = k + MetaDataByteLength;
// 4 data Length 4 or 8 bytes
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
bytes[k++] = dsLengthBytes[i];
// 5 exactMidBytes exactMidBytes_size bytes
for (i = 0; i < tdps->exactMidBytes_size; i++)
bytes[k++] = tdps->exactMidBytes[i];
*size = totalByteLength;
}
else
{
//
// not same format
//
size_t residualMidBitsLength = tdps->residualMidBits == NULL ? 0 : tdps->residualMidBits_size;
// version(1) + samebyte(1)
size_t totalByteLength = 1 + 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 4 + 4 + 4 + 1 + 8
+ exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE
+ tdps->leadNumArray_size
+ tdps->exactMidBytes_size
+ residualMidBitsLength;
if (tdps->ifAdtFse == 0) {
totalByteLength += tdps->typeArray_size + exe_params->SZ_SIZE_TYPE;
} else {
totalByteLength += tdps->FseCode_size + tdps->transCodeBits_size + exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE;
}
//*bytes = (unsigned char *)malloc(sizeof(unsigned char)*totalByteLength); // comment by tickduan
if(totalByteLength >= tdps->dataSeriesLength * sizeof(float))
{
*size = 0;
return false;
}
convertTDPStoBytes_float(tdps, bytes, dsLengthBytes, sameByte);
*size = totalByteLength;
return true;
}
return true;
}
/**
* to free the memory used in the compression
* */
void free_TightDataPointStorageF(TightDataPointStorageF *tdps)
{
if(tdps->leadNumArray!=NULL)
free(tdps->leadNumArray);
if(tdps->FseCode!=NULL)
free(tdps->FseCode);
if(tdps->transCodeBits!=NULL)
free(tdps->transCodeBits);
if(tdps->exactMidBytes!=NULL)
free(tdps->exactMidBytes);
if(tdps->residualMidBits!=NULL)
free(tdps->residualMidBits);
if(tdps->typeArray)
free(tdps->typeArray);
free(tdps);
}
/**
* to free the memory used in the decompression
* */
void free_TightDataPointStorageF2(TightDataPointStorageF *tdps)
{
free(tdps);
}

203
utils/TSZ/sz/src/TypeManager.c Normal file
View File

@ -0,0 +1,203 @@
/**
* @file TypeManager.c
* @author Sheng Di
* @date May, 2016
* @brief TypeManager is used to manage the type array: parsing of the bytes and other types in between.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#include <stdio.h>
#include <stdlib.h>
#include "DynamicByteArray.h"
#include "sz.h"
//int convertIntArray2ByteArray_fast_8b()
size_t convertIntArray2ByteArray_fast_1b(unsigned char* intArray, size_t intArrayLength, unsigned char **result)
{
size_t byteLength = 0;
size_t i, j;
if(intArrayLength%8==0)
byteLength = intArrayLength/8;
else
byteLength = intArrayLength/8+1;
if(byteLength>0)
*result = (unsigned char*)malloc(byteLength*sizeof(unsigned char));
else
*result = NULL;
size_t n = 0;
int tmp, type;
for(i = 0;i<byteLength;i++)
{
tmp = 0;
for(j = 0;j<8&&n<intArrayLength;j++)
{
type = intArray[n];
if(type == 1)
tmp = (tmp | (1 << (7-j)));
n++;
}
(*result)[i] = (unsigned char)tmp;
}
return byteLength;
}
size_t convertIntArray2ByteArray_fast_1b_to_result(unsigned char* intArray, size_t intArrayLength, unsigned char *result)
{
size_t byteLength = 0;
size_t i, j;
if(intArrayLength%8==0)
byteLength = intArrayLength/8;
else
byteLength = intArrayLength/8+1;
size_t n = 0;
int tmp, type;
for(i = 0;i<byteLength;i++)
{
tmp = 0;
for(j = 0;j<8&&n<intArrayLength;j++)
{
type = intArray[n];
if(type == 1)
tmp = (tmp | (1 << (7-j)));
n++;
}
result[i] = (unsigned char)tmp;
}
return byteLength;
}
void convertByteArray2IntArray_fast_2b(size_t stepLength, unsigned char* byteArray, size_t byteArrayLength, unsigned char **intArray)
{
if(stepLength > byteArrayLength*4)
{
printf("Error: stepLength > byteArray.length*4\n");
printf("stepLength=%zu, byteArray.length=%zu\n", stepLength, byteArrayLength);
exit(0);
}
if(stepLength>0)
*intArray = (unsigned char*)malloc(stepLength*sizeof(unsigned char));
else
*intArray = NULL;
size_t i, n = 0;
for (i = 0; i < byteArrayLength; i++) {
unsigned char tmp = byteArray[i];
(*intArray)[n++] = (tmp & 0xC0) >> 6;
if(n==stepLength)
break;
(*intArray)[n++] = (tmp & 0x30) >> 4;
if(n==stepLength)
break;
(*intArray)[n++] = (tmp & 0x0C) >> 2;
if(n==stepLength)
break;
(*intArray)[n++] = tmp & 0x03;
if(n==stepLength)
break;
}
}
/**
* little endian
* [01|10|11|00|....]-->[01|10|11|00][....]
* @param timeStepType
* @return
*/
size_t convertIntArray2ByteArray_fast_2b(unsigned char* timeStepType, size_t timeStepTypeLength, unsigned char **result)
{
size_t i, j, byteLength = 0;
if(timeStepTypeLength%4==0)
byteLength = timeStepTypeLength*2/8;
else
byteLength = timeStepTypeLength*2/8+1;
if(byteLength>0)
*result = (unsigned char*)malloc(byteLength*sizeof(unsigned char));
else
*result = NULL;
size_t n = 0;
for(i = 0;i<byteLength;i++)
{
int tmp = 0;
for(j = 0;j<4&&n<timeStepTypeLength;j++)
{
int type = timeStepType[n];
switch(type)
{
case 0:
break;
case 1:
tmp = (tmp | (1 << (6-j*2)));
break;
case 2:
tmp = (tmp | (2 << (6-j*2)));
break;
case 3:
tmp = (tmp | (3 << (6-j*2)));
break;
default:
printf("Error: wrong timestep type...: type[%zu]=%d\n", n, type);
exit(0);
}
n++;
}
(*result)[i] = (unsigned char)tmp;
}
return byteLength;
}
INLINE int getLeftMovingSteps(size_t k, unsigned char resiBitLength)
{
return 8 - k%8 - resiBitLength;
}
/**
*
* @param timeStepType is the resiMidBits
* @param resiBitLength is the length of resiMidBits for each element, (the number of resiBitLength == the # of unpredictable elements
* @return
*/
size_t convertIntArray2ByteArray_fast_dynamic(unsigned char* timeStepType, unsigned char resiBitLength, size_t nbEle, unsigned char **bytes)
{
size_t i = 0, j = 0, k = 0;
int value;
DynamicByteArray* dba;
new_DBA(&dba, 1024);
int tmp = 0, leftMovSteps = 0;
for(j = 0;j<nbEle;j++)
{
if(resiBitLength==0)
continue;
value = timeStepType[i];
leftMovSteps = getLeftMovingSteps(k, resiBitLength);
if(leftMovSteps < 0)
{
tmp = tmp | (value >> (-leftMovSteps));
addDBA_Data(dba, (unsigned char)tmp);
tmp = 0 | (value << (8+leftMovSteps));
}
else if(leftMovSteps > 0)
{
tmp = tmp | (value << leftMovSteps);
}
else //==0
{
tmp = tmp | value;
addDBA_Data(dba, (unsigned char)tmp);
tmp = 0;
}
i++;
k += resiBitLength;
}
if(leftMovSteps != 0)
addDBA_Data(dba, (unsigned char)tmp);
convertDBAtoBytes(dba, bytes);
size_t size = dba->size;
free_DBA(dba);
return size;
}

355
utils/TSZ/sz/src/conf.c Normal file
View File

@ -0,0 +1,355 @@
/**
* @file conf.c
* @author Sheng Di (sdi1@anl.gov or disheng222@gmail.com)
* @date 2015.
* @brief Configuration loading functions for the SZ library.
* (C) 2015 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#include <math.h>
#include "string.h"
#include "sz.h"
#include "iniparser.h"
#include "Huffman.h"
//
// set default value
//
void setDefaulParams(sz_exedata* exedata, sz_params* params)
{
// sz_params
if(params)
{
// first important
params->errorBoundMode = SZ_ABS;
params->absErrBound = 1E-8;
params->absErrBoundDouble = 1E-16;
params->max_quant_intervals = 500;
params->quantization_intervals = 100;
params->losslessCompressor = ZSTD_COMPRESSOR; //other option: GZIP_COMPRESSOR;
// second important
params->sol_ID = SZ;
params->maxRangeRadius = params->max_quant_intervals/2;
params->predThreshold = 0.99;
params->sampleDistance = 100;
params->szMode = SZ_BEST_COMPRESSION;
// other
params->psnr = 90;
params->relBoundRatio = 1E-8;
params->accelerate_pw_rel_compression = 1;
params->pw_relBoundRatio = 1E-3;
params->segment_size = 36;
params->pwr_type = SZ_PWR_MIN_TYPE;
params->snapshotCmprStep = 5;
params->withRegression = SZ_WITH_LINEAR_REGRESSION;
params->randomAccess = 0; //0: no random access , 1: support random access
params->protectValueRange = 0;
params->plus_bits = 3;
}
// sz_exedata
if(exedata)
{
exedata->optQuantMode = 1;
exedata->SZ_SIZE_TYPE = 4;
if(params)
{
exedata->intvCapacity = params->maxRangeRadius*2;
exedata->intvRadius = params->maxRangeRadius;
}
else
{
exedata->intvCapacity = 500;
exedata->intvRadius = 200;
}
}
}
unsigned int roundUpToPowerOf2(unsigned int base)
{
base -= 1;
base = base | (base >> 1);
base = base | (base >> 2);
base = base | (base >> 4);
base = base | (base >> 8);
base = base | (base >> 16);
return base + 1;
}
void updateQuantizationInfo(int quant_intervals)
{
exe_params->intvCapacity = quant_intervals;
exe_params->intvRadius = quant_intervals/2;
}
double computeABSErrBoundFromPSNR(double psnr, double threshold, double value_range)
{
double v1 = psnr + 10 * log10(1-2.0/3.0*threshold);
double v2 = v1/(-20);
double v3 = pow(10, v2);
return value_range * v3;
}
double computeABSErrBoundFromNORM_ERR(double normErr, size_t nbEle)
{
return sqrt(3.0/nbEle)*normErr;
}
/*-------------------------------------------------------------------------*/
/**
*
*
* @return the status of loading conf. file: 1 (success) or 0 (error code);
* */
int SZ_ReadConf(const char* sz_cfgFile) {
// Check access to SZ configuration file and load dictionary
//record the setting in confparams_cpr
if(confparams_cpr == NULL)
confparams_cpr = (sz_params*)malloc(sizeof(sz_params));
if(exe_params == NULL)
exe_params = (sz_exedata*)malloc(sizeof(sz_exedata));
int x = 1;
char sol_name[256];
char *modeBuf;
char *errBoundMode;
char *endianTypeString;
dictionary *ini;
char *par;
char *y = (char*)&x;
if(*y==1)
sysEndianType = LITTLE_ENDIAN_SYSTEM;
else //=0
sysEndianType = BIG_ENDIAN_SYSTEM;
// default option
if(sz_cfgFile == NULL)
{
dataEndianType = LITTLE_ENDIAN_DATA;
setDefaulParams(exe_params, confparams_cpr);
return SZ_SUCCESS;
}
//printf("[SZ] Reading SZ configuration file (%s) ...\n", sz_cfgFile);
ini = iniparser_load(sz_cfgFile);
if (ini == NULL)
{
printf("[SZ] Iniparser failed to parse the conf. file.\n");
return SZ_FAILED;
}
endianTypeString = iniparser_getstring(ini, "ENV:dataEndianType", "LITTLE_ENDIAN_DATA");
if(strcmp(endianTypeString, "LITTLE_ENDIAN_DATA")==0)
dataEndianType = LITTLE_ENDIAN_DATA;
else if(strcmp(endianTypeString, "BIG_ENDIAN_DATA")==0)
dataEndianType = BIG_ENDIAN_DATA;
else
{
printf("Error: Wrong dataEndianType: please set it correctly in sz.config.\n");
iniparser_freedict(ini);
return SZ_FAILED;
}
// Reading/setting detection parameters
par = iniparser_getstring(ini, "ENV:sol_name", NULL);
snprintf(sol_name, 256, "%s", par);
if(strcmp(sol_name, "SZ")==0)
confparams_cpr->sol_ID = SZ;
else if(strcmp(sol_name, "PASTRI")==0)
confparams_cpr->sol_ID = PASTRI;
else if(strcmp(sol_name, "SZ_Transpose")==0)
confparams_cpr->sol_ID = SZ_Transpose;
else{
printf("[SZ] Error: wrong solution name (please check sz.config file), sol=%s\n", sol_name);
iniparser_freedict(ini);
return SZ_FAILED;
}
if(confparams_cpr->sol_ID==SZ || confparams_cpr->sol_ID==SZ_Transpose)
{
int max_quant_intervals = iniparser_getint(ini, "PARAMETER:max_quant_intervals", 65536);
confparams_cpr->max_quant_intervals = max_quant_intervals;
int quantization_intervals = (int)iniparser_getint(ini, "PARAMETER:quantization_intervals", 0);
confparams_cpr->quantization_intervals = quantization_intervals;
if(quantization_intervals>0)
{
updateQuantizationInfo(quantization_intervals);
confparams_cpr->max_quant_intervals = max_quant_intervals = quantization_intervals;
exe_params->optQuantMode = 0;
}
else //==0
{
confparams_cpr->maxRangeRadius = max_quant_intervals/2;
exe_params->intvCapacity = confparams_cpr->maxRangeRadius*2;
exe_params->intvRadius = confparams_cpr->maxRangeRadius;
exe_params->optQuantMode = 1;
}
if(quantization_intervals%2!=0)
{
printf("Error: quantization_intervals must be an even number!\n");
iniparser_freedict(ini);
return SZ_FAILED;
}
confparams_cpr->predThreshold = (float)iniparser_getdouble(ini, "PARAMETER:predThreshold", 0);
confparams_cpr->sampleDistance = (int)iniparser_getint(ini, "PARAMETER:sampleDistance", 0);
modeBuf = iniparser_getstring(ini, "PARAMETER:szMode", NULL);
if(modeBuf==NULL)
{
printf("[SZ] Error: Null szMode setting (please check sz.config file)\n");
iniparser_freedict(ini);
return SZ_FAILED;
}
else if(strcmp(modeBuf, "SZ_BEST_SPEED")==0)
confparams_cpr->szMode = SZ_BEST_SPEED;
else if(strcmp(modeBuf, "SZ_DEFAULT_COMPRESSION")==0)
confparams_cpr->szMode = SZ_DEFAULT_COMPRESSION;
else if(strcmp(modeBuf, "SZ_BEST_COMPRESSION")==0)
confparams_cpr->szMode = SZ_BEST_COMPRESSION;
else
{
printf("[SZ] Error: Wrong szMode setting (please check sz.config file)\n");
iniparser_freedict(ini);
return SZ_FAILED;
}
modeBuf = iniparser_getstring(ini, "PARAMETER:losslessCompressor", "ZSTD_COMPRESSOR");
if(strcmp(modeBuf, "GZIP_COMPRESSOR")==0)
confparams_cpr->losslessCompressor = GZIP_COMPRESSOR;
else if(strcmp(modeBuf, "ZSTD_COMPRESSOR")==0)
confparams_cpr->losslessCompressor = ZSTD_COMPRESSOR;
else
{
printf("[SZ] Error: Wrong losslessCompressor setting (please check sz.config file)\n");\
printf("No Such a lossless compressor: %s\n", modeBuf);
iniparser_freedict(ini);
return SZ_FAILED;
}
modeBuf = iniparser_getstring(ini, "PARAMETER:withLinearRegression", "YES");
if(strcmp(modeBuf, "YES")==0 || strcmp(modeBuf, "yes")==0)
confparams_cpr->withRegression = SZ_WITH_LINEAR_REGRESSION;
else
confparams_cpr->withRegression = SZ_NO_REGRESSION;
modeBuf = iniparser_getstring(ini, "PARAMETER:protectValueRange", "YES");
if(strcmp(modeBuf, "YES")==0)
confparams_cpr->protectValueRange = 1;
else
confparams_cpr->protectValueRange = 0;
confparams_cpr->randomAccess = (int)iniparser_getint(ini, "PARAMETER:randomAccess", 0);
//TODO
confparams_cpr->snapshotCmprStep = (int)iniparser_getint(ini, "PARAMETER:snapshotCmprStep", 5);
errBoundMode = iniparser_getstring(ini, "PARAMETER:errorBoundMode", NULL);
if(errBoundMode==NULL)
{
printf("[SZ] Error: Null error bound setting (please check sz.config file)\n");
iniparser_freedict(ini);
return SZ_FAILED;
}
else if(strcmp(errBoundMode,"ABS")==0||strcmp(errBoundMode,"abs")==0)
confparams_cpr->errorBoundMode=SZ_ABS;
else if(strcmp(errBoundMode, "REL")==0||strcmp(errBoundMode,"rel")==0)
confparams_cpr->errorBoundMode=REL;
else if(strcmp(errBoundMode, "VR_REL")==0||strcmp(errBoundMode, "vr_rel")==0)
confparams_cpr->errorBoundMode=REL;
else if(strcmp(errBoundMode, "ABS_AND_REL")==0||strcmp(errBoundMode, "abs_and_rel")==0)
confparams_cpr->errorBoundMode=ABS_AND_REL;
else if(strcmp(errBoundMode, "ABS_OR_REL")==0||strcmp(errBoundMode, "abs_or_rel")==0)
confparams_cpr->errorBoundMode=ABS_OR_REL;
else if(strcmp(errBoundMode, "PW_REL")==0||strcmp(errBoundMode, "pw_rel")==0)
confparams_cpr->errorBoundMode=PW_REL;
else if(strcmp(errBoundMode, "PSNR")==0||strcmp(errBoundMode, "psnr")==0)
confparams_cpr->errorBoundMode=PSNR;
else if(strcmp(errBoundMode, "ABS_AND_PW_REL")==0||strcmp(errBoundMode, "abs_and_pw_rel")==0)
confparams_cpr->errorBoundMode=ABS_AND_PW_REL;
else if(strcmp(errBoundMode, "ABS_OR_PW_REL")==0||strcmp(errBoundMode, "abs_or_pw_rel")==0)
confparams_cpr->errorBoundMode=ABS_OR_PW_REL;
else if(strcmp(errBoundMode, "REL_AND_PW_REL")==0||strcmp(errBoundMode, "rel_and_pw_rel")==0)
confparams_cpr->errorBoundMode=REL_AND_PW_REL;
else if(strcmp(errBoundMode, "REL_OR_PW_REL")==0||strcmp(errBoundMode, "rel_or_pw_rel")==0)
confparams_cpr->errorBoundMode=REL_OR_PW_REL;
else if(strcmp(errBoundMode, "NORM")==0||strcmp(errBoundMode, "norm")==0)
confparams_cpr->errorBoundMode=NORM;
else
{
printf("[SZ] Error: Wrong error bound mode (please check sz.config file)\n");
iniparser_freedict(ini);
return SZ_FAILED;
}
confparams_cpr->absErrBound = (double)iniparser_getdouble(ini, "PARAMETER:absErrBound", 0);
confparams_cpr->relBoundRatio = (double)iniparser_getdouble(ini, "PARAMETER:relBoundRatio", 0);
confparams_cpr->psnr = (double)iniparser_getdouble(ini, "PARAMETER:psnr", 0);
confparams_cpr->normErr = (double)iniparser_getdouble(ini, "PARAMETER:normErr", 0);
confparams_cpr->pw_relBoundRatio = (double)iniparser_getdouble(ini, "PARAMETER:pw_relBoundRatio", 0);
confparams_cpr->segment_size = (int)iniparser_getint(ini, "PARAMETER:segment_size", 0);
confparams_cpr->accelerate_pw_rel_compression = (int)iniparser_getint(ini, "PARAMETER:accelerate_pw_rel_compression", 1);
modeBuf = iniparser_getstring(ini, "PARAMETER:pwr_type", "MIN");
if(strcmp(modeBuf, "MIN")==0)
confparams_cpr->pwr_type = SZ_PWR_MIN_TYPE;
else if(strcmp(modeBuf, "AVG")==0)
confparams_cpr->pwr_type = SZ_PWR_AVG_TYPE;
else if(strcmp(modeBuf, "MAX")==0)
confparams_cpr->pwr_type = SZ_PWR_MAX_TYPE;
else if(modeBuf!=NULL)
{
printf("[SZ] Error: Wrong pwr_type setting (please check sz.config file).\n");
iniparser_freedict(ini);
return SZ_FAILED;
}
else //by default
confparams_cpr->pwr_type = SZ_PWR_AVG_TYPE;
//initialization for Huffman encoding
//SZ_Reset();
}
iniparser_freedict(ini);
return SZ_SUCCESS;
}
/*-------------------------------------------------------------------------*/
/**
@brief It reads and tests the configuration given.
@return integer 1 if successfull.
This function reads the configuration file. Then test that the
configuration parameters are correct (including directories).
**/
/*-------------------------------------------------------------------------*/
int SZ_LoadConf(const char* sz_cfgFile) {
int res = SZ_ReadConf(sz_cfgFile);
if (res != SZ_SUCCESS)
{
printf("[SZ] ERROR: Impossible to read configuration.\n");
return SZ_FAILED;
}
return SZ_SUCCESS;
}

653
utils/TSZ/sz/src/dataCompression.c Normal file
View File

@ -0,0 +1,653 @@
/**
* @file double_compression.c
* @author Sheng Di, Dingwen Tao, Xin Liang, Xiangyu Zou, Tao Lu, Wen Xia, Xuan Wang, Weizhe Zhang
* @date April, 2016
* @brief Compression Technique for double array
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "sz.h"
#include "DynamicByteArray.h"
#include "DynamicIntArray.h"
#include "TightDataPointStorageD.h"
#include "CompressElement.h"
#include "dataCompression.h"
float computeRangeSize_float(float* oriData, size_t size, float* valueRangeSize, float* medianValue)
{
size_t i = 0;
float min = oriData[0];
float max = min;
for(i=1;i<size;i++)
{
float data = oriData[i];
if(min>data)
min = data;
else if(max<data)
max = data;
}
*valueRangeSize = max - min;
*medianValue = min + *valueRangeSize/2;
return min;
}
double computeRangeSize_double(double* oriData, size_t size, double* valueRangeSize, double* medianValue)
{
size_t i = 0;
double min = oriData[0];
double max = min;
for(i=1;i<size;i++)
{
double data = oriData[i];
if(min>data)
min = data;
else if(max<data)
max = data;
}
*valueRangeSize = max - min;
*medianValue = min + *valueRangeSize/2;
return min;
}
double min_d(double a, double b)
{
if(a<b)
return a;
else
return b;
}
double max_d(double a, double b)
{
if(a>b)
return a;
else
return b;
}
float min_f(float a, float b)
{
if(a<b)
return a;
else
return b;
}
float max_f(float a, float b)
{
if(a>b)
return a;
else
return b;
}
double getRealPrecision_double(double valueRangeSize, int errBoundMode, double absErrBound, double relBoundRatio, int *status)
{
int state = SZ_SUCCESS;
double precision = 0;
if(errBoundMode==SZ_ABS||errBoundMode==ABS_OR_PW_REL||errBoundMode==ABS_AND_PW_REL)
precision = absErrBound;
else if(errBoundMode==REL||errBoundMode==REL_OR_PW_REL||errBoundMode==REL_AND_PW_REL)
precision = relBoundRatio*valueRangeSize;
else if(errBoundMode==ABS_AND_REL)
precision = min_d(absErrBound, relBoundRatio*valueRangeSize);
else if(errBoundMode==ABS_OR_REL)
precision = max_d(absErrBound, relBoundRatio*valueRangeSize);
else if(errBoundMode==PW_REL)
precision = 0;
else
{
printf("Error: error-bound-mode is incorrect!\n");
state = SZ_BERR;
}
*status = state;
return precision;
}
double getRealPrecision_float(float valueRangeSize, int errBoundMode, double absErrBound, double relBoundRatio, int *status)
{
int state = SZ_SUCCESS;
double precision = 0;
if(errBoundMode==SZ_ABS||errBoundMode==ABS_OR_PW_REL||errBoundMode==ABS_AND_PW_REL)
precision = absErrBound;
else if(errBoundMode==REL||errBoundMode==REL_OR_PW_REL||errBoundMode==REL_AND_PW_REL)
precision = relBoundRatio*valueRangeSize;
else if(errBoundMode==ABS_AND_REL)
precision = min_f(absErrBound, relBoundRatio*valueRangeSize);
else if(errBoundMode==ABS_OR_REL)
precision = max_f(absErrBound, relBoundRatio*valueRangeSize);
else if(errBoundMode==PW_REL)
precision = 0;
else
{
printf("Error: error-bound-mode is incorrect!\n");
state = SZ_BERR;
}
*status = state;
return precision;
}
double getRealPrecision_int(long valueRangeSize, int errBoundMode, double absErrBound, double relBoundRatio, int *status)
{
int state = SZ_SUCCESS;
double precision = 0;
if(errBoundMode==SZ_ABS||errBoundMode==ABS_OR_PW_REL||errBoundMode==ABS_AND_PW_REL)
precision = absErrBound;
else if(errBoundMode==REL||errBoundMode==REL_OR_PW_REL||errBoundMode==REL_AND_PW_REL)
precision = relBoundRatio*valueRangeSize;
else if(errBoundMode==ABS_AND_REL)
precision = min_f(absErrBound, relBoundRatio*valueRangeSize);
else if(errBoundMode==ABS_OR_REL)
precision = max_f(absErrBound, relBoundRatio*valueRangeSize);
else if(errBoundMode==PW_REL)
precision = -1;
else
{
printf("Error: error-bound-mode is incorrect!\n");
state = SZ_BERR;
}
*status = state;
return precision;
}
void symTransform_8bytes(unsigned char data[8])
{
unsigned char tmp = data[0];
data[0] = data[7];
data[7] = tmp;
tmp = data[1];
data[1] = data[6];
data[6] = tmp;
tmp = data[2];
data[2] = data[5];
data[5] = tmp;
tmp = data[3];
data[3] = data[4];
data[4] = tmp;
}
INLINE void symTransform_2bytes(unsigned char data[2])
{
unsigned char tmp = data[0];
data[0] = data[1];
data[1] = tmp;
}
INLINE void symTransform_4bytes(unsigned char data[4])
{
unsigned char tmp = data[0];
data[0] = data[3];
data[3] = tmp;
tmp = data[1];
data[1] = data[2];
data[2] = tmp;
}
INLINE void compressSingleFloatValue(FloatValueCompressElement *vce, float oriValue, float precision, float medianValue,
int reqLength, int reqBytesLength, int resiBitsLength)
{
lfloat diffVal;
diffVal.value = oriValue - medianValue;
// calc ignore bit count
int ignBitCount = 32 - reqLength;
if(ignBitCount<0)
ignBitCount = 0;
intToBytes_bigEndian(vce->curBytes, diffVal.ivalue);
// truncate diff value tail bit with ignBitCount
diffVal.ivalue = (diffVal.ivalue >> ignBitCount) << ignBitCount;
// save to vce
vce->data = diffVal.value + medianValue;
vce->curValue = diffVal.ivalue;
vce->reqBytesLength = reqBytesLength;
vce->resiBitsLength = resiBitsLength;
}
void compressSingleDoubleValue(DoubleValueCompressElement *vce, double tgtValue, double precision, double medianValue,
int reqLength, int reqBytesLength, int resiBitsLength)
{
double normValue = tgtValue - medianValue;
ldouble lfBuf;
lfBuf.value = normValue;
int ignBytesLength = 64 - reqLength;
if(ignBytesLength<0)
ignBytesLength = 0;
long tmp_long = lfBuf.lvalue;
longToBytes_bigEndian(vce->curBytes, tmp_long);
lfBuf.lvalue = (lfBuf.lvalue >> ignBytesLength)<<ignBytesLength;
//double tmpValue = lfBuf.value;
vce->data = lfBuf.value+medianValue;
vce->curValue = tmp_long;
vce->reqBytesLength = reqBytesLength;
vce->resiBitsLength = resiBitsLength;
}
int compIdenticalLeadingBytesCount_double(unsigned char* preBytes, unsigned char* curBytes)
{
int i, n = 0;
for(i=0;i<8;i++)
if(preBytes[i]==curBytes[i])
n++;
else
break;
if(n>3) n = 3;
return n;
}
INLINE int compIdenticalLeadingBytesCount_float(unsigned char* preBytes, unsigned char* curBytes)
{
int i, n = 0;
for(i=0;i<4;i++)
if(preBytes[i]==curBytes[i])
n++;
else
break;
if(n>3) n = 3;
return n;
}
//TODO double-check the correctness...
INLINE void addExactData(DynamicByteArray *exactMidByteArray, DynamicIntArray *exactLeadNumArray,
DynamicIntArray *resiBitArray, LossyCompressionElement *lce)
{
int i;
int leadByteLength = lce->leadingZeroBytes;
addDIA_Data(exactLeadNumArray, leadByteLength);
unsigned char* intMidBytes = lce->integerMidBytes;
int integerMidBytesLength = lce->integerMidBytes_Length;
int resMidBitsLength = lce->resMidBitsLength;
if(intMidBytes!=NULL||resMidBitsLength!=0)
{
if(intMidBytes!=NULL)
for(i = 0;i<integerMidBytesLength;i++)
addDBA_Data(exactMidByteArray, intMidBytes[i]);
if(resMidBitsLength!=0)
addDIA_Data(resiBitArray, lce->residualMidBits);
}
}
/**
* @deprecated
* @return: the length of the coefficient array.
* */
int getPredictionCoefficients(int layers, int dimension, int **coeff_array, int *status)
{
size_t size = 0;
switch(dimension)
{
case 1:
switch(layers)
{
case 1:
*coeff_array = (int*)malloc(sizeof(int));
(*coeff_array)[0] = 1;
size = 1;
break;
case 2:
*coeff_array = (int*)malloc(2*sizeof(int));
(*coeff_array)[0] = 2;
(*coeff_array)[1] = -1;
size = 2;
break;
case 3:
*coeff_array = (int*)malloc(3*sizeof(int));
(*coeff_array)[0] = 3;
(*coeff_array)[1] = -3;
(*coeff_array)[2] = 1;
break;
}
break;
case 2:
switch(layers)
{
case 1:
break;
case 2:
break;
case 3:
break;
}
break;
case 3:
switch(layers)
{
case 1:
break;
case 2:
break;
case 3:
break;
}
break;
default:
printf("Error: dimension must be no greater than 3 in the current version.\n");
*status = SZ_DERR;
}
*status = SZ_SUCCESS;
return size;
}
int computeBlockEdgeSize_2D(int segmentSize)
{
int i = 1;
for(i=1; i<segmentSize;i++)
{
if(i*i>segmentSize)
break;
}
return i;
//return (int)(sqrt(segmentSize)+1);
}
int computeBlockEdgeSize_3D(int segmentSize)
{
int i = 1;
for(i=1; i<segmentSize;i++)
{
if(i*i*i>segmentSize)
break;
}
return i;
//return (int)(pow(segmentSize, 1.0/3)+1);
}
//The following functions are float-precision version of dealing with the unpredictable data points
int generateLossyCoefficients_float(float* oriData, double precision, size_t nbEle, int* reqBytesLength, int* resiBitsLength, float* medianValue, float* decData)
{
float valueRangeSize;
computeRangeSize_float(oriData, nbEle, &valueRangeSize, medianValue);
short radExpo = getExponent_float(valueRangeSize/2);
int reqLength;
computeReqLength_float(precision, radExpo, &reqLength, medianValue);
*reqBytesLength = reqLength/8;
*resiBitsLength = reqLength%8;
size_t i = 0;
for(i = 0;i < nbEle;i++)
{
float normValue = oriData[i] - *medianValue;
lfloat lfBuf;
lfBuf.value = normValue;
int ignBytesLength = 32 - reqLength;
if(ignBytesLength<0)
ignBytesLength = 0;
lfBuf.ivalue = (lfBuf.ivalue >> ignBytesLength) << ignBytesLength;
//float tmpValue = lfBuf.value;
decData[i] = lfBuf.value + *medianValue;
}
return reqLength;
}
/**
* @param float* oriData: inplace argument (input / output)
*
* */
int compressExactDataArray_float(float* oriData, double precision, size_t nbEle, unsigned char** leadArray, unsigned char** midArray, unsigned char** resiArray,
int reqLength, int reqBytesLength, int resiBitsLength, float medianValue)
{
//allocate memory for coefficient compression arrays
DynamicIntArray *exactLeadNumArray;
new_DIA(&exactLeadNumArray, DynArrayInitLen);
DynamicByteArray *exactMidByteArray;
new_DBA(&exactMidByteArray, DynArrayInitLen);
DynamicIntArray *resiBitArray;
new_DIA(&resiBitArray, DynArrayInitLen);
unsigned char preDataBytes[4] = {0,0,0,0};
//allocate memory for vce and lce
FloatValueCompressElement *vce = (FloatValueCompressElement*)malloc(sizeof(FloatValueCompressElement));
LossyCompressionElement *lce = (LossyCompressionElement*)malloc(sizeof(LossyCompressionElement));
size_t i = 0;
for(i = 0;i < nbEle;i++)
{
compressSingleFloatValue(vce, oriData[i], precision, medianValue, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Float(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,4);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
oriData[i] = vce->data;
}
convertDIAtoInts(exactLeadNumArray, leadArray);
convertDBAtoBytes(exactMidByteArray,midArray);
convertDIAtoInts(resiBitArray, resiArray);
size_t midArraySize = exactMidByteArray->size;
free(vce);
free(lce);
free_DIA(exactLeadNumArray);
free_DBA(exactMidByteArray);
free_DIA(resiBitArray);
return midArraySize;
}
void decompressExactDataArray_float(unsigned char* leadNum, unsigned char* exactMidBytes, unsigned char* residualMidBits, size_t nbEle, int reqLength, float medianValue, float** decData)
{
*decData = (float*)malloc(nbEle*sizeof(float));
size_t i = 0, j = 0, k = 0, l = 0, p = 0, curByteIndex = 0;
float exactData = 0;
unsigned char preBytes[4] = {0,0,0,0};
unsigned char curBytes[4];
int resiBits;
unsigned char leadingNum;
int reqBytesLength = reqLength/8;
int resiBitsLength = reqLength%8;
for(i = 0; i<nbEle;i++)
{
// compute resiBits
resiBits = 0;
if (resiBitsLength != 0) {
int kMod8 = k % 8;
int rightMovSteps = getRightMovingSteps(kMod8, resiBitsLength);
if (rightMovSteps > 0) {
int code = getRightMovingCode(kMod8, resiBitsLength);
resiBits = (residualMidBits[p] & code) >> rightMovSteps;
} else if (rightMovSteps < 0) {
int code1 = getLeftMovingCode(kMod8);
int code2 = getRightMovingCode(kMod8, resiBitsLength);
int leftMovSteps = -rightMovSteps;
rightMovSteps = 8 - leftMovSteps;
resiBits = (residualMidBits[p] & code1) << leftMovSteps;
p++;
resiBits = resiBits
| ((residualMidBits[p] & code2) >> rightMovSteps);
} else // rightMovSteps == 0
{
int code = getRightMovingCode(kMod8, resiBitsLength);
resiBits = (residualMidBits[p] & code);
p++;
}
k += resiBitsLength;
}
// recover the exact data
memset(curBytes, 0, 4);
leadingNum = leadNum[l++];
memcpy(curBytes, preBytes, leadingNum);
for (j = leadingNum; j < reqBytesLength; j++)
curBytes[j] = exactMidBytes[curByteIndex++];
if (resiBitsLength != 0) {
unsigned char resiByte = (unsigned char) (resiBits << (8 - resiBitsLength));
curBytes[reqBytesLength] = resiByte;
}
exactData = bytesToFloat(curBytes);
(*decData)[i] = exactData + medianValue;
memcpy(preBytes,curBytes,4);
}
}
//double-precision version of dealing with unpredictable data points in sz 2.0
int generateLossyCoefficients_double(double* oriData, double precision, size_t nbEle, int* reqBytesLength, int* resiBitsLength, double* medianValue, double* decData)
{
double valueRangeSize;
computeRangeSize_double(oriData, nbEle, &valueRangeSize, medianValue);
short radExpo = getExponent_double(valueRangeSize/2);
int reqLength;
computeReqLength_double(precision, radExpo, &reqLength, medianValue);
*reqBytesLength = reqLength/8;
*resiBitsLength = reqLength%8;
size_t i = 0;
for(i = 0;i < nbEle;i++)
{
double normValue = oriData[i] - *medianValue;
ldouble ldBuf;
ldBuf.value = normValue;
int ignBytesLength = 64 - reqLength;
if(ignBytesLength<0)
ignBytesLength = 0;
ldBuf.lvalue = (ldBuf.lvalue >> ignBytesLength) << ignBytesLength;
decData[i] = ldBuf.value + *medianValue;
}
return reqLength;
}
/**
* @param double* oriData: inplace argument (input / output)
*
* */
int compressExactDataArray_double(double* oriData, double precision, size_t nbEle, unsigned char** leadArray, unsigned char** midArray, unsigned char** resiArray,
int reqLength, int reqBytesLength, int resiBitsLength, double medianValue)
{
//allocate memory for coefficient compression arrays
DynamicIntArray *exactLeadNumArray;
new_DIA(&exactLeadNumArray, DynArrayInitLen);
DynamicByteArray *exactMidByteArray;
new_DBA(&exactMidByteArray, DynArrayInitLen);
DynamicIntArray *resiBitArray;
new_DIA(&resiBitArray, DynArrayInitLen);
unsigned char preDataBytes[8] = {0,0,0,0,0,0,0,0};
//allocate memory for vce and lce
DoubleValueCompressElement *vce = (DoubleValueCompressElement*)malloc(sizeof(DoubleValueCompressElement));
LossyCompressionElement *lce = (LossyCompressionElement*)malloc(sizeof(LossyCompressionElement));
size_t i = 0;
for(i = 0;i < nbEle;i++)
{
compressSingleDoubleValue(vce, oriData[i], precision, medianValue, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,8);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
oriData[i] = vce->data;
}
convertDIAtoInts(exactLeadNumArray, leadArray);
convertDBAtoBytes(exactMidByteArray,midArray);
convertDIAtoInts(resiBitArray, resiArray);
size_t midArraySize = exactMidByteArray->size;
free(vce);
free(lce);
free_DIA(exactLeadNumArray);
free_DBA(exactMidByteArray);
free_DIA(resiBitArray);
return midArraySize;
}
void decompressExactDataArray_double(unsigned char* leadNum, unsigned char* exactMidBytes, unsigned char* residualMidBits, size_t nbEle, int reqLength, double medianValue, double** decData)
{
*decData = (double*)malloc(nbEle*sizeof(double));
size_t i = 0, j = 0, k = 0, l = 0, p = 0, curByteIndex = 0;
double exactData = 0;
unsigned char preBytes[8] = {0,0,0,0,0,0,0,0};
unsigned char curBytes[8];
int resiBits;
unsigned char leadingNum;
int reqBytesLength = reqLength/8;
int resiBitsLength = reqLength%8;
for(i = 0; i<nbEle;i++)
{
// compute resiBits
resiBits = 0;
if (resiBitsLength != 0) {
int kMod8 = k % 8;
int rightMovSteps = getRightMovingSteps(kMod8, resiBitsLength);
if (rightMovSteps > 0) {
int code = getRightMovingCode(kMod8, resiBitsLength);
resiBits = (residualMidBits[p] & code) >> rightMovSteps;
} else if (rightMovSteps < 0) {
int code1 = getLeftMovingCode(kMod8);
int code2 = getRightMovingCode(kMod8, resiBitsLength);
int leftMovSteps = -rightMovSteps;
rightMovSteps = 8 - leftMovSteps;
resiBits = (residualMidBits[p] & code1) << leftMovSteps;
p++;
resiBits = resiBits
| ((residualMidBits[p] & code2) >> rightMovSteps);
} else // rightMovSteps == 0
{
int code = getRightMovingCode(kMod8, resiBitsLength);
resiBits = (residualMidBits[p] & code);
p++;
}
k += resiBitsLength;
}
// recover the exact data
memset(curBytes, 0, 8);
leadingNum = leadNum[l++];
memcpy(curBytes, preBytes, leadingNum);
for (j = leadingNum; j < reqBytesLength; j++)
curBytes[j] = exactMidBytes[curByteIndex++];
if (resiBitsLength != 0) {
unsigned char resiByte = (unsigned char) (resiBits << (8 - resiBitsLength));
curBytes[reqBytesLength] = resiByte;
}
exactData = bytesToDouble(curBytes);
(*decData)[i] = exactData + medianValue;
memcpy(preBytes,curBytes,8);
}
}

397
utils/TSZ/sz/src/dictionary.c Normal file
View File

@ -0,0 +1,397 @@
/*-------------------------------------------------------------------------*/
/**
@file dictionary.c
@author N. Devillard
@brief Implements a dictionary for string variables.
This module implements a simple dictionary object, i.e. a list
of string/string associations. This object is useful to store e.g.
informations retrieved from a configuration file (ini files).
*/
/*--------------------------------------------------------------------------*/
/*---------------------------------------------------------------------------
Includes
---------------------------------------------------------------------------*/
#include "dictionary.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
/** Maximum value size for integers and doubles. */
#define MAXVALSZ 1024
/** Minimal allocated number of entries in a dictionary */
#define DICTMINSZ 128
/** Invalid key token */
#define DICT_INVALID_KEY ((char*)-1)
/*---------------------------------------------------------------------------
Private functions
---------------------------------------------------------------------------*/
/* Doubles the allocated size associated to a pointer */
/* 'size' is the current allocated size. */
static void * mem_double(void * ptr, int size)
{
void * newptr ;
newptr = calloc(2*size, 1);
if (newptr==NULL) {
return NULL ;
}
memcpy(newptr, ptr, size);
free(ptr);
return newptr ;
}
/*-------------------------------------------------------------------------*/
/**
@brief Duplicate a string
@param s String to duplicate
@return Pointer to a newly allocated string, to be freed with free()
This is a replacement for strdup(). This implementation is provided
for systems that do not have it.
*/
/*--------------------------------------------------------------------------*/
static char * xstrdup(const char * s)
{
char * t ;
if (!s)
return NULL ;
t = (char*)malloc(strlen(s)+1) ;
if (t) {
strcpy(t,s);
}
return t ;
}
/*---------------------------------------------------------------------------
Function codes
---------------------------------------------------------------------------*/
/*-------------------------------------------------------------------------*/
/**
@brief Compute the hash key for a string.
@param key Character string to use for key.
@return 1 unsigned int on at least 32 bits.
This hash function has been taken from an Article in Dr Dobbs Journal.
This is normally a collision-free function, distributing keys evenly.
The key is stored anyway in the struct so that collision can be avoided
by comparing the key itself in last resort.
*/
/*--------------------------------------------------------------------------*/
unsigned dictionary_hash(const char * key)
{
int len ;
unsigned hash ;
int i ;
len = strlen(key);
for (hash=0, i=0 ; i<len ; i++) {
hash += (unsigned)key[i] ;
hash += (hash<<10);
hash ^= (hash>>6) ;
}
hash += (hash <<3);
hash ^= (hash >>11);
hash += (hash <<15);
return hash ;
}
/*-------------------------------------------------------------------------*/
/**
@brief Create a new dictionary object.
@param size Optional initial size of the dictionary.
@return 1 newly allocated dictionary objet.
This function allocates a new dictionary object of given size and returns
it. If you do not know in advance (roughly) the number of entries in the
dictionary, give size=0.
*/
/*--------------------------------------------------------------------------*/
dictionary * dictionary_new(int size)
{
dictionary * d ;
/* If no size was specified, allocate space for DICTMINSZ */
if (size<DICTMINSZ) size=DICTMINSZ ;
if (!(d = (dictionary *)calloc(1, sizeof(dictionary)))) {
return NULL;
}
d->size = size ;
d->val = (char **)calloc(size, sizeof(char*));
d->key = (char **)calloc(size, sizeof(char*));
d->hash = (unsigned int *)calloc(size, sizeof(unsigned));
return d ;
}
/*-------------------------------------------------------------------------*/
/**
@brief Delete a dictionary object
@param d dictionary object to deallocate.
@return void
Deallocate a dictionary object and all memory associated to it.
*/
/*--------------------------------------------------------------------------*/
void dictionary_del(dictionary * d)
{
int i ;
if (d==NULL) return ;
for (i=0 ; i<d->size ; i++) {
if (d->key[i]!=NULL)
free(d->key[i]);
if (d->val[i]!=NULL)
free(d->val[i]);
}
free(d->val);
free(d->key);
free(d->hash);
free(d);
return ;
}
/*-------------------------------------------------------------------------*/
/**
@brief Get a value from a dictionary.
@param d dictionary object to search.
@param key Key to look for in the dictionary.
@param def Default value to return if key not found.
@return 1 pointer to internally allocated character string.
This function locates a key in a dictionary and returns a pointer to its
value, or the passed 'def' pointer if no such key can be found in
dictionary. The returned character pointer points to data internal to the
dictionary object, you should not try to free it or modify it.
*/
/*--------------------------------------------------------------------------*/
char * dictionary_get(dictionary * d, const char * key, char * def)
{
unsigned hash ;
int i ;
hash = dictionary_hash(key);
for (i=0 ; i<d->size ; i++) {
if (d->key[i]==NULL)
continue ;
/* Compare hash */
if (hash==d->hash[i]) {
/* Compare string, to avoid hash collisions */
if (!strcmp(key, d->key[i])) {
return d->val[i] ;
}
}
}
return def ;
}
/*-------------------------------------------------------------------------*/
/**
@brief Set a value in a dictionary.
@param d dictionary object to modify.
@param key Key to modify or add.
@param val Value to add.
@return int 0 if Ok, anything else otherwise
If the given key is found in the dictionary, the associated value is
replaced by the provided one. If the key cannot be found in the
dictionary, it is added to it.
It is Ok to provide a NULL value for val, but NULL values for the dictionary
or the key are considered as errors: the function will return immediately
in such a case.
Notice that if you dictionary_set a variable to NULL, a call to
dictionary_get will return a NULL value: the variable will be found, and
its value (NULL) is returned. In other words, setting the variable
content to NULL is equivalent to deleting the variable from the
dictionary. It is not possible (in this implementation) to have a key in
the dictionary without value.
This function returns non-zero in case of failure.
*/
/*--------------------------------------------------------------------------*/
int dictionary_set(dictionary * d, const char * key, const char * val)
{
int i ;
unsigned hash ;
if (d==NULL || key==NULL) return -1 ;
/* Compute hash for this key */
hash = dictionary_hash(key) ;
/* Find if value is already in dictionary */
if (d->n>0) {
for (i=0 ; i<d->size ; i++) {
if (d->key[i]==NULL)
continue ;
if (hash==d->hash[i]) { /* Same hash value */
if (!strcmp(key, d->key[i])) { /* Same key */
/* Found a value: modify and return */
if (d->val[i]!=NULL)
free(d->val[i]);
d->val[i] = val ? xstrdup(val) : NULL ;
/* Value has been modified: return */
return 0 ;
}
}
}
}
/* Add a new value */
/* See if dictionary needs to grow */
if (d->n==d->size) {
/* Reached maximum size: reallocate dictionary */
d->val = (char **)mem_double(d->val, d->size * sizeof(char*)) ;
d->key = (char **)mem_double(d->key, d->size * sizeof(char*)) ;
d->hash = (unsigned int *)mem_double(d->hash, d->size * sizeof(unsigned)) ;
if ((d->val==NULL) || (d->key==NULL) || (d->hash==NULL)) {
/* Cannot grow dictionary */
return -1 ;
}
/* Double size */
d->size *= 2 ;
}
/* Insert key in the first empty slot. Start at d->n and wrap at
d->size. Because d->n < d->size this will necessarily
terminate. */
for (i=d->n ; d->key[i] ; ) {
if(++i == d->size) i = 0;
}
/* Copy key */
d->key[i] = xstrdup(key);
d->val[i] = val ? xstrdup(val) : NULL ;
d->hash[i] = hash;
d->n ++ ;
return 0 ;
}
/*-------------------------------------------------------------------------*/
/**
@brief Delete a key in a dictionary
@param d dictionary object to modify.
@param key Key to remove.
@return void
This function deletes a key in a dictionary. Nothing is done if the
key cannot be found.
*/
/*--------------------------------------------------------------------------*/
void dictionary_unset(dictionary * d, const char * key)
{
unsigned hash ;
int i ;
if (key == NULL) {
return;
}
hash = dictionary_hash(key);
for (i=0 ; i<d->size ; i++) {
if (d->key[i]==NULL)
continue ;
/* Compare hash */
if (hash==d->hash[i]) {
/* Compare string, to avoid hash collisions */
if (!strcmp(key, d->key[i])) {
/* Found key */
break ;
}
}
}
if (i>=d->size)
/* Key not found */
return ;
free(d->key[i]);
d->key[i] = NULL ;
if (d->val[i]!=NULL) {
free(d->val[i]);
d->val[i] = NULL ;
}
d->hash[i] = 0 ;
d->n -- ;
return ;
}
/*-------------------------------------------------------------------------*/
/**
@brief Dump a dictionary to an opened file pointer.
@param d Dictionary to dump
@param f Opened file pointer.
@return void
Dumps a dictionary onto an opened file pointer. Key pairs are printed out
as @c [Key]=[Value], one per line. It is Ok to provide stdout or stderr as
output file pointers.
*/
/*--------------------------------------------------------------------------*/
void dictionary_dump(dictionary * d, FILE * out)
{
int i ;
if (d==NULL || out==NULL) return ;
if (d->n<1) {
fprintf(out, "empty dictionary\n");
return ;
}
for (i=0 ; i<d->size ; i++) {
if (d->key[i]) {
fprintf(out, "%20s\t[%s]\n",
d->key[i],
d->val[i] ? d->val[i] : "UNDEF");
}
}
return ;
}
/* Test code */
#ifdef TESTDIC
#define NVALS 20000
int main(int argc, char *argv[])
{
dictionary * d ;
char * val ;
int i ;
char cval[90] ;
/* Allocate dictionary */
printf("allocating...\n");
d = dictionary_new(0);
/* Set values in dictionary */
printf("setting %d values...\n", NVALS);
for (i=0 ; i<NVALS ; i++) {
sprintf(cval, "%04d", i);
dictionary_set(d, cval, "salut");
}
printf("getting %d values...\n", NVALS);
for (i=0 ; i<NVALS ; i++) {
sprintf(cval, "%04d", i);
val = dictionary_get(d, cval, DICT_INVALID_KEY);
if (val==DICT_INVALID_KEY) {
printf("cannot get value for key [%s]\n", cval);
}
}
printf("unsetting %d values...\n", NVALS);
for (i=0 ; i<NVALS ; i++) {
sprintf(cval, "%04d", i);
dictionary_unset(d, cval);
}
if (d->n != 0) {
printf("error deleting values\n");
}
printf("deallocating...\n");
dictionary_del(d);
return 0 ;
}
#endif
/* vim: set ts=4 et sw=4 tw=75 */

774
utils/TSZ/sz/src/iniparser.c Normal file
View File

@ -0,0 +1,774 @@
/*-------------------------------------------------------------------------*/
/**
@file iniparser.c
@author N. Devillard
@brief Parser for ini files.
*/
/*--------------------------------------------------------------------------*/
/*---------------------------- Includes ------------------------------------*/
#include <ctype.h>
#include "iniparser.h"
/*---------------------------- Defines -------------------------------------*/
#define ASCIILINESZ (1024)
#define INI_INVALID_KEY ((char*)-1)
/*---------------------------------------------------------------------------
Private to this module
---------------------------------------------------------------------------*/
/**
* This enum stores the status for each parsed line (internal use only).
*/
typedef enum _line_status_ {
LINE_UNPROCESSED,
LINE_ERROR,
LINE_EMPTY,
LINE_COMMENT,
LINE_SECTION,
LINE_VALUE
} line_status ;
/*-------------------------------------------------------------------------*/
/**
@brief Convert a string to lowercase.
@param s String to convert.
@return ptr to statically allocated string.
This function returns a pointer to a statically allocated string
containing a lowercased version of the input string. Do not free
or modify the returned string! Since the returned string is statically
allocated, it will be modified at each function call (not re-entrant).
*/
/*--------------------------------------------------------------------------*/
static char * strlwc(const char * s)
{
static char l[ASCIILINESZ+1];
int i ;
if (s==NULL) return NULL ;
memset(l, 0, ASCIILINESZ+1);
i=0 ;
while (s[i] && i<ASCIILINESZ) {
l[i] = (char)tolower((int)s[i]);
i++ ;
}
l[ASCIILINESZ]=(char)0;
return l ;
}
/*-------------------------------------------------------------------------*/
/**
@brief Remove blanks at the beginning and the end of a string.
@param s String to parse.
@return ptr to statically allocated string.
This function returns a pointer to a statically allocated string,
which is identical to the input string, except that all blank
characters at the end and the beg. of the string have been removed.
Do not free or modify the returned string! Since the returned string
is statically allocated, it will be modified at each function call
(not re-entrant).
*/
/*--------------------------------------------------------------------------*/
static char * strstrip(const char * s)
{
static char l[ASCIILINESZ+1];
char * last;
if (s==NULL) return NULL ;
while (isspace((int)*s) && *s) s++;
memset(l, 0, ASCIILINESZ+1);
strncpy(l, s, ASCIILINESZ);
last = l + strlen(l);
while (last > l) {
if (!isspace((int)*(last-1)))
break ;
last -- ;
}
*last = (char)0;
return (char*)l ;
}
/*-------------------------------------------------------------------------*/
/**
@brief Get number of sections in a dictionary
@param d Dictionary to examine
@return int Number of sections found in dictionary
This function returns the number of sections found in a dictionary.
The test to recognize sections is done on the string stored in the
dictionary: a section name is given as "section" whereas a key is
stored as "section:key", thus the test looks for entries that do not
contain a colon.
This clearly fails in the case a section name contains a colon, but
this should simply be avoided.
This function returns -1 in case of error.
*/
/*--------------------------------------------------------------------------*/
int iniparser_getnsec(dictionary * d)
{
int i ;
int nsec ;
if (d==NULL) return -1 ;
nsec=0 ;
for (i=0 ; i<d->size ; i++) {
if (d->key[i]==NULL)
continue ;
if (strchr(d->key[i], ':')==NULL) {
nsec ++ ;
}
}
return nsec ;
}
/*-------------------------------------------------------------------------*/
/**
@brief Get name for section n in a dictionary.
@param d Dictionary to examine
@param n Section number (from 0 to nsec-1).
@return Pointer to char string
This function locates the n-th section in a dictionary and returns
its name as a pointer to a string statically allocated inside the
dictionary. Do not free or modify the returned string!
This function returns NULL in case of error.
*/
/*--------------------------------------------------------------------------*/
char * iniparser_getsecname(dictionary * d, int n)
{
int i ;
int foundsec ;
if (d==NULL || n<0) return NULL ;
foundsec=0 ;
for (i=0 ; i<d->size ; i++) {
if (d->key[i]==NULL)
continue ;
if (strchr(d->key[i], ':')==NULL) {
foundsec++ ;
if (foundsec>n)
break ;
}
}
if (foundsec<=n) {
return NULL ;
}
return d->key[i] ;
}
/*-------------------------------------------------------------------------*/
/**
@brief Dump a dictionary to an opened file pointer.
@param d Dictionary to dump.
@param f Opened file pointer to dump to.
@return void
This function prints out the contents of a dictionary, one element by
line, onto the provided file pointer. It is OK to specify @c stderr
or @c stdout as output files. This function is meant for debugging
purposes mostly.
*/
/*--------------------------------------------------------------------------*/
void iniparser_dump(dictionary * d, FILE * f)
{
int i ;
if (d==NULL || f==NULL) return ;
for (i=0 ; i<d->size ; i++) {
if (d->key[i]==NULL)
continue ;
if (d->val[i]!=NULL) {
fprintf(f, "[%s]=[%s]\n", d->key[i], d->val[i]);
} else {
fprintf(f, "[%s]=UNDEF\n", d->key[i]);
}
}
return ;
}
/*-------------------------------------------------------------------------*/
/**
@brief Save a dictionary to a loadable ini file
@param d Dictionary to dump
@param f Opened file pointer to dump to
@return void
This function dumps a given dictionary into a loadable ini file.
It is Ok to specify @c stderr or @c stdout as output files.
*/
/*--------------------------------------------------------------------------*/
void iniparser_dump_ini(dictionary * d, FILE * f)
{
int i ;
int nsec ;
char * secname ;
if (d==NULL || f==NULL) return ;
nsec = iniparser_getnsec(d);
if (nsec<1) {
/* No section in file: dump all keys as they are */
for (i=0 ; i<d->size ; i++) {
if (d->key[i]==NULL)
continue ;
fprintf(f, "%s = %s\n", d->key[i], d->val[i]);
}
return ;
}
for (i=0 ; i<nsec ; i++) {
secname = iniparser_getsecname(d, i) ;
iniparser_dumpsection_ini(d, secname, f) ;
}
fprintf(f, "\n");
return ;
}
/*-------------------------------------------------------------------------*/
/**
@brief Save a dictionary section to a loadable ini file
@param d Dictionary to dump
@param s Section name of dictionary to dump
@param f Opened file pointer to dump to
@return void
This function dumps a given section of a given dictionary into a loadable ini
file. It is Ok to specify @c stderr or @c stdout as output files.
*/
/*--------------------------------------------------------------------------*/
void iniparser_dumpsection_ini(dictionary * d, char * s, FILE * f)
{
int j ;
char keym[ASCIILINESZ+1];
int seclen ;
if (d==NULL || f==NULL) return ;
if (! iniparser_find_entry(d, s)) return ;
seclen = (int)strlen(s);
fprintf(f, "\n[%s]\n", s);
sprintf(keym, "%s:", s);
for (j=0 ; j<d->size ; j++) {
if (d->key[j]==NULL)
continue ;
if (!strncmp(d->key[j], keym, seclen+1)) {
fprintf(f,
"%-30s = %s\n",
d->key[j]+seclen+1,
d->val[j] ? d->val[j] : "");
}
}
fprintf(f, "\n");
return ;
}
/*-------------------------------------------------------------------------*/
/**
@brief Get the number of keys in a section of a dictionary.
@param d Dictionary to examine
@param s Section name of dictionary to examine
@return Number of keys in section
*/
/*--------------------------------------------------------------------------*/
int iniparser_getsecnkeys(dictionary * d, char * s)
{
int seclen, nkeys ;
char keym[ASCIILINESZ+1];
int j ;
nkeys = 0;
if (d==NULL) return nkeys;
if (! iniparser_find_entry(d, s)) return nkeys;
seclen = (int)strlen(s);
sprintf(keym, "%s:", s);
for (j=0 ; j<d->size ; j++) {
if (d->key[j]==NULL)
continue ;
if (!strncmp(d->key[j], keym, seclen+1))
nkeys++;
}
return nkeys;
}
/*-------------------------------------------------------------------------*/
/**
@brief Get the number of keys in a section of a dictionary.
@param d Dictionary to examine
@param s Section name of dictionary to examine
@return pointer to statically allocated character strings
This function queries a dictionary and finds all keys in a given section.
Each pointer in the returned char pointer-to-pointer is pointing to
a string allocated in the dictionary; do not free or modify them.
This function returns NULL in case of error.
*/
/*--------------------------------------------------------------------------*/
char ** iniparser_getseckeys(dictionary * d, char * s)
{
char **keys;
int i, j ;
char keym[ASCIILINESZ+1];
int seclen, nkeys ;
keys = NULL;
if (d==NULL) return keys;
if (! iniparser_find_entry(d, s)) return keys;
nkeys = iniparser_getsecnkeys(d, s);
keys = (char**) malloc(nkeys*sizeof(char*));
seclen = (int)strlen(s);
sprintf(keym, "%s:", s);
i = 0;
for (j=0 ; j<d->size ; j++) {
if (d->key[j]==NULL)
continue ;
if (!strncmp(d->key[j], keym, seclen+1)) {
keys[i] = d->key[j];
i++;
}
}
return keys;
}
/*-------------------------------------------------------------------------*/
/**
@brief Get the string associated to a key
@param d Dictionary to search
@param key Key string to look for
@param def Default value to return if key not found.
@return pointer to statically allocated character string
This function queries a dictionary for a key. A key as read from an
ini file is given as "section:key". If the key cannot be found,
the pointer passed as 'def' is returned.
The returned char pointer is pointing to a string allocated in
the dictionary, do not free or modify it.
*/
/*--------------------------------------------------------------------------*/
char * iniparser_getstring(dictionary * d, const char * key, char * def)
{
char * lc_key ;
char * sval ;
if (d==NULL || key==NULL)
return def ;
lc_key = strlwc(key);
sval = dictionary_get(d, lc_key, def);
return sval ;
}
/*-------------------------------------------------------------------------*/
/**
@brief Get the string associated to a key, convert to an int
@param d Dictionary to search
@param key Key string to look for
@param notfound Value to return in case of error
@return integer
This function queries a dictionary for a key. A key as read from an
ini file is given as "section:key". If the key cannot be found,
the notfound value is returned.
Supported values for integers include the usual C notation
so decimal, octal (starting with 0) and hexadecimal (starting with 0x)
are supported. Examples:
"42" -> 42
"042" -> 34 (octal -> decimal)
"0x42" -> 66 (hexa -> decimal)
Warning: the conversion may overflow in various ways. Conversion is
totally outsourced to strtol(), see the associated man page for overflow
handling.
Credits: Thanks to A. Becker for suggesting strtol()
*/
/*--------------------------------------------------------------------------*/
int iniparser_getint(dictionary * d, const char * key, int notfound)
{
char * str ;
str = iniparser_getstring(d, key, INI_INVALID_KEY);
if (str==INI_INVALID_KEY) return notfound ;
return (int)strtol(str, NULL, 0);
}
/*-------------------------------------------------------------------------*/
/**
@brief Get the string associated to a key, convert to a long
@param d Dictionary to search
@param key Key string to look for
@param notfound Value to return in case of error
@return long
Credits: This function bases completely on int iniparser_getint and was
slightly modified to return long instead of int.
*/
/*--------------------------------------------------------------------------*/
long iniparser_getlint(dictionary * d, const char * key, int notfound)
{
char * str ;
str = iniparser_getstring(d, key, INI_INVALID_KEY);
if (str==INI_INVALID_KEY) return notfound ;
return strtol(str, NULL, 0);
}
/*-------------------------------------------------------------------------*/
/**
@brief Get the string associated to a key, convert to a double
@param d Dictionary to search
@param key Key string to look for
@param notfound Value to return in case of error
@return double
This function queries a dictionary for a key. A key as read from an
ini file is given as "section:key". If the key cannot be found,
the notfound value is returned.
*/
/*--------------------------------------------------------------------------*/
double iniparser_getdouble(dictionary * d, const char * key, double notfound)
{
char * str ;
str = iniparser_getstring(d, key, INI_INVALID_KEY);
if (str==INI_INVALID_KEY) return notfound ;
return atof(str);
}
/*-------------------------------------------------------------------------*/
/**
@brief Get the string associated to a key, convert to a boolean
@param d Dictionary to search
@param key Key string to look for
@param notfound Value to return in case of error
@return integer
This function queries a dictionary for a key. A key as read from an
ini file is given as "section:key". If the key cannot be found,
the notfound value is returned.
A true boolean is found if one of the following is matched:
- A string starting with 'y'
- A string starting with 'Y'
- A string starting with 't'
- A string starting with 'T'
- A string starting with '1'
A false boolean is found if one of the following is matched:
- A string starting with 'n'
- A string starting with 'N'
- A string starting with 'f'
- A string starting with 'F'
- A string starting with '0'
The notfound value returned if no boolean is identified, does not
necessarily have to be 0 or 1.
*/
/*--------------------------------------------------------------------------*/
int iniparser_getboolean(dictionary * d, const char * key, int notfound)
{
char * c ;
int ret ;
c = iniparser_getstring(d, key, INI_INVALID_KEY);
if (c==INI_INVALID_KEY) return notfound ;
if (c[0]=='y' || c[0]=='Y' || c[0]=='1' || c[0]=='t' || c[0]=='T') {
ret = 1 ;
} else if (c[0]=='n' || c[0]=='N' || c[0]=='0' || c[0]=='f' || c[0]=='F') {
ret = 0 ;
} else {
ret = notfound ;
}
return ret;
}
/*-------------------------------------------------------------------------*/
/**
@brief Finds out if a given entry exists in a dictionary
@param ini Dictionary to search
@param entry Name of the entry to look for
@return integer 1 if entry exists, 0 otherwise
Finds out if a given entry exists in the dictionary. Since sections
are stored as keys with NULL associated values, this is the only way
of querying for the presence of sections in a dictionary.
*/
/*--------------------------------------------------------------------------*/
int iniparser_find_entry(
dictionary * ini,
const char * entry
)
{
int found=0 ;
if (iniparser_getstring(ini, entry, INI_INVALID_KEY)!=INI_INVALID_KEY) {
found = 1 ;
}
return found ;
}
/*-------------------------------------------------------------------------*/
/**
@brief Set an entry in a dictionary.
@param ini Dictionary to modify.
@param entry Entry to modify (entry name)
@param val New value to associate to the entry.
@return int 0 if Ok, -1 otherwise.
If the given entry can be found in the dictionary, it is modified to
contain the provided value. If it cannot be found, -1 is returned.
It is Ok to set val to NULL.
*/
/*--------------------------------------------------------------------------*/
int iniparser_set(dictionary * ini, const char * entry, const char * val)
{
return dictionary_set(ini, strlwc(entry), val) ;
}
/*-------------------------------------------------------------------------*/
/**
@brief Delete an entry in a dictionary
@param ini Dictionary to modify
@param entry Entry to delete (entry name)
@return void
If the given entry can be found, it is deleted from the dictionary.
*/
/*--------------------------------------------------------------------------*/
void iniparser_unset(dictionary * ini, const char * entry)
{
dictionary_unset(ini, strlwc(entry));
}
/*-------------------------------------------------------------------------*/
/**
@brief Load a single line from an INI file
@param input_line Input line, may be concatenated multi-line input
@param section Output space to store section
@param key Output space to store key
@param value Output space to store value
@return line_status value
*/
/*--------------------------------------------------------------------------*/
static line_status iniparser_line(
const char * input_line,
char * section,
char * key,
char * value)
{
line_status sta ;
char line[ASCIILINESZ+1];
int len ;
memset(line, 0, ASCIILINESZ + 1);
len = (int)strlen(strstrip(input_line));
if (len > ASCIILINESZ)
len = ASCIILINESZ;
strncpy(line, strstrip(input_line), len);
len = (int)strlen(line);
sta = LINE_UNPROCESSED ;
if (len<1) {
/* Empty line */
sta = LINE_EMPTY ;
} else if (line[0]=='#' || line[0]==';') {
/* Comment line */
sta = LINE_COMMENT ;
} else if (line[0]=='[' && line[len-1]==']') {
/* Section name */
sscanf(line, "[%[^]]", section);
strcpy(section, strstrip(section));
strcpy(section, strlwc(section));
sta = LINE_SECTION ;
} else if (sscanf (line, "%[^=] = \"%[^\"]\"", key, value) == 2
|| sscanf (line, "%[^=] = '%[^\']'", key, value) == 2
|| sscanf (line, "%[^=] = %[^;#]", key, value) == 2) {
/* Usual key=value, with or without comments */
strcpy(key, strstrip(key));
strcpy(key, strlwc(key));
strcpy(value, strstrip(value));
/*
* sscanf cannot handle '' or "" as empty values
* this is done here
*/
if (!strcmp(value, "\"\"") || (!strcmp(value, "''"))) {
value[0]=0 ;
}
sta = LINE_VALUE ;
} else if (sscanf(line, "%[^=] = %[;#]", key, value)==2
|| sscanf(line, "%[^=] %[=]", key, value) == 2) {
/*
* Special cases:
* key=
* key=;
* key=#
*/
strcpy(key, strstrip(key));
strcpy(key, strlwc(key));
value[0]=0 ;
sta = LINE_VALUE ;
} else {
/* Generate syntax error */
sta = LINE_ERROR ;
printf("===== > %s ===> %s\n", input_line, line);
}
return sta ;
}
/*-------------------------------------------------------------------------*/
/**
@brief Parse an ini file and return an allocated dictionary object
@param ininame Name of the ini file to read.
@return Pointer to newly allocated dictionary
This is the parser for ini files. This function is called, providing
the name of the file to be read. It returns a dictionary object that
should not be accessed directly, but through accessor functions
instead.
The returned dictionary must be freed using iniparser_freedict().
*/
/*--------------------------------------------------------------------------*/
dictionary * iniparser_load(const char * ininame)
{
FILE * in ;
char line [ASCIILINESZ+1] ;
char section [ASCIILINESZ+1] ;
char key [ASCIILINESZ+1] ;
char tmp [2*ASCIILINESZ+2] ;
char val [ASCIILINESZ+1] ;
int last=0 ;
int len ;
int lineno=0 ;
int errs=0;
dictionary * dict ;
if ((in=fopen(ininame, "r"))==NULL) {
fprintf(stderr, "iniparser: cannot open %s\n", ininame);
return NULL ;
}
dict = dictionary_new(0) ;
if (!dict) {
fclose(in);
return NULL ;
}
memset(line, 0, ASCIILINESZ);
memset(section, 0, ASCIILINESZ);
memset(key, 0, ASCIILINESZ);
memset(val, 0, ASCIILINESZ);
last=0 ;
while (fgets(line+last, ASCIILINESZ-last, in)!=NULL) {
lineno++ ;
len = (int)strlen(line)-1;
if (len==0)
continue;
/* Safety check against buffer overflows */
if (line[len]!='\n') {
fprintf(stderr,
"iniparser: input line too long in %s (%d)\n",
ininame,
lineno);
dictionary_del(dict);
fclose(in);
return NULL ;
}
/* Get rid of \n and spaces at end of line */
while ((len>=0) &&
((line[len]=='\n') || (isspace(line[len])))) {
line[len]=0 ;
len-- ;
}
/* Detect multi-line */
if (line[len]=='\\') {
/* Multi-line value */
last=len ;
continue ;
} else {
last=0 ;
}
switch (iniparser_line(line, section, key, val)) {
case LINE_EMPTY:
case LINE_COMMENT:
break ;
case LINE_SECTION:
errs = dictionary_set(dict, section, NULL);
break ;
case LINE_VALUE:
sprintf(tmp, "%s:%s", section, key);
errs = dictionary_set(dict, tmp, val) ;
break ;
case LINE_ERROR:
fprintf(stderr, "iniparser: syntax error in %s (%d):\n",
ininame,
lineno);
fprintf(stderr, "-> %s\n", line);
errs++ ;
break;
default:
break ;
}
memset(line, 0, ASCIILINESZ);
last=0;
if (errs<0) {
fprintf(stderr, "iniparser: memory allocation failure\n");
break ;
}
}
if (errs) {
dictionary_del(dict);
dict = NULL ;
}
fclose(in);
return dict ;
}
/*-------------------------------------------------------------------------*/
/**
@brief Free all memory associated to an ini dictionary
@param d Dictionary to free
@return void
Free all memory associated to an ini dictionary.
It is mandatory to call this function before the dictionary object
gets out of the current context.
*/
/*--------------------------------------------------------------------------*/
void iniparser_freedict(dictionary * d)
{
dictionary_del(d);
}
/* vim: set ts=4 et sw=4 tw=75 */

212
utils/TSZ/sz/src/sz.c Normal file
View File

@ -0,0 +1,212 @@
/**
* @file sz.c
* @author Sheng Di and Dingwen Tao
* @date Aug, 2016
* @brief SZ_Init, Compression and Decompression functions
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "sz.h"
#include "CompressElement.h"
#include "DynamicByteArray.h"
#include "TightDataPointStorageD.h"
#include "TightDataPointStorageF.h"
#include "conf.h"
#include "utility.h"
//#include "CurveFillingCompressStorage.h"
unsigned char versionNumber = DATA_FROMAT_VER1;
int SZ_SIZE_TYPE_DEFUALT = 4;
int dataEndianType = LITTLE_ENDIAN_DATA; //*endian type of the data read from disk
int sysEndianType = LITTLE_ENDIAN_SYSTEM ; //*sysEndianType is actually set automatically.
//the confparams should be separate between compression and decopmression, in case of mutual-affection when calling compression/decompression alternatively
sz_params *confparams_cpr = NULL; //used for compression
sz_exedata *exe_params = NULL;
int SZ_Init(const char *configFilePath)
{
// check CPU EndianType
int x = 1;
char *y = (char*)&x;
if(*y==1)
sysEndianType = LITTLE_ENDIAN_SYSTEM;
else //=0
sysEndianType = BIG_ENDIAN_SYSTEM;
int loadFileResult = SZ_LoadConf(configFilePath);
if(loadFileResult==SZ_FAILED)
return SZ_FAILED;
exe_params->SZ_SIZE_TYPE = SZ_SIZE_TYPE_DEFUALT;
return SZ_SUCCESS;
}
int SZ_Init_Params(sz_params *params)
{
SZ_Init(NULL);
if(params->losslessCompressor!=GZIP_COMPRESSOR && params->losslessCompressor!=ZSTD_COMPRESSOR)
params->losslessCompressor = ZSTD_COMPRESSOR;
if(params->max_quant_intervals > 0)
params->maxRangeRadius = params->max_quant_intervals/2;
memcpy(confparams_cpr, params, sizeof(sz_params));
if(params->quantization_intervals%2!=0)
{
printf("Error: quantization_intervals must be an even number!\n");
return SZ_FAILED;
}
return SZ_SUCCESS;
}
/*-------------------------------------------------------------------------*/
/**
@brief Perform Compression
@param data data to be compressed
@param outSize the size (in bytes) after compression
@param r5,r4,r3,r2,r1 the sizes of each dimension (supporting only 5 dimensions at most in this version.
@return compressed data (in binary stream) or NULL(0) if any errors
**/
/*-------------------------------------------------------------------------*/
//
// compress output data to outData and return outSize
//
size_t SZ_compress_args(int dataType, void *data, size_t r1, unsigned char* outData, sz_params* params)
{
size_t outSize = 0;
if(dataType==SZ_FLOAT)
{
SZ_compress_args_float((float *)data, r1, outData, &outSize, params);
}
else if(dataType==SZ_DOUBLE)
{
SZ_compress_args_double((double *)data, r1, outData, &outSize, params);
}
else
{
printf("Error: dataType can only be SZ_FLOAT, SZ_DOUBLE .\n");
return 0;
}
return outSize;
}
//
// decompress output data to outData and return outSize
//
size_t SZ_decompress(int dataType, unsigned char *bytes, size_t byteLength, size_t r1, unsigned char* outData)
{
sz_exedata de_exe;
memset(&de_exe, 0, sizeof(sz_exedata));
sz_params de_params;
memset(&de_params, 0, sizeof(sz_params));
size_t outSize = 0;
if(dataType == SZ_FLOAT)
{
int status = SZ_decompress_args_float((float*)outData, r1, bytes, byteLength, 0, NULL, &de_exe, &de_params);
if(status == SZ_SUCCESS)
{
return r1*sizeof(float);
}
return 0;
}
else if(dataType == SZ_DOUBLE)
{
int status = SZ_decompress_args_double((double*)outData, r1, bytes, byteLength, 0, NULL, &de_exe, &de_params);
if(status == SZ_SUCCESS)
{
return r1*sizeof(double);
}
return 0;
}
else
{
printf("Error: data type cannot be the types other than SZ_FLOAT or SZ_DOUBLE\n");
}
return outSize;
}
void SZ_Finalize()
{
if(confparams_cpr!=NULL)
{
free(confparams_cpr);
confparams_cpr = NULL;
}
if(exe_params!=NULL)
{
free(exe_params);
exe_params = NULL;
}
}
#ifdef WINDOWS
#include <windows.h>
int gettimeofday(struct timeval *tp, void *tzp) {
time_t clock;
struct tm tm;
SYSTEMTIME wtm;
GetLocalTime(&wtm);
tm.tm_year = wtm.wYear - 1900;
tm.tm_mon = wtm.wMonth - 1;
tm.tm_mday = wtm.wDay;
tm.tm_hour = wtm.wHour;
tm.tm_min = wtm.wMinute;
tm.tm_sec = wtm.wSecond;
tm. tm_isdst = -1;
clock = mktime(&tm);
tp->tv_sec = clock;
tp->tv_usec = wtm.wMilliseconds * 1000;
return 0;
}
#else
#include <sys/time.h>
#endif
struct timeval costStart; /*only used for recording the cost*/
double totalCost = 0;
void cost_start()
{
totalCost = 0;
gettimeofday(&costStart, NULL);
}
double cost_end(const char* tag)
{
double elapsed;
struct timeval costEnd;
gettimeofday(&costEnd, NULL);
elapsed = ((costEnd.tv_sec*1000000+costEnd.tv_usec)-(costStart.tv_sec*1000000+costStart.tv_usec))/1000000.0;
totalCost += elapsed;
double use_ms = totalCost*1000;
printf(" timecost %s : %.3f ms\n", tag, use_ms);
return use_ms;
}
void show_rate(int in_len, int out_len)
{
float rate=100*(float)out_len/(float)in_len;
printf(" in_len=%d out_len=%d compress rate=%.4f%%\n", in_len, out_len, rate);
}

415
utils/TSZ/sz/src/sz_double.c Normal file
View File

@ -0,0 +1,415 @@
/**
* @file sz_double.c
* @author Sheng Di, Dingwen Tao, Xin Liang, Xiangyu Zou, Tao Lu, Wen Xia, Xuan Wang, Weizhe Zhang
* @date Aug, 2016
* @brief SZ_Init, Compression and Decompression functions
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include "sz.h"
#include "CompressElement.h"
#include "DynamicByteArray.h"
#include "DynamicIntArray.h"
#include "TightDataPointStorageD.h"
#include "sz_double.h"
#include "szd_double.h"
#include "utility.h"
unsigned char* SZ_skip_compress_double(double* data, size_t dataLength, size_t* outSize)
{
*outSize = dataLength*sizeof(double);
unsigned char* out = (unsigned char*)malloc(dataLength*sizeof(double));
memcpy(out, data, dataLength*sizeof(double));
return out;
}
INLINE void computeReqLength_double(double realPrecision, short radExpo, int* reqLength, double* medianValue)
{
short reqExpo = getPrecisionReqLength_double(realPrecision);
*reqLength = 12+radExpo - reqExpo; //radExpo-reqExpo == reqMantiLength
if(*reqLength<12)
*reqLength = 12;
if(*reqLength>64)
{
*reqLength = 64;
*medianValue = 0;
}
}
unsigned int optimize_intervals_double_1D(double *oriData, size_t dataLength, double realPrecision)
{
size_t i = 0, radiusIndex;
double pred_value = 0, pred_err;
size_t *intervals = (size_t*)malloc(confparams_cpr->maxRangeRadius*sizeof(size_t));
memset(intervals, 0, confparams_cpr->maxRangeRadius*sizeof(size_t));
size_t totalSampleSize = dataLength/confparams_cpr->sampleDistance;
for(i=2;i<dataLength;i++)
{
if(i%confparams_cpr->sampleDistance==0)
{
//pred_value = 2*oriData[i-1] - oriData[i-2];
pred_value = oriData[i-1];
pred_err = fabs(pred_value - oriData[i]);
radiusIndex = (unsigned long)((pred_err/realPrecision+1)/2);
if(radiusIndex>=confparams_cpr->maxRangeRadius)
radiusIndex = confparams_cpr->maxRangeRadius - 1;
intervals[radiusIndex]++;
}
}
//compute the appropriate number
size_t targetCount = totalSampleSize*confparams_cpr->predThreshold;
size_t sum = 0;
for(i=0;i<confparams_cpr->maxRangeRadius;i++)
{
sum += intervals[i];
if(sum>targetCount)
break;
}
if(i>=confparams_cpr->maxRangeRadius)
i = confparams_cpr->maxRangeRadius-1;
unsigned int accIntervals = 2*(i+1);
unsigned int powerOf2 = roundUpToPowerOf2(accIntervals);
if(powerOf2<32)
powerOf2 = 32;
free(intervals);
//printf("accIntervals=%d, powerOf2=%d\n", accIntervals, powerOf2);
return powerOf2;
}
TightDataPointStorageD* SZ_compress_double_1D_MDQ(double *oriData,
size_t dataLength, double realPrecision, double valueRangeSize, double medianValue_d)
{
unsigned int quantization_intervals;
if(exe_params->optQuantMode==1)
quantization_intervals = optimize_intervals_double_1D_opt(oriData, dataLength, realPrecision);
else
quantization_intervals = exe_params->intvCapacity;
//updateQuantizationInfo(quantization_intervals);
int intvRadius = quantization_intervals/2;
size_t i;
int reqLength;
double medianValue = medianValue_d;
short radExpo = getExponent_double(valueRangeSize/2);
computeReqLength_double(realPrecision, radExpo, &reqLength, &medianValue);
int* type = (int*) malloc(dataLength*sizeof(int));
double* spaceFillingValue = oriData; //
DynamicIntArray *exactLeadNumArray;
new_DIA(&exactLeadNumArray, DynArrayInitLen);
DynamicByteArray *exactMidByteArray;
new_DBA(&exactMidByteArray, DynArrayInitLen);
DynamicIntArray *resiBitArray;
new_DIA(&resiBitArray, DynArrayInitLen);
unsigned char preDataBytes[8];
longToBytes_bigEndian(preDataBytes, 0);
int reqBytesLength = reqLength/8;
int resiBitsLength = reqLength%8;
double last3CmprsData[3] = {0};
DoubleValueCompressElement *vce = (DoubleValueCompressElement*)malloc(sizeof(DoubleValueCompressElement));
LossyCompressionElement *lce = (LossyCompressionElement*)malloc(sizeof(LossyCompressionElement));
//add the first data
type[0] = 0;
compressSingleDoubleValue(vce, spaceFillingValue[0], realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,8);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
listAdd_double(last3CmprsData, vce->data);
//add the second data
type[1] = 0;
compressSingleDoubleValue(vce, spaceFillingValue[1], realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,8);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
listAdd_double(last3CmprsData, vce->data);
int state;
double checkRadius;
double curData;
double pred = last3CmprsData[0];
double predAbsErr;
checkRadius = (quantization_intervals-1)*realPrecision;
double interval = 2*realPrecision;
double recip_realPrecision = 1/realPrecision;
for(i=2; i < dataLength; i++)
{
//printf("%.30G\n",last3CmprsData[0]);
curData = spaceFillingValue[i];
//pred = 2*last3CmprsData[0] - last3CmprsData[1];
//pred = last3CmprsData[0];
predAbsErr = fabs(curData - pred);
if(predAbsErr<checkRadius)
{
state = (predAbsErr*recip_realPrecision+1)*0.5;
if(curData>=pred)
{
type[i] = intvRadius+state;
pred = pred + state*interval;
}
else //curData<pred
{
type[i] = intvRadius-state;
pred = pred - state*interval;
}
//listAdd_double(last3CmprsData, pred);
continue;
}
//unpredictable data processing
type[i] = 0;
compressSingleDoubleValue(vce, curData, realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,8);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
//listAdd_double(last3CmprsData, vce->data);
pred = vce->data;
}//end of for
size_t exactDataNum = exactLeadNumArray->size;
TightDataPointStorageD* tdps;
new_TightDataPointStorageD(&tdps, dataLength, exactDataNum,
type, exactMidByteArray->array, exactMidByteArray->size,
exactLeadNumArray->array,
resiBitArray->array, resiBitArray->size,
resiBitsLength,
realPrecision, medianValue, (char)reqLength, quantization_intervals, 0);
// printf("exactDataNum=%d, expSegmentsInBytes_size=%d, exactMidByteArray->size=%d\n",
// exactDataNum, expSegmentsInBytes_size, exactMidByteArray->size);
//free memory
free_DIA(exactLeadNumArray);
free_DIA(resiBitArray);
free(type);
free(vce);
free(lce);
free(exactMidByteArray); //exactMidByteArray->array has been released in free_TightDataPointStorageF(tdps);
return tdps;
}
void SZ_compress_args_double_StoreOriData(double* oriData, size_t dataLength, unsigned char* newByteData, size_t *outSize)
{
int doubleSize = sizeof(double);
size_t k = 0, i;
size_t totalByteLength = 1 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + 1 + doubleSize*dataLength;
/*No need to malloc because newByteData should always already be allocated with no less totalByteLength.*/
//*newByteData = (unsigned char*)malloc(totalByteLength);
unsigned char dsLengthBytes[8];
newByteData[k++] = versionNumber;
if(exe_params->SZ_SIZE_TYPE==4)//1
newByteData[k++] = 16; //00010000
else
newByteData[k++] = 80; //01010000: 01000000 indicates the SZ_SIZE_TYPE=8
convertSZParamsToBytes(confparams_cpr, &(newByteData[k]), exe_params->optQuantMode);
k = k + MetaDataByteLength_double;
sizeToBytes(dsLengthBytes,dataLength, exe_params->SZ_SIZE_TYPE);
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)//ST: 4 or 8
newByteData[k++] = dsLengthBytes[i];
if(sysEndianType==BIG_ENDIAN_SYSTEM)
memcpy(newByteData+4+MetaDataByteLength_double+exe_params->SZ_SIZE_TYPE, oriData, dataLength*doubleSize);
else
{
unsigned char* p = newByteData+4+MetaDataByteLength_double+exe_params->SZ_SIZE_TYPE;
for(i=0;i<dataLength;i++,p+=doubleSize)
doubleToBytes(p, oriData[i]);
}
*outSize = totalByteLength;
}
bool SZ_compress_args_double_NoCkRngeNoGzip_1D(unsigned char* newByteData, double *oriData,
size_t dataLength, double realPrecision, size_t *outSize, double valueRangeSize, double medianValue_d)
{
TightDataPointStorageD* tdps = NULL;
tdps = SZ_compress_double_1D_MDQ(oriData, dataLength, realPrecision, valueRangeSize, medianValue_d);
if(!convertTDPStoFlatBytes_double(tdps, newByteData, outSize))
{
free_TightDataPointStorageD(tdps);
return false;
}
//if(*outSize>3 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + 1 + sizeof(double)*dataLength)
// SZ_compress_args_double_StoreOriData(oriData, dataLength, newByteData, outSize);
free_TightDataPointStorageD(tdps);
return true;
}
void SZ_compress_args_double_withinRange(unsigned char* newByteData, double *oriData, size_t dataLength, size_t *outSize)
{
TightDataPointStorageD* tdps = (TightDataPointStorageD*) malloc(sizeof(TightDataPointStorageD));
memset(tdps, 0, sizeof(TightDataPointStorageD));
tdps->leadNumArray = NULL;
tdps->residualMidBits = NULL;
tdps->allSameData = 1;
tdps->dataSeriesLength = dataLength;
tdps->exactMidBytes = (unsigned char*)malloc(sizeof(unsigned char)*8);
tdps->isLossless = 0;
double value = oriData[0];
doubleToBytes(tdps->exactMidBytes, value);
tdps->exactMidBytes_size = 8;
size_t tmpOutSize;
//unsigned char *tmpByteData;
convertTDPStoFlatBytes_double(tdps, newByteData, &tmpOutSize);
//convertTDPStoFlatBytes_double(tdps, &tmpByteData, &tmpOutSize);
//*newByteData = (unsigned char*)malloc(sizeof(unsigned char)*16); //for floating-point data (1+3+4+4)
//memcpy(*newByteData, tmpByteData, 16);
*outSize = tmpOutSize;//12==3+1+8(double_size)+MetaDataByteLength_double
free_TightDataPointStorageD(tdps);
}
int SZ_compress_args_double(double *oriData, size_t r1, unsigned char* newByteData, size_t *outSize, sz_params* params)
{
int status = SZ_SUCCESS;
size_t dataLength = r1;
double valueRangeSize = 0, medianValue = 0;
// check at least elements count
if(dataLength <= MIN_NUM_OF_ELEMENTS)
{
printf("warning, double input elements count=%d less than %d, so need not do compress.\n", (int)dataLength, MIN_NUM_OF_ELEMENTS);
return SZ_LITTER_ELEMENT;
}
double min = computeRangeSize_double(oriData, dataLength, &valueRangeSize, &medianValue);
double max = min+valueRangeSize;
params->dmin = min;
params->dmax = max;
double realPrecision = 0;
if(params->errorBoundMode==PSNR)
{
params->errorBoundMode = SZ_ABS;
realPrecision = params->absErrBoundDouble = computeABSErrBoundFromPSNR(params->psnr, (double)params->predThreshold, valueRangeSize);
}
else if(params->errorBoundMode==NORM) //norm error = sqrt(sum((xi-xi_)^2))
{
params->errorBoundMode = SZ_ABS;
realPrecision = params->absErrBoundDouble = computeABSErrBoundFromNORM_ERR(params->normErr, dataLength);
//printf("realPrecision=%lf\n", realPrecision);
}
else
{
realPrecision = getRealPrecision_double(valueRangeSize, params->errorBoundMode, params->absErrBoundDouble, params->relBoundRatio, &status);
params->absErrBoundDouble = realPrecision;
}
if(valueRangeSize <= realPrecision)
{
SZ_compress_args_double_withinRange(newByteData, oriData, dataLength, outSize);
}
else
{
size_t tmpOutSize = 0;
unsigned char* tmpByteData = newByteData;
bool twoStage = params->szMode != SZ_BEST_SPEED;
if(twoStage)
{
tmpByteData = (unsigned char*)malloc(r1*sizeof(double)*1.2);
}
if(!SZ_compress_args_double_NoCkRngeNoGzip_1D(tmpByteData, oriData, r1, realPrecision, &tmpOutSize, valueRangeSize, medianValue))
{
if(twoStage)
free(tmpByteData);
return SZ_ALGORITHM_ERR;
}
//if(tmpOutSize>=dataLength*sizeof(double) + 3 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + 1)
// SZ_compress_args_double_StoreOriData(oriData, dataLength, tmpByteData, &tmpOutSize);
//
//Call Gzip to do the further compression.
//
if(twoStage)
{
*outSize = sz_lossless_compress(params->losslessCompressor, tmpByteData, tmpOutSize, newByteData);
free(tmpByteData);
}
else
{
*outSize = tmpOutSize;
}
}
return status;
}
unsigned int optimize_intervals_double_1D_opt(double *oriData, size_t dataLength, double realPrecision)
{
size_t i = 0, radiusIndex;
double pred_value = 0, pred_err;
size_t *intervals = (size_t*)malloc(confparams_cpr->maxRangeRadius*sizeof(size_t));
memset(intervals, 0, confparams_cpr->maxRangeRadius*sizeof(size_t));
size_t totalSampleSize = 0;
double * data_pos = oriData + 2;
while(data_pos - oriData < dataLength){
totalSampleSize++;
pred_value = data_pos[-1];
pred_err = fabs(pred_value - *data_pos);
radiusIndex = (unsigned long)((pred_err/realPrecision+1)/2);
if(radiusIndex>=confparams_cpr->maxRangeRadius)
radiusIndex = confparams_cpr->maxRangeRadius - 1;
intervals[radiusIndex]++;
data_pos += confparams_cpr->sampleDistance;
}
//compute the appropriate number
size_t targetCount = totalSampleSize*confparams_cpr->predThreshold;
size_t sum = 0;
for(i=0;i<confparams_cpr->maxRangeRadius;i++)
{
sum += intervals[i];
if(sum>targetCount)
break;
}
if(i>=confparams_cpr->maxRangeRadius)
i = confparams_cpr->maxRangeRadius-1;
unsigned int accIntervals = 2*(i+1);
unsigned int powerOf2 = roundUpToPowerOf2(accIntervals);
if(powerOf2<32)
powerOf2 = 32;
free(intervals);
return powerOf2;
}

Some files were not shown because too many files have changed in this diff Show More