From d287b1b34c54ae881ca7d49f6ef7931723f03ee9 Mon Sep 17 00:00:00 2001 From: tickduan <417921451@qq.com> Date: Mon, 12 Jul 2021 18:09:05 +0800 Subject: [PATCH] modify intervals default value from 300 to 100 --- deps/SZ/sz/include/ByteToolkit.h | 2 +- deps/SZ/sz/include/sz.h | 4 ++++ deps/SZ/sz/src/DynamicByteArray.c | 5 ++++- deps/SZ/sz/src/conf.c | 4 ++-- src/common/src/tglobal.c | 4 ++-- src/kit/taospack/taospack.c | 36 +++++++++++++++---------------- 6 files changed, 31 insertions(+), 24 deletions(-) diff --git a/deps/SZ/sz/include/ByteToolkit.h b/deps/SZ/sz/include/ByteToolkit.h index d35fbcf809..82d060483c 100644 --- a/deps/SZ/sz/include/ByteToolkit.h +++ b/deps/SZ/sz/include/ByteToolkit.h @@ -20,7 +20,7 @@ int bytesToInt_bigEndian(unsigned char* bytes); void intToBytes_bigEndian(unsigned char *b, unsigned int num); long bytesToLong_bigEndian(unsigned char* b); -void longToBytes_bigEndian(unsigned char *b, unsigned long num); +void longToBytes_bigEndian(unsigned char *b, long num); short getExponent_float(float value); short getPrecisionReqLength_float(float precision); diff --git a/deps/SZ/sz/include/sz.h b/deps/SZ/sz/include/sz.h index 0aca9a5cd1..735c38ea49 100644 --- a/deps/SZ/sz/include/sz.h +++ b/deps/SZ/sz/include/sz.h @@ -43,6 +43,10 @@ extern "C" { #endif +void cost_start(); +double cost_end(const char* tag); +void show_rate( int in_len, int out_len); + //typedef char int8_t; //typedef unsigned char uint8_t; //typedef short int16_t; diff --git a/deps/SZ/sz/src/DynamicByteArray.c b/deps/SZ/sz/src/DynamicByteArray.c index ba69c1365a..1787f69cf8 100644 --- a/deps/SZ/sz/src/DynamicByteArray.c +++ b/deps/SZ/sz/src/DynamicByteArray.c @@ -26,7 +26,10 @@ void convertDBAtoBytes(DynamicByteArray *dba, unsigned char** bytes) if(size>0) *bytes = (unsigned char*)malloc(size * sizeof(unsigned char)); else - *bytes = NULL; + { + *bytes = NULL; + return ; + } memcpy(*bytes, dba->array, size*sizeof(unsigned char)); } diff --git a/deps/SZ/sz/src/conf.c b/deps/SZ/sz/src/conf.c index 6501f728a1..91ce9790b5 100644 --- a/deps/SZ/sz/src/conf.c +++ b/deps/SZ/sz/src/conf.c @@ -25,8 +25,8 @@ void setDefaulParams(sz_exedata* exedata, sz_params* params) params->errorBoundMode = SZ_ABS; params->absErrBound = 1E-8; params->absErrBoundDouble = 1E-16; - params->max_quant_intervals = 800; - params->quantization_intervals = 500; + params->max_quant_intervals = 500; + params->quantization_intervals = 100; params->losslessCompressor = ZSTD_COMPRESSOR; //other option: GZIP_COMPRESSOR; // second important diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 9dbc34798e..86f0da8bb5 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -251,8 +251,8 @@ char lossyColumns[32] = ""; // "float|double" means all float and double column // below option can take effect when tsLossyColumns not empty double fPrecision = 1E-8; // float column precision double dPrecision = 1E-16; // double column precision -uint32_t maxIntervals = 800; // max intervals -uint32_t intervals = 500; // intervals +uint32_t maxIntervals = 500; // max intervals +uint32_t intervals = 100; // intervals char Compressor[32] = "ZSTD_COMPRESSOR"; // ZSTD_COMPRESSOR or GZIP_COMPRESSOR diff --git a/src/kit/taospack/taospack.c b/src/kit/taospack/taospack.c index 9d7a205bf4..b87d0ad927 100644 --- a/src/kit/taospack/taospack.c +++ b/src/kit/taospack/taospack.c @@ -137,6 +137,8 @@ float* read_float(const char* inFile, int* pcount){ int fi = 0; while(fgets(buf, sizeof(buf), pfin) != NULL) { // get item + if(buf[0] == 0 || strcmp(buf, " ") == 0) + continue; floats[fi] = atof(buf); //printf(" buff=%s float=%.50f \n ", buf, floats[fi]); if ( ++fi == malloc_cnt ) { @@ -257,7 +259,7 @@ bool DoDouble(double* doubles, int cnt, int algorithm) { return true; } -bool DoFloat(float* floats, int cnt, int algorithm) { +bool DoFloat(float* floats, int cnt, int algorithm, bool lossy) { // compress const char* input = (const char*)floats; int input_len = cnt * sizeof(float); @@ -268,11 +270,8 @@ bool DoFloat(float* floats, int cnt, int algorithm) { cost_start(); int ret_len = 0; - if(algorithm == 2) - ret_len = tsCompressFloat(input, input_len, cnt, output, output_len, algorithm, buff, buff_len); - else - ret_len = tsCompressFloatLossy(input, input_len, cnt, output, output_len, algorithm, buff, buff_len); - + ret_len = tsCompressFloat(input, input_len, cnt, output, output_len, algorithm, buff, buff_len); + if(ret_len == -1) { printf(" compress error.\n"); return 0; @@ -289,20 +288,15 @@ bool DoFloat(float* floats, int cnt, int algorithm) { float* ft2 = (float*)malloc(input_len); cost_start(); int code = 0; - - if(algorithm == 2) - code = tsDecompressFloat(output, ret_len, cnt, (char*)ft2, input_len, algorithm, buff, buff_len); - else - code = tsDecompressFloatLossy(output, ret_len, cnt, (char*)ft2, input_len, algorithm, buff, buff_len); - - + code = tsDecompressFloat(output, ret_len, cnt, (char*)ft2, input_len, algorithm, buff, buff_len); + double use_ms2 = cost_end("Decompress"); printf(" Decompress return length=%d \n", code); // compare same float same_rate = check_same(floats, ft2, cnt); - printf("\n ------------------ count:%d <%s> ---------------- \n", cnt, algorithm == 2?"TD":"SZ"); + printf("\n ------------------ count:%d <%s> ---------------- \n", cnt, lossy?"SZ":"TD"); printf(" Compress Rate ......... [%.2f%%] \n", rate); double speed1 = (cnt*sizeof(float)*1000/1024/1024)/use_ms1; printf(" Compress Time ......... [%.4fms] speed=%.1f MB/s\n", use_ms1, speed1); @@ -320,7 +314,7 @@ bool DoFloat(float* floats, int cnt, int algorithm) { } -bool testFile(const char* inFile, char algorithm){ +bool testFile(const char* inFile, char algorithm, bool lossy){ // check valid if(inFile == NULL || inFile[0] == 0 ){ printf(" inFile is NULL or EMPTY.\n"); @@ -333,7 +327,7 @@ bool testFile(const char* inFile, char algorithm){ return false; } - DoFloat(floats, cnt, algorithm); + DoFloat(floats, cnt, algorithm, lossy); free(floats); return true; @@ -688,10 +682,15 @@ extern bool lossyFloat; // int main(int argc, char *argv[]) { printf("welcome to use taospack tools v1.3\n"); + + printf(" sizeof(int)=%d\n", (int)sizeof(int)); + printf(" sizeof(long)=%d\n", (int)sizeof(long)); + printf(" sizeof(short)=%d\n",(int)sizeof(short)); strcpy(lossyColumns, "float|double"); + bool lossy = true; tsCompressInit(); - //lossyFloat = lossyDouble = true; + lossyFloat = lossyDouble = true; // //tsCompressExit(); @@ -707,6 +706,7 @@ int main(int argc, char *argv[]) { } if(strcmp(argv[1], "-tw") == 0) { algo = TWO_STAGE_COMP; + lossy = false; lossyFloat = lossyDouble = false; } @@ -730,7 +730,7 @@ int main(int argc, char *argv[]) { return 0; } - bool ret = testFile(argv[2], algo); + bool ret = testFile(argv[2], algo, lossy); printf(" test file %s. \n", ret ? "ok" : "err"); return 1;