From 68fe671cbc826d8392c84d77516eb6e345e66bce Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 1 Feb 2024 21:04:38 +0800 Subject: [PATCH 01/11] add tsz compress --- tests/army/community/cluster/snapshot.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/army/community/cluster/snapshot.py b/tests/army/community/cluster/snapshot.py index 5b5457be75..26bb6b9377 100644 --- a/tests/army/community/cluster/snapshot.py +++ b/tests/army/community/cluster/snapshot.py @@ -29,7 +29,11 @@ from frame import * class TDTestCase(TBase): updatecfgDict = { - "countAlwaysReturnValue" : "0" + "countAlwaysReturnValue" : "0", + "lossyColumns" : "float|double", + "fPrecision" : "0.000000001", + "dPrecision" : "0.00000000000000001", + "ifAdtFse" : "1" } def insertData(self): From b852993b8d9ce074f660a225dc657a31fd95b457 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 4 Feb 2024 13:22:39 +0800 Subject: [PATCH 02/11] remove TD_TSZ macro define --- include/util/tcompression.h | 1 - source/dnode/mgmt/node_mgmt/src/dmMgmt.c | 6 - source/util/src/tcompression.c | 24 +--- .../army/community/storage/oneStageComp.json | 66 ++++++++++ tests/army/community/storage/oneStageComp.py | 123 ++++++++++++++++++ tests/parallel_test/cases.task | 2 +- 6 files changed, 192 insertions(+), 30 deletions(-) create mode 100644 tests/army/community/storage/oneStageComp.json create mode 100644 tests/army/community/storage/oneStageComp.py diff --git a/include/util/tcompression.h b/include/util/tcompression.h index 75ddbb12e7..79fe9b613b 100644 --- a/include/util/tcompression.h +++ b/include/util/tcompression.h @@ -55,7 +55,6 @@ extern "C" { #define HEAD_MODE(x) x % 2 #define HEAD_ALGO(x) x / 2 -#ifdef TD_TSZ extern bool lossyFloat; extern bool lossyDouble; int32_t tsCompressInit(char* lossyColumns, float fPrecision, double dPrecision, uint32_t maxIntervals, uint32_t intervals, diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index 6cbf31b15f..2eab045251 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -19,10 +19,8 @@ #include "index.h" #include "qworker.h" #include "tstream.h" -#ifdef TD_TSZ #include "tcompression.h" #include "tglobal.h" -#endif static bool dmRequireNode(SDnode *pDnode, SMgmtWrapper *pWrapper) { SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper); @@ -47,10 +45,8 @@ int32_t dmInitDnode(SDnode *pDnode) { goto _OVER; } -#ifdef TD_TSZ // compress module init tsCompressInit(tsLossyColumns, tsFPrecision, tsDPrecision, tsMaxRange, tsCurRange, (int)tsIfAdtFse, tsCompressor); -#endif pDnode->wrappers[DNODE].func = dmGetMgmtFunc(); pDnode->wrappers[MNODE].func = mmGetMgmtFunc(); @@ -119,10 +115,8 @@ void dmCleanupDnode(SDnode *pDnode) { indexCleanup(); taosConvDestroy(); -#ifdef TD_TSZ // compress destroy tsCompressExit(); -#endif dDebug("dnode is closed, ptr:%p", pDnode); } diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 3cc00ddc7f..5cb92a4d47 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -54,9 +54,7 @@ #include "tlog.h" #include "ttypes.h" -#ifdef TD_TSZ #include "td_sz.h" -#endif static const int32_t TEST_NUMBER = 1; #define is_bigendian() ((*(char *)&TEST_NUMBER) == 0) @@ -64,7 +62,6 @@ static const int32_t TEST_NUMBER = 1; #define safeInt64Add(a, b) (((a >= 0) && (b <= INT64_MAX - a)) || ((a < 0) && (b >= INT64_MIN - a))) -#ifdef TD_TSZ bool lossyFloat = false; bool lossyDouble = false; @@ -83,7 +80,6 @@ int32_t tsCompressInit(char* lossyColumns, float fPrecision, double dPrecision, // exit call void tsCompressExit() { tdszExit(); } -#endif /* * Compress Integer (Simple8B). @@ -906,7 +902,6 @@ int32_t tsDecompressFloatImp(const char *const input, const int32_t nelements, c return nelements * FLOAT_BYTES; } -#ifdef TD_TSZ // // ---------- float double lossy ----------- // @@ -977,6 +972,7 @@ int32_t tsDecompressDoubleLossyImp(const char *input, int32_t compressedSize, co } #endif +#ifdef BUILD_NO_CALL /************************************************************************* * STREAM COMPRESSION *************************************************************************/ @@ -2120,7 +2116,7 @@ int32_t tCompressEnd(SCompressor *pCmprsor, const uint8_t **ppOut, int32_t *nOut int32_t tCompress(SCompressor *pCmprsor, const void *pData, int64_t nData) { return DATA_TYPE_INFO[pCmprsor->type].cmprFn(pCmprsor, pData, nData); } - +#endif /************************************************************************* * REGULAR COMPRESSION *************************************************************************/ @@ -2154,13 +2150,11 @@ int32_t tsDecompressTimestamp(void *pIn, int32_t nIn, int32_t nEle, void *pOut, // Float ===================================================== int32_t tsCompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, int32_t nBuf) { -#ifdef TD_TSZ // lossy mode if (lossyFloat) { return tsCompressFloatLossyImp(pIn, nEle, pOut); // lossless mode } else { -#endif if (cmprAlg == ONE_STAGE_COMP) { return tsCompressFloatImp(pIn, nEle, pOut); } else if (cmprAlg == TWO_STAGE_COMP) { @@ -2170,19 +2164,15 @@ int32_t tsCompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_ ASSERTS(0, "compress algo invalid"); return -1; } -#ifdef TD_TSZ } -#endif } int32_t tsDecompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, int32_t nBuf) { -#ifdef TD_TSZ if (HEAD_ALGO(((uint8_t *)pIn)[0]) == ALGO_SZ_LOSSY) { // decompress lossy return tsDecompressFloatLossyImp(pIn, nIn, nEle, pOut); } else { -#endif // decompress lossless if (cmprAlg == ONE_STAGE_COMP) { return tsDecompressFloatImp(pIn, nEle, pOut); @@ -2193,20 +2183,16 @@ int32_t tsDecompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int3 ASSERTS(0, "compress algo invalid"); return -1; } -#ifdef TD_TSZ } -#endif } // Double ===================================================== int32_t tsCompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, int32_t nBuf) { -#ifdef TD_TSZ if (lossyDouble) { // lossy mode return tsCompressDoubleLossyImp(pIn, nEle, pOut); } else { -#endif // lossless mode if (cmprAlg == ONE_STAGE_COMP) { return tsCompressDoubleImp(pIn, nEle, pOut); @@ -2217,19 +2203,15 @@ int32_t tsCompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32 ASSERTS(0, "compress algo invalid"); return -1; } -#ifdef TD_TSZ } -#endif } int32_t tsDecompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, int32_t nBuf) { -#ifdef TD_TSZ if (HEAD_ALGO(((uint8_t *)pIn)[0]) == ALGO_SZ_LOSSY) { // decompress lossy return tsDecompressDoubleLossyImp(pIn, nIn, nEle, pOut); } else { -#endif // decompress lossless if (cmprAlg == ONE_STAGE_COMP) { return tsDecompressDoubleImp(pIn, nEle, pOut); @@ -2240,9 +2222,7 @@ int32_t tsDecompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int ASSERTS(0, "compress algo invalid"); return -1; } -#ifdef TD_TSZ } -#endif } // Binary ===================================================== diff --git a/tests/army/community/storage/oneStageComp.json b/tests/army/community/storage/oneStageComp.json new file mode 100644 index 0000000000..12fa51db83 --- /dev/null +++ b/tests/army/community/storage/oneStageComp.json @@ -0,0 +1,66 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "connection_pool_size": 8, + "num_of_records_per_req": 3000, + "prepared_rand": 3000, + "thread_count": 2, + "create_table_thread_count": 1, + "confirm_parameter_prompt": "no", + "databases": [ + { + "dbinfo": { + "name": "db", + "drop": "yes", + "vgroups": 2, + "replica": 3, + "wal_retention_period": 10, + "wal_retention_size": 100, + "keep": "60d,120d,365d", + "stt_trigger": 1, + "wal_level": 2, + "WAL_FSYNC_PERIOD": 3300, + "cachemode": "last_value", + "TABLE_PREFIX":1, + "comp": 1 + }, + "super_tables": [ + { + "name": "stb", + "child_table_exists": "no", + "childtable_count": 10, + "insert_rows": 100000, + "childtable_prefix": "d", + "insert_mode": "taosc", + "timestamp_step": 1000, + "start_timestamp":"2023-01-01 00:00:00", + "columns": [ + { "type": "bool", "name": "bc","max": 1,"min": 1}, + { "type": "float", "name": "fc" ,"max": 101,"min": 101}, + { "type": "double", "name": "dc" ,"max": 102,"min": 102}, + { "type": "tinyint", "name": "ti" ,"max": 103,"min": 103}, + { "type": "smallint", "name": "si" ,"max": 104,"min": 104}, + { "type": "int", "name": "ic" ,"max": 105,"min": 105}, + { "type": "bigint", "name": "bi" ,"max": 106,"min": 106}, + { "type": "utinyint", "name": "uti","max": 107,"min": 107}, + { "type": "usmallint", "name": "usi","max": 108,"min": 108}, + { "type": "uint", "name": "ui" ,"max": 109,"min": 109}, + { "type": "ubigint", "name": "ubi","max": 110,"min": 110}, + { "type": "binary", "name": "bin", "len": 16}, + { "type": "nchar", "name": "nch", "len": 32} + ], + "tags": [ + {"type": "tinyint", "name": "groupid","max": 100,"min": 100}, + {"name": "location","type": "binary", "len": 16, "values": + ["San Francisco", "Los Angles", "San Diego", "San Jose", "Palo Alto", "Campbell", "Mountain View","Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + } + ] + } + ] +} diff --git a/tests/army/community/storage/oneStageComp.py b/tests/army/community/storage/oneStageComp.py new file mode 100644 index 0000000000..f3718bc716 --- /dev/null +++ b/tests/army/community/storage/oneStageComp.py @@ -0,0 +1,123 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import time +import random + +import taos +import frame +import frame.etool + + +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame import * + + +class TDTestCase(TBase): + updatecfgDict = { + "compressMsgSize" : "100", + } + + def insertData(self): + tdLog.info(f"insert data.") + # taosBenchmark run + jfile = etool.curFile(__file__, "oneStageComp.json") + etool.benchMark(json=jfile) + + tdSql.execute(f"use {self.db}") + # set insert data information + self.childtable_count = 10 + self.insert_rows = 100000 + self.timestamp_step = 1000 + + + + def checkColValueCorrect(self): + tdLog.info(f"do action.") + self.flushDb() + + # check all columns correct + cnt = self.insert_rows * self.childtable_count + sql = "select * from stb where bc!=1" + tdSql.checkRows(0) + sql = "select * from stb where fc=101" + tdSql.checkRows(cnt) + sql = "select * from stb where dc!=102" + tdSql.checkRows(0) + sql = "select * from stb where ti!=103" + tdSql.checkRows(0) + sql = "select * from stb where si!=104" + tdSql.checkRows(0) + sql = "select * from stb where ic!=105" + tdSql.checkRows(0) + sql = "select * from stb where b!i=106" + tdSql.checkRows(0) + sql = "select * from stb where uti!=107" + tdSql.checkRows(0) + sql = "select * from stb where usi!=108" + tdSql.checkRows(0) + sql = "select * from stb where ui!=109" + tdSql.checkRows(0) + sql = "select * from stb where ubi!=110" + tdSql.checkRows(0) + + def insertNull(self): + # insert 6 lines + sql = "insert into d0(ts) values(now) (now + 1s) (now + 2s) (now + 3s) (now + 4s) (now + 5s)" + tdSql.execute(sql) + + self.flushDb() + self.trimDb() + + # check all columns correct + cnt = self.insert_rows * self.childtable_count + sql = "select * from stb where bc!=1" + tdSql.checkRows(6) + sql = "select * from stb where bc=1" + tdSql.checkRows(cnt) + sql = "select * from stb where usi!=108" + tdSql.checkRows(6) + + # run + def run(self): + tdLog.debug(f"start to excute {__file__}") + + # insert data + self.insertData() + + # check insert data correct + self.checkInsertCorrect() + + # save + self.snapshotAgg() + + # do action + self.checkColValueCorrect() + + # check save agg result correct + self.checkAggCorrect() + + # insert null + self.insertNull() + + + tdLog.success(f"{__file__} successfully executed") + + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 91a0ac46e5..9ea03b4e6b 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -23,7 +23,7 @@ fi ,,y,army,./pytest.sh python3 ./test.py -f community/query/query_basic.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f community/cluster/splitVgroupByLearner.py -N 3 ,,n,army,python3 ./test.py -f community/cmdline/fullopt.py - +,,y,army,./pytest.sh python3 ./test.py -f community/storage/oneStageComp.py -N 3 -L 3 -D 1 # From 8559e7061ce55f1c12582bdd1659beec0870743e Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 4 Feb 2024 13:26:36 +0800 Subject: [PATCH 03/11] fix: , replace | with float,double --- tests/army/community/cluster/snapshot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/army/community/cluster/snapshot.py b/tests/army/community/cluster/snapshot.py index 26bb6b9377..3f9a497f16 100644 --- a/tests/army/community/cluster/snapshot.py +++ b/tests/army/community/cluster/snapshot.py @@ -30,7 +30,7 @@ from frame import * class TDTestCase(TBase): updatecfgDict = { "countAlwaysReturnValue" : "0", - "lossyColumns" : "float|double", + "lossyColumns" : "float,double", "fPrecision" : "0.000000001", "dPrecision" : "0.00000000000000001", "ifAdtFse" : "1" From 656af515ceed48eea2e85beff44dcf549128672f Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 4 Feb 2024 13:34:14 +0800 Subject: [PATCH 04/11] fix: build error --- include/util/tcompression.h | 1 - 1 file changed, 1 deletion(-) diff --git a/include/util/tcompression.h b/include/util/tcompression.h index 79fe9b613b..9948102494 100644 --- a/include/util/tcompression.h +++ b/include/util/tcompression.h @@ -91,7 +91,6 @@ static FORCE_INLINE int32_t tsDecompressDoubleLossy(const char *const input, int return tsDecompressDoubleLossyImp(input, compressedSize, nelements, output); } -#endif /************************************************************************* * REGULAR COMPRESSION From 3ae02aaac24826c2c8514c73a1b02386c2ccc9cd Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 4 Feb 2024 13:38:13 +0800 Subject: [PATCH 05/11] fix: build error1 --- source/util/src/tcompression.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 5cb92a4d47..f605c60ccd 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -970,7 +970,6 @@ int32_t tsDecompressDoubleLossyImp(const char *input, int32_t compressedSize, co // decompressed with sz return tdszDecompress(SZ_DOUBLE, input + 1, compressedSize - 1, nelements, output); } -#endif #ifdef BUILD_NO_CALL /************************************************************************* From b4621bb37c8186c0a342cb9e27b0f4563a885005 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 4 Feb 2024 13:57:16 +0800 Subject: [PATCH 06/11] fix: restore TD_TSZ macro define --- include/util/tcompression.h | 2 ++ source/dnode/mgmt/node_mgmt/src/dmMgmt.c | 6 ++++++ source/util/src/tcompression.c | 22 ++++++++++++++++++++++ 3 files changed, 30 insertions(+) diff --git a/include/util/tcompression.h b/include/util/tcompression.h index 9948102494..75ddbb12e7 100644 --- a/include/util/tcompression.h +++ b/include/util/tcompression.h @@ -55,6 +55,7 @@ extern "C" { #define HEAD_MODE(x) x % 2 #define HEAD_ALGO(x) x / 2 +#ifdef TD_TSZ extern bool lossyFloat; extern bool lossyDouble; int32_t tsCompressInit(char* lossyColumns, float fPrecision, double dPrecision, uint32_t maxIntervals, uint32_t intervals, @@ -91,6 +92,7 @@ static FORCE_INLINE int32_t tsDecompressDoubleLossy(const char *const input, int return tsDecompressDoubleLossyImp(input, compressedSize, nelements, output); } +#endif /************************************************************************* * REGULAR COMPRESSION diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index 2eab045251..6cbf31b15f 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -19,8 +19,10 @@ #include "index.h" #include "qworker.h" #include "tstream.h" +#ifdef TD_TSZ #include "tcompression.h" #include "tglobal.h" +#endif static bool dmRequireNode(SDnode *pDnode, SMgmtWrapper *pWrapper) { SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper); @@ -45,8 +47,10 @@ int32_t dmInitDnode(SDnode *pDnode) { goto _OVER; } +#ifdef TD_TSZ // compress module init tsCompressInit(tsLossyColumns, tsFPrecision, tsDPrecision, tsMaxRange, tsCurRange, (int)tsIfAdtFse, tsCompressor); +#endif pDnode->wrappers[DNODE].func = dmGetMgmtFunc(); pDnode->wrappers[MNODE].func = mmGetMgmtFunc(); @@ -115,8 +119,10 @@ void dmCleanupDnode(SDnode *pDnode) { indexCleanup(); taosConvDestroy(); +#ifdef TD_TSZ // compress destroy tsCompressExit(); +#endif dDebug("dnode is closed, ptr:%p", pDnode); } diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index f605c60ccd..656e2706f2 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -54,7 +54,9 @@ #include "tlog.h" #include "ttypes.h" +#ifdef TD_TSZ #include "td_sz.h" +#endif static const int32_t TEST_NUMBER = 1; #define is_bigendian() ((*(char *)&TEST_NUMBER) == 0) @@ -62,6 +64,7 @@ static const int32_t TEST_NUMBER = 1; #define safeInt64Add(a, b) (((a >= 0) && (b <= INT64_MAX - a)) || ((a < 0) && (b >= INT64_MIN - a))) +#ifdef TD_TSZ bool lossyFloat = false; bool lossyDouble = false; @@ -80,6 +83,7 @@ int32_t tsCompressInit(char* lossyColumns, float fPrecision, double dPrecision, // exit call void tsCompressExit() { tdszExit(); } +#endif /* * Compress Integer (Simple8B). @@ -902,6 +906,7 @@ int32_t tsDecompressFloatImp(const char *const input, const int32_t nelements, c return nelements * FLOAT_BYTES; } +#ifdef TD_TSZ // // ---------- float double lossy ----------- // @@ -970,6 +975,7 @@ int32_t tsDecompressDoubleLossyImp(const char *input, int32_t compressedSize, co // decompressed with sz return tdszDecompress(SZ_DOUBLE, input + 1, compressedSize - 1, nelements, output); } +#endif #ifdef BUILD_NO_CALL /************************************************************************* @@ -2149,11 +2155,13 @@ int32_t tsDecompressTimestamp(void *pIn, int32_t nIn, int32_t nEle, void *pOut, // Float ===================================================== int32_t tsCompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, int32_t nBuf) { +#ifdef TD_TSZ // lossy mode if (lossyFloat) { return tsCompressFloatLossyImp(pIn, nEle, pOut); // lossless mode } else { +#endif if (cmprAlg == ONE_STAGE_COMP) { return tsCompressFloatImp(pIn, nEle, pOut); } else if (cmprAlg == TWO_STAGE_COMP) { @@ -2163,15 +2171,19 @@ int32_t tsCompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_ ASSERTS(0, "compress algo invalid"); return -1; } +#ifdef TD_TSZ } +#endif } int32_t tsDecompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, int32_t nBuf) { +#ifdef TD_TSZ if (HEAD_ALGO(((uint8_t *)pIn)[0]) == ALGO_SZ_LOSSY) { // decompress lossy return tsDecompressFloatLossyImp(pIn, nIn, nEle, pOut); } else { +#endif // decompress lossless if (cmprAlg == ONE_STAGE_COMP) { return tsDecompressFloatImp(pIn, nEle, pOut); @@ -2182,16 +2194,20 @@ int32_t tsDecompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int3 ASSERTS(0, "compress algo invalid"); return -1; } +#ifdef TD_TSZ } +#endif } // Double ===================================================== int32_t tsCompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, int32_t nBuf) { +#ifdef TD_TSZ if (lossyDouble) { // lossy mode return tsCompressDoubleLossyImp(pIn, nEle, pOut); } else { +#endif // lossless mode if (cmprAlg == ONE_STAGE_COMP) { return tsCompressDoubleImp(pIn, nEle, pOut); @@ -2202,15 +2218,19 @@ int32_t tsCompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32 ASSERTS(0, "compress algo invalid"); return -1; } +#ifdef TD_TSZ } +#endif } int32_t tsDecompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, int32_t nBuf) { +#ifdef TD_TSZ if (HEAD_ALGO(((uint8_t *)pIn)[0]) == ALGO_SZ_LOSSY) { // decompress lossy return tsDecompressDoubleLossyImp(pIn, nIn, nEle, pOut); } else { +#endif // decompress lossless if (cmprAlg == ONE_STAGE_COMP) { return tsDecompressDoubleImp(pIn, nEle, pOut); @@ -2221,7 +2241,9 @@ int32_t tsDecompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int ASSERTS(0, "compress algo invalid"); return -1; } +#ifdef TD_TSZ } +#endif } // Binary ===================================================== From 0029c348c5e049ce90d8ef23e8fe644e8e7dfb6e Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 4 Feb 2024 14:56:24 +0800 Subject: [PATCH 07/11] fix: float and double tsz check --- tests/army/community/cluster/snapshot.json | 4 ++-- tests/army/community/cluster/snapshot.py | 17 +++++++++++++++++ utils/TSZ/sz/src/sz_double.c | 8 +++++--- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/tests/army/community/cluster/snapshot.json b/tests/army/community/cluster/snapshot.json index d4f6f00d37..f664278a72 100644 --- a/tests/army/community/cluster/snapshot.json +++ b/tests/army/community/cluster/snapshot.json @@ -35,8 +35,8 @@ "start_timestamp":"now-12d", "columns": [ { "type": "bool", "name": "bc"}, - { "type": "float", "name": "fc" }, - { "type": "double", "name": "dc"}, + { "type": "float", "name": "fc", "min": 100, "min": 100}, + { "type": "double", "name": "dc", "min": 200, "min": 200}, { "type": "tinyint", "name": "ti"}, { "type": "smallint", "name": "si" }, { "type": "int", "name": "ic" }, diff --git a/tests/army/community/cluster/snapshot.py b/tests/army/community/cluster/snapshot.py index 3f9a497f16..6648efa027 100644 --- a/tests/army/community/cluster/snapshot.py +++ b/tests/army/community/cluster/snapshot.py @@ -52,6 +52,16 @@ class TDTestCase(TBase): sql = f"create table {self.db}.ta(ts timestamp, age int) tags(area int)" tdSql.execute(sql) + def checkFloatDouble(self): + sql = f"select count(*) from {self.db}.{self.stb} where fc!=100" + tdSql.checkFirstValue(sql, 0) + sql = f"select count(*) from {self.db}.{self.stb} where dc!=200" + tdSql.checkFirstValue(sql, 0) + sql = f"select avg(fc) from {self.db}.{self.stb}" + tdSql.checkFirstValue(sql, 100) + sql = f"select avg(dc) from {self.db}.{self.stb}" + tdSql.checkFirstValue(sql, 200) + def doAction(self): tdLog.info(f"do action.") self.flushDb() @@ -89,6 +99,9 @@ class TDTestCase(TBase): # check insert data correct self.checkInsertCorrect() + # check float double value ok + tdSql.checkFloatDouble() + # save self.snapshotAgg() @@ -101,6 +114,10 @@ class TDTestCase(TBase): # check insert correct again self.checkInsertCorrect() + # check float double value ok + tdSql.checkFloatDouble() + + tdLog.success(f"{__file__} successfully executed") diff --git a/utils/TSZ/sz/src/sz_double.c b/utils/TSZ/sz/src/sz_double.c index 1adfdf3b56..0510fc612d 100644 --- a/utils/TSZ/sz/src/sz_double.c +++ b/utils/TSZ/sz/src/sz_double.c @@ -385,9 +385,11 @@ unsigned int optimize_intervals_double_1D_opt(double *oriData, size_t dataLength totalSampleSize++; pred_value = data_pos[-1]; pred_err = fabs(pred_value - *data_pos); - radiusIndex = (unsigned long)((pred_err/realPrecision+1)/2); - if(radiusIndex>=confparams_cpr->maxRangeRadius) - radiusIndex = confparams_cpr->maxRangeRadius - 1; + double dbri = (unsigned long)((pred_err/realPrecision+1)/2); + if(dbri >= (double)confparams_cpr->maxRangeRadius) + radiusIndex = confparams_cpr->maxRangeRadius - 1; + else + radiusIndex = dbri; intervals[radiusIndex]++; data_pos += confparams_cpr->sampleDistance; From d51e651c17ef19b74b9325b3ebf95164e10601e3 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 4 Feb 2024 15:00:44 +0800 Subject: [PATCH 08/11] fix: case right --- tests/army/community/cluster/snapshot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/army/community/cluster/snapshot.py b/tests/army/community/cluster/snapshot.py index 6648efa027..6fd2218344 100644 --- a/tests/army/community/cluster/snapshot.py +++ b/tests/army/community/cluster/snapshot.py @@ -100,7 +100,7 @@ class TDTestCase(TBase): self.checkInsertCorrect() # check float double value ok - tdSql.checkFloatDouble() + self.checkFloatDouble() # save self.snapshotAgg() @@ -115,7 +115,7 @@ class TDTestCase(TBase): self.checkInsertCorrect() # check float double value ok - tdSql.checkFloatDouble() + self.checkFloatDouble() tdLog.success(f"{__file__} successfully executed") From c5f5aca4ce19f3bb0128b46a20e1d0f69303a7dd Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 4 Feb 2024 17:35:34 +0800 Subject: [PATCH 09/11] fix: fix case passed --- tests/army/community/cluster/snapshot.json | 4 ++-- tests/army/community/cluster/snapshot.py | 8 +++++--- tests/army/community/storage/oneStageComp.json | 4 ++-- tests/army/community/storage/oneStageComp.py | 11 +++++++++++ 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/tests/army/community/cluster/snapshot.json b/tests/army/community/cluster/snapshot.json index f664278a72..4855c23260 100644 --- a/tests/army/community/cluster/snapshot.json +++ b/tests/army/community/cluster/snapshot.json @@ -35,8 +35,8 @@ "start_timestamp":"now-12d", "columns": [ { "type": "bool", "name": "bc"}, - { "type": "float", "name": "fc", "min": 100, "min": 100}, - { "type": "double", "name": "dc", "min": 200, "min": 200}, + { "type": "float", "name": "fc", "min": 100, "max": 100}, + { "type": "double", "name": "dc", "min": 200, "max": 200}, { "type": "tinyint", "name": "ti"}, { "type": "smallint", "name": "si" }, { "type": "int", "name": "ic" }, diff --git a/tests/army/community/cluster/snapshot.py b/tests/army/community/cluster/snapshot.py index 6fd2218344..b4c4d3c4c8 100644 --- a/tests/army/community/cluster/snapshot.py +++ b/tests/army/community/cluster/snapshot.py @@ -53,10 +53,12 @@ class TDTestCase(TBase): tdSql.execute(sql) def checkFloatDouble(self): - sql = f"select count(*) from {self.db}.{self.stb} where fc!=100" - tdSql.checkFirstValue(sql, 0) + sql = f"select * from {self.db}.{self.stb} where fc!=100" + tdSql.query(sql) + tdSql.checkRows(0) sql = f"select count(*) from {self.db}.{self.stb} where dc!=200" - tdSql.checkFirstValue(sql, 0) + tdSql.query(sql) + tdSql.checkRows(0) sql = f"select avg(fc) from {self.db}.{self.stb}" tdSql.checkFirstValue(sql, 100) sql = f"select avg(dc) from {self.db}.{self.stb}" diff --git a/tests/army/community/storage/oneStageComp.json b/tests/army/community/storage/oneStageComp.json index 12fa51db83..f64fda3824 100644 --- a/tests/army/community/storage/oneStageComp.json +++ b/tests/army/community/storage/oneStageComp.json @@ -24,7 +24,7 @@ "stt_trigger": 1, "wal_level": 2, "WAL_FSYNC_PERIOD": 3300, - "cachemode": "last_value", + "cachemodel": "'last_value'", "TABLE_PREFIX":1, "comp": 1 }, @@ -37,7 +37,7 @@ "childtable_prefix": "d", "insert_mode": "taosc", "timestamp_step": 1000, - "start_timestamp":"2023-01-01 00:00:00", + "start_timestamp":"now-360d", "columns": [ { "type": "bool", "name": "bc","max": 1,"min": 1}, { "type": "float", "name": "fc" ,"max": 101,"min": 101}, diff --git a/tests/army/community/storage/oneStageComp.py b/tests/army/community/storage/oneStageComp.py index f3718bc716..35a7717449 100644 --- a/tests/army/community/storage/oneStageComp.py +++ b/tests/army/community/storage/oneStageComp.py @@ -53,26 +53,37 @@ class TDTestCase(TBase): # check all columns correct cnt = self.insert_rows * self.childtable_count sql = "select * from stb where bc!=1" + tdSql.query(sql) tdSql.checkRows(0) sql = "select * from stb where fc=101" + tdSql.query(sql) tdSql.checkRows(cnt) sql = "select * from stb where dc!=102" + tdSql.query(sql) tdSql.checkRows(0) sql = "select * from stb where ti!=103" + tdSql.query(sql) tdSql.checkRows(0) sql = "select * from stb where si!=104" + tdSql.query(sql) tdSql.checkRows(0) sql = "select * from stb where ic!=105" + tdSql.query(sql) tdSql.checkRows(0) sql = "select * from stb where b!i=106" + tdSql.query(sql) tdSql.checkRows(0) sql = "select * from stb where uti!=107" + tdSql.query(sql) tdSql.checkRows(0) sql = "select * from stb where usi!=108" + tdSql.query(sql) tdSql.checkRows(0) sql = "select * from stb where ui!=109" + tdSql.query(sql) tdSql.checkRows(0) sql = "select * from stb where ubi!=110" + tdSql.query(sql) tdSql.checkRows(0) def insertNull(self): From cb4cb936d32a09ef8390dce3c84dbb27c825e447 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 4 Feb 2024 17:48:24 +0800 Subject: [PATCH 10/11] fix: case passed --- tests/army/community/storage/oneStageComp.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/army/community/storage/oneStageComp.py b/tests/army/community/storage/oneStageComp.py index 35a7717449..8860220673 100644 --- a/tests/army/community/storage/oneStageComp.py +++ b/tests/army/community/storage/oneStageComp.py @@ -70,7 +70,7 @@ class TDTestCase(TBase): sql = "select * from stb where ic!=105" tdSql.query(sql) tdSql.checkRows(0) - sql = "select * from stb where b!i=106" + sql = "select * from stb where bi!=106" tdSql.query(sql) tdSql.checkRows(0) sql = "select * from stb where uti!=107" @@ -97,10 +97,16 @@ class TDTestCase(TBase): # check all columns correct cnt = self.insert_rows * self.childtable_count sql = "select * from stb where bc!=1" + tdSql.query(sql) + tdSql.checkRows(0) + sql = "select * from stb where bc is null" + tdSql.query(sql) tdSql.checkRows(6) sql = "select * from stb where bc=1" + tdSql.query(sql) tdSql.checkRows(cnt) sql = "select * from stb where usi!=108" + tdSql.query(sql) tdSql.checkRows(6) # run From 8360d6a12b126b4a82152f50898ed288fc87f5c5 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 4 Feb 2024 18:01:27 +0800 Subject: [PATCH 11/11] fix: check null value --- tests/army/community/storage/oneStageComp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/army/community/storage/oneStageComp.py b/tests/army/community/storage/oneStageComp.py index 8860220673..9a2c7cfcd6 100644 --- a/tests/army/community/storage/oneStageComp.py +++ b/tests/army/community/storage/oneStageComp.py @@ -105,7 +105,7 @@ class TDTestCase(TBase): sql = "select * from stb where bc=1" tdSql.query(sql) tdSql.checkRows(cnt) - sql = "select * from stb where usi!=108" + sql = "select * from stb where usi is null" tdSql.query(sql) tdSql.checkRows(6)