From b852993b8d9ce074f660a225dc657a31fd95b457 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 4 Feb 2024 13:22:39 +0800 Subject: [PATCH] remove TD_TSZ macro define --- include/util/tcompression.h | 1 - source/dnode/mgmt/node_mgmt/src/dmMgmt.c | 6 - source/util/src/tcompression.c | 24 +--- .../army/community/storage/oneStageComp.json | 66 ++++++++++ tests/army/community/storage/oneStageComp.py | 123 ++++++++++++++++++ tests/parallel_test/cases.task | 2 +- 6 files changed, 192 insertions(+), 30 deletions(-) create mode 100644 tests/army/community/storage/oneStageComp.json create mode 100644 tests/army/community/storage/oneStageComp.py diff --git a/include/util/tcompression.h b/include/util/tcompression.h index 75ddbb12e7..79fe9b613b 100644 --- a/include/util/tcompression.h +++ b/include/util/tcompression.h @@ -55,7 +55,6 @@ extern "C" { #define HEAD_MODE(x) x % 2 #define HEAD_ALGO(x) x / 2 -#ifdef TD_TSZ extern bool lossyFloat; extern bool lossyDouble; int32_t tsCompressInit(char* lossyColumns, float fPrecision, double dPrecision, uint32_t maxIntervals, uint32_t intervals, diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index 6cbf31b15f..2eab045251 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -19,10 +19,8 @@ #include "index.h" #include "qworker.h" #include "tstream.h" -#ifdef TD_TSZ #include "tcompression.h" #include "tglobal.h" -#endif static bool dmRequireNode(SDnode *pDnode, SMgmtWrapper *pWrapper) { SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper); @@ -47,10 +45,8 @@ int32_t dmInitDnode(SDnode *pDnode) { goto _OVER; } -#ifdef TD_TSZ // compress module init tsCompressInit(tsLossyColumns, tsFPrecision, tsDPrecision, tsMaxRange, tsCurRange, (int)tsIfAdtFse, tsCompressor); -#endif pDnode->wrappers[DNODE].func = dmGetMgmtFunc(); pDnode->wrappers[MNODE].func = mmGetMgmtFunc(); @@ -119,10 +115,8 @@ void dmCleanupDnode(SDnode *pDnode) { indexCleanup(); taosConvDestroy(); -#ifdef TD_TSZ // compress destroy tsCompressExit(); -#endif dDebug("dnode is closed, ptr:%p", pDnode); } diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 3cc00ddc7f..5cb92a4d47 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -54,9 +54,7 @@ #include "tlog.h" #include "ttypes.h" -#ifdef TD_TSZ #include "td_sz.h" -#endif static const int32_t TEST_NUMBER = 1; #define is_bigendian() ((*(char *)&TEST_NUMBER) == 0) @@ -64,7 +62,6 @@ static const int32_t TEST_NUMBER = 1; #define safeInt64Add(a, b) (((a >= 0) && (b <= INT64_MAX - a)) || ((a < 0) && (b >= INT64_MIN - a))) -#ifdef TD_TSZ bool lossyFloat = false; bool lossyDouble = false; @@ -83,7 +80,6 @@ int32_t tsCompressInit(char* lossyColumns, float fPrecision, double dPrecision, // exit call void tsCompressExit() { tdszExit(); } -#endif /* * Compress Integer (Simple8B). @@ -906,7 +902,6 @@ int32_t tsDecompressFloatImp(const char *const input, const int32_t nelements, c return nelements * FLOAT_BYTES; } -#ifdef TD_TSZ // // ---------- float double lossy ----------- // @@ -977,6 +972,7 @@ int32_t tsDecompressDoubleLossyImp(const char *input, int32_t compressedSize, co } #endif +#ifdef BUILD_NO_CALL /************************************************************************* * STREAM COMPRESSION *************************************************************************/ @@ -2120,7 +2116,7 @@ int32_t tCompressEnd(SCompressor *pCmprsor, const uint8_t **ppOut, int32_t *nOut int32_t tCompress(SCompressor *pCmprsor, const void *pData, int64_t nData) { return DATA_TYPE_INFO[pCmprsor->type].cmprFn(pCmprsor, pData, nData); } - +#endif /************************************************************************* * REGULAR COMPRESSION *************************************************************************/ @@ -2154,13 +2150,11 @@ int32_t tsDecompressTimestamp(void *pIn, int32_t nIn, int32_t nEle, void *pOut, // Float ===================================================== int32_t tsCompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, int32_t nBuf) { -#ifdef TD_TSZ // lossy mode if (lossyFloat) { return tsCompressFloatLossyImp(pIn, nEle, pOut); // lossless mode } else { -#endif if (cmprAlg == ONE_STAGE_COMP) { return tsCompressFloatImp(pIn, nEle, pOut); } else if (cmprAlg == TWO_STAGE_COMP) { @@ -2170,19 +2164,15 @@ int32_t tsCompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_ ASSERTS(0, "compress algo invalid"); return -1; } -#ifdef TD_TSZ } -#endif } int32_t tsDecompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, int32_t nBuf) { -#ifdef TD_TSZ if (HEAD_ALGO(((uint8_t *)pIn)[0]) == ALGO_SZ_LOSSY) { // decompress lossy return tsDecompressFloatLossyImp(pIn, nIn, nEle, pOut); } else { -#endif // decompress lossless if (cmprAlg == ONE_STAGE_COMP) { return tsDecompressFloatImp(pIn, nEle, pOut); @@ -2193,20 +2183,16 @@ int32_t tsDecompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int3 ASSERTS(0, "compress algo invalid"); return -1; } -#ifdef TD_TSZ } -#endif } // Double ===================================================== int32_t tsCompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, int32_t nBuf) { -#ifdef TD_TSZ if (lossyDouble) { // lossy mode return tsCompressDoubleLossyImp(pIn, nEle, pOut); } else { -#endif // lossless mode if (cmprAlg == ONE_STAGE_COMP) { return tsCompressDoubleImp(pIn, nEle, pOut); @@ -2217,19 +2203,15 @@ int32_t tsCompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32 ASSERTS(0, "compress algo invalid"); return -1; } -#ifdef TD_TSZ } -#endif } int32_t tsDecompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, int32_t nBuf) { -#ifdef TD_TSZ if (HEAD_ALGO(((uint8_t *)pIn)[0]) == ALGO_SZ_LOSSY) { // decompress lossy return tsDecompressDoubleLossyImp(pIn, nIn, nEle, pOut); } else { -#endif // decompress lossless if (cmprAlg == ONE_STAGE_COMP) { return tsDecompressDoubleImp(pIn, nEle, pOut); @@ -2240,9 +2222,7 @@ int32_t tsDecompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int ASSERTS(0, "compress algo invalid"); return -1; } -#ifdef TD_TSZ } -#endif } // Binary ===================================================== diff --git a/tests/army/community/storage/oneStageComp.json b/tests/army/community/storage/oneStageComp.json new file mode 100644 index 0000000000..12fa51db83 --- /dev/null +++ b/tests/army/community/storage/oneStageComp.json @@ -0,0 +1,66 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "connection_pool_size": 8, + "num_of_records_per_req": 3000, + "prepared_rand": 3000, + "thread_count": 2, + "create_table_thread_count": 1, + "confirm_parameter_prompt": "no", + "databases": [ + { + "dbinfo": { + "name": "db", + "drop": "yes", + "vgroups": 2, + "replica": 3, + "wal_retention_period": 10, + "wal_retention_size": 100, + "keep": "60d,120d,365d", + "stt_trigger": 1, + "wal_level": 2, + "WAL_FSYNC_PERIOD": 3300, + "cachemode": "last_value", + "TABLE_PREFIX":1, + "comp": 1 + }, + "super_tables": [ + { + "name": "stb", + "child_table_exists": "no", + "childtable_count": 10, + "insert_rows": 100000, + "childtable_prefix": "d", + "insert_mode": "taosc", + "timestamp_step": 1000, + "start_timestamp":"2023-01-01 00:00:00", + "columns": [ + { "type": "bool", "name": "bc","max": 1,"min": 1}, + { "type": "float", "name": "fc" ,"max": 101,"min": 101}, + { "type": "double", "name": "dc" ,"max": 102,"min": 102}, + { "type": "tinyint", "name": "ti" ,"max": 103,"min": 103}, + { "type": "smallint", "name": "si" ,"max": 104,"min": 104}, + { "type": "int", "name": "ic" ,"max": 105,"min": 105}, + { "type": "bigint", "name": "bi" ,"max": 106,"min": 106}, + { "type": "utinyint", "name": "uti","max": 107,"min": 107}, + { "type": "usmallint", "name": "usi","max": 108,"min": 108}, + { "type": "uint", "name": "ui" ,"max": 109,"min": 109}, + { "type": "ubigint", "name": "ubi","max": 110,"min": 110}, + { "type": "binary", "name": "bin", "len": 16}, + { "type": "nchar", "name": "nch", "len": 32} + ], + "tags": [ + {"type": "tinyint", "name": "groupid","max": 100,"min": 100}, + {"name": "location","type": "binary", "len": 16, "values": + ["San Francisco", "Los Angles", "San Diego", "San Jose", "Palo Alto", "Campbell", "Mountain View","Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + } + ] + } + ] +} diff --git a/tests/army/community/storage/oneStageComp.py b/tests/army/community/storage/oneStageComp.py new file mode 100644 index 0000000000..f3718bc716 --- /dev/null +++ b/tests/army/community/storage/oneStageComp.py @@ -0,0 +1,123 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import time +import random + +import taos +import frame +import frame.etool + + +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame import * + + +class TDTestCase(TBase): + updatecfgDict = { + "compressMsgSize" : "100", + } + + def insertData(self): + tdLog.info(f"insert data.") + # taosBenchmark run + jfile = etool.curFile(__file__, "oneStageComp.json") + etool.benchMark(json=jfile) + + tdSql.execute(f"use {self.db}") + # set insert data information + self.childtable_count = 10 + self.insert_rows = 100000 + self.timestamp_step = 1000 + + + + def checkColValueCorrect(self): + tdLog.info(f"do action.") + self.flushDb() + + # check all columns correct + cnt = self.insert_rows * self.childtable_count + sql = "select * from stb where bc!=1" + tdSql.checkRows(0) + sql = "select * from stb where fc=101" + tdSql.checkRows(cnt) + sql = "select * from stb where dc!=102" + tdSql.checkRows(0) + sql = "select * from stb where ti!=103" + tdSql.checkRows(0) + sql = "select * from stb where si!=104" + tdSql.checkRows(0) + sql = "select * from stb where ic!=105" + tdSql.checkRows(0) + sql = "select * from stb where b!i=106" + tdSql.checkRows(0) + sql = "select * from stb where uti!=107" + tdSql.checkRows(0) + sql = "select * from stb where usi!=108" + tdSql.checkRows(0) + sql = "select * from stb where ui!=109" + tdSql.checkRows(0) + sql = "select * from stb where ubi!=110" + tdSql.checkRows(0) + + def insertNull(self): + # insert 6 lines + sql = "insert into d0(ts) values(now) (now + 1s) (now + 2s) (now + 3s) (now + 4s) (now + 5s)" + tdSql.execute(sql) + + self.flushDb() + self.trimDb() + + # check all columns correct + cnt = self.insert_rows * self.childtable_count + sql = "select * from stb where bc!=1" + tdSql.checkRows(6) + sql = "select * from stb where bc=1" + tdSql.checkRows(cnt) + sql = "select * from stb where usi!=108" + tdSql.checkRows(6) + + # run + def run(self): + tdLog.debug(f"start to excute {__file__}") + + # insert data + self.insertData() + + # check insert data correct + self.checkInsertCorrect() + + # save + self.snapshotAgg() + + # do action + self.checkColValueCorrect() + + # check save agg result correct + self.checkAggCorrect() + + # insert null + self.insertNull() + + + tdLog.success(f"{__file__} successfully executed") + + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 91a0ac46e5..9ea03b4e6b 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -23,7 +23,7 @@ fi ,,y,army,./pytest.sh python3 ./test.py -f community/query/query_basic.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f community/cluster/splitVgroupByLearner.py -N 3 ,,n,army,python3 ./test.py -f community/cmdline/fullopt.py - +,,y,army,./pytest.sh python3 ./test.py -f community/storage/oneStageComp.py -N 3 -L 3 -D 1 #