xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

remove TD_TSZ macro define

This commit is contained in:
Alex Duan 2024-02-04 13:22:39 +08:00
parent 68fe671cbc
commit b852993b8d
6 changed files with 192 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
include/util/tcompression.h
View File

@ -55,7 +55,6 @@ extern "C" {
#define HEAD_MODE(x) x % 2
#define HEAD_ALGO(x) x / 2
#ifdef TD_TSZ
extern bool lossyFloat;
extern bool lossyDouble;
int32_t tsCompressInit(char* lossyColumns, float fPrecision, double dPrecision, uint32_t maxIntervals, uint32_t intervals,

6
source/dnode/mgmt/node_mgmt/src/dmMgmt.c
View File

@ -19,10 +19,8 @@
#include "index.h"
#include "qworker.h"
#include "tstream.h"
#ifdef TD_TSZ
#include "tcompression.h"
#include "tglobal.h"
#endif
static bool dmRequireNode(SDnode *pDnode, SMgmtWrapper *pWrapper) {
SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper);
@ -47,10 +45,8 @@ int32_t dmInitDnode(SDnode *pDnode) {
goto _OVER;
}
#ifdef TD_TSZ
// compress module init
tsCompressInit(tsLossyColumns, tsFPrecision, tsDPrecision, tsMaxRange, tsCurRange, (int)tsIfAdtFse, tsCompressor);
#endif
pDnode->wrappers[DNODE].func = dmGetMgmtFunc();
pDnode->wrappers[MNODE].func = mmGetMgmtFunc();
@ -119,10 +115,8 @@ void dmCleanupDnode(SDnode *pDnode) {
indexCleanup();
taosConvDestroy();
#ifdef TD_TSZ
// compress destroy
tsCompressExit();
#endif
dDebug("dnode is closed, ptr:%p", pDnode);
}

24
source/util/src/tcompression.c
View File

@ -54,9 +54,7 @@
#include "tlog.h"
#include "ttypes.h"
#ifdef TD_TSZ
#include "td_sz.h"
#endif
static const int32_t TEST_NUMBER = 1;
#define is_bigendian() ((*(char *)&TEST_NUMBER) == 0)
@ -64,7 +62,6 @@ static const int32_t TEST_NUMBER = 1;
#define safeInt64Add(a, b) (((a >= 0) && (b <= INT64_MAX - a)) || ((a < 0) && (b >= INT64_MIN - a)))
#ifdef TD_TSZ
bool lossyFloat = false;
bool lossyDouble = false;
@ -83,7 +80,6 @@ int32_t tsCompressInit(char* lossyColumns, float fPrecision, double dPrecision,
// exit call
void tsCompressExit() { tdszExit(); }
#endif
/*
* Compress Integer (Simple8B).
@ -906,7 +902,6 @@ int32_t tsDecompressFloatImp(const char *const input, const int32_t nelements, c
return nelements * FLOAT_BYTES;
}
#ifdef TD_TSZ
//
// ---------- float double lossy -----------
//
@ -977,6 +972,7 @@ int32_t tsDecompressDoubleLossyImp(const char *input, int32_t compressedSize, co
}
#endif
#ifdef BUILD_NO_CALL
/*************************************************************************
* STREAM COMPRESSION
*************************************************************************/
@ -2120,7 +2116,7 @@ int32_t tCompressEnd(SCompressor *pCmprsor, const uint8_t **ppOut, int32_t *nOut
int32_t tCompress(SCompressor *pCmprsor, const void *pData, int64_t nData) {
return DATA_TYPE_INFO[pCmprsor->type].cmprFn(pCmprsor, pData, nData);
}
#endif
/*************************************************************************
* REGULAR COMPRESSION
*************************************************************************/
@ -2154,13 +2150,11 @@ int32_t tsDecompressTimestamp(void *pIn, int32_t nIn, int32_t nEle, void *pOut,
// Float =====================================================
int32_t tsCompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t nBuf) {
#ifdef TD_TSZ
// lossy mode
if (lossyFloat) {
return tsCompressFloatLossyImp(pIn, nEle, pOut);
// lossless mode
} else {
#endif
if (cmprAlg == ONE_STAGE_COMP) {
return tsCompressFloatImp(pIn, nEle, pOut);
} else if (cmprAlg == TWO_STAGE_COMP) {
@ -2170,19 +2164,15 @@ int32_t tsCompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_
ASSERTS(0, "compress algo invalid");
return -1;
}
#ifdef TD_TSZ
}
#endif
}
int32_t tsDecompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t nBuf) {
#ifdef TD_TSZ
if (HEAD_ALGO(((uint8_t *)pIn)[0]) == ALGO_SZ_LOSSY) {
// decompress lossy
return tsDecompressFloatLossyImp(pIn, nIn, nEle, pOut);
} else {
#endif
// decompress lossless
if (cmprAlg == ONE_STAGE_COMP) {
return tsDecompressFloatImp(pIn, nEle, pOut);
@ -2193,20 +2183,16 @@ int32_t tsDecompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int3
ASSERTS(0, "compress algo invalid");
return -1;
}
#ifdef TD_TSZ
}
#endif
}
// Double =====================================================
int32_t tsCompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t nBuf) {
#ifdef TD_TSZ
if (lossyDouble) {
// lossy mode
return tsCompressDoubleLossyImp(pIn, nEle, pOut);
} else {
#endif
// lossless mode
if (cmprAlg == ONE_STAGE_COMP) {
return tsCompressDoubleImp(pIn, nEle, pOut);
@ -2217,19 +2203,15 @@ int32_t tsCompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32
ASSERTS(0, "compress algo invalid");
return -1;
}
#ifdef TD_TSZ
}
#endif
}
int32_t tsDecompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t nBuf) {
#ifdef TD_TSZ
if (HEAD_ALGO(((uint8_t *)pIn)[0]) == ALGO_SZ_LOSSY) {
// decompress lossy
return tsDecompressDoubleLossyImp(pIn, nIn, nEle, pOut);
} else {
#endif
// decompress lossless
if (cmprAlg == ONE_STAGE_COMP) {
return tsDecompressDoubleImp(pIn, nEle, pOut);
@ -2240,9 +2222,7 @@ int32_t tsDecompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int
ASSERTS(0, "compress algo invalid");
return -1;
}
#ifdef TD_TSZ
}
#endif
}
// Binary =====================================================

66
tests/army/community/storage/oneStageComp.json Normal file
View File

@ -0,0 +1,66 @@
{
"filetype": "insert",
"cfgdir": "/etc/taos",
"host": "127.0.0.1",
"port": 6030,
"user": "root",
"password": "taosdata",
"connection_pool_size": 8,
"num_of_records_per_req": 3000,
"prepared_rand": 3000,
"thread_count": 2,
"create_table_thread_count": 1,
"confirm_parameter_prompt": "no",
"databases": [
{
"dbinfo": {
"name": "db",
"drop": "yes",
"vgroups": 2,
"replica": 3,
"wal_retention_period": 10,
"wal_retention_size": 100,
"keep": "60d,120d,365d",
"stt_trigger": 1,
"wal_level": 2,
"WAL_FSYNC_PERIOD": 3300,
"cachemode": "last_value",
"TABLE_PREFIX":1,
"comp": 1
},
"super_tables": [
{
"name": "stb",
"child_table_exists": "no",
"childtable_count": 10,
"insert_rows": 100000,
"childtable_prefix": "d",
"insert_mode": "taosc",
"timestamp_step": 1000,
"start_timestamp":"2023-01-01 00:00:00",
"columns": [
{ "type": "bool", "name": "bc","max": 1,"min": 1},
{ "type": "float", "name": "fc" ,"max": 101,"min": 101},
{ "type": "double", "name": "dc" ,"max": 102,"min": 102},
{ "type": "tinyint", "name": "ti" ,"max": 103,"min": 103},
{ "type": "smallint", "name": "si" ,"max": 104,"min": 104},
{ "type": "int", "name": "ic" ,"max": 105,"min": 105},
{ "type": "bigint", "name": "bi" ,"max": 106,"min": 106},
{ "type": "utinyint", "name": "uti","max": 107,"min": 107},
{ "type": "usmallint", "name": "usi","max": 108,"min": 108},
{ "type": "uint", "name": "ui" ,"max": 109,"min": 109},
{ "type": "ubigint", "name": "ubi","max": 110,"min": 110},
{ "type": "binary", "name": "bin", "len": 16},
{ "type": "nchar", "name": "nch", "len": 32}
],
"tags": [
{"type": "tinyint", "name": "groupid","max": 100,"min": 100},
{"name": "location","type": "binary", "len": 16, "values":
["San Francisco", "Los Angles", "San Diego", "San Jose", "Palo Alto", "Campbell", "Mountain View","Sunnyvale", "Santa Clara", "Cupertino"]
}
]
}
]
}
]
}

123
tests/army/community/storage/oneStageComp.py Normal file
View File

@ -0,0 +1,123 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import time
import random
import taos
import frame
import frame.etool
from frame.log import *
from frame.cases import *
from frame.sql import *
from frame.caseBase import *
from frame import *
class TDTestCase(TBase):
updatecfgDict = {
"compressMsgSize" : "100",
}
def insertData(self):
tdLog.info(f"insert data.")
# taosBenchmark run
jfile = etool.curFile(__file__, "oneStageComp.json")
etool.benchMark(json=jfile)
tdSql.execute(f"use {self.db}")
# set insert data information
self.childtable_count = 10
self.insert_rows = 100000
self.timestamp_step = 1000
def checkColValueCorrect(self):
tdLog.info(f"do action.")
self.flushDb()
# check all columns correct
cnt = self.insert_rows * self.childtable_count
sql = "select * from stb where bc!=1"
tdSql.checkRows(0)
sql = "select * from stb where fc=101"
tdSql.checkRows(cnt)
sql = "select * from stb where dc!=102"
tdSql.checkRows(0)
sql = "select * from stb where ti!=103"
tdSql.checkRows(0)
sql = "select * from stb where si!=104"
tdSql.checkRows(0)
sql = "select * from stb where ic!=105"
tdSql.checkRows(0)
sql = "select * from stb where b!i=106"
tdSql.checkRows(0)
sql = "select * from stb where uti!=107"
tdSql.checkRows(0)
sql = "select * from stb where usi!=108"
tdSql.checkRows(0)
sql = "select * from stb where ui!=109"
tdSql.checkRows(0)
sql = "select * from stb where ubi!=110"
tdSql.checkRows(0)
def insertNull(self):
# insert 6 lines
sql = "insert into d0(ts) values(now) (now + 1s) (now + 2s) (now + 3s) (now + 4s) (now + 5s)"
tdSql.execute(sql)
self.flushDb()
self.trimDb()
# check all columns correct
cnt = self.insert_rows * self.childtable_count
sql = "select * from stb where bc!=1"
tdSql.checkRows(6)
sql = "select * from stb where bc=1"
tdSql.checkRows(cnt)
sql = "select * from stb where usi!=108"
tdSql.checkRows(6)
# run
def run(self):
tdLog.debug(f"start to excute {__file__}")
# insert data
self.insertData()
# check insert data correct
self.checkInsertCorrect()
# save
self.snapshotAgg()
# do action
self.checkColValueCorrect()
# check save agg result correct
self.checkAggCorrect()
# insert null
self.insertNull()
tdLog.success(f"{__file__} successfully executed")
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())

2
tests/parallel_test/cases.task
View File

@ -23,7 +23,7 @@ fi
,,y,army,./pytest.sh python3 ./test.py -f community/query/query_basic.py -N 3
,,y,army,./pytest.sh python3 ./test.py -f community/cluster/splitVgroupByLearner.py -N 3
,,n,army,python3 ./test.py -f community/cmdline/fullopt.py
,,y,army,./pytest.sh python3 ./test.py -f community/storage/oneStageComp.py -N 3 -L 3 -D 1
#