xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #24716 from taosdata/coverage/TD-28559-3.0 

add tsz compress
This commit is contained in:
Alex Duan 2024-02-04 19:58:43 +08:00 committed by GitHub
parent 75603934a1 8360d6a12b
commit 83808943a5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 240 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
source/util/src/tcompression.c
View File

@ -977,6 +977,7 @@ int32_t tsDecompressDoubleLossyImp(const char *input, int32_t compressedSize, co
}
#endif
#ifdef BUILD_NO_CALL
/*************************************************************************
* STREAM COMPRESSION
*************************************************************************/
@ -2120,7 +2121,7 @@ int32_t tCompressEnd(SCompressor *pCmprsor, const uint8_t **ppOut, int32_t *nOut
int32_t tCompress(SCompressor *pCmprsor, const void *pData, int64_t nData) {
return DATA_TYPE_INFO[pCmprsor->type].cmprFn(pCmprsor, pData, nData);
}
#endif
/*************************************************************************
* REGULAR COMPRESSION
*************************************************************************/

4
tests/army/community/cluster/snapshot.json
View File

@ -35,8 +35,8 @@
"start_timestamp":"now-12d",
"columns": [
{ "type": "bool", "name": "bc"},
{ "type": "float", "name": "fc" },
{ "type": "double", "name": "dc"},
{ "type": "float", "name": "fc", "min": 100, "max": 100},
{ "type": "double", "name": "dc", "min": 200, "max": 200},
{ "type": "tinyint", "name": "ti"},
{ "type": "smallint", "name": "si" },
{ "type": "int", "name": "ic" },

25
tests/army/community/cluster/snapshot.py
View File

@ -29,7 +29,11 @@ from frame import *
class TDTestCase(TBase):
updatecfgDict = {
"countAlwaysReturnValue" : "0"
"countAlwaysReturnValue" : "0",
"lossyColumns" : "float,double",
"fPrecision" : "0.000000001",
"dPrecision" : "0.00000000000000001",
"ifAdtFse" : "1"
}
def insertData(self):
@ -48,6 +52,18 @@ class TDTestCase(TBase):
sql = f"create table {self.db}.ta(ts timestamp, age int) tags(area int)"
tdSql.execute(sql)
def checkFloatDouble(self):
sql = f"select * from {self.db}.{self.stb} where fc!=100"
tdSql.query(sql)
tdSql.checkRows(0)
sql = f"select count(*) from {self.db}.{self.stb} where dc!=200"
tdSql.query(sql)
tdSql.checkRows(0)
sql = f"select avg(fc) from {self.db}.{self.stb}"
tdSql.checkFirstValue(sql, 100)
sql = f"select avg(dc) from {self.db}.{self.stb}"
tdSql.checkFirstValue(sql, 200)
def doAction(self):
tdLog.info(f"do action.")
self.flushDb()
@ -85,6 +101,9 @@ class TDTestCase(TBase):
# check insert data correct
self.checkInsertCorrect()
# check float double value ok
self.checkFloatDouble()
# save
self.snapshotAgg()
@ -97,6 +116,10 @@ class TDTestCase(TBase):
# check insert correct again
self.checkInsertCorrect()
# check float double value ok
self.checkFloatDouble()
tdLog.success(f"{__file__} successfully executed")

66
tests/army/community/storage/oneStageComp.json Normal file
View File

@ -0,0 +1,66 @@
{
"filetype": "insert",
"cfgdir": "/etc/taos",
"host": "127.0.0.1",
"port": 6030,
"user": "root",
"password": "taosdata",
"connection_pool_size": 8,
"num_of_records_per_req": 3000,
"prepared_rand": 3000,
"thread_count": 2,
"create_table_thread_count": 1,
"confirm_parameter_prompt": "no",
"databases": [
{
"dbinfo": {
"name": "db",
"drop": "yes",
"vgroups": 2,
"replica": 3,
"wal_retention_period": 10,
"wal_retention_size": 100,
"keep": "60d,120d,365d",
"stt_trigger": 1,
"wal_level": 2,
"WAL_FSYNC_PERIOD": 3300,
"cachemodel": "'last_value'",
"TABLE_PREFIX":1,
"comp": 1
},
"super_tables": [
{
"name": "stb",
"child_table_exists": "no",
"childtable_count": 10,
"insert_rows": 100000,
"childtable_prefix": "d",
"insert_mode": "taosc",
"timestamp_step": 1000,
"start_timestamp":"now-360d",
"columns": [
{ "type": "bool", "name": "bc","max": 1,"min": 1},
{ "type": "float", "name": "fc" ,"max": 101,"min": 101},
{ "type": "double", "name": "dc" ,"max": 102,"min": 102},
{ "type": "tinyint", "name": "ti" ,"max": 103,"min": 103},
{ "type": "smallint", "name": "si" ,"max": 104,"min": 104},
{ "type": "int", "name": "ic" ,"max": 105,"min": 105},
{ "type": "bigint", "name": "bi" ,"max": 106,"min": 106},
{ "type": "utinyint", "name": "uti","max": 107,"min": 107},
{ "type": "usmallint", "name": "usi","max": 108,"min": 108},
{ "type": "uint", "name": "ui" ,"max": 109,"min": 109},
{ "type": "ubigint", "name": "ubi","max": 110,"min": 110},
{ "type": "binary", "name": "bin", "len": 16},
{ "type": "nchar", "name": "nch", "len": 32}
],
"tags": [
{"type": "tinyint", "name": "groupid","max": 100,"min": 100},
{"name": "location","type": "binary", "len": 16, "values":
["San Francisco", "Los Angles", "San Diego", "San Jose", "Palo Alto", "Campbell", "Mountain View","Sunnyvale", "Santa Clara", "Cupertino"]
}
]
}
]
}
]
}

140
tests/army/community/storage/oneStageComp.py Normal file
View File

@ -0,0 +1,140 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import time
import random
import taos
import frame
import frame.etool
from frame.log import *
from frame.cases import *
from frame.sql import *
from frame.caseBase import *
from frame import *
class TDTestCase(TBase):
updatecfgDict = {
"compressMsgSize" : "100",
}
def insertData(self):
tdLog.info(f"insert data.")
# taosBenchmark run
jfile = etool.curFile(__file__, "oneStageComp.json")
etool.benchMark(json=jfile)
tdSql.execute(f"use {self.db}")
# set insert data information
self.childtable_count = 10
self.insert_rows = 100000
self.timestamp_step = 1000
def checkColValueCorrect(self):
tdLog.info(f"do action.")
self.flushDb()
# check all columns correct
cnt = self.insert_rows * self.childtable_count
sql = "select * from stb where bc!=1"
tdSql.query(sql)
tdSql.checkRows(0)
sql = "select * from stb where fc=101"
tdSql.query(sql)
tdSql.checkRows(cnt)
sql = "select * from stb where dc!=102"
tdSql.query(sql)
tdSql.checkRows(0)
sql = "select * from stb where ti!=103"
tdSql.query(sql)
tdSql.checkRows(0)
sql = "select * from stb where si!=104"
tdSql.query(sql)
tdSql.checkRows(0)
sql = "select * from stb where ic!=105"
tdSql.query(sql)
tdSql.checkRows(0)
sql = "select * from stb where bi!=106"
tdSql.query(sql)
tdSql.checkRows(0)
sql = "select * from stb where uti!=107"
tdSql.query(sql)
tdSql.checkRows(0)
sql = "select * from stb where usi!=108"
tdSql.query(sql)
tdSql.checkRows(0)
sql = "select * from stb where ui!=109"
tdSql.query(sql)
tdSql.checkRows(0)
sql = "select * from stb where ubi!=110"
tdSql.query(sql)
tdSql.checkRows(0)
def insertNull(self):
# insert 6 lines
sql = "insert into d0(ts) values(now) (now + 1s) (now + 2s) (now + 3s) (now + 4s) (now + 5s)"
tdSql.execute(sql)
self.flushDb()
self.trimDb()
# check all columns correct
cnt = self.insert_rows * self.childtable_count
sql = "select * from stb where bc!=1"
tdSql.query(sql)
tdSql.checkRows(0)
sql = "select * from stb where bc is null"
tdSql.query(sql)
tdSql.checkRows(6)
sql = "select * from stb where bc=1"
tdSql.query(sql)
tdSql.checkRows(cnt)
sql = "select * from stb where usi is null"
tdSql.query(sql)
tdSql.checkRows(6)
# run
def run(self):
tdLog.debug(f"start to excute {__file__}")
# insert data
self.insertData()
# check insert data correct
self.checkInsertCorrect()
# save
self.snapshotAgg()
# do action
self.checkColValueCorrect()
# check save agg result correct
self.checkAggCorrect()
# insert null
self.insertNull()
tdLog.success(f"{__file__} successfully executed")
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())

2
tests/parallel_test/cases.task
View File

@ -23,7 +23,7 @@ fi
,,y,army,./pytest.sh python3 ./test.py -f community/query/query_basic.py -N 3
,,y,army,./pytest.sh python3 ./test.py -f community/cluster/splitVgroupByLearner.py -N 3
,,n,army,python3 ./test.py -f community/cmdline/fullopt.py
,,y,army,./pytest.sh python3 ./test.py -f community/storage/oneStageComp.py -N 3 -L 3 -D 1
#

8
utils/TSZ/sz/src/sz_double.c
View File

@ -385,9 +385,11 @@ unsigned int optimize_intervals_double_1D_opt(double *oriData, size_t dataLength
totalSampleSize++;
pred_value = data_pos[-1];
pred_err = fabs(pred_value - *data_pos);
radiusIndex = (unsigned long)((pred_err/realPrecision+1)/2);
if(radiusIndex>=confparams_cpr->maxRangeRadius)
radiusIndex = confparams_cpr->maxRangeRadius - 1;
double dbri = (unsigned long)((pred_err/realPrecision+1)/2);
if(dbri >= (double)confparams_cpr->maxRangeRadius)
radiusIndex = confparams_cpr->maxRangeRadius - 1;
else
radiusIndex = dbri;
intervals[radiusIndex]++;
data_pos += confparams_cpr->sampleDistance;