Merge pull request #24716 from taosdata/coverage/TD-28559-3.0

add tsz compress

source/util/src/tcompression.c

@@ -977,6 +977,7 @@ int32_t tsDecompressDoubleLossyImp(const char *input, int32_t compressedSize, co
 }
 #endif
 
+#ifdef BUILD_NO_CALL
 /*************************************************************************
  *                  STREAM COMPRESSION
  *************************************************************************/
@@ -2120,7 +2121,7 @@ int32_t tCompressEnd(SCompressor *pCmprsor, const uint8_t **ppOut, int32_t *nOut
 int32_t tCompress(SCompressor *pCmprsor, const void *pData, int64_t nData) {
   return DATA_TYPE_INFO[pCmprsor->type].cmprFn(pCmprsor, pData, nData);
 }
-
+#endif
 /*************************************************************************
  *                  REGULAR COMPRESSION
  *************************************************************************/

tests/army/community/cluster/snapshot.json

@@ -35,8 +35,8 @@
                     "start_timestamp":"now-12d",
                     "columns": [
                         { "type": "bool",        "name": "bc"},
-                        { "type": "float",       "name": "fc" },
+                        { "type": "float",       "name": "fc", "min": 100, "max": 100},
-                        { "type": "double",      "name": "dc"},
+                        { "type": "double",      "name": "dc", "min": 200, "max": 200},
                         { "type": "tinyint",     "name": "ti"},
                         { "type": "smallint",    "name": "si" },
                         { "type": "int",         "name": "ic" },

tests/army/community/cluster/snapshot.py

@@ -29,7 +29,11 @@ from frame import *
 
 class TDTestCase(TBase):
     updatecfgDict = {
-        "countAlwaysReturnValue" : "0"
+        "countAlwaysReturnValue" : "0",
+        "lossyColumns"           : "float,double",
+        "fPrecision"             : "0.000000001",
+        "dPrecision"             : "0.00000000000000001",
+        "ifAdtFse"               : "1"
     }
 
     def insertData(self):
@@ -48,6 +52,18 @@ class TDTestCase(TBase):
         sql = f"create table {self.db}.ta(ts timestamp, age int) tags(area int)"
         tdSql.execute(sql)
 
+    def checkFloatDouble(self):
+        sql = f"select * from {self.db}.{self.stb} where fc!=100"
+        tdSql.query(sql)
+        tdSql.checkRows(0)
+        sql = f"select count(*) from {self.db}.{self.stb} where dc!=200"
+        tdSql.query(sql)
+        tdSql.checkRows(0)
+        sql = f"select avg(fc) from {self.db}.{self.stb}"
+        tdSql.checkFirstValue(sql, 100)
+        sql = f"select avg(dc) from {self.db}.{self.stb}"
+        tdSql.checkFirstValue(sql, 200)
+
     def doAction(self):
         tdLog.info(f"do action.")
         self.flushDb()
@@ -85,6 +101,9 @@ class TDTestCase(TBase):
         # check insert data correct
         self.checkInsertCorrect()
 
+        # check float double value ok
+        self.checkFloatDouble()
+
         # save
         self.snapshotAgg()
 
@@ -97,6 +116,10 @@ class TDTestCase(TBase):
         # check insert correct again
         self.checkInsertCorrect()
 
+        # check float double value ok
+        self.checkFloatDouble()
+
+
         tdLog.success(f"{__file__} successfully executed")
 
         

tests/army/community/storage/oneStageComp.json

@@ -0,0 +1,66 @@
+{
+    "filetype": "insert",
+    "cfgdir": "/etc/taos",
+    "host": "127.0.0.1",
+    "port": 6030,
+    "user": "root",
+    "password": "taosdata",
+    "connection_pool_size": 8,
+    "num_of_records_per_req": 3000,
+    "prepared_rand": 3000,
+    "thread_count": 2,
+    "create_table_thread_count": 1,
+    "confirm_parameter_prompt": "no",
+    "databases": [
+        {
+            "dbinfo": {
+                "name": "db",
+                "drop": "yes",
+                "vgroups": 2,
+                "replica": 3,
+                "wal_retention_period": 10,
+                "wal_retention_size": 100,
+                "keep": "60d,120d,365d",
+                "stt_trigger": 1,
+                "wal_level": 2,
+                "WAL_FSYNC_PERIOD": 3300,
+                "cachemodel": "'last_value'",
+                "TABLE_PREFIX":1,
+                "comp": 1
+            },
+            "super_tables": [
+                {
+                    "name": "stb",
+                    "child_table_exists": "no",
+                    "childtable_count": 10,
+                    "insert_rows": 100000,
+                    "childtable_prefix": "d",
+                    "insert_mode": "taosc",
+                    "timestamp_step": 1000,
+                    "start_timestamp":"now-360d",
+                    "columns": [
+                        { "type": "bool",        "name": "bc","max": 1,"min": 1},
+                        { "type": "float",       "name": "fc" ,"max": 101,"min": 101},
+                        { "type": "double",      "name": "dc" ,"max": 102,"min": 102},
+                        { "type": "tinyint",     "name": "ti" ,"max": 103,"min": 103},
+                        { "type": "smallint",    "name": "si" ,"max": 104,"min": 104},
+                        { "type": "int",         "name": "ic" ,"max": 105,"min": 105},
+                        { "type": "bigint",      "name": "bi" ,"max": 106,"min": 106},
+                        { "type": "utinyint",    "name": "uti","max": 107,"min": 107},
+                        { "type": "usmallint",   "name": "usi","max": 108,"min": 108},
+                        { "type": "uint",        "name": "ui" ,"max": 109,"min": 109},
+                        { "type": "ubigint",     "name": "ubi","max": 110,"min": 110},
+                        { "type": "binary",      "name": "bin", "len": 16},
+                        { "type": "nchar",       "name": "nch", "len": 32}
+                    ],
+                    "tags": [
+                        {"type": "tinyint", "name": "groupid","max": 100,"min": 100},
+                        {"name": "location","type": "binary", "len": 16, "values":
+                           ["San Francisco", "Los Angles", "San Diego", "San Jose", "Palo Alto", "Campbell", "Mountain View","Sunnyvale", "Santa Clara", "Cupertino"]
+                        }
+                    ]
+                }
+            ]
+        }
+    ]
+}

tests/army/community/storage/oneStageComp.py

@@ -0,0 +1,140 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import time
+import random
+
+import taos
+import frame
+import frame.etool
+
+
+from frame.log import *
+from frame.cases import *
+from frame.sql import *
+from frame.caseBase import *
+from frame import *
+
+
+class TDTestCase(TBase):
+    updatecfgDict = {
+        "compressMsgSize" : "100",
+    }
+
+    def insertData(self):
+        tdLog.info(f"insert data.")
+        # taosBenchmark run
+        jfile = etool.curFile(__file__, "oneStageComp.json")
+        etool.benchMark(json=jfile)
+
+        tdSql.execute(f"use {self.db}")
+        # set insert data information
+        self.childtable_count = 10
+        self.insert_rows      = 100000
+        self.timestamp_step   = 1000
+    
+
+
+    def checkColValueCorrect(self):
+        tdLog.info(f"do action.")
+        self.flushDb()
+
+        # check all columns correct
+        cnt = self.insert_rows * self.childtable_count
+        sql = "select * from stb where bc!=1"
+        tdSql.query(sql)
+        tdSql.checkRows(0)
+        sql = "select * from stb where fc=101"
+        tdSql.query(sql)
+        tdSql.checkRows(cnt)
+        sql = "select * from stb where dc!=102"
+        tdSql.query(sql)
+        tdSql.checkRows(0)
+        sql = "select * from stb where ti!=103"
+        tdSql.query(sql)
+        tdSql.checkRows(0)
+        sql = "select * from stb where si!=104"
+        tdSql.query(sql)
+        tdSql.checkRows(0)
+        sql = "select * from stb where ic!=105"
+        tdSql.query(sql)
+        tdSql.checkRows(0)
+        sql = "select * from stb where bi!=106"
+        tdSql.query(sql)
+        tdSql.checkRows(0)
+        sql = "select * from stb where uti!=107"
+        tdSql.query(sql)
+        tdSql.checkRows(0)
+        sql = "select * from stb where usi!=108"
+        tdSql.query(sql)
+        tdSql.checkRows(0)
+        sql = "select * from stb where ui!=109"
+        tdSql.query(sql)
+        tdSql.checkRows(0)
+        sql = "select * from stb where ubi!=110"
+        tdSql.query(sql)
+        tdSql.checkRows(0)
+    
+    def insertNull(self):
+        # insert 6 lines
+        sql = "insert into d0(ts) values(now) (now + 1s) (now + 2s) (now + 3s) (now + 4s) (now + 5s)"
+        tdSql.execute(sql)
+
+        self.flushDb()
+        self.trimDb()
+
+        # check all columns correct
+        cnt = self.insert_rows * self.childtable_count
+        sql = "select * from stb where bc!=1"
+        tdSql.query(sql)
+        tdSql.checkRows(0)
+        sql = "select * from stb where bc is null"
+        tdSql.query(sql)
+        tdSql.checkRows(6)
+        sql = "select * from stb where bc=1"
+        tdSql.query(sql)
+        tdSql.checkRows(cnt)
+        sql = "select * from stb where usi is null"
+        tdSql.query(sql)
+        tdSql.checkRows(6)
+
+    # run
+    def run(self):
+        tdLog.debug(f"start to excute {__file__}")
+
+        # insert data
+        self.insertData()
+
+        # check insert data correct
+        self.checkInsertCorrect()
+
+        # save
+        self.snapshotAgg()
+
+        # do action
+        self.checkColValueCorrect()
+
+        # check save agg result correct
+        self.checkAggCorrect()
+
+        # insert null 
+        self.insertNull()
+
+
+        tdLog.success(f"{__file__} successfully executed")
+
+        
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tests/parallel_test/cases.task

@@ -23,7 +23,7 @@ fi
 ,,y,army,./pytest.sh python3 ./test.py -f community/query/query_basic.py -N 3
 ,,y,army,./pytest.sh python3 ./test.py -f community/cluster/splitVgroupByLearner.py -N 3
 ,,n,army,python3 ./test.py -f community/cmdline/fullopt.py
-
+,,y,army,./pytest.sh python3 ./test.py -f community/storage/oneStageComp.py -N 3 -L 3 -D 1
 
 
 #

utils/TSZ/sz/src/sz_double.c

@@ -385,9 +385,11 @@ unsigned int optimize_intervals_double_1D_opt(double *oriData, size_t dataLength
 		totalSampleSize++;
 		pred_value = data_pos[-1];
 		pred_err = fabs(pred_value - *data_pos);
-		radiusIndex = (unsigned long)((pred_err/realPrecision+1)/2);
+		double dbri = (unsigned long)((pred_err/realPrecision+1)/2);
-		if(radiusIndex>=confparams_cpr->maxRangeRadius)
+		if(dbri >= (double)confparams_cpr->maxRangeRadius)
-			radiusIndex = confparams_cpr->maxRangeRadius - 1;			
+			radiusIndex = confparams_cpr->maxRangeRadius - 1;
+		else
+		    radiusIndex = dbri;
 		intervals[radiusIndex]++;
 
 		data_pos += confparams_cpr->sampleDistance;