From 53a128b2a3a469b06ae28ff013397b8d015433c8 Mon Sep 17 00:00:00 2001 From: wu champion Date: Mon, 22 Mar 2021 17:02:53 +0800 Subject: [PATCH 1/7] fix case,add delete file when processing over --- tests/pytest/query/query1970YearsAf.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/pytest/query/query1970YearsAf.py b/tests/pytest/query/query1970YearsAf.py index d2eead241f..441f835e89 100644 --- a/tests/pytest/query/query1970YearsAf.py +++ b/tests/pytest/query/query1970YearsAf.py @@ -18,7 +18,6 @@ import json import subprocess import datetime - from util.log import * from util.sql import * from util.cases import * @@ -71,7 +70,7 @@ class TDTestCase: "update": 0 } - # 设置创建的超级表格式 + # set stable schema stable1 = { "name": "stb2", "child_table_exists": "no", @@ -83,7 +82,7 @@ class TDTestCase: "insert_rows": 5000, "multi_thread_write_one_tbl": "no", "number_of_tbl_in_one_sql": 0, - "rows_per_tbl": 1000, + "rows_per_tbl": 1, "max_sql_len": 65480, "disorder_ratio": 0, "disorder_range": 1000, @@ -117,7 +116,7 @@ class TDTestCase: ] } - # 创建不同的超级表格式并添加至super_tables + # create different stables like stable1 and add to list super_tables super_tables = [] super_tables.append(stable1) database = { @@ -235,7 +234,7 @@ class TDTestCase: tdLog.info("==========step2:query join") self.sqlsquery() - # 进行数据落盘后检查 + # after wal and sync, check again tdSql.query("show dnodes") index = tdSql.getData(0, 0) tdDnodes.stop(index) @@ -246,8 +245,9 @@ class TDTestCase: def stop(self): tdSql.close() + rm_cmd = f"rm -f /tmp/insert* > /dev/null 2>&1" + _ = subprocess.check_output(rm_cmd, shell=True).decode("utf-8") tdLog.success(f"{__file__} successfully executed") - tdCases.addLinux(__file__, TDTestCase()) tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file From 7c749e24c8814f03eb147048158efa1ae7b7bfd9 Mon Sep 17 00:00:00 2001 From: wu champion Date: Fri, 19 Mar 2021 18:35:26 +0800 Subject: [PATCH 2/7] [TD-1380] fix the case of query1970YearsAf.py --- tests/pytest/query/query1970YearsAf.py | 253 +++++++++++++++++++++++++ 1 file changed, 253 insertions(+) create mode 100644 tests/pytest/query/query1970YearsAf.py diff --git a/tests/pytest/query/query1970YearsAf.py b/tests/pytest/query/query1970YearsAf.py new file mode 100644 index 0000000000..9902f6908b --- /dev/null +++ b/tests/pytest/query/query1970YearsAf.py @@ -0,0 +1,253 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import taos +import sys +import os +import json +import subprocess +import datetime + + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.dnodes import TDDnode + +class TDTestCase: + + def __init__(self): + self.path = "" + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def getcfgPath(self, path): + binPath = os.path.dirname(os.path.realpath(__file__)) + binPath = binPath + "/../../../debug/" + tdLog.debug(f"binPath {binPath}") + binPath = os.path.realpath(binPath) + tdLog.debug(f"binPath real path {binPath}") + if path == "": + self.path = os.path.abspath(binPath + "../../") + else: + self.path = os.path.realpath(path) + return self.path + + def getCfgDir(self): + self.getcfgPath(self.path) + self.cfgDir = f"{self.path}/sim/psim/cfg" + return self.cfgDir + + def creatcfg(self): + dbinfo = { + "name": "db", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 16, + "blocks": 8, + "precision": "ms", + "keep": 36500, + "minRows": 100, + "maxRows": 4096, + "comp": 2, + "walLevel": 1, + "cachelast": 0, + "quorum": 1, + "fsync": 3000, + "update": 0 + } + + # 设置创建的超级表格式 + stable1 = { + "name": "stb2", + "child_table_exists": "no", + "childtable_count": 10, + "childtable_prefix": "t", + "auto_create_table": "no", + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 5000, + "multi_thread_write_one_tbl": "no", + "number_of_tbl_in_one_sql": 0, + "rows_per_tbl": 1000, + "max_sql_len": 65480, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 20000, + "start_timestamp": "1969-12-31 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [ + {"type": "INT", "count": 2}, + {"type": "DOUBLE", "count": 2}, + {"type": "BIGINT", "count": 2}, + {"type": "FLOAT", "count": 2}, + {"type": "SMALLINT", "count": 2}, + {"type": "TINYINT", "count": 2}, + {"type": "BOOL", "count": 2}, + {"type": "NCHAR", "len": 3, "count": 1}, + {"type": "BINARY", "len": 8, "count": 1} + + ], + "tags": [ + {"type": "INT", "count": 2}, + {"type": "DOUBLE", "count": 2}, + {"type": "BIGINT", "count": 2}, + {"type": "FLOAT", "count": 2}, + {"type": "SMALLINT", "count": 2}, + {"type": "TINYINT", "count": 2}, + {"type": "BOOL", "count": 2}, + {"type": "NCHAR", "len": 3, "count": 1}, + {"type": "BINARY", "len": 8, "count": 1} + ] + } + + # 创建不同的超级表格式并添加至super_tables + super_tables = [] + super_tables.append(stable1) + database = { + "dbinfo": dbinfo, + "super_tables": super_tables + } + + cfgdir = self.getCfgDir() + create_table = { + "filetype": "insert", + "cfgdir": cfgdir, + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "/tmp/insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "num_of_records_per_req": 100, + "databases": [database] + } + return create_table + + def createinsertfile(self): + create_table = self.creatcfg() + date = datetime.datetime.now().strftime("%Y%m%d%H%M") + file_create_table = f"/tmp/insert_{date}.json" + + with open(file_create_table, 'w') as f: + json.dump(create_table, f) + return file_create_table + + def inserttable(self, filepath): + create_table_cmd = f"taosdemo -f {filepath} > /tmp/1.log 2>&1" + _ = subprocess.check_output(create_table_cmd, shell=True).decode("utf-8") + + def sqlsquery(self): + # stable query + tdSql.query( + "select * from stb2 where stb2.ts < '1970-01-01 00:00:00.000' " + ) + tdSql.checkRows(43200) + + tdSql.query( + "select * from stb2 where stb2.ts >= '1970-01-01 00:00:00.000' " + ) + tdSql.checkRows(6800) + + tdSql.query( + "select * from stb2 where stb2.ts > '1969-12-31 23:00:00.000' and stb2.ts <'1970-01-01 01:00:00.000' " + ) + tdSql.checkRows(3590) + + # child-table query + tdSql.query( + "select * from t0 where t0.ts < '1970-01-01 00:00:00.000' " + ) + tdSql.checkRows(4320) + + tdSql.query( + "select * from t1 where t1.ts >= '1970-01-01 00:00:00.000' " + ) + tdSql.checkRows(680) + + tdSql.query( + "select * from t9 where t9.ts > '1969-12-31 22:00:00.000' and t9.ts <'1970-01-01 02:00:00.000' " + ) + tdSql.checkRows(719) + + tdSql.query( + "select * from t0,t1 where t0.ts=t1.ts and t1.ts >= '1970-01-01 00:00:00.000' " + ) + tdSql.checkRows(680) + + tdSql.query( + "select diff(col1) from t0 where t0.ts >= '1970-01-01 00:00:00.000' " + ) + tdSql.checkRows(679) + + tdSql.query( + "select t0,col1 from stb2 where stb2.ts < '1970-01-01 00:00:00.000' order by ts" + ) + tdSql.checkRows(43200) + + # query with timestamp in 'where ...' + tdSql.query( + "select * from stb2 where stb2.ts > -28800000 " + ) + tdSql.checkRows(6790) + + tdSql.query( + "select * from stb2 where stb2.ts > -28800000 and stb2.ts < '1970-01-01 08:00:00.000' " + ) + tdSql.checkRows(6790) + + tdSql.query( + "select * from stb2 where stb2.ts < -28800000 and stb2.ts > '1969-12-31 22:00:00.000' " + ) + tdSql.checkRows(3590) + + def run(self): + s = 'reset query cache' + tdSql.execute(s) + s = 'create database if not exists db' + tdSql.execute(s) + s = 'use db' + tdSql.execute(s) + + tdLog.info("==========step1:create table stable and child table,then insert data automatically") + insertfile = self.createinsertfile() + self.inserttable(insertfile) + + tdLog.info("==========step2:query join") + self.sqlsquery() + + # 进行数据落盘后检查 + tdSql.query("show dnodes") + index = tdSql.getData(0, 0) + tdDnodes.stop(index) + tdDnodes.start(index) + + tdLog.info("==========step3: query join again") + self.sqlsquery() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file From dff5c11f290453a349bdb885e00128586700bece Mon Sep 17 00:00:00 2001 From: wu champion Date: Sat, 20 Mar 2021 16:10:45 +0800 Subject: [PATCH 3/7] [test] add test case and bug case --- tests/pytest/fulltest.sh | 4 ++ tests/pytest/query/bug3351.py | 74 ++++++++++++++++++++++++++ tests/pytest/query/query1970YearsAf.py | 4 +- 3 files changed, 80 insertions(+), 2 deletions(-) create mode 100644 tests/pytest/query/bug3351.py diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 5037f2c399..aa8e679e81 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -196,6 +196,10 @@ python3 ./test.py -f query/bug2119.py python3 ./test.py -f query/isNullTest.py python3 ./test.py -f query/queryWithTaosdKilled.py python3 ./test.py -f query/floatCompare.py +python3 ./test.py -f query/query1970YearsAf.py +python3 ./test.py -f query/bug3351.py + + #stream python3 ./test.py -f stream/metric_1.py diff --git a/tests/pytest/query/bug3351.py b/tests/pytest/query/bug3351.py new file mode 100644 index 0000000000..288d071a69 --- /dev/null +++ b/tests/pytest/query/bug3351.py @@ -0,0 +1,74 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + tdSql.execute("drop database if exists db") + tdSql.execute("create database if not exists db keep 36500") + tdSql.execute("use db") + tdLog.printNoPrefix("==========step1:create table && insert data") + + tdSql.execute( + "create table stb1 (ts timestamp, c1 int) TAGS(t1 int)" + ) + tdSql.execute("create table t0 using stb1 tags(1)") + tdSql.execute("insert into t0 values (-865000000, 1)") + tdSql.execute("insert into t0 values (-864000000, 2)") + tdSql.execute("insert into t0 values (-863000000, 3)") + tdSql.execute("insert into t0 values (-15230000, 4)") + tdSql.execute("insert into t0 values (-15220000, 5)") + tdSql.execute("insert into t0 values (-15210000, 6)") + + tdLog.printNoPrefix("==========step2:query") + # bug1:when ts > -864000000, return 0 rows; + # bug2:when ts = -15220000, return 0 rows. + tdSql.query('select * from t0 where ts < -864000000') + tdSql.checkRows(1) + tdSql.query('select * from t0 where ts <= -864000000') + tdSql.checkRows(2) + tdSql.query('select * from t0 where ts = -864000000') + tdSql.checkRows(1) + tdSql.query('select * from t0 where ts > -864000000') + tdSql.checkRows(4) + tdSql.query('select * from t0 where ts >= -864000000') + tdSql.checkRows(5) + tdSql.query('select * from t0 where ts < -15220000') + tdSql.checkRows(4) + tdSql.query('select * from t0 where ts <= -15220000') + tdSql.checkRows(5) + tdSql.query('select * from t0 where ts = -15220000') + tdSql.checkRows(1) + tdSql.query('select * from t0 where ts > -15220000') + tdSql.checkRows(1) + tdSql.query('select * from t0 where ts >= -15220000') + tdSql.checkRows(2) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/pytest/query/query1970YearsAf.py b/tests/pytest/query/query1970YearsAf.py index 9902f6908b..d2eead241f 100644 --- a/tests/pytest/query/query1970YearsAf.py +++ b/tests/pytest/query/query1970YearsAf.py @@ -88,7 +88,7 @@ class TDTestCase: "disorder_ratio": 0, "disorder_range": 1000, "timestamp_step": 20000, - "start_timestamp": "1969-12-31 00:00:00.000", + "start_timestamp": "1969-12-30 23:59:40.000", "sample_format": "csv", "sample_file": "./sample.csv", "tags_file": "", @@ -153,7 +153,7 @@ class TDTestCase: return file_create_table def inserttable(self, filepath): - create_table_cmd = f"taosdemo -f {filepath} > /tmp/1.log 2>&1" + create_table_cmd = f"taosdemo -f {filepath} > /dev/null 2>&1" _ = subprocess.check_output(create_table_cmd, shell=True).decode("utf-8") def sqlsquery(self): From ef248d09fd248775dfea4a88136354b42dd54f1e Mon Sep 17 00:00:00 2001 From: wu champion Date: Mon, 22 Mar 2021 17:02:53 +0800 Subject: [PATCH 4/7] fix case,add delete file when processing over --- tests/pytest/query/query1970YearsAf.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/pytest/query/query1970YearsAf.py b/tests/pytest/query/query1970YearsAf.py index d2eead241f..441f835e89 100644 --- a/tests/pytest/query/query1970YearsAf.py +++ b/tests/pytest/query/query1970YearsAf.py @@ -18,7 +18,6 @@ import json import subprocess import datetime - from util.log import * from util.sql import * from util.cases import * @@ -71,7 +70,7 @@ class TDTestCase: "update": 0 } - # 设置创建的超级表格式 + # set stable schema stable1 = { "name": "stb2", "child_table_exists": "no", @@ -83,7 +82,7 @@ class TDTestCase: "insert_rows": 5000, "multi_thread_write_one_tbl": "no", "number_of_tbl_in_one_sql": 0, - "rows_per_tbl": 1000, + "rows_per_tbl": 1, "max_sql_len": 65480, "disorder_ratio": 0, "disorder_range": 1000, @@ -117,7 +116,7 @@ class TDTestCase: ] } - # 创建不同的超级表格式并添加至super_tables + # create different stables like stable1 and add to list super_tables super_tables = [] super_tables.append(stable1) database = { @@ -235,7 +234,7 @@ class TDTestCase: tdLog.info("==========step2:query join") self.sqlsquery() - # 进行数据落盘后检查 + # after wal and sync, check again tdSql.query("show dnodes") index = tdSql.getData(0, 0) tdDnodes.stop(index) @@ -246,8 +245,9 @@ class TDTestCase: def stop(self): tdSql.close() + rm_cmd = f"rm -f /tmp/insert* > /dev/null 2>&1" + _ = subprocess.check_output(rm_cmd, shell=True).decode("utf-8") tdLog.success(f"{__file__} successfully executed") - tdCases.addLinux(__file__, TDTestCase()) tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file From 2de96282490234d89195ad401d001bc71ab71bb5 Mon Sep 17 00:00:00 2001 From: wu champion Date: Tue, 23 Mar 2021 13:40:29 +0800 Subject: [PATCH 5/7] fix the case --- tests/pytest/query/query1970YearsAf.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/pytest/query/query1970YearsAf.py b/tests/pytest/query/query1970YearsAf.py index 441f835e89..9cc8948c2f 100644 --- a/tests/pytest/query/query1970YearsAf.py +++ b/tests/pytest/query/query1970YearsAf.py @@ -243,10 +243,11 @@ class TDTestCase: tdLog.info("==========step3: query join again") self.sqlsquery() - def stop(self): - tdSql.close() rm_cmd = f"rm -f /tmp/insert* > /dev/null 2>&1" _ = subprocess.check_output(rm_cmd, shell=True).decode("utf-8") + + def stop(self): + tdSql.close() tdLog.success(f"{__file__} successfully executed") tdCases.addLinux(__file__, TDTestCase()) From 965d553964aea78aeb2e59b3bad9811ea08a86f5 Mon Sep 17 00:00:00 2001 From: wu champion Date: Tue, 23 Mar 2021 13:59:29 +0800 Subject: [PATCH 6/7] fix the case:add delete tmp file step --- tests/pytest/query/query1970YearsAf.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/pytest/query/query1970YearsAf.py b/tests/pytest/query/query1970YearsAf.py index 9cc8948c2f..ea83d8669d 100644 --- a/tests/pytest/query/query1970YearsAf.py +++ b/tests/pytest/query/query1970YearsAf.py @@ -87,7 +87,7 @@ class TDTestCase: "disorder_ratio": 0, "disorder_range": 1000, "timestamp_step": 20000, - "start_timestamp": "1969-12-30 23:59:40.000", + "start_timestamp": "1969-12-31 00:00:00.000", "sample_format": "csv", "sample_file": "./sample.csv", "tags_file": "", @@ -172,7 +172,7 @@ class TDTestCase: ) tdSql.checkRows(3590) - # child-table query + # child-tables query tdSql.query( "select * from t0 where t0.ts < '1970-01-01 00:00:00.000' " ) @@ -243,6 +243,7 @@ class TDTestCase: tdLog.info("==========step3: query join again") self.sqlsquery() + # delete temporary file rm_cmd = f"rm -f /tmp/insert* > /dev/null 2>&1" _ = subprocess.check_output(rm_cmd, shell=True).decode("utf-8") From 5f27c38a0b0753d1ff9bcab9fbf64dd378d923ab Mon Sep 17 00:00:00 2001 From: wu champion <72908628+wu-champion@users.noreply.github.com> Date: Tue, 23 Mar 2021 14:10:25 +0800 Subject: [PATCH 7/7] Update query1970YearsAf.py --- tests/pytest/query/query1970YearsAf.py | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/tests/pytest/query/query1970YearsAf.py b/tests/pytest/query/query1970YearsAf.py index 307180a29c..b3f3ff7a82 100644 --- a/tests/pytest/query/query1970YearsAf.py +++ b/tests/pytest/query/query1970YearsAf.py @@ -87,11 +87,7 @@ class TDTestCase: "disorder_ratio": 0, "disorder_range": 1000, "timestamp_step": 20000, -<<<<<<< HEAD "start_timestamp": "1969-12-31 00:00:00.000", -======= - "start_timestamp": "1969-12-30 23:59:40.000", ->>>>>>> 53a128b2a3a469b06ae28ff013397b8d015433c8 "sample_format": "csv", "sample_file": "./sample.csv", "tags_file": "", @@ -176,11 +172,7 @@ class TDTestCase: ) tdSql.checkRows(3590) -<<<<<<< HEAD # child-tables query -======= - # child-table query ->>>>>>> 53a128b2a3a469b06ae28ff013397b8d015433c8 tdSql.query( "select * from t0 where t0.ts < '1970-01-01 00:00:00.000' " ) @@ -251,20 +243,13 @@ class TDTestCase: tdLog.info("==========step3: query join again") self.sqlsquery() -<<<<<<< HEAD # delete temporary file rm_cmd = f"rm -f /tmp/insert* > /dev/null 2>&1" _ = subprocess.check_output(rm_cmd, shell=True).decode("utf-8") def stop(self): tdSql.close() -======= - def stop(self): - tdSql.close() - rm_cmd = f"rm -f /tmp/insert* > /dev/null 2>&1" - _ = subprocess.check_output(rm_cmd, shell=True).decode("utf-8") ->>>>>>> 53a128b2a3a469b06ae28ff013397b8d015433c8 tdLog.success(f"{__file__} successfully executed") tdCases.addLinux(__file__, TDTestCase()) -tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file +tdCases.addWindows(__file__, TDTestCase())