From 240d2ee8ad428cb1b08fe1f58d3ae4d427f6548e Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Mon, 6 Nov 2023 16:52:11 +0800 Subject: [PATCH 1/3] case : check file can be delete after drop table --- tests/system-test/0-others/delete_check.py | 154 +++++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 tests/system-test/0-others/delete_check.py diff --git a/tests/system-test/0-others/delete_check.py b/tests/system-test/0-others/delete_check.py new file mode 100644 index 0000000000..ea5b77a51c --- /dev/null +++ b/tests/system-test/0-others/delete_check.py @@ -0,0 +1,154 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * +from util.autogen import * + +import random +import time +import traceback +import os +from os import path + + +class TDTestCase: + # init + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + + # autoGen + self.autoGen = AutoGen() + # init cluster path + selfPath = os.path.dirname(os.path.realpath(__file__)) + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + self.projDir = f"{projPath}sim/" + tdLog.info(f" init projPath={self.projDir}") + + def compactDatbase(self): + # compact database + tdSql.execute(f"compact database {self.dbname}", show=True) + waitSeconds = 20 + if self.waitTranslation(waitSeconds) == False: + tdLog.exit(f"translation can not finish after wait {waitSeconds} seconds") + return + + # check tsdb folder empty + def check_tsdb_dir(self, tsdbDir): + for vfile in os.listdir(tsdbDir): + fileName, ext = os.path.splitext(vfile) + pathFile = os.path.join(tsdbDir, vfile) + ext = ext.lower() + tdLog.info(f"check exist file {pathFile} ...") + if ext == ".head" or ext == ".data" or ext == ".stt" or ext == ".sma": + tdLog.info(f"found {pathFile} not to be deleted ...") + real = True + for i in range(50): + tdLog.info(f"i={i} compact and try again ...") + self.compactDatbase() + if os.path.exists(pathFile) is False: + real = False + break + else: + time.sleep(0.5) + tdLog.info(f"file real exist {pathFile} , sleep 500ms and try") + + if real is False: + continue + fileStat = os.stat(pathFile) + tdLog.exit(f" check file can not be deleted. file={pathFile} file size={fileStat.st_size}") + + return True + + # check vnode tsdb folder empty + def check_filedelete(self): + # put all vnode to list + for dnode in os.listdir(self.projDir): + vnodesDir = self.projDir + f"{dnode}/data/vnode/" + if os.path.isdir(vnodesDir) == False or dnode[:5] != "dnode": + continue + print(f"vnodesDir={vnodesDir}") + # enum all vnode + for vnode in os.listdir(vnodesDir): + vnodeDir = path.join(vnodesDir, vnode) + print(f"vnodeDir={vnodeDir}") + if os.path.isdir(vnodesDir): + tsdbDir = path.join(vnodeDir, "tsdb") + if path.exists(tsdbDir) : + self.check_tsdb_dir(tsdbDir) + + def waitTranslation(self, waitSeconds): + # wait end + for i in range(waitSeconds): + sql ="show transactions;" + rows = tdSql.query(sql) + if rows == 0: + return True + tdLog.info(f"i={i} wait for translation finish ...") + time.sleep(1) + + return False + + # run + def run(self): + # seed + random.seed(int(time.time())) + self.dbname = "deletecheck" + stbname = "meters" + childname= "d" + child_cnt = 2 + batch_size = 8000 + insert_rows = 200015 + start_ts = 1600000000000 + + self.autoGen.create_db(self.dbname) + + loop = 10 + for i in range(loop): + self.autoGen.create_stable(stbname, 4, 10, 4, 8) + self.autoGen.create_child(stbname, childname, child_cnt) + self.autoGen.set_batch_size(batch_size) + self.autoGen.insert_data(insert_rows) + self.autoGen.set_start_ts(start_ts) + + if i % 2 == 1: + tdSql.execute(f"flush database {self.dbname}", show=True) + + # drop stable + tdSql.execute(f"drop table {self.dbname}.{stbname} ", show = True) + + self.compactDatbase() + + # check file delete + self.check_filedelete() + tdLog.info(f"loop = {i+1} / {loop} check file delete ok after drop table successfully.") + + start_ts += i*100000000 + + + # stop + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From c6f4c0124d709c6aae607fd490f6bd7b4a9d1e65 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Mon, 6 Nov 2023 16:56:14 +0800 Subject: [PATCH 2/3] case : add delete_check.py to cases.task --- tests/parallel_test/cases.task | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 21dcd16441..2c861b4aa5 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -232,6 +232,8 @@ e ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/splitVGroupRep1.py -N 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/splitVGroupRep3.py -N 3 ,,n,system-test,python3 ./test.py -f 0-others/timeRangeWise.py -N 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/delete_check.py + ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_database.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_replica.py -N 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py From ecbe5e605138a0d73eeafaa8d5b789eb52d68a12 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 7 Nov 2023 16:38:26 +0800 Subject: [PATCH 3/3] case: reduce loop count from 10 to 3 --- tests/pytest/util/autogen.py | 2 -- tests/system-test/0-others/delete_check.py | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/pytest/util/autogen.py b/tests/pytest/util/autogen.py index 6f5c89d15b..e1227a680f 100644 --- a/tests/pytest/util/autogen.py +++ b/tests/pytest/util/autogen.py @@ -133,8 +133,6 @@ class AutoGen: if batch_size == 1 or (i > 0 and i % batch_size == 0) : sql = f"insert into {child_name} values {values}" tdSql.execute(sql) - if batch_size > 40: - tdLog.info(f" insert data i={i}") values = "" # end batch diff --git a/tests/system-test/0-others/delete_check.py b/tests/system-test/0-others/delete_check.py index ea5b77a51c..ec589295e4 100644 --- a/tests/system-test/0-others/delete_check.py +++ b/tests/system-test/0-others/delete_check.py @@ -117,12 +117,12 @@ class TDTestCase: childname= "d" child_cnt = 2 batch_size = 8000 - insert_rows = 200015 + insert_rows = 100015 start_ts = 1600000000000 self.autoGen.create_db(self.dbname) - loop = 10 + loop = 3 for i in range(loop): self.autoGen.create_stable(stbname, 4, 10, 4, 8) self.autoGen.create_child(stbname, childname, child_cnt)