[TD-4552]<test>: add testcase that compact wal file with replica

tests/pytest/wal/insertDataDb1Replica2.json

@@ -0,0 +1,87 @@
+{
+    "filetype": "insert",
+    "cfgdir": "/etc/taos",
+    "host": "127.0.0.1",
+    "port": 6030,
+    "user": "root",
+    "password": "taosdata",
+    "thread_count": 4,
+    "thread_count_create_tbl": 4,
+    "result_file": "./insert_res.txt",
+    "confirm_parameter_prompt": "no",
+    "insert_interval": 0,
+    "interlace_rows": 10,
+    "num_of_records_per_req": 1000,
+    "max_sql_len": 1024000,
+    "databases": [{
+        "dbinfo": {
+            "name": "db1",
+            "drop": "yes",
+            "replica": 2,
+            "days": 10,
+            "cache": 50,
+            "blocks": 8,
+            "precision": "ms",
+            "keep": 365,
+            "minRows": 100,
+            "maxRows": 4096,
+            "comp":2,
+            "walLevel":1,
+            "cachelast":0,
+            "quorum":1,
+            "fsync":3000,
+            "update": 0
+        },
+        "super_tables": [{
+            "name": "stb0",
+            "child_table_exists":"no",
+            "childtable_count": 1000,
+            "childtable_prefix": "stb00_",
+            "auto_create_table": "no",
+            "batch_create_tbl_num": 100,
+            "data_source": "rand",
+            "insert_mode": "taosc",
+            "insert_rows": 100,
+            "childtable_limit": 0,
+            "childtable_offset":0,
+            "interlace_rows": 0,
+            "insert_interval":0,
+            "max_sql_len": 1024000,
+            "disorder_ratio": 0,
+            "disorder_range": 1000,
+            "timestamp_step": 1,
+            "start_timestamp": "2020-10-01 00:00:00.000",
+            "sample_format": "csv",
+            "sample_file": "./sample.csv",
+            "tags_file": "",
+            "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}],
+            "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}]
+        },
+        {
+            "name": "stb1",
+            "child_table_exists":"no",
+            "childtable_count": 1000,
+            "childtable_prefix": "stb01_",
+            "auto_create_table": "no",
+            "batch_create_tbl_num": 10,
+            "data_source": "rand",
+            "insert_mode": "taosc",
+            "insert_rows": 200,
+            "childtable_limit": 0,
+            "childtable_offset":0,
+            "interlace_rows": 0,
+            "insert_interval":0,
+            "max_sql_len": 1024000,
+            "disorder_ratio": 0,
+            "disorder_range": 1000,
+            "timestamp_step": 1,
+            "start_timestamp": "2020-10-01 00:00:00.000",
+            "sample_format": "csv",
+            "sample_file": "./sample.csv",
+            "tags_file": "",
+            "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}],
+            "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}]
+        }]
+    }]
+}
+

tests/pytest/wal/insertDataDb2NewstabReplica2.json

@@ -0,0 +1,86 @@
+{
+    "filetype": "insert",
+    "cfgdir": "/etc/taos",
+    "host": "127.0.0.1",
+    "port": 6030,
+    "user": "root",
+    "password": "taosdata",
+    "thread_count": 4,
+    "thread_count_create_tbl": 4,
+    "result_file": "./insert_res.txt",
+    "confirm_parameter_prompt": "no",
+    "insert_interval": 0,
+    "interlace_rows": 0,
+    "num_of_records_per_req": 3000,
+    "max_sql_len": 1024000,
+    "databases": [{
+        "dbinfo": {
+            "name": "db2",
+            "drop": "no",
+            "replica": 2,
+            "days": 10,
+            "cache": 50,
+            "blocks": 8,
+            "precision": "ms",
+            "keep": 365,
+            "minRows": 100,
+            "maxRows": 4096,
+            "comp":2,
+            "walLevel":1,
+            "cachelast":0,
+            "quorum":1,
+            "fsync":3000,
+            "update": 0
+        },
+        "super_tables": [{
+            "name": "stb0",
+            "child_table_exists":"no",
+            "childtable_count": 1,
+            "childtable_prefix": "stb0_",
+            "auto_create_table": "no",
+            "batch_create_tbl_num": 100,
+            "data_source": "rand",
+            "insert_mode": "taosc",
+            "insert_rows": 0,
+            "childtable_limit": -1,
+            "childtable_offset":0,
+            "interlace_rows": 0,
+            "insert_interval":0,
+            "max_sql_len": 1024000,
+            "disorder_ratio": 0,
+            "disorder_range": 1000,
+            "timestamp_step": 1,
+            "start_timestamp": "2020-10-01 00:00:00.000",
+            "sample_format": "csv",
+            "sample_file": "./sample.csv",
+            "tags_file": "",
+            "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}],
+            "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}]
+        },
+        {
+            "name": "stb1",
+            "child_table_exists":"yes",
+            "childtable_count": 1,
+            "childtable_prefix": "stb01_",
+            "auto_create_table": "no",
+            "batch_create_tbl_num": 10,
+            "data_source": "rand",
+            "insert_mode": "taosc",
+            "insert_rows": 10,
+            "childtable_limit": -1,
+            "childtable_offset":0,
+            "interlace_rows": 0,
+            "insert_interval":0,
+            "max_sql_len": 1024000,
+            "disorder_ratio": 0,
+            "disorder_range": 1000,
+            "timestamp_step": 1,
+            "start_timestamp": "2020-11-01 00:00:00.000",
+            "sample_format": "csv",
+            "sample_file": "./sample.csv",
+            "tags_file": "",
+            "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}],
+            "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}]
+        }]
+    }]
+}

tests/pytest/wal/insertDataDb2Replica2.json

@@ -0,0 +1,86 @@
+{
+    "filetype": "insert",
+    "cfgdir": "/etc/taos",
+    "host": "127.0.0.1",
+    "port": 6030,
+    "user": "root",
+    "password": "taosdata",
+    "thread_count": 4,
+    "thread_count_create_tbl": 4,
+    "result_file": "./insert_res.txt",
+    "confirm_parameter_prompt": "no",
+    "insert_interval": 0,
+    "interlace_rows": 0,
+    "num_of_records_per_req": 3000,
+    "max_sql_len": 1024000,
+    "databases": [{
+        "dbinfo": {
+            "name": "db2",
+            "drop": "yes",
+            "replica": 2,
+            "days": 10,
+            "cache": 50,
+            "blocks": 8,
+            "precision": "ms",
+            "keep": 365,
+            "minRows": 100,
+            "maxRows": 4096,
+            "comp":2,
+            "walLevel":1,
+            "cachelast":0,
+            "quorum":1,
+            "fsync":3000,
+            "update": 0
+        },
+        "super_tables": [{
+            "name": "stb0",
+            "child_table_exists":"no",
+            "childtable_count": 2000,
+            "childtable_prefix": "stb0_",
+            "auto_create_table": "no",
+            "batch_create_tbl_num": 100,
+            "data_source": "rand",
+            "insert_mode": "taosc",
+            "insert_rows": 2000,
+            "childtable_limit": 0,
+            "childtable_offset":0,
+            "interlace_rows": 0,
+            "insert_interval":0,
+            "max_sql_len": 1024000,
+            "disorder_ratio": 0,
+            "disorder_range": 1000,
+            "timestamp_step": 1,
+            "start_timestamp": "2020-10-01 00:00:00.000",
+            "sample_format": "csv",
+            "sample_file": "./sample.csv",
+            "tags_file": "",
+            "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}],
+            "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}]
+        },
+        {
+            "name": "stb1",
+            "child_table_exists":"no",
+            "childtable_count": 2,
+            "childtable_prefix": "stb1_",
+            "auto_create_table": "no",
+            "batch_create_tbl_num": 10,
+            "data_source": "rand",
+            "insert_mode": "taosc",
+            "insert_rows": 5,
+            "childtable_limit": 0,
+            "childtable_offset":0,
+            "interlace_rows": 0,
+            "insert_interval":0,
+            "max_sql_len": 1024000,
+            "disorder_ratio": 0,
+            "disorder_range": 1000,
+            "timestamp_step": 1,
+            "start_timestamp": "2020-10-01 00:00:00.000",
+            "sample_format": "csv",
+            "sample_file": "./sample.csv",
+            "tags_file": "",
+            "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}],
+            "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}]
+        }]
+    }]
+}

tests/pytest/wal/sdbComp.py

@@ -108,10 +108,10 @@ class TDTestCase:
         tdSql.query("select count(*) from stb2_0")
         tdSql.checkData(0, 0, 2)
    
-
+        # delete useless file
-
+        testcaseFilename = os.path.split(__file__)[-1]
         os.system("rm -rf ./insert_res.txt")
-        os.system("rm -rf wal/sdbComp.py.sql")        
+        os.system("rm -rf wal/%s.sql" % testcaseFilename )       
         
         
         

tests/pytest/wal/sdbCompCluster.py

@@ -61,7 +61,6 @@ class TwoClients:
         print(conn1)
         cur1 = conn1.cursor()
         tdSql.init(cur1, True)
-        # tdSql.init(cur2, True)
 
         # new db and insert data
         os.system("rm -rf  /var/lib/taos/mnode_bak/")
@@ -87,17 +86,12 @@ class TwoClients:
         tdSql.execute("alter table stb2_0 add column col2 binary(4)")
         tdSql.execute("alter table stb2_0 drop column col1")
         tdSql.execute("insert into stb2_0 values(1614218422000,8638,'R')")
-        
-        conn2 = taos.connect(host=self.host, user=self.user, password=self.password, config=self.config )
-        print(conn2)
-        cur2 = conn2.cursor()
-        tdSql.init(cur2, True)
 
         # stop taosd and compact wal file
-        os.system("ps -ef |grep taosd |grep -v 'grep' |awk '{print $2}'|xargs kill -9")
+        os.system("ps -ef |grep taosd |grep -v 'grep' |awk '{print $2}'|xargs kill -2")
-        sleep(2)
+        sleep(10)
         os.system("nohup taosd  --compact-mnode-wal  -c /etc/taos & ")
-        sleep(5)
+        sleep(10)
         os.system("nohup /usr/bin/taosd > /dev/null 2>&1 &")
         sleep(4)
         tdSql.execute("reset query cache")
@@ -105,7 +99,17 @@ class TwoClients:
         print(query_pid2)
         assert os.path.exists(walFilePath) , "%s is not generated " % walFilePath    
 
+        # new taos connecting to server
+        conn2 = taos.connect(host=self.host, user=self.user, password=self.password, config=self.config )
+        print(conn2)
+        cur2 = conn2.cursor()
+        tdSql.init(cur2, True)
+
         # use new wal file to start up tasod 
+        tdSql.query("show databases")
+        for i in range(tdSql.queryRows):
+            if tdSql.queryResult[i][0]=="db2":
+                assert tdSql.queryResult[i][4]==1 , "replica is wrong"
         tdSql.execute("use db2")
         tdSql.query("select count (tbname) from stb0")
         tdSql.checkData(0, 0, 1)
@@ -117,7 +121,14 @@ class TwoClients:
         tdSql.checkData(0, 0, 2)
         tdSql.query("select count(*) from stb2_0")
         tdSql.checkData(0, 0, 2)
-   
+        tdSql.query("select * from stb2_0")
+        tdSql.checkData(1, 2, 'R')
+        
+        # delete useless file       
+        testcaseFilename = os.path.split(__file__)[-1]
+        os.system("rm -rf ./insert_res.txt")
+        os.system("rm -rf wal/%s.sql" % testcaseFilename )       
+
 clients = TwoClients()
 clients.initConnection()
 # clients.getBuildPath()

tests/pytest/wal/sdbCompClusterReplica2.py

@@ -0,0 +1,136 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import os
+import sys
+sys.path.insert(0, os.getcwd())
+from util.log import *
+from util.sql import *
+from util.dnodes import *
+import taos
+import threading
+
+ 
+class TwoClients:
+    def initConnection(self):
+        self.host = "chenhaoran02"
+        self.user = "root"
+        self.password = "taosdata"
+        self.config = "/etc/taos/"     
+        self.port =6030 
+        self.rowNum = 10
+        self.ts = 1537146000000  
+
+    def getBuildPath(self):
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+
+        for root, dirs, files in os.walk(projPath):
+            if ("taosd" in files):
+                rootRealPath = os.path.dirname(os.path.realpath(root))
+                if ("packaging" not in rootRealPath):
+                    buildPath = root[:len(root)-len("/build/bin")]
+                    break
+        return buildPath
+
+    def run(self):
+        buildPath = self.getBuildPath()
+        if (buildPath == ""):
+            tdLog.exit("taosd not found!")
+        else:
+            tdLog.info("taosd found in %s" % buildPath)
+        binPath = buildPath+ "/build/bin/"
+        walFilePath = "/var/lib/taos/mnode_bak/wal/"
+        
+        # new taos client
+        conn1 = taos.connect(host=self.host, user=self.user, password=self.password, config=self.config )
+        print(conn1)
+        cur1 = conn1.cursor()
+        tdSql.init(cur1, True)
+
+        # new db and insert data
+        os.system("rm -rf  /var/lib/taos/mnode_bak/")
+        os.system("rm -rf  /var/lib/taos/mnode_temp/")
+        tdSql.execute("drop database if exists db2")
+        os.system("%staosdemo -f wal/insertDataDb1Replica2.json -y " % binPath)
+        tdSql.execute("drop database if exists db1") 
+        os.system("%staosdemo -f wal/insertDataDb2Replica2.json -y " % binPath)
+        tdSql.execute("drop table if exists db2.stb0") 
+        os.system("%staosdemo -f wal/insertDataDb2NewstabReplica2.json -y " % binPath)
+        query_pid1 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1])
+        print(query_pid1)
+        tdSql.execute("use db2")
+        tdSql.execute("drop table if exists stb1_0")
+        tdSql.execute("drop table if exists stb1_1")
+        tdSql.execute("insert into stb0_0 values(1614218412000,8637,78.861045,'R','bf3')(1614218422000,8637,98.861045,'R','bf3')")
+        tdSql.execute("alter table db2.stb0 add column col4 int")
+        tdSql.execute("alter table db2.stb0 drop column col2")
+        tdSql.execute("alter table db2.stb0  add tag t3 int")        
+        tdSql.execute("alter table db2.stb0 drop tag t1")
+        tdSql.execute("create table  if not exists stb2_0 (ts timestamp, col0 int, col1 float)  ")
+        tdSql.execute("insert into stb2_0 values(1614218412000,8637,78.861045)")
+        tdSql.execute("alter table stb2_0 add column col2 binary(4)")
+        tdSql.execute("alter table stb2_0 drop column col1")
+        tdSql.execute("insert into stb2_0 values(1614218422000,8638,'R')")
+        
+
+        # stop taosd and compact wal file
+        os.system("ps -ef |grep taosd |grep -v 'grep' |awk '{print $2}'|xargs kill -2")
+        sleep(10)
+        os.system("nohup taosd  --compact-mnode-wal  -c /etc/taos & ")
+        sleep(10)
+        os.system("nohup /usr/bin/taosd > /dev/null 2>&1 &")
+        sleep(4)
+        tdSql.execute("reset query cache")
+        query_pid2 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1])
+        print(query_pid2)
+        assert os.path.exists(walFilePath) , "%s is not generated " % walFilePath    
+
+        # new taos connecting to server
+        conn2 = taos.connect(host=self.host, user=self.user, password=self.password, config=self.config )
+        print(conn2)
+        cur2 = conn2.cursor()
+        tdSql.init(cur2, True)
+
+        # use new wal file to start up tasod 
+        tdSql.query("show databases")
+        for i in range(tdSql.queryRows):
+            if tdSql.queryResult[i][0]=="db2":
+                assert tdSql.queryResult[i][4]==2 , "replica is wrong"
+        tdSql.execute("use db2")
+        tdSql.query("select count (tbname) from stb0")
+        tdSql.checkData(0, 0, 1)
+        tdSql.query("select count (tbname) from stb1")
+        tdSql.checkRows(0)
+        tdSql.query("select count(*) from stb0_0")
+        tdSql.checkData(0, 0, 2)
+        tdSql.query("select count(*) from stb0")
+        tdSql.checkData(0, 0, 2)
+        tdSql.query("select count(*) from stb2_0")
+        tdSql.checkData(0, 0, 2)
+        tdSql.query("select * from stb2_0")
+        tdSql.checkData(1, 2, 'R')
+
+        # delete useless file
+        testcaseFilename = os.path.split(__file__)[-1]
+        os.system("rm -rf ./insert_res.txt")
+        os.system("rm -rf wal/%s.sql" % testcaseFilename )       
+   
+clients = TwoClients()
+clients.initConnection()
+# clients.getBuildPath()
+clients.run()