test: add case for TS-2811 wide column

tests/parallel_test/cases.task

@@ -140,6 +140,7 @@
 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/update_data_muti_rows.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/db_tb_name_check.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/InsertFuturets.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/insert_wide_column.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/information_schema.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py

tests/pytest/util/autogen.py

@@ -0,0 +1,173 @@
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+import threading
+import random
+import string
+import time
+
+
+#
+# Auto Gen class
+#
+class AutoGen:
+    def __init__(self):
+        self.ts = 1600000000000
+        self.batch_size = 100
+        seed = time.clock_gettime(time.CLOCK_REALTIME)
+        random.seed(seed)
+
+    # set start ts
+    def set_start_ts(self, ts):
+        self.ts = ts
+
+    # set batch size
+    def set_batch_size(self, batch_size):
+        self.batch_size = batch_size
+
+    #  _columns_sql
+    def gen_columns_sql(self, pre, cnt, binary_len, nchar_len):
+        types = [ 
+            'timestamp',
+            'tinyint',
+            'smallint',
+            'tinyint unsigned',
+            'smallint unsigned',
+            'int',
+            'bigint',
+            'int unsigned',
+            'bigint unsigned',
+            'float',
+            'double',
+            'bool',
+            f'varchar({binary_len})',
+            f'nchar({nchar_len})'
+        ]
+
+        sqls = ""
+        metas = []
+        for i in range(cnt):
+            colname = f"{pre}{i}"
+            if i < len(types):
+               sel = i
+            else:
+               sel = random.randint(0, len(types)-1)
+            coltype = types[sel]
+            sql = f"{colname} {coltype}"
+            if sqls != "":
+                sqls += ","
+            sqls += sql
+            metas.append(sel)
+        
+        return metas, sqls;    
+
+    # gen tags data
+    def gen_data(self, i, marr):
+        datas = ""   
+        for c in marr:
+            data = ""
+            if c == 0 : # timestamp
+                data = "%d" % (self.ts + i)
+            elif c <= 4 : # small
+                data = "%d"%(i%128)
+            elif c <= 8 : # int
+                data = f"{i}"
+            elif c <= 10 : # float
+                data = "%f"%(i+i/1000)
+            elif c <= 11 : # bool
+                data = "%d"%(i%2)
+            elif c == 12 : # binary
+                data = '"' + self.random_string(self.bin_len) + '"'
+            elif c == 13 : # binary
+                data = '"' + self.random_string(self.nch_len) + '"'
+
+            if datas != "":
+                datas += ","
+            datas += data
+        
+        return datas
+
+    # generate specail wide random string
+    def random_string(self, count):
+        letters = string.ascii_letters
+        return ''.join(random.choice(letters) for i in range(count))
+
+    # create db
+    def create_db(self, dbname):
+        self.dbname  = dbname
+        tdSql.execute(f'create database {dbname}')
+        tdSql.execute(f'use {dbname}')
+        
+    # create table or stable
+    def create_stable(self, stbname, tag_cnt, column_cnt, binary_len, nchar_len):
+        self.bin_len = binary_len
+        self.nch_len = nchar_len
+        self.stbname = stbname
+        self.mtags, tags = self.gen_columns_sql("t", tag_cnt, binary_len, nchar_len)
+        self.mcols, cols = self.gen_columns_sql("c", column_cnt - 1, binary_len, nchar_len)
+
+        sql = f"create table {stbname} (ts timestamp, {cols}) tags({tags})"
+        tdSql.execute(sql)
+
+        # create stream 
+        sql = f"create stream m_{stbname} fill_history 1 into st_{stbname} as select count(*) from {stbname} interval(100a);"
+        tdLog.info(f" create stable {stbname} ok. column={column_cnt} tag={tag_cnt} ")
+
+
+    # create child table 
+    def create_child(self, stbname, prename, cnt):
+        self.child_cnt = cnt
+        self.child_name = prename
+        for i in range(cnt):
+            tags_data = self.gen_data(i, self.mtags)
+            sql = f"create table {prename}{i} using {stbname} tags({tags_data})"
+            tdSql.execute(sql)
+
+        tdLog.info(f"create child tables {cnt} ok")
+
+    def insert_data_child(self, child_name, cnt, batch_size, step):        
+        values = ""
+        print("insert child data")
+        ts = self.ts
+
+        # loop do
+        for i in range(cnt):
+            value = self.gen_data(i, self.mcols)
+            ts += step
+            values += f"({ts},{value}) "
+            if batch_size == 1 or (i > 0 and i % batch_size == 0) :
+                sql = f"insert into {child_name} values {values}"
+                tdSql.execute(sql)
+                if batch_size > 40:
+                   tdLog.info(f" insert data i={i}")
+                values = ""
+
+        # end batch
+        if values != "":
+            sql = f"insert into {child_name} values {values}"
+            tdSql.execute(sql)
+            tdLog.info(f" insert data i={i}")
+            values = ""
+
+        tdLog.info(f" insert child data {child_name} finished, insert rows={cnt}")    
+
+    # insert data 
+    def insert_data(self, cnt):
+        for i in range(self.child_cnt):
+            name = f"{self.child_name}{i}"
+            self.insert_data_child(name, cnt, self.batch_size, 1)
+
+        tdLog.info(f" insert data ok, child table={self.child_cnt} insert rows={cnt}")
+
+    # insert same timestamp to all childs
+    def insert_samets(self, cnt):
+        for i in range(self.child_cnt):
+            name = f"{self.child_name}{i}"
+            self.insert_data_child(name, cnt, self.batch_size, 0)
+
+        tdLog.info(f" insert same timestamp ok, child table={self.child_cnt} insert rows={cnt}")         
+
+

tests/system-test/1-insert/insert_wide_coulum.py

@@ -0,0 +1,76 @@
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.autogen import *
+import threading
+import random
+import string
+import time
+
+
+#
+# Test Main class
+#
+class TDTestCase:
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+        self.autoGen = AutoGen()
+
+    def query_test(self, stbname):
+        sql = f"select count(*) from {stbname}"
+        tdSql.execute(sql)
+        sql = f"select * from {stbname} order by ts desc;"
+        tdSql.execute(sql)
+        sql = f"select * from (select * from {stbname} where c1=c2 or c3=c4 or c5=c6) order by ts desc;"
+        tdSql.execute(sql)
+
+        tdLog.info(" test query ok!")
+
+
+    def test_db(self, dbname, stbname, childname, tag_cnt, column_cnt, child_cnt, insert_rows, binary_len, nchar_len):    
+        self.autoGen.create_db(dbname)
+        self.autoGen.create_stable(stbname, tag_cnt, column_cnt, binary_len, nchar_len)
+        self.autoGen.create_child(stbname, childname, child_cnt)
+        self.autoGen.insert_data(insert_rows)
+        self.autoGen.insert_samets(insert_rows)
+        self.query_test(stbname)
+
+    def run(self):  
+        dbname = "test"
+        stbname = "st"
+        childname = "d"
+        child_cnt = 20
+        insert_rows = 1000
+        tag_cnt    = 15
+        column_cnt = 25
+        binary_len = 30
+        nchar_len =  45
+        self.autoGen.set_batch_size(500)
+        
+        # normal
+        self.test_db(dbname, stbname, childname, tag_cnt, column_cnt, child_cnt, insert_rows, binary_len, nchar_len)
+
+        # max
+        dbname = "test_max_col"
+        child_cnt = 3
+        insert_rows = 100
+        tag_cnt = 128
+        binary_len = 3
+        nchar_len = 4
+        column_cnt = 4096 - tag_cnt
+        self.autoGen.set_batch_size(1)
+        self.test_db(dbname, stbname, childname, tag_cnt, column_cnt, child_cnt, insert_rows, binary_len, nchar_len)
+        
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())