xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #20416 from taosdata/enh/changeTagIndexCase 

change tag index case
This commit is contained in:
Shengliang Guan 2023-03-12 20:46:48 +08:00 committed by GitHub
parent c011e7d75f bc13432661
commit 5a957eeba2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 99 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

194
tests/system-test/0-others/tag_index_advance.py
View File

@ -11,45 +11,42 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE import os
import taos import random
import socket
import string
import subprocess
import sys import sys
import time import time
import os from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE
import socket
import subprocess
import random
import string
import random
import taos
from util.log import *
from util.sql import *
from util.cases import * from util.cases import *
from util.common import * from util.common import *
from util.dnodes import *
from util.dnodes import TDDnode, TDDnodes
from util.log import *
from util.sql import *
from util.sqlset import * from util.sqlset import *
from util.dnodes import *
from util.dnodes import TDDnodes
from util.dnodes import TDDnode
# #
# -------------- util -------------------------- # -------------- util --------------------------
# #
def pathSize(path): def pathSize(path):
total_size = 0 total_size = 0
for dirpath, dirnames, filenames in os.walk(path): for dirpath, dirnames, filenames in os.walk(path):
for i in filenames: for i in filenames:
#use join to concatenate all the components of path # use join to concatenate all the components of path
f = os.path.join(dirpath, i) f = os.path.join(dirpath, i)
#use getsize to generate size in bytes and add it to the total size # use getsize to generate size in bytes and add it to the total size
total_size += os.path.getsize(f) total_size += os.path.getsize(f)
#print(dirpath) # print(dirpath)
print(" %s %.02f MB"%(path, total_size/1024/1024)) print(" %s %.02f MB" % (path, total_size/1024/1024))
return total_size return total_size
''' '''
total = 0 total = 0
with os.scandir(path) as it: with os.scandir(path) as it:
@ -67,24 +64,27 @@ def pathSize(path):
# --------------- cluster ------------------------ # --------------- cluster ------------------------
# #
class MyDnodes(TDDnodes): class MyDnodes(TDDnodes):
def __init__(self ,dnodes_lists): def __init__(self, dnodes_lists):
super(MyDnodes,self).__init__() super(MyDnodes, self).__init__()
self.dnodes = dnodes_lists # dnode must be TDDnode instance self.dnodes = dnodes_lists # dnode must be TDDnode instance
self.simDeployed = False self.simDeployed = False
class TagCluster: class TagCluster:
noConn = True noConn = True
def init(self, conn, logSql, replicaVar=1): def init(self, conn, logSql, replicaVar=1):
tdLog.debug(f"start to excute {__file__}") tdLog.debug(f"start to excute {__file__}")
self.TDDnodes = None self.TDDnodes = None
self.depoly_cluster(5) self.depoly_cluster(5)
self.master_dnode = self.TDDnodes.dnodes[0] self.master_dnode = self.TDDnodes.dnodes[0]
self.host=self.master_dnode.cfgDict["fqdn"] self.host = self.master_dnode.cfgDict["fqdn"]
conn1 = taos.connect(self.master_dnode.cfgDict["fqdn"] , config=self.master_dnode.cfgDir) conn1 = taos.connect(
self.master_dnode.cfgDict["fqdn"], config=self.master_dnode.cfgDir)
tdSql.init(conn1.cursor()) tdSql.init(conn1.cursor())
def getBuildPath(self): def getBuildPath(self):
selfPath = os.path.dirname(os.path.realpath(__file__)) selfPath = os.path.dirname(os.path.realpath(__file__))
@ -101,8 +101,7 @@ class TagCluster:
break break
return buildPath return buildPath
def depoly_cluster(self, dnodes_nums):
def depoly_cluster(self ,dnodes_nums):
testCluster = False testCluster = False
valgrind = 0 valgrind = 0
@ -126,7 +125,7 @@ class TagCluster:
self.TDDnodes.setAsan(tdDnodes.getAsan()) self.TDDnodes.setAsan(tdDnodes.getAsan())
self.TDDnodes.stopAll() self.TDDnodes.stopAll()
for dnode in self.TDDnodes.dnodes: for dnode in self.TDDnodes.dnodes:
self.TDDnodes.deploy(dnode.index,{}) self.TDDnodes.deploy(dnode.index, {})
for dnode in self.TDDnodes.dnodes: for dnode in self.TDDnodes.dnodes:
self.TDDnodes.starttaosd(dnode.index) self.TDDnodes.starttaosd(dnode.index)
@ -136,7 +135,8 @@ class TagCluster:
sql = "" sql = ""
for dnode in self.TDDnodes.dnodes[1:]: for dnode in self.TDDnodes.dnodes[1:]:
# print(dnode.cfgDict) # print(dnode.cfgDict)
dnode_id = dnode.cfgDict["fqdn"] + ":" +dnode.cfgDict["serverPort"] dnode_id = dnode.cfgDict["fqdn"] + \
":" + dnode.cfgDict["serverPort"]
if dnode_first_host == "": if dnode_first_host == "":
dnode_first_host = dnode.cfgDict["firstEp"].split(":")[0] dnode_first_host = dnode.cfgDict["firstEp"].split(":")[0]
dnode_first_port = dnode.cfgDict["firstEp"].split(":")[-1] dnode_first_port = dnode.cfgDict["firstEp"].split(":")[-1]
@ -145,18 +145,17 @@ class TagCluster:
cmd = f"{self.getBuildPath()}/build/bin/taos -h {dnode_first_host} -P {dnode_first_port} -s " cmd = f"{self.getBuildPath()}/build/bin/taos -h {dnode_first_host} -P {dnode_first_port} -s "
cmd += f'"{sql}"' cmd += f'"{sql}"'
print(cmd) print(cmd)
os.system(cmd) os.system(cmd)
time.sleep(2) time.sleep(2)
tdLog.info(" create cluster done! ") tdLog.info(" create cluster done! ")
def getConnection(self, dnode): def getConnection(self, dnode):
host = dnode.cfgDict["fqdn"] host = dnode.cfgDict["fqdn"]
port = dnode.cfgDict["serverPort"] port = dnode.cfgDict["serverPort"]
config_dir = dnode.cfgDir config_dir = dnode.cfgDir
return taos.connect(host=host, port=int(port), config=config_dir) return taos.connect(host=host, port=int(port), config=config_dir)
def run(self): def run(self):
tdLog.info(" create cluster ok.") tdLog.info(" create cluster ok.")
@ -168,22 +167,22 @@ class TagCluster:
class PerfDB: class PerfDB:
def __init__(self): def __init__(self):
self.sqls = [] self.sqls = []
self.spends= [] self.spends = []
# execute # execute
def execute(self, sql): def execute(self, sql):
print(f" perfdb execute {sql}") print(f" perfdb execute {sql}")
stime = time.time() stime = time.time()
ret = tdSql.execute(sql, 1) ret = tdSql.execute(sql, 1)
spend = time.time() - stime spend = time.time() - stime
self.sqls.append(sql) self.sqls.append(sql)
self.spends.append(spend) self.spends.append(spend)
return ret return ret
# query # query
def query(self, sql): def query(self, sql):
print(f" perfdb query {sql}") print(f" perfdb query {sql}")
start = time.time() start = time.time()
ret = tdSql.query(sql, None, 1) ret = tdSql.query(sql, None, 1)
spend = time.time() - start spend = time.time() - start
@ -203,9 +202,9 @@ class TDTestCase:
self.tagCluster = TagCluster() self.tagCluster = TagCluster()
self.tagCluster.init(conn, logSql, replicaVar) self.tagCluster.init(conn, logSql, replicaVar)
self.lenBinary = 64 self.lenBinary = 64
self.lenNchar = 32 self.lenNchar = 32
# column # column
self.column_dict = { self.column_dict = {
'ts': 'timestamp', 'ts': 'timestamp',
'col1': 'tinyint', 'col1': 'tinyint',
@ -252,14 +251,14 @@ class TDTestCase:
# query # query
def query(self, sql): def query(self, sql):
return self.dbs[self.cur].query(sql) return self.dbs[self.cur].query(sql)
def set_stb_sql(self,stbname,column_dict,tag_dict): def set_stb_sql(self, stbname, column_dict, tag_dict):
column_sql = '' column_sql = ''
tag_sql = '' tag_sql = ''
for k,v in column_dict.items(): for k, v in column_dict.items():
column_sql += f"{k} {v}, " column_sql += f"{k} {v}, "
for k,v in tag_dict.items(): for k, v in tag_dict.items():
tag_sql += f"{k} {v}, " tag_sql += f"{k} {v}, "
create_stb_sql = f'create stable {stbname} ({column_sql[:-2]}) tags ({tag_sql[:-2]})' create_stb_sql = f'create stable {stbname} ({column_sql[:-2]}) tags ({tag_sql[:-2]})'
return create_stb_sql return create_stb_sql
@ -268,37 +267,41 @@ class TDTestCase:
def create_database(self, dbname, vgroups, replica): def create_database(self, dbname, vgroups, replica):
sql = f'create database {dbname} vgroups {vgroups} replica {replica}' sql = f'create database {dbname} vgroups {vgroups} replica {replica}'
tdSql.execute(sql) tdSql.execute(sql)
#tdSql.execute(sql) # tdSql.execute(sql)
tdSql.execute(f'use {dbname}') tdSql.execute(f'use {dbname}')
# create stable and child tables # create stable and child tables
def create_table(self, stbname, tbname, count): def create_table(self, stbname, tbname, count):
# create stable # create stable
create_table_sql = self.set_stb_sql(stbname, self.column_dict, self.tag_dict) create_table_sql = self.set_stb_sql(
stbname, self.column_dict, self.tag_dict)
tdSql.execute(create_table_sql) tdSql.execute(create_table_sql)
# create child table # create child table
tdLog.info(f" start create {count} child tables.") tdLog.info(f" start create {count} child tables.")
for i in range(count): batchSql = ""
ti = i % 128 batchSize = 5000
binTxt = self.random_string(self.lenBinary) for i in range(int(count/batchSize)):
tags = f'{ti},{ti},{i},{i},{ti},{ti},{i},{i},{i}.000{i},{i}.000{i},true,"{binTxt}","nch{i}",now' batchSql = "create table"
sql = f'create table {tbname}{i} using {stbname} tags({tags})' for j in range(batchSize):
tdSql.execute(sql) ti = (i * batchSize + j) % 128
if i > 0 and i % 1000 == 0: binTxt = self.random_string(self.lenBinary)
tdLog.info(f" child table count = {i}") idx = i * batchSize + j
tags = f'{ti},{ti},{idx},{idx},{ti},{ti},{idx},{idx},{idx}.000{idx},{idx}.000{idx},true,"{binTxt}","nch{idx}",now'
sql = f'{tbname}{idx} using {stbname} tags({tags})'
batchSql = batchSql + " " + sql
tdSql.execute(batchSql)
tdLog.info(f" child table count = {i * batchSize}")
tdLog.info(f" end create {count} child tables.")
# create stable and child tables # create stable and child tables
def create_tagidx(self, stbname): def create_tagidx(self, stbname):
cnt = -1 cnt = -1
for key in self.tag_dict.keys(): for key in self.tag_dict.keys():
# first tag have default index, so skip # first tag have default index, so skip
if cnt == -1: if cnt == -1:
cnt = 0 cnt = 0
continue; continue
sql = f'create index idx_{key} on {stbname} ({key})' sql = f'create index idx_{key} on {stbname} ({key})'
tdLog.info(f" sql={sql}") tdLog.info(f" sql={sql}")
tdSql.execute(sql) tdSql.execute(sql)
@ -309,11 +312,11 @@ class TDTestCase:
def insert_data(self, tbname): def insert_data(self, tbname):
# d1 insert 3 rows # d1 insert 3 rows
for i in range(3): for i in range(3):
sql = f'insert into {tbname}1(ts,col1) values(now+{i}s,{i});' sql = f'insert into {tbname}1(ts,col1) values(now+{i}s,{i});'
tdSql.execute(sql) tdSql.execute(sql)
# d20 insert 4 # d20 insert 4
for i in range(4): for i in range(4):
sql = f'insert into {tbname}20(ts,col1) values(now+{i}s,{i});' sql = f'insert into {tbname}20(ts,col1) values(now+{i}s,{i});'
tdSql.execute(sql) tdSql.execute(sql)
# check show indexs # check show indexs
@ -376,17 +379,17 @@ class TDTestCase:
self.query(sql) self.query(sql)
tdSql.checkRows(4) tdSql.checkRows(4)
# drop child table # drop child table
def drop_tables(self, tbname, count): def drop_tables(self, tbname, count):
# table d1 and d20 have verify data , so can not drop # table d1 and d20 have verify data , so can not drop
start = random.randint(21, count/2) start = random.randint(21, count/2)
end = random.randint(count/2 + 1, count - 1) end = random.randint(count/2 + 1, count - 1)
for i in range(start, end): for i in range(start, end):
sql = f'drop table {tbname}{i}' sql = f'drop table {tbname}{i}'
tdSql.execute(sql) tdSql.execute(sql)
cnt = end - start + 1 cnt = end - start + 1
tdLog.info(f' drop table from {start} to {end} count={cnt}') tdLog.info(f' drop table from {start} to {end} count={cnt}')
# drop tag index # drop tag index
def drop_tagidx(self, dbname, stbname): def drop_tagidx(self, dbname, stbname):
@ -396,11 +399,11 @@ class TDTestCase:
# first tag have default index, so skip # first tag have default index, so skip
if cnt == -1: if cnt == -1:
cnt = 0 cnt = 0
continue; continue
sql = f'drop index idx_{key}' sql = f'drop index idx_{key}'
tdSql.execute(sql) tdSql.execute(sql)
cnt += 1 cnt += 1
# check idx result is 0 # check idx result is 0
sql = f'select index_name,column_name from information_schema.ins_indexes where db_name="{dbname}"' sql = f'select index_name,column_name from information_schema.ins_indexes where db_name="{dbname}"'
tdSql.query(sql) tdSql.query(sql)
@ -408,17 +411,19 @@ class TDTestCase:
tdLog.info(f' drop {cnt} tag indexs ok.') tdLog.info(f' drop {cnt} tag indexs ok.')
# show performance # show performance
def show_performance(self, count) : def show_performance(self, count):
db = self.dbs[0] db = self.dbs[0]
db1 = self.dbs[1] db1 = self.dbs[1]
cnt = len(db.sqls) cnt = len(db.sqls)
cnt1 = len(db1.sqls) cnt1 = len(db1.sqls)
if cnt != len(db1.sqls): if cnt != len(db1.sqls):
tdLog.info(f" datebase sql count not equal. cnt={cnt} cnt1={cnt1}\n") tdLog.info(
f" datebase sql count not equal. cnt={cnt} cnt1={cnt1}\n")
return False return False
tdLog.info(f" database sql cnt ={cnt}") tdLog.info(f" database sql cnt ={cnt}")
print(f" ----------------- performance (child tables = {count})--------------------") print(
f" ----------------- performance (child tables = {count})--------------------")
print(" No time(index) time(no-index) diff(col3-col2) rate(col2/col3) sql") print(" No time(index) time(no-index) diff(col3-col2) rate(col2/col3) sql")
for i in range(cnt): for i in range(cnt):
key = db.sqls[i] key = db.sqls[i]
@ -427,12 +432,13 @@ class TDTestCase:
value1 = db1.spends[i] value1 = db1.spends[i]
diff = value1 - value diff = value1 - value
rate = value/value1*100 rate = value/value1*100
print(" %d %.3fs %.3fs %.3fs %d%% %s"%(i+1, value, value1, diff, rate, key)) print(" %d %.3fs %.3fs %.3fs %d%% %s" % (
i+1, value, value1, diff, rate, key))
print(" --------------------- end ------------------------") print(" --------------------- end ------------------------")
return True return True
def show_diskspace(self): def show_diskspace(self):
#calc # calc
selfPath = os.path.dirname(os.path.realpath(__file__)) selfPath = os.path.dirname(os.path.realpath(__file__))
projPath = "" projPath = ""
if ("community" in selfPath): if ("community" in selfPath):
@ -451,43 +457,41 @@ class TDTestCase:
idx_size = vnode2_size + vnode3_size idx_size = vnode2_size + vnode3_size
noidx_size = vnode4_size + vnode5_size noidx_size = vnode4_size + vnode5_size
print(" index = %.02f M"%(idx_size/1024/1024)) print(" index = %.02f M" % (idx_size/1024/1024))
print(" no-index = %.02f M"%(noidx_size/1024/1024)) print(" no-index = %.02f M" % (noidx_size/1024/1024))
print(" index/no-index = %.2f multiple"%(idx_size/noidx_size)) print(" index/no-index = %.2f multiple" % (idx_size/noidx_size))
print(" -------------------- end ------------------------") print(" -------------------- end ------------------------")
# main # main
def testdb(self, dbname, stable, tbname, count, createidx): def testdb(self, dbname, stable, tbname, count, createidx):
# cur # cur
if createidx: if createidx:
self.cur = 0 self.cur = 0
else : else:
self.cur = 1 self.cur = 1
# do # do
self.create_database(dbname, 2, 1) self.create_database(dbname, 2, 1)
self.create_table(stable, tbname, count) self.create_table(stable, tbname, count)
if(createidx): if (createidx):
self.create_tagidx(stable) self.create_tagidx(stable)
self.insert_data(tbname) self.insert_data(tbname)
if(createidx): if (createidx):
self.show_tagidx(dbname,stable) self.show_tagidx(dbname, stable)
self.query_tagidx(stable) self.query_tagidx(stable)
#self.drop_tables(tbname, count) # self.drop_tables(tbname, count)
#if(createidx): # if(createidx):
# self.drop_tagidx(dbname, stable) # self.drop_tagidx(dbname, stable)
# query after delete , expect no crash # query after delete , expect no crash
#self.query_tagidx(stable) # self.query_tagidx(stable)
tdSql.execute(f'flush database {dbname}') tdSql.execute(f'flush database {dbname}')
# run # run
def run(self): def run(self):
self.tagCluster.run() self.tagCluster.run()
# var # var
dbname = "tagindex" dbname = "tagindex"
dbname1 = dbname + "1" dbname1 = dbname + "1"
@ -511,10 +515,10 @@ class TDTestCase:
self.show_diskspace() self.show_diskspace()
def stop(self): def stop(self):
self.tagCluster.stop() self.tagCluster.stop()
tdLog.success("%s successfully executed" % __file__) tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase()) tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase()) tdCases.addLinux(__file__, TDTestCase())