test: add sma profermance test
This commit is contained in:
tomchon
parent
de2b09faeb
commit
154498ddb5
|
|
@ -0,0 +1,291 @@
|
||||||
|
###################################################################
|
||||||
|
# Copyright (c) 2016 by TAOS Technologies, Inc.
|
||||||
|
# All rights reserved.
|
||||||
|
#
|
||||||
|
# This file is proprietary and confidential to TAOS Technologies.
|
||||||
|
# No part of this file may be reproduced, stored, transmitted,
|
||||||
|
# disclosed or used in any form or by any means other than as
|
||||||
|
# expressly provided by the written permission from Jianhui Tao
|
||||||
|
#
|
||||||
|
###################################################################
|
||||||
|
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
import sys
|
||||||
|
import threading
|
||||||
|
import multiprocessing as mp
|
||||||
|
from numpy.lib.function_base import insert
|
||||||
|
import taos
|
||||||
|
from util.log import *
|
||||||
|
from util.cases import *
|
||||||
|
from util.sql import *
|
||||||
|
import numpy as np
|
||||||
|
import datetime as dt
|
||||||
|
import time
|
||||||
|
# constant define
|
||||||
|
WAITS = 5 # wait seconds
|
||||||
|
|
||||||
|
class TDTestCase:
|
||||||
|
#
|
||||||
|
# --------------- main frame -------------------
|
||||||
|
#
|
||||||
|
|
||||||
|
def caseDescription(self):
|
||||||
|
'''
|
||||||
|
limit and offset keyword function test cases;
|
||||||
|
case1: limit offset base function test
|
||||||
|
case2: offset return valid
|
||||||
|
'''
|
||||||
|
return
|
||||||
|
|
||||||
|
def getBuildPath(self):
|
||||||
|
selfPath = os.path.dirname(os.path.realpath(__file__))
|
||||||
|
|
||||||
|
if ("community" in selfPath):
|
||||||
|
projPath = selfPath[:selfPath.find("community")]
|
||||||
|
else:
|
||||||
|
projPath = selfPath[:selfPath.find("tests")]
|
||||||
|
|
||||||
|
for root, dirs, files in os.walk(projPath):
|
||||||
|
if ("taosd" in files):
|
||||||
|
rootRealPath = os.path.dirname(os.path.realpath(root))
|
||||||
|
if ("packaging" not in rootRealPath):
|
||||||
|
buildPath = root[:len(root)-len("/build/bin")]
|
||||||
|
break
|
||||||
|
return buildPath
|
||||||
|
|
||||||
|
# init
|
||||||
|
def init(self, conn, logSql):
|
||||||
|
tdLog.debug("start to execute %s" % __file__)
|
||||||
|
tdSql.init(conn.cursor())
|
||||||
|
# tdSql.prepare()
|
||||||
|
# self.create_tables();
|
||||||
|
self.ts = 1500000000000
|
||||||
|
|
||||||
|
|
||||||
|
# run case
|
||||||
|
def run(self):
|
||||||
|
|
||||||
|
# test base case
|
||||||
|
self.test_case1()
|
||||||
|
tdLog.debug(" LIMIT test_case1 ............ [OK]")
|
||||||
|
|
||||||
|
# test advance case
|
||||||
|
# self.test_case2()
|
||||||
|
# tdLog.debug(" LIMIT test_case2 ............ [OK]")
|
||||||
|
|
||||||
|
# stop
|
||||||
|
def stop(self):
|
||||||
|
tdSql.close()
|
||||||
|
tdLog.success("%s successfully executed" % __file__)
|
||||||
|
|
||||||
|
# --------------- case -------------------
|
||||||
|
|
||||||
|
# create tables
|
||||||
|
def create_tables(self,dbname,stbname,count):
|
||||||
|
tdSql.execute("use %s" %dbname)
|
||||||
|
tdSql.execute("create stable %s(ts timestamp, c1 int, c2 binary(10)) tags(t1 int)"%stbname)
|
||||||
|
pre_create = "create table"
|
||||||
|
sql = pre_create
|
||||||
|
tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname))
|
||||||
|
# print(time.time())
|
||||||
|
exeStartTime=time.time()
|
||||||
|
for i in range(count):
|
||||||
|
sql += " %s_%d using %s tags(%d)"%(stbname,i,stbname,i+1)
|
||||||
|
if i >0 and i%3000 == 0:
|
||||||
|
tdSql.execute(sql)
|
||||||
|
sql = pre_create
|
||||||
|
# print(time.time())
|
||||||
|
# end sql
|
||||||
|
if sql != pre_create:
|
||||||
|
tdSql.execute(sql)
|
||||||
|
exeEndTime=time.time()
|
||||||
|
spendTime=exeEndTime-exeStartTime
|
||||||
|
speedCreate=count/spendTime
|
||||||
|
tdLog.debug("spent %.2fs to create 1 stable and %d table, create speed is %.2f table/s... [OK]"% (spendTime,count,speedCreate))
|
||||||
|
return
|
||||||
|
|
||||||
|
def newcur(self,host,cfg):
|
||||||
|
user = "root"
|
||||||
|
password = "taosdata"
|
||||||
|
port =6030
|
||||||
|
con=taos.connect(host=host, user=user, password=password, config=cfg ,port=port)
|
||||||
|
cur=con.cursor()
|
||||||
|
print(cur)
|
||||||
|
return cur
|
||||||
|
|
||||||
|
def new_create_tables(self,dbname,vgroups,stbname,tcountStart,tcountStop):
|
||||||
|
host = "chenhaoran02"
|
||||||
|
buildPath = self.getBuildPath()
|
||||||
|
config = buildPath+ "../sim/dnode1/cfg/"
|
||||||
|
|
||||||
|
tsql=self.newcur(host,config)
|
||||||
|
tsql.execute("create database %s vgroups %d"%(dbname,vgroups))
|
||||||
|
tsql.execute("use %s" %dbname)
|
||||||
|
tsql.execute("create stable %s(ts timestamp, c1 int, c2 binary(10)) tags(t1 int)"%stbname)
|
||||||
|
|
||||||
|
pre_create = "create table"
|
||||||
|
sql = pre_create
|
||||||
|
tcountStop=int(tcountStop)
|
||||||
|
tcountStart=int(tcountStart)
|
||||||
|
count=tcountStop-tcountStart
|
||||||
|
|
||||||
|
tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname))
|
||||||
|
# print(time.time())
|
||||||
|
exeStartTime=time.time()
|
||||||
|
# print(type(tcountStop),type(tcountStart))
|
||||||
|
for i in range(tcountStart,tcountStop):
|
||||||
|
sql += " %s_%d using %s tags(%d)"%(stbname,i,stbname,i+1)
|
||||||
|
if i >0 and i%20000 == 0:
|
||||||
|
# print(sql)
|
||||||
|
tsql.execute(sql)
|
||||||
|
sql = pre_create
|
||||||
|
# print(time.time())
|
||||||
|
# end sql
|
||||||
|
if sql != pre_create:
|
||||||
|
# print(sql)
|
||||||
|
tsql.execute(sql)
|
||||||
|
exeEndTime=time.time()
|
||||||
|
spendTime=exeEndTime-exeStartTime
|
||||||
|
speedCreate=count/spendTime
|
||||||
|
# tdLog.debug("spent %.2fs to create 1 stable and %d table, create speed is %.2f table/s... [OK]"% (spendTime,count,speedCreate))
|
||||||
|
return
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# insert data
|
||||||
|
def insert_data(self, dbname, stbname, ts_start, tcountStart,tcountStop,rowCount):
|
||||||
|
tdSql.execute("use %s" %dbname)
|
||||||
|
pre_insert = "insert into "
|
||||||
|
sql = pre_insert
|
||||||
|
tcount=tcountStop-tcountStart
|
||||||
|
allRows=tcount*rowCount
|
||||||
|
tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbname, allRows))
|
||||||
|
exeStartTime=time.time()
|
||||||
|
for i in range(tcountStart,tcountStop):
|
||||||
|
sql += " %s_%d values "%(stbname,i)
|
||||||
|
for j in range(rowCount):
|
||||||
|
sql += "(%d, %d, 'taos_%d') "%(ts_start + j*1000, j, j)
|
||||||
|
if j >0 and j%5000 == 0:
|
||||||
|
# print(sql)
|
||||||
|
tdSql.execute(sql)
|
||||||
|
sql = "insert into %s_%d values " %(stbname,i)
|
||||||
|
# end sql
|
||||||
|
if sql != pre_insert:
|
||||||
|
# print(sql)
|
||||||
|
tdSql.execute(sql)
|
||||||
|
exeEndTime=time.time()
|
||||||
|
spendTime=exeEndTime-exeStartTime
|
||||||
|
speedInsert=allRows/spendTime
|
||||||
|
# tdLog.debug("spent %.2fs to INSERT %d rows , insert rate is %.2f rows/s... [OK]"% (spendTime,allRows,speedInsert))
|
||||||
|
|
||||||
|
tdLog.debug("INSERT TABLE DATA ............ [OK]")
|
||||||
|
return
|
||||||
|
|
||||||
|
|
||||||
|
# test case1 base
|
||||||
|
def test_case1(self):
|
||||||
|
tdLog.debug("-----create database and tables test------- ")
|
||||||
|
tdSql.execute("drop database if exists db1")
|
||||||
|
tdSql.execute("drop database if exists db4")
|
||||||
|
tdSql.execute("drop database if exists db6")
|
||||||
|
tdSql.execute("drop database if exists db8")
|
||||||
|
tdSql.execute("drop database if exists db12")
|
||||||
|
tdSql.execute("drop database if exists db16")
|
||||||
|
|
||||||
|
#create database and tables;
|
||||||
|
|
||||||
|
# tdSql.execute("create database db11 vgroups 1")
|
||||||
|
# # self.create_tables("db1", "stb1", 30*10000)
|
||||||
|
# tdSql.execute("use db1")
|
||||||
|
# tdSql.execute("create stable stb1(ts timestamp, c1 int, c2 binary(10)) tags(t1 int)")
|
||||||
|
|
||||||
|
# tdSql.execute("create database db12 vgroups 1")
|
||||||
|
# # self.create_tables("db1", "stb1", 30*10000)
|
||||||
|
# tdSql.execute("use db1")
|
||||||
|
|
||||||
|
# t1 = threading.Thread(target=self.new_create_tables("db1", "stb1", 15*10000), args=(1,))
|
||||||
|
# t2 = threading.Thread(target=self.new_create_tables("db1", "stb1", 15*10000), args=(2,))
|
||||||
|
# t1 = mp.Process(target=self.new_create_tables, args=("db1", "stb1", 0,count/2,))
|
||||||
|
# t2 = mp.Process(target=self.new_create_tables, args=("db1", "stb1", count/2,count,))
|
||||||
|
|
||||||
|
count=50000
|
||||||
|
vgroups=1
|
||||||
|
threads = []
|
||||||
|
threadNumbers=2
|
||||||
|
for i in range(threadNumbers):
|
||||||
|
threads.append(mp.Process(target=self.new_create_tables, args=("db1%d"%i, vgroups, "stb1", 0,count,)))
|
||||||
|
start_time = time.time()
|
||||||
|
for tr in threads:
|
||||||
|
tr.start()
|
||||||
|
for tr in threads:
|
||||||
|
tr.join()
|
||||||
|
end_time = time.time()
|
||||||
|
spendTime=end_time-start_time
|
||||||
|
speedCreate=count/spendTime
|
||||||
|
tdLog.debug("spent %.2fs to create 1 stable and %d table, create speed is %.2f table/s... [OK]"% (spendTime,count,speedCreate))
|
||||||
|
# self.new_create_tables("db1", "stb1", 15*10000)
|
||||||
|
# self.new_create_tables("db1", "stb1", 15*10000)
|
||||||
|
|
||||||
|
# tdSql.execute("create database db4 vgroups 4")
|
||||||
|
# self.create_tables("db4", "stb4", 30*10000)
|
||||||
|
|
||||||
|
# tdSql.execute("create database db6 vgroups 6")
|
||||||
|
# self.create_tables("db6", "stb6", 30*10000)
|
||||||
|
|
||||||
|
# tdSql.execute("create database db8 vgroups 8")
|
||||||
|
# self.create_tables("db8", "stb8", 30*10000)
|
||||||
|
|
||||||
|
# tdSql.execute("create database db12 vgroups 12")
|
||||||
|
# self.create_tables("db12", "stb12", 30*10000)
|
||||||
|
|
||||||
|
# tdSql.execute("create database db16 vgroups 16")
|
||||||
|
# self.create_tables("db16", "stb16", 30*10000)
|
||||||
|
return
|
||||||
|
|
||||||
|
# test case2 base:insert data
|
||||||
|
def test_case2(self):
|
||||||
|
|
||||||
|
tdLog.debug("-----insert data test------- ")
|
||||||
|
# drop database
|
||||||
|
tdSql.execute("drop database if exists db1")
|
||||||
|
tdSql.execute("drop database if exists db4")
|
||||||
|
tdSql.execute("drop database if exists db6")
|
||||||
|
tdSql.execute("drop database if exists db8")
|
||||||
|
tdSql.execute("drop database if exists db12")
|
||||||
|
tdSql.execute("drop database if exists db16")
|
||||||
|
|
||||||
|
#create database and tables;
|
||||||
|
|
||||||
|
tdSql.execute("create database db1 vgroups 1")
|
||||||
|
self.create_tables("db1", "stb1", 1*100)
|
||||||
|
self.insert_data("db1", "stb1", self.ts, 1*50,1*10000)
|
||||||
|
|
||||||
|
|
||||||
|
tdSql.execute("create database db4 vgroups 4")
|
||||||
|
self.create_tables("db4", "stb4", 1*100)
|
||||||
|
self.insert_data("db4", "stb4", self.ts, 1*100,1*10000)
|
||||||
|
|
||||||
|
tdSql.execute("create database db6 vgroups 6")
|
||||||
|
self.create_tables("db6", "stb6", 1*100)
|
||||||
|
self.insert_data("db6", "stb6", self.ts, 1*100,1*10000)
|
||||||
|
|
||||||
|
tdSql.execute("create database db8 vgroups 8")
|
||||||
|
self.create_tables("db8", "stb8", 1*100)
|
||||||
|
self.insert_data("db8", "stb8", self.ts, 1*100,1*10000)
|
||||||
|
|
||||||
|
tdSql.execute("create database db12 vgroups 12")
|
||||||
|
self.create_tables("db12", "stb12", 1*100)
|
||||||
|
self.insert_data("db12", "stb12", self.ts, 1*100,1*10000)
|
||||||
|
|
||||||
|
tdSql.execute("create database db16 vgroups 16")
|
||||||
|
self.create_tables("db16", "stb16", 1*100)
|
||||||
|
self.insert_data("db16", "stb16", self.ts, 1*100,1*10000)
|
||||||
|
|
||||||
|
return
|
||||||
|
|
||||||
|
#
|
||||||
|
# add case with filename
|
||||||
|
#
|
||||||
|
tdCases.addWindows(__file__, TDTestCase())
|
||||||
|
tdCases.addLinux(__file__, TDTestCase())
|
||||||
|
|
@ -27,7 +27,9 @@ class TDTestCase:
|
||||||
#
|
#
|
||||||
# --------------- main frame -------------------
|
# --------------- main frame -------------------
|
||||||
#
|
#
|
||||||
|
# updatecfgDict = {'debugFlag': 135}
|
||||||
|
# updatecfgDict = {'fqdn': 135}
|
||||||
|
|
||||||
def caseDescription(self):
|
def caseDescription(self):
|
||||||
'''
|
'''
|
||||||
limit and offset keyword function test cases;
|
limit and offset keyword function test cases;
|
||||||
|
|
@ -48,7 +50,7 @@ class TDTestCase:
|
||||||
# run case
|
# run case
|
||||||
def run(self):
|
def run(self):
|
||||||
# insert data
|
# insert data
|
||||||
self.insert_data("t1", self.ts, 1000*10000)
|
self.insert_data1("t1", self.ts, 1000*10000)
|
||||||
self.insert_data1("t4", self.ts, 1000*10000)
|
self.insert_data1("t4", self.ts, 1000*10000)
|
||||||
# test base case
|
# test base case
|
||||||
# self.test_case1()
|
# self.test_case1()
|
||||||
|
|
@ -70,11 +72,9 @@ class TDTestCase:
|
||||||
# create table
|
# create table
|
||||||
def create_tables(self):
|
def create_tables(self):
|
||||||
# super table
|
# super table
|
||||||
tdSql.execute("create table st(ts timestamp, i1 int) tags(area int)");
|
tdSql.execute("create table st(ts timestamp, i1 int,i2 int) tags(area int)");
|
||||||
# child table
|
# child table
|
||||||
tdSql.execute("create table t1 using st tags(1)");
|
tdSql.execute("create table t1 using st tags(1)");
|
||||||
tdSql.execute("create table t2 using st tags(2)");
|
|
||||||
tdSql.execute("create table t3 using st tags(3)");
|
|
||||||
|
|
||||||
tdSql.execute("create table st1(ts timestamp, i1 int ,i2 int) tags(area int) sma(i2) ");
|
tdSql.execute("create table st1(ts timestamp, i1 int ,i2 int) tags(area int) sma(i2) ");
|
||||||
tdSql.execute("create table t4 using st1 tags(1)");
|
tdSql.execute("create table t4 using st1 tags(1)");
|
||||||
|
|
@ -122,66 +122,7 @@ class TDTestCase:
|
||||||
# # base no where
|
# # base no where
|
||||||
# sql = "select * from t1 limit 10"
|
# sql = "select * from t1 limit 10"
|
||||||
# tdSql.waitedQuery(sql, 10, WAITS)
|
# tdSql.waitedQuery(sql, 10, WAITS)
|
||||||
# tdSql.checkData(0, 1, 0)
|
|
||||||
# tdSql.checkData(9, 1, 9)
|
|
||||||
|
|
||||||
# # have where
|
|
||||||
# sql = "select * from t1 where ts>='2017-07-14 10:40:01' and ts<'2017-07-14 10:40:06' limit 10"
|
|
||||||
# tdSql.waitedQuery(sql, 5, WAITS)
|
|
||||||
# tdSql.checkData(0, 1, 1)
|
|
||||||
# tdSql.checkData(4, 1, 5)
|
|
||||||
|
|
||||||
# #
|
|
||||||
# # offset base function
|
|
||||||
# #
|
|
||||||
# # no where
|
|
||||||
# sql = "select * from t1 limit 10 offset 5"
|
|
||||||
# tdSql.waitedQuery(sql, 10, WAITS)
|
|
||||||
# tdSql.checkData(0, 1, 5)
|
|
||||||
# tdSql.checkData(9, 1, 14)
|
|
||||||
|
|
||||||
# # have where only ts
|
|
||||||
# sql = "select * from t1 where ts>='2017-07-14 10:40:10' and ts<'2017-07-14 10:40:20' limit 10 offset 5"
|
|
||||||
# tdSql.waitedQuery(sql, 5, WAITS)
|
|
||||||
# tdSql.checkData(0, 1, 15)
|
|
||||||
# tdSql.checkData(4, 1, 19)
|
|
||||||
|
|
||||||
# # have where with other column condition
|
|
||||||
# sql = "select * from t1 where i1>=1 and i1<11 limit 10 offset 5"
|
|
||||||
# tdSql.waitedQuery(sql, 5, WAITS)
|
|
||||||
# tdSql.checkData(0, 1, 6)
|
|
||||||
# tdSql.checkData(4, 1, 10)
|
|
||||||
|
|
||||||
# # have where with ts and other column condition
|
|
||||||
# sql = "select * from t1 where ts>='2017-07-14 10:40:10' and ts<'2017-07-14 10:40:50' and i1>=20 and i1<=25 limit 10 offset 5"
|
|
||||||
# tdSql.waitedQuery(sql, 1, WAITS)
|
|
||||||
# tdSql.checkData(0, 1, 25)
|
|
||||||
|
|
||||||
# return
|
|
||||||
|
|
||||||
# # test advance
|
|
||||||
# def test_case2(self):
|
|
||||||
# #
|
|
||||||
# # OFFSET merge file data with memory data
|
|
||||||
# #
|
|
||||||
|
|
||||||
# # offset
|
|
||||||
# sql = "select * from t1 limit 10 offset 72000"
|
|
||||||
# tdSql.waitedQuery(sql, 10, WAITS)
|
|
||||||
# tdSql.checkData(0, 1, 72000)
|
|
||||||
|
|
||||||
# # each insert one row into NO.0 NO.2 NO.7 blocks
|
|
||||||
# sql = "insert into t1 values (%d, 0) (%d, 2) (%d, 7)"%(self.ts+1, self.ts + 2*3300*1000+1, self.ts + 7*3300*1000+1)
|
|
||||||
# tdSql.execute(sql)
|
|
||||||
# # query result
|
|
||||||
# sql = "select * from t1 limit 10 offset 72000"
|
|
||||||
# tdSql.waitedQuery(sql, 10, WAITS)
|
|
||||||
# tdSql.checkData(0, 1, 72000 - 3)
|
|
||||||
|
|
||||||
# # have where
|
|
||||||
# sql = "select * from t1 where ts>='2017-07-14 10:40:10' and ts<'2017-07-22 18:40:10' limit 10 offset 72000"
|
|
||||||
# tdSql.waitedQuery(sql, 10, WAITS)
|
|
||||||
# tdSql.checkData(0, 1, 72000 - 3 + 10 + 1)
|
|
||||||
|
|
||||||
#
|
#
|
||||||
# add case with filename
|
# add case with filename
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue