From 6aa38d1cc908f21a7fd6168fcd3840d92bd04ab0 Mon Sep 17 00:00:00 2001 From: tomchon Date: Sun, 24 Apr 2022 20:39:10 +0800 Subject: [PATCH 1/3] test: add sma profermance test --- tests/system-test/2-query/smaTest.py | 190 +++++++++++++++++++++++++++ 1 file changed, 190 insertions(+) create mode 100644 tests/system-test/2-query/smaTest.py diff --git a/tests/system-test/2-query/smaTest.py b/tests/system-test/2-query/smaTest.py new file mode 100644 index 0000000000..6f78a6e6bd --- /dev/null +++ b/tests/system-test/2-query/smaTest.py @@ -0,0 +1,190 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys + +from numpy.lib.function_base import insert +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + +# constant define +WAITS = 5 # wait seconds + +class TDTestCase: + # + # --------------- main frame ------------------- + # + + def caseDescription(self): + ''' + limit and offset keyword function test cases; + case1: limit offset base function test + case2: offset return valid + ''' + return + + # init + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + tdSql.prepare() + self.create_tables(); + self.ts = 1500000000000 + + + # run case + def run(self): + # insert data + self.insert_data("t1", self.ts, 1000*10000) + self.insert_data1("t4", self.ts, 1000*10000) + # test base case + # self.test_case1() + tdLog.debug(" LIMIT test_case1 ............ [OK]") + # test advance case + # self.test_case2() + tdLog.debug(" LIMIT test_case2 ............ [OK]") + + + # stop + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + # + # --------------- case ------------------- + # + + # create table + def create_tables(self): + # super table + tdSql.execute("create table st(ts timestamp, i1 int) tags(area int)"); + # child table + tdSql.execute("create table t1 using st tags(1)"); + tdSql.execute("create table t2 using st tags(2)"); + tdSql.execute("create table t3 using st tags(3)"); + + tdSql.execute("create table st1(ts timestamp, i1 int ,i2 int) tags(area int) sma(i2) "); + tdSql.execute("create table t4 using st1 tags(1)"); + + return + + # insert data1 + def insert_data(self, tbname, ts_start, count): + pre_insert = "insert into %s values"%tbname + sql = pre_insert + tdLog.debug("doing insert table %s rows=%d ..."%(tbname, count)) + for i in range(count): + sql += " (%d,%d)"%(ts_start + i*1000, i ) + if i >0 and i%30000 == 0: + tdSql.execute(sql) + sql = pre_insert + # end sql + if sql != pre_insert: + tdSql.execute(sql) + + tdLog.debug("INSERT TABLE DATA ............ [OK]") + return + + def insert_data1(self, tbname, ts_start, count): + pre_insert = "insert into %s values"%tbname + sql = pre_insert + tdLog.debug("doing insert table %s rows=%d ..."%(tbname, count)) + for i in range(count): + sql += " (%d,%d,%d)"%(ts_start + i*1000, i , i+1) + if i >0 and i%30000 == 0: + tdSql.execute(sql) + sql = pre_insert + # end sql + if sql != pre_insert: + tdSql.execute(sql) + + tdLog.debug("INSERT TABLE DATA ............ [OK]") + return + + # test case1 base + # def test_case1(self): + # # + # # limit base function + # # + # # base no where + # sql = "select * from t1 limit 10" + # tdSql.waitedQuery(sql, 10, WAITS) + # tdSql.checkData(0, 1, 0) + # tdSql.checkData(9, 1, 9) + + # # have where + # sql = "select * from t1 where ts>='2017-07-14 10:40:01' and ts<'2017-07-14 10:40:06' limit 10" + # tdSql.waitedQuery(sql, 5, WAITS) + # tdSql.checkData(0, 1, 1) + # tdSql.checkData(4, 1, 5) + + # # + # # offset base function + # # + # # no where + # sql = "select * from t1 limit 10 offset 5" + # tdSql.waitedQuery(sql, 10, WAITS) + # tdSql.checkData(0, 1, 5) + # tdSql.checkData(9, 1, 14) + + # # have where only ts + # sql = "select * from t1 where ts>='2017-07-14 10:40:10' and ts<'2017-07-14 10:40:20' limit 10 offset 5" + # tdSql.waitedQuery(sql, 5, WAITS) + # tdSql.checkData(0, 1, 15) + # tdSql.checkData(4, 1, 19) + + # # have where with other column condition + # sql = "select * from t1 where i1>=1 and i1<11 limit 10 offset 5" + # tdSql.waitedQuery(sql, 5, WAITS) + # tdSql.checkData(0, 1, 6) + # tdSql.checkData(4, 1, 10) + + # # have where with ts and other column condition + # sql = "select * from t1 where ts>='2017-07-14 10:40:10' and ts<'2017-07-14 10:40:50' and i1>=20 and i1<=25 limit 10 offset 5" + # tdSql.waitedQuery(sql, 1, WAITS) + # tdSql.checkData(0, 1, 25) + + # return + + # # test advance + # def test_case2(self): + # # + # # OFFSET merge file data with memory data + # # + + # # offset + # sql = "select * from t1 limit 10 offset 72000" + # tdSql.waitedQuery(sql, 10, WAITS) + # tdSql.checkData(0, 1, 72000) + + # # each insert one row into NO.0 NO.2 NO.7 blocks + # sql = "insert into t1 values (%d, 0) (%d, 2) (%d, 7)"%(self.ts+1, self.ts + 2*3300*1000+1, self.ts + 7*3300*1000+1) + # tdSql.execute(sql) + # # query result + # sql = "select * from t1 limit 10 offset 72000" + # tdSql.waitedQuery(sql, 10, WAITS) + # tdSql.checkData(0, 1, 72000 - 3) + + # # have where + # sql = "select * from t1 where ts>='2017-07-14 10:40:10' and ts<'2017-07-22 18:40:10' limit 10 offset 72000" + # tdSql.waitedQuery(sql, 10, WAITS) + # tdSql.checkData(0, 1, 72000 - 3 + 10 + 1) + +# +# add case with filename +# +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From 154498ddb5068a8383478caeb1387f9e2c6622de Mon Sep 17 00:00:00 2001 From: tomchon Date: Thu, 5 May 2022 22:17:25 +0800 Subject: [PATCH 2/3] test: add sma profermance test --- .../1-insert/insertWithMoreVgroup.py | 291 ++++++++++++++++++ tests/system-test/2-query/smaTest.py | 69 +---- 2 files changed, 296 insertions(+), 64 deletions(-) create mode 100644 tests/system-test/1-insert/insertWithMoreVgroup.py diff --git a/tests/system-test/1-insert/insertWithMoreVgroup.py b/tests/system-test/1-insert/insertWithMoreVgroup.py new file mode 100644 index 0000000000..a7d17bc41e --- /dev/null +++ b/tests/system-test/1-insert/insertWithMoreVgroup.py @@ -0,0 +1,291 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import threading +import multiprocessing as mp +from numpy.lib.function_base import insert +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np +import datetime as dt +import time +# constant define +WAITS = 5 # wait seconds + +class TDTestCase: + # + # --------------- main frame ------------------- + # + + def caseDescription(self): + ''' + limit and offset keyword function test cases; + case1: limit offset base function test + case2: offset return valid + ''' + return + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + # init + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + # tdSql.prepare() + # self.create_tables(); + self.ts = 1500000000000 + + + # run case + def run(self): + + # test base case + self.test_case1() + tdLog.debug(" LIMIT test_case1 ............ [OK]") + + # test advance case + # self.test_case2() + # tdLog.debug(" LIMIT test_case2 ............ [OK]") + + # stop + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + # --------------- case ------------------- + + # create tables + def create_tables(self,dbname,stbname,count): + tdSql.execute("use %s" %dbname) + tdSql.execute("create stable %s(ts timestamp, c1 int, c2 binary(10)) tags(t1 int)"%stbname) + pre_create = "create table" + sql = pre_create + tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname)) + # print(time.time()) + exeStartTime=time.time() + for i in range(count): + sql += " %s_%d using %s tags(%d)"%(stbname,i,stbname,i+1) + if i >0 and i%3000 == 0: + tdSql.execute(sql) + sql = pre_create + # print(time.time()) + # end sql + if sql != pre_create: + tdSql.execute(sql) + exeEndTime=time.time() + spendTime=exeEndTime-exeStartTime + speedCreate=count/spendTime + tdLog.debug("spent %.2fs to create 1 stable and %d table, create speed is %.2f table/s... [OK]"% (spendTime,count,speedCreate)) + return + + def newcur(self,host,cfg): + user = "root" + password = "taosdata" + port =6030 + con=taos.connect(host=host, user=user, password=password, config=cfg ,port=port) + cur=con.cursor() + print(cur) + return cur + + def new_create_tables(self,dbname,vgroups,stbname,tcountStart,tcountStop): + host = "chenhaoran02" + buildPath = self.getBuildPath() + config = buildPath+ "../sim/dnode1/cfg/" + + tsql=self.newcur(host,config) + tsql.execute("create database %s vgroups %d"%(dbname,vgroups)) + tsql.execute("use %s" %dbname) + tsql.execute("create stable %s(ts timestamp, c1 int, c2 binary(10)) tags(t1 int)"%stbname) + + pre_create = "create table" + sql = pre_create + tcountStop=int(tcountStop) + tcountStart=int(tcountStart) + count=tcountStop-tcountStart + + tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname)) + # print(time.time()) + exeStartTime=time.time() + # print(type(tcountStop),type(tcountStart)) + for i in range(tcountStart,tcountStop): + sql += " %s_%d using %s tags(%d)"%(stbname,i,stbname,i+1) + if i >0 and i%20000 == 0: + # print(sql) + tsql.execute(sql) + sql = pre_create + # print(time.time()) + # end sql + if sql != pre_create: + # print(sql) + tsql.execute(sql) + exeEndTime=time.time() + spendTime=exeEndTime-exeStartTime + speedCreate=count/spendTime + # tdLog.debug("spent %.2fs to create 1 stable and %d table, create speed is %.2f table/s... [OK]"% (spendTime,count,speedCreate)) + return + + + + # insert data + def insert_data(self, dbname, stbname, ts_start, tcountStart,tcountStop,rowCount): + tdSql.execute("use %s" %dbname) + pre_insert = "insert into " + sql = pre_insert + tcount=tcountStop-tcountStart + allRows=tcount*rowCount + tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbname, allRows)) + exeStartTime=time.time() + for i in range(tcountStart,tcountStop): + sql += " %s_%d values "%(stbname,i) + for j in range(rowCount): + sql += "(%d, %d, 'taos_%d') "%(ts_start + j*1000, j, j) + if j >0 and j%5000 == 0: + # print(sql) + tdSql.execute(sql) + sql = "insert into %s_%d values " %(stbname,i) + # end sql + if sql != pre_insert: + # print(sql) + tdSql.execute(sql) + exeEndTime=time.time() + spendTime=exeEndTime-exeStartTime + speedInsert=allRows/spendTime + # tdLog.debug("spent %.2fs to INSERT %d rows , insert rate is %.2f rows/s... [OK]"% (spendTime,allRows,speedInsert)) + + tdLog.debug("INSERT TABLE DATA ............ [OK]") + return + + + # test case1 base + def test_case1(self): + tdLog.debug("-----create database and tables test------- ") + tdSql.execute("drop database if exists db1") + tdSql.execute("drop database if exists db4") + tdSql.execute("drop database if exists db6") + tdSql.execute("drop database if exists db8") + tdSql.execute("drop database if exists db12") + tdSql.execute("drop database if exists db16") + + #create database and tables; + + # tdSql.execute("create database db11 vgroups 1") + # # self.create_tables("db1", "stb1", 30*10000) + # tdSql.execute("use db1") + # tdSql.execute("create stable stb1(ts timestamp, c1 int, c2 binary(10)) tags(t1 int)") + + # tdSql.execute("create database db12 vgroups 1") + # # self.create_tables("db1", "stb1", 30*10000) + # tdSql.execute("use db1") + + # t1 = threading.Thread(target=self.new_create_tables("db1", "stb1", 15*10000), args=(1,)) + # t2 = threading.Thread(target=self.new_create_tables("db1", "stb1", 15*10000), args=(2,)) + # t1 = mp.Process(target=self.new_create_tables, args=("db1", "stb1", 0,count/2,)) + # t2 = mp.Process(target=self.new_create_tables, args=("db1", "stb1", count/2,count,)) + + count=50000 + vgroups=1 + threads = [] + threadNumbers=2 + for i in range(threadNumbers): + threads.append(mp.Process(target=self.new_create_tables, args=("db1%d"%i, vgroups, "stb1", 0,count,))) + start_time = time.time() + for tr in threads: + tr.start() + for tr in threads: + tr.join() + end_time = time.time() + spendTime=end_time-start_time + speedCreate=count/spendTime + tdLog.debug("spent %.2fs to create 1 stable and %d table, create speed is %.2f table/s... [OK]"% (spendTime,count,speedCreate)) + # self.new_create_tables("db1", "stb1", 15*10000) + # self.new_create_tables("db1", "stb1", 15*10000) + + # tdSql.execute("create database db4 vgroups 4") + # self.create_tables("db4", "stb4", 30*10000) + + # tdSql.execute("create database db6 vgroups 6") + # self.create_tables("db6", "stb6", 30*10000) + + # tdSql.execute("create database db8 vgroups 8") + # self.create_tables("db8", "stb8", 30*10000) + + # tdSql.execute("create database db12 vgroups 12") + # self.create_tables("db12", "stb12", 30*10000) + + # tdSql.execute("create database db16 vgroups 16") + # self.create_tables("db16", "stb16", 30*10000) + return + + # test case2 base:insert data + def test_case2(self): + + tdLog.debug("-----insert data test------- ") + # drop database + tdSql.execute("drop database if exists db1") + tdSql.execute("drop database if exists db4") + tdSql.execute("drop database if exists db6") + tdSql.execute("drop database if exists db8") + tdSql.execute("drop database if exists db12") + tdSql.execute("drop database if exists db16") + + #create database and tables; + + tdSql.execute("create database db1 vgroups 1") + self.create_tables("db1", "stb1", 1*100) + self.insert_data("db1", "stb1", self.ts, 1*50,1*10000) + + + tdSql.execute("create database db4 vgroups 4") + self.create_tables("db4", "stb4", 1*100) + self.insert_data("db4", "stb4", self.ts, 1*100,1*10000) + + tdSql.execute("create database db6 vgroups 6") + self.create_tables("db6", "stb6", 1*100) + self.insert_data("db6", "stb6", self.ts, 1*100,1*10000) + + tdSql.execute("create database db8 vgroups 8") + self.create_tables("db8", "stb8", 1*100) + self.insert_data("db8", "stb8", self.ts, 1*100,1*10000) + + tdSql.execute("create database db12 vgroups 12") + self.create_tables("db12", "stb12", 1*100) + self.insert_data("db12", "stb12", self.ts, 1*100,1*10000) + + tdSql.execute("create database db16 vgroups 16") + self.create_tables("db16", "stb16", 1*100) + self.insert_data("db16", "stb16", self.ts, 1*100,1*10000) + + return + +# +# add case with filename +# +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/2-query/smaTest.py b/tests/system-test/2-query/smaTest.py index 6f78a6e6bd..67824cc3a3 100644 --- a/tests/system-test/2-query/smaTest.py +++ b/tests/system-test/2-query/smaTest.py @@ -27,7 +27,9 @@ class TDTestCase: # # --------------- main frame ------------------- # - + # updatecfgDict = {'debugFlag': 135} + # updatecfgDict = {'fqdn': 135} + def caseDescription(self): ''' limit and offset keyword function test cases; @@ -48,7 +50,7 @@ class TDTestCase: # run case def run(self): # insert data - self.insert_data("t1", self.ts, 1000*10000) + self.insert_data1("t1", self.ts, 1000*10000) self.insert_data1("t4", self.ts, 1000*10000) # test base case # self.test_case1() @@ -70,11 +72,9 @@ class TDTestCase: # create table def create_tables(self): # super table - tdSql.execute("create table st(ts timestamp, i1 int) tags(area int)"); + tdSql.execute("create table st(ts timestamp, i1 int,i2 int) tags(area int)"); # child table tdSql.execute("create table t1 using st tags(1)"); - tdSql.execute("create table t2 using st tags(2)"); - tdSql.execute("create table t3 using st tags(3)"); tdSql.execute("create table st1(ts timestamp, i1 int ,i2 int) tags(area int) sma(i2) "); tdSql.execute("create table t4 using st1 tags(1)"); @@ -122,66 +122,7 @@ class TDTestCase: # # base no where # sql = "select * from t1 limit 10" # tdSql.waitedQuery(sql, 10, WAITS) - # tdSql.checkData(0, 1, 0) - # tdSql.checkData(9, 1, 9) - # # have where - # sql = "select * from t1 where ts>='2017-07-14 10:40:01' and ts<'2017-07-14 10:40:06' limit 10" - # tdSql.waitedQuery(sql, 5, WAITS) - # tdSql.checkData(0, 1, 1) - # tdSql.checkData(4, 1, 5) - - # # - # # offset base function - # # - # # no where - # sql = "select * from t1 limit 10 offset 5" - # tdSql.waitedQuery(sql, 10, WAITS) - # tdSql.checkData(0, 1, 5) - # tdSql.checkData(9, 1, 14) - - # # have where only ts - # sql = "select * from t1 where ts>='2017-07-14 10:40:10' and ts<'2017-07-14 10:40:20' limit 10 offset 5" - # tdSql.waitedQuery(sql, 5, WAITS) - # tdSql.checkData(0, 1, 15) - # tdSql.checkData(4, 1, 19) - - # # have where with other column condition - # sql = "select * from t1 where i1>=1 and i1<11 limit 10 offset 5" - # tdSql.waitedQuery(sql, 5, WAITS) - # tdSql.checkData(0, 1, 6) - # tdSql.checkData(4, 1, 10) - - # # have where with ts and other column condition - # sql = "select * from t1 where ts>='2017-07-14 10:40:10' and ts<'2017-07-14 10:40:50' and i1>=20 and i1<=25 limit 10 offset 5" - # tdSql.waitedQuery(sql, 1, WAITS) - # tdSql.checkData(0, 1, 25) - - # return - - # # test advance - # def test_case2(self): - # # - # # OFFSET merge file data with memory data - # # - - # # offset - # sql = "select * from t1 limit 10 offset 72000" - # tdSql.waitedQuery(sql, 10, WAITS) - # tdSql.checkData(0, 1, 72000) - - # # each insert one row into NO.0 NO.2 NO.7 blocks - # sql = "insert into t1 values (%d, 0) (%d, 2) (%d, 7)"%(self.ts+1, self.ts + 2*3300*1000+1, self.ts + 7*3300*1000+1) - # tdSql.execute(sql) - # # query result - # sql = "select * from t1 limit 10 offset 72000" - # tdSql.waitedQuery(sql, 10, WAITS) - # tdSql.checkData(0, 1, 72000 - 3) - - # # have where - # sql = "select * from t1 where ts>='2017-07-14 10:40:10' and ts<'2017-07-22 18:40:10' limit 10 offset 72000" - # tdSql.waitedQuery(sql, 10, WAITS) - # tdSql.checkData(0, 1, 72000 - 3 + 10 + 1) # # add case with filename From b72849aaeb61786929ddd9780e43575ef8b97c00 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 6 May 2022 14:04:37 +0800 Subject: [PATCH 3/3] fix: let the shell parse statements like create table `abd\` --- tools/shell/src/shellEngine.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index befcfc0731..08fe2cbc32 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -141,6 +141,9 @@ int32_t shellRunCommand(char *command) { *p++ = '\\'; } break; + default: + *p++ = '\\'; + break; } *p++ = c; esc = false;