xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #17705 from taosdata/test3.0/lihui 

test:modify test cases of tmq for poll delay
This commit is contained in:
Hui Li 2022-10-27 18:56:44 +08:00 committed by GitHub
parent 4e14b375ff 7797027da2
commit 7820ca2a9f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 82 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
tests/system-test/7-tmq/tmqDnodeRestart.py
View File

@ -87,7 +87,7 @@ class TDTestCase:
'rowsPerTbl': 1000, 'rowsPerTbl': 1000,
'batchNum': 100, 'batchNum': 100,
'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'startTs': 1640966400000, # 2022-01-01 00:00:00.000
'pollDelay': 10, 'pollDelay': 30,
'showMsg': 1, 'showMsg': 1,
'showRow': 1, 'showRow': 1,
'snapshot': 0} 'snapshot': 0}
@ -173,7 +173,7 @@ class TDTestCase:
'rowsPerTbl': 1000, 'rowsPerTbl': 1000,
'batchNum': 100, 'batchNum': 100,
'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'startTs': 1640966400000, # 2022-01-01 00:00:00.000
'pollDelay': 20, 'pollDelay': 30,
'showMsg': 1, 'showMsg': 1,
'showRow': 1, 'showRow': 1,
'snapshot': 0} 'snapshot': 0}

41
tests/system-test/7-tmq/tmqDnodeRestart1.py
View File

@ -40,9 +40,9 @@ class TDTestCase:
'ctbStartIdx': 0, 'ctbStartIdx': 0,
'ctbNum': 100, 'ctbNum': 100,
'rowsPerTbl': 1000, 'rowsPerTbl': 1000,
'batchNum': 10, 'batchNum': 100,
'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'startTs': 1640966400000, # 2022-01-01 00:00:00.000
'pollDelay': 3, 'pollDelay': 30,
'showMsg': 1, 'showMsg': 1,
'showRow': 1, 'showRow': 1,
'snapshot': 0} 'snapshot': 0}
@ -87,7 +87,7 @@ class TDTestCase:
'rowsPerTbl': 1000, 'rowsPerTbl': 1000,
'batchNum': 100, 'batchNum': 100,
'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'startTs': 1640966400000, # 2022-01-01 00:00:00.000
'pollDelay': 10, 'pollDelay': 30,
'showMsg': 1, 'showMsg': 1,
'showRow': 1, 'showRow': 1,
'snapshot': 0} 'snapshot': 0}
@ -101,6 +101,7 @@ class TDTestCase:
topicFromStb = 'topic_stb' topicFromStb = 'topic_stb'
queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
sqlString = "create topic %s as stable %s.%s" %(topicFromStb, paraDict['dbName'], paraDict['stbName']) sqlString = "create topic %s as stable %s.%s" %(topicFromStb, paraDict['dbName'], paraDict['stbName'])
#sqlString = "create topic %s as %s" %(topicFromStb, queryString)
tdLog.info("create topic sql: %s"%sqlString) tdLog.info("create topic sql: %s"%sqlString)
tdSql.execute(sqlString) tdSql.execute(sqlString)
@ -111,7 +112,7 @@ class TDTestCase:
ifManualCommit = 0 ifManualCommit = 0
keyList = 'group.id:cgrp1,\ keyList = 'group.id:cgrp1,\
enable.auto.commit:true,\ enable.auto.commit:true,\
auto.commit.interval.ms:1000,\ auto.commit.interval.ms:200,\
auto.offset.reset:latest' auto.offset.reset:latest'
tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
@ -119,7 +120,15 @@ class TDTestCase:
tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
# time.sleep(3) # time.sleep(3)
tmqCom.getStartCommitNotifyFromTmqsim() tmqCom.getStartCommitNotifyFromTmqsim('cdb',1)
tdLog.info("create some new child table and insert data for latest mode")
paraDict["batchNum"] = 100
paraDict["ctbPrefix"] = 'newCtb'
paraDict["ctbNum"] = 10
paraDict["rowsPerTbl"] = 10
tmqCom.insert_data_with_autoCreateTbl(tdSql,paraDict["dbName"],paraDict["stbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"])
tdLog.info("================= restart dnode ===========================") tdLog.info("================= restart dnode ===========================")
tdDnodes.stoptaosd(1) tdDnodes.stoptaosd(1)
tdDnodes.starttaosd(1) tdDnodes.starttaosd(1)
@ -132,8 +141,7 @@ class TDTestCase:
for i in range(expectRows): for i in range(expectRows):
totalConsumeRows += resultList[i] totalConsumeRows += resultList[i]
tdSql.query(queryString) totalRowsFromQury = paraDict["ctbNum"] * paraDict["rowsPerTbl"]
totalRowsFromQury = tdSql.getRows()
tdLog.info("act consume rows: %d, act query rows: %d"%(totalConsumeRows, totalRowsFromQury)) tdLog.info("act consume rows: %d, act query rows: %d"%(totalConsumeRows, totalRowsFromQury))
if (totalConsumeRows < totalRowsFromQury): if (totalConsumeRows < totalRowsFromQury):
@ -161,7 +169,7 @@ class TDTestCase:
'rowsPerTbl': 1000, 'rowsPerTbl': 1000,
'batchNum': 100, 'batchNum': 100,
'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'startTs': 1640966400000, # 2022-01-01 00:00:00.000
'pollDelay': 10, 'pollDelay': 30,
'showMsg': 1, 'showMsg': 1,
'showRow': 1, 'showRow': 1,
'snapshot': 0} 'snapshot': 0}
@ -185,19 +193,25 @@ class TDTestCase:
ifManualCommit = 0 ifManualCommit = 0
keyList = 'group.id:cgrp1,\ keyList = 'group.id:cgrp1,\
enable.auto.commit:true,\ enable.auto.commit:true,\
auto.commit.interval.ms:1000,\ auto.commit.interval.ms:200,\
auto.offset.reset:latest' auto.offset.reset:latest'
tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
tdLog.info("start consume processor") tdLog.info("start consume processor")
tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
# time.sleep(3) tmqCom.getStartCommitNotifyFromTmqsim('cdb',1)
tmqCom.getStartCommitNotifyFromTmqsim()
tdLog.info("create some new child table and insert data for latest mode")
paraDict["batchNum"] = 100
paraDict["ctbPrefix"] = 'newCtb'
paraDict["ctbNum"] = 10
paraDict["rowsPerTbl"] = 10
tmqCom.insert_data_with_autoCreateTbl(tdSql,paraDict["dbName"],paraDict["stbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"])
tdLog.info("================= restart dnode ===========================") tdLog.info("================= restart dnode ===========================")
tdDnodes.stoptaosd(1) tdDnodes.stoptaosd(1)
tdDnodes.starttaosd(1) tdDnodes.starttaosd(1)
# time.sleep(3)
tdLog.info(" restart taosd end and wait to check consume result") tdLog.info(" restart taosd end and wait to check consume result")
expectRows = 1 expectRows = 1
@ -206,8 +220,7 @@ class TDTestCase:
for i in range(expectRows): for i in range(expectRows):
totalConsumeRows += resultList[i] totalConsumeRows += resultList[i]
tdSql.query(queryString) totalRowsFromQury = paraDict["ctbNum"] * paraDict["rowsPerTbl"]
totalRowsFromQury = tdSql.getRows()
tdLog.info("act consume rows: %d, act query rows: %d"%(totalConsumeRows, totalRowsFromQury)) tdLog.info("act consume rows: %d, act query rows: %d"%(totalConsumeRows, totalRowsFromQury))
if (totalConsumeRows < totalRowsFromQury): if (totalConsumeRows < totalRowsFromQury):

81
tests/system-test/7-tmq/tmqError.py
View File

@ -11,6 +11,8 @@ from util.log import *
from util.sql import * from util.sql import *
from util.cases import * from util.cases import *
from util.dnodes import * from util.dnodes import *
sys.path.append("./7-tmq")
from tmqCommon import *
class actionType(Enum): class actionType(Enum):
CREATE_DATABASE = 0 CREATE_DATABASE = 0
@ -193,32 +195,41 @@ class TDTestCase:
and restart a consumption process to complete a consumption and restart a consumption process to complete a consumption
''' '''
tdLog.printNoPrefix("======== test case 1: ") tdLog.printNoPrefix("======== test case 1: ")
tmqCom.initConsumerTable()
self.initConsumerTable() #self.initConsumerTable()
# create and start thread # create and start thread
parameterDict = {'cfg': '', \ paraDict = {'dbName': 'dbt',
'actionType': 0, \ 'dropFlag': 1,
'dbName': 'db3', \ 'event': '',
'dropFlag': 1, \ 'vgroups': 4,
'vgroups': 4, \ 'stbName': 'stb',
'replica': 1, \ 'colPrefix': 'c',
'stbName': 'stb1', \ 'tagPrefix': 't',
'ctbNum': 10, \ 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
'rowsPerTbl': 20000, \ 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
'batchNum': 100, \ 'ctbPrefix': 'ctb',
'startTs': 1640966400000} # 2022-01-01 00:00:00.000 'ctbStartIdx': 0,
parameterDict['cfg'] = cfgPath 'ctbNum': 10,
'rowsPerTbl': 20000,
'batchNum': 1000,
'startTs': 1640966400000, # 2022-01-01 00:00:00.000
'pollDelay': 30,
'showMsg': 1,
'showRow': 1,
'snapshot': 0}
paraDict['cfg'] = cfgPath
self.create_database(tdSql, parameterDict["dbName"]) self.create_database(tdSql, paraDict["dbName"])
self.create_stable(tdSql, parameterDict["dbName"], parameterDict["stbName"]) self.create_stable(tdSql, paraDict["dbName"], paraDict["stbName"])
self.create_ctables(tdSql, parameterDict["dbName"], parameterDict["stbName"], parameterDict["ctbNum"]) self.create_ctables(tdSql, paraDict["dbName"], paraDict["stbName"], paraDict["ctbNum"])
self.insert_data(tdSql,parameterDict["dbName"],parameterDict["stbName"],parameterDict["ctbNum"],parameterDict["rowsPerTbl"],parameterDict["batchNum"]) self.insert_data(tdSql,paraDict["dbName"],paraDict["stbName"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"])
tdLog.info("create topics from stb1") tdLog.info("create topics from stb1")
topicFromStb1 = 'topic_stb1' topicFromStb1 = 'topic_stb1'
tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb1, parameterDict['dbName'], parameterDict['stbName'])) tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb1, paraDict['dbName'], paraDict['stbName']))
consumerId = 0 consumerId = 0
# expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] # expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"]
expectrowcnt = 90000000000 expectrowcnt = 90000000000
@ -229,41 +240,45 @@ class TDTestCase:
enable.auto.commit:false,\ enable.auto.commit:false,\
auto.commit.interval.ms:6000,\ auto.commit.interval.ms:6000,\
auto.offset.reset:earliest' auto.offset.reset:earliest'
self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) #self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
tdLog.info("start consume processor") tdLog.info("start consume processor")
pollDelay = 9000000 # Forever loop pollDelay = 9000000 # Forever loop
showMsg = 1 showMsg = 1
showRow = 1 showRow = 1
self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) #self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow)
tdLog.info("start consume processor")
tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
time.sleep(3) #time.sleep(3)
tmqCom.getStartConsumeNotifyFromTmqsim()
tdLog.info("================= stop dnode, and remove data file, then start dnode ===========================") tdLog.info("================= stop dnode, and remove data file, then start dnode ===========================")
tdDnodes.stop(1) tdDnodes.stop(1)
# time.sleep(5) time.sleep(5)
dataPath = buildPath + "/../sim/dnode1/data/*" dataPath = buildPath + "/../sim/dnode1/data/*"
shellCmd = 'rm -rf ' + dataPath shellCmd = 'rm -rf ' + dataPath
tdLog.info(shellCmd) tdLog.info(shellCmd)
os.system(shellCmd) os.system(shellCmd)
#tdDnodes.start(1) #tdDnodes.start(1)
tdDnodes.starttaosd(1) tdDnodes.starttaosd(1)
time.sleep(2) time.sleep(5)
######### redo to consume ######### redo to consume
self.initConsumerTable() self.initConsumerTable()
self.create_database(tdSql, parameterDict["dbName"]) self.create_database(tdSql, paraDict["dbName"])
self.create_stable(tdSql, parameterDict["dbName"], parameterDict["stbName"]) self.create_stable(tdSql, paraDict["dbName"], paraDict["stbName"])
self.create_ctables(tdSql, parameterDict["dbName"], parameterDict["stbName"], parameterDict["ctbNum"]) self.create_ctables(tdSql, paraDict["dbName"], paraDict["stbName"], paraDict["ctbNum"])
self.insert_data(tdSql,parameterDict["dbName"],parameterDict["stbName"],parameterDict["ctbNum"],parameterDict["rowsPerTbl"],parameterDict["batchNum"]) self.insert_data(tdSql,paraDict["dbName"],paraDict["stbName"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"])
tdLog.info("create topics from stb1") tdLog.info("create topics from stb1")
topicFromStb1 = 'topic_stb1' topicFromStb1 = 'topic_stb1'
tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb1, parameterDict['dbName'], parameterDict['stbName'])) tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb1, paraDict['dbName'], paraDict['stbName']))
consumerId = 0 consumerId = 0
expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"]
topicList = topicFromStb1 topicList = topicFromStb1
ifcheckdata = 0 ifcheckdata = 0
ifManualCommit = 0 ifManualCommit = 0
@ -271,13 +286,17 @@ class TDTestCase:
enable.auto.commit:false,\ enable.auto.commit:false,\
auto.commit.interval.ms:6000,\ auto.commit.interval.ms:6000,\
auto.offset.reset:earliest' auto.offset.reset:earliest'
self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) #self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
tdLog.info("start consume processor") tdLog.info("start consume processor")
pollDelay = 20 pollDelay = 20
showMsg = 1 showMsg = 1
showRow = 1 showRow = 1
self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) paraDict['pollDelay'] = 20
#self.startTmqSimProcess(buildPath,cfgPath,pollDelay,paraDict["dbName"],showMsg, showRow)
tdLog.info("start consume processor")
tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
expectRows = 1 expectRows = 1
resultList = self.selectConsumeResult(expectRows) resultList = self.selectConsumeResult(expectRows)

4
tests/system-test/7-tmq/tmqShow.py
View File

@ -41,7 +41,7 @@ class TDTestCase:
'rowsPerTbl': 4000, 'rowsPerTbl': 4000,
'batchNum': 15, 'batchNum': 15,
'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'startTs': 1640966400000, # 2022-01-01 00:00:00.000
'pollDelay': 20, 'pollDelay': 30,
'showMsg': 1, 'showMsg': 1,
'showRow': 1} 'showRow': 1}
@ -124,7 +124,7 @@ class TDTestCase:
tdLog.info("async insert data") tdLog.info("async insert data")
pThread = tmqCom.asyncInsertData(paraDict) pThread = tmqCom.asyncInsertData(paraDict)
tmqCom.getStartConsumeNotifyFromTmqsim(); tmqCom.getStartConsumeNotifyFromTmqsim()
#time.sleep(5) #time.sleep(5)
tdLog.info("check show consumers") tdLog.info("check show consumers")
tdSql.query("show consumers") tdSql.query("show consumers")