Merge pull request #12381 from taosdata/test-v3.0/lihui
test: add test case for tmq
This commit is contained in:
Hui Li
GitHub
commit
d538f9b879
|
|
@ -13,14 +13,12 @@ from util.dnodes import *
|
|||
|
||||
class TDTestCase:
|
||||
hostname = socket.gethostname()
|
||||
rpcDebugFlagVal = '143'
|
||||
clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''}
|
||||
clientCfgDict["rpcDebugFlag"] = rpcDebugFlagVal
|
||||
|
||||
updatecfgDict = {'clientCfg': {}, 'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''}
|
||||
updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal
|
||||
|
||||
print ("===================: ", updatecfgDict)
|
||||
#rpcDebugFlagVal = '143'
|
||||
#clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''}
|
||||
#clientCfgDict["rpcDebugFlag"] = rpcDebugFlagVal
|
||||
#updatecfgDict = {'clientCfg': {}, 'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''}
|
||||
#updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal
|
||||
#print ("===================: ", updatecfgDict)
|
||||
|
||||
def init(self, conn, logSql):
|
||||
tdLog.debug(f"start to excute {__file__}")
|
||||
|
|
@ -43,27 +41,35 @@ class TDTestCase:
|
|||
break
|
||||
return buildPath
|
||||
|
||||
def create_tables(self,dbName,vgroups,stbName,ctbNum,rowsPerTbl):
|
||||
tdSql.execute("create database if not exists %s vgroups %d"%(dbName, vgroups))
|
||||
tdSql.execute("use %s" %dbName)
|
||||
tdSql.execute("create table %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName)
|
||||
def newcur(self,cfg,host,port):
|
||||
user = "root"
|
||||
password = "taosdata"
|
||||
con=taos.connect(host=host, user=user, password=password, config=cfg ,port=port)
|
||||
cur=con.cursor()
|
||||
print(cur)
|
||||
return cur
|
||||
|
||||
def create_tables(self,tsql, dbName,vgroups,stbName,ctbNum,rowsPerTbl):
|
||||
tsql.execute("create database if not exists %s vgroups %d"%(dbName, vgroups))
|
||||
tsql.execute("use %s" %dbName)
|
||||
tsql.execute("create table %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName)
|
||||
pre_create = "create table"
|
||||
sql = pre_create
|
||||
#tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname))
|
||||
for i in range(ctbNum):
|
||||
sql += " %s_%d using %s tags(%d)"%(stbName,i,stbName,i+1)
|
||||
if (i > 0) and (i%100 == 0):
|
||||
tdSql.execute(sql)
|
||||
tsql.execute(sql)
|
||||
sql = pre_create
|
||||
if sql != pre_create:
|
||||
tdSql.execute(sql)
|
||||
tsql.execute(sql)
|
||||
|
||||
tdLog.debug("complete to create database[%s], stable[%s] and %d child tables" %(dbName, stbName, ctbNum))
|
||||
return
|
||||
|
||||
def insert_data(self,dbName,stbName,ctbNum,rowsPerTbl,startTs):
|
||||
def insert_data(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs):
|
||||
tdLog.debug("start to insert data ............")
|
||||
tdSql.execute("use %s" %dbName)
|
||||
tsql.execute("use %s" %dbName)
|
||||
pre_insert = "insert into "
|
||||
sql = pre_insert
|
||||
|
||||
|
|
@ -72,31 +78,38 @@ class TDTestCase:
|
|||
sql += " %s_%d values "%(stbName,i)
|
||||
for j in range(rowsPerTbl):
|
||||
sql += "(%d, %d, 'tmqrow_%d') "%(startTs + j, j, j)
|
||||
if (j > 0) and (j%2000 == 0):
|
||||
tdSql.execute(sql)
|
||||
sql = "insert into %s_%d values " %(stbName,i)
|
||||
if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)):
|
||||
tsql.execute(sql)
|
||||
if j < rowsPerTbl - 1:
|
||||
sql = "insert into %s_%d values " %(stbName,i)
|
||||
else:
|
||||
sql = "insert into "
|
||||
#end sql
|
||||
if sql != pre_insert:
|
||||
# print(sql)
|
||||
print("sql:%s"%sql)
|
||||
tdSql.execute(sql)
|
||||
#print("insert sql:%s"%sql)
|
||||
tsql.execute(sql)
|
||||
tdLog.debug("insert data ............ [OK]")
|
||||
return
|
||||
|
||||
def prepareEnv(self, **parameterDict):
|
||||
print ("input parameters:")
|
||||
print (parameterDict)
|
||||
self.create_tables(parameterDict["dbName"],\
|
||||
# create new connector for my thread
|
||||
tsql=self.newcur(parameterDict['cfg'], 'localhost', 6030)
|
||||
self.create_tables(tsql,\
|
||||
parameterDict["dbName"],\
|
||||
parameterDict["vgroups"],\
|
||||
parameterDict["stbName"],\
|
||||
parameterDict["ctbNum"],\
|
||||
parameterDict["rowsPerTbl"])
|
||||
|
||||
self.insert_data(parameterDict["dbName"],\
|
||||
parameterDict["stbName"],\
|
||||
parameterDict["ctbNum"],\
|
||||
parameterDict["rowsPerTbl"],\
|
||||
parameterDict["startTs"])
|
||||
self.insert_data(tsql,\
|
||||
parameterDict["dbName"],\
|
||||
parameterDict["stbName"],\
|
||||
parameterDict["ctbNum"],\
|
||||
parameterDict["rowsPerTbl"],\
|
||||
parameterDict["batchNum"],\
|
||||
parameterDict["startTs"])
|
||||
return
|
||||
|
||||
def run(self):
|
||||
|
|
@ -113,18 +126,115 @@ class TDTestCase:
|
|||
tdLog.printNoPrefix("======== test scenario 1: ")
|
||||
tdLog.info("step 1: create database, stb, ctb and insert data")
|
||||
# create and start thread
|
||||
parameterDict = {'dbName': 'db', \
|
||||
parameterDict = {'cfg': '', \
|
||||
'dbName': 'db', \
|
||||
'vgroups': 1, \
|
||||
'stbName': 'stb', \
|
||||
'ctbNum': 10, \
|
||||
'rowsPerTbl': 10, \
|
||||
'rowsPerTbl': 10000, \
|
||||
'batchNum': 10, \
|
||||
'startTs': 1640966400000} # 2022-01-01 00:00:00.000
|
||||
parameterDict['cfg'] = cfgPath
|
||||
prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict)
|
||||
prepareEnvThread.start()
|
||||
time.sleep(2)
|
||||
|
||||
# wait stb ready
|
||||
while 1:
|
||||
#tdSql.query("show %s.stables"%parameterDict['dbName'])
|
||||
tdSql.query("show db.stables")
|
||||
#print (self.queryResult)
|
||||
#print (tdSql.getRows())
|
||||
if tdSql.getRows() == 1:
|
||||
break
|
||||
else:
|
||||
time.sleep(1)
|
||||
|
||||
tdLog.info("create topics from super table")
|
||||
topicFromStb = 'topic_stb_column'
|
||||
topicFromCtb = 'topic_ctb_column'
|
||||
|
||||
tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb, parameterDict['dbName'], parameterDict['stbName']))
|
||||
tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s_0" %(topicFromCtb, parameterDict['dbName'], parameterDict['stbName']))
|
||||
|
||||
time.sleep(1)
|
||||
tdSql.query("show topics")
|
||||
print ("======================================")
|
||||
#print (self.queryResult)
|
||||
#tdSql.checkRows(2)
|
||||
topic1 = tdSql.getData(0 , 0)
|
||||
topic2 = tdSql.getData(1 , 0)
|
||||
print (topic1)
|
||||
print (topic2)
|
||||
|
||||
print (topicFromStb)
|
||||
print (topicFromCtb)
|
||||
#tdLog.info("show topics: %s, %s"%topic1, topic2)
|
||||
#if topic1 != topicFromStb or topic1 != topicFromCtb:
|
||||
# tdLog.exit("topic error1")
|
||||
#if topic2 != topicFromStb or topic2 != topicFromCtb:
|
||||
# tdLog.exit("topic error2")
|
||||
|
||||
tdLog.info("create consume info table and consume result table")
|
||||
cdbName = parameterDict["dbName"]
|
||||
#tdSql.query("create database %s"%cdbName)
|
||||
#tdSql.query("use %s"%cdbName)
|
||||
tdSql.query("create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)")
|
||||
tdSql.query("create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)")
|
||||
|
||||
consumerId = 0
|
||||
expectmsgcnt = (parameterDict["rowsPerTbl"] / parameterDict["batchNum"] ) * parameterDict["ctbNum"]
|
||||
expectmsgcnt1 = expectmsgcnt + parameterDict["ctbNum"]
|
||||
topicList = topicFromStb
|
||||
ifcheckdata = 0
|
||||
keyList = 'group.id:cgrp1,\
|
||||
enable.auto.commit:false,\
|
||||
auto.commit.interval.ms:6000,\
|
||||
auto.offset.reset:earliest'
|
||||
sql = "insert into consumeinfo values "
|
||||
sql += "(now, %d, '%s', '%s', %d, %d)"%(consumerId, topicList, keyList, expectmsgcnt1, ifcheckdata)
|
||||
tdSql.query(sql)
|
||||
|
||||
tdLog.info("check stb if there are data")
|
||||
while 1:
|
||||
tdSql.query("select count(*) from %s"%parameterDict["stbName"])
|
||||
#tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3))
|
||||
countOfStb = tdSql.getData(0, 0)
|
||||
if countOfStb != 0:
|
||||
tdLog.info("count from stb: %d"%countOfStb)
|
||||
break
|
||||
else:
|
||||
time.sleep(1)
|
||||
|
||||
tdLog.info("start consume processor")
|
||||
pollDelay = 5
|
||||
showMsg = 1
|
||||
showRow = 1
|
||||
|
||||
shellCmd = 'nohup ' + buildPath + '/build/bin/tmq_sim -c ' + cfgPath
|
||||
shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, parameterDict["dbName"], showMsg, showRow, cdbName)
|
||||
shellCmd += "> /dev/null 2>&1 &"
|
||||
tdLog.info(shellCmd)
|
||||
os.system(shellCmd)
|
||||
|
||||
# wait for data ready
|
||||
prepareEnvThread.join()
|
||||
|
||||
tdLog.info("insert process end, and start to check consume result")
|
||||
while 1:
|
||||
tdSql.query("select * from consumeresult")
|
||||
#tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3))
|
||||
if tdSql.getRows() == 1:
|
||||
break
|
||||
else:
|
||||
time.sleep(5)
|
||||
|
||||
expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"]
|
||||
|
||||
tdSql.checkData(0 , 1, consumerId)
|
||||
tdSql.checkData(0 , 2, expectmsgcnt)
|
||||
tdSql.checkData(0 , 3, expectrowcnt)
|
||||
|
||||
tdLog.printNoPrefix("======== test scenario 2: ")
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -51,3 +51,7 @@ python3 ./test.py -f 2-query/arcsin.py
|
|||
python3 ./test.py -f 2-query/arccos.py
|
||||
python3 ./test.py -f 2-query/arctan.py
|
||||
# python3 ./test.py -f 2-query/query_cols_tags_and_or.py
|
||||
|
||||
python3 ./test.py -f 7-tmq/basic5.py
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -37,6 +37,10 @@ typedef struct {
|
|||
TdThread thread;
|
||||
int32_t consumerId;
|
||||
|
||||
int32_t autoCommitIntervalMs; // 1000 ms
|
||||
char autoCommit[8]; // true, false
|
||||
char autoOffsetRest[16]; // none, earliest, latest
|
||||
|
||||
int32_t ifCheckData;
|
||||
int64_t expectMsgCnt;
|
||||
|
||||
|
|
@ -120,6 +124,9 @@ void saveConfigToLogFile() {
|
|||
|
||||
for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) {
|
||||
taosFprintfFile(g_fp, "# consumer %d info:\n", g_stConfInfo.stThreads[i].consumerId);
|
||||
taosFprintfFile(g_fp, " auto commit: %s\n", g_stConfInfo.stThreads[i].autoCommit);
|
||||
taosFprintfFile(g_fp, " auto commit interval ms: %d\n", g_stConfInfo.stThreads[i].autoCommitIntervalMs);
|
||||
taosFprintfFile(g_fp, " auto offset rest: %s\n", g_stConfInfo.stThreads[i].autoOffsetRest);
|
||||
taosFprintfFile(g_fp, " Topics: ");
|
||||
for (int j = 0; j < g_stConfInfo.stThreads[i].numOfTopic; j++) {
|
||||
taosFprintfFile(g_fp, "%s, ", g_stConfInfo.stThreads[i].topics[j]);
|
||||
|
|
@ -251,7 +258,7 @@ void build_consumer(SThreadInfo* pInfo) {
|
|||
tmq_conf_set(conf, "td.connect.user", "root");
|
||||
tmq_conf_set(conf, "td.connect.pass", "taosdata");
|
||||
|
||||
tmq_conf_set(conf, "td.connect.db", g_stConfInfo.dbName);
|
||||
//tmq_conf_set(conf, "td.connect.db", g_stConfInfo.dbName);
|
||||
|
||||
tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print, NULL);
|
||||
|
||||
|
|
@ -272,6 +279,9 @@ void build_consumer(SThreadInfo* pInfo) {
|
|||
// tmq_conf_set(conf, "auto.offset.reset", "latest");
|
||||
|
||||
pInfo->tmq = tmq_consumer_new(conf, NULL, 0);
|
||||
|
||||
tmq_conf_destroy(conf);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -324,9 +334,11 @@ void loop_consume(SThreadInfo* pInfo) {
|
|||
totalMsgs++;
|
||||
|
||||
if (totalMsgs >= pInfo->expectMsgCnt) {
|
||||
taosFprintfFile(g_fp, "==== totalMsgs >= pInfo->expectMsgCnt, so break\n");
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
taosFprintfFile(g_fp, "==== delay over time, so break\n");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
|
@ -355,6 +367,9 @@ void* consumeThreadFunc(void* param) {
|
|||
exit(-1);
|
||||
}
|
||||
|
||||
tmq_list_destroy(pInfo->topicList);
|
||||
pInfo->topicList = NULL;
|
||||
|
||||
loop_consume(pInfo);
|
||||
|
||||
tmq_commit(pInfo->tmq, NULL, 0);
|
||||
|
|
@ -442,6 +457,11 @@ int32_t getConsumeInfo() {
|
|||
while ((row = taos_fetch_row(pRes))) {
|
||||
int32_t* lengths = taos_fetch_lengths(pRes);
|
||||
|
||||
// set default value
|
||||
g_stConfInfo.stThreads[numOfThread].autoCommitIntervalMs = 5000;
|
||||
memcpy(g_stConfInfo.stThreads[numOfThread].autoCommit, "true", strlen("true"));
|
||||
memcpy(g_stConfInfo.stThreads[numOfThread].autoOffsetRest, "earlieast", strlen("earlieast"));
|
||||
|
||||
for (int i = 0; i < num_fields; ++i) {
|
||||
if (row[i] == NULL || 0 == i) {
|
||||
continue;
|
||||
|
|
@ -457,6 +477,12 @@ int32_t getConsumeInfo() {
|
|||
g_stConfInfo.stThreads[numOfThread].expectMsgCnt = *((int64_t*)row[i]);
|
||||
} else if ((5 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) {
|
||||
g_stConfInfo.stThreads[numOfThread].ifCheckData = *((int32_t*)row[i]);
|
||||
} else if ((6 == i) && (fields[i].type == TSDB_DATA_TYPE_BINARY)) {
|
||||
memcpy(g_stConfInfo.stThreads[numOfThread].autoCommit, row[i], lengths[i]);
|
||||
} else if ((7 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) {
|
||||
g_stConfInfo.stThreads[numOfThread].autoCommitIntervalMs = *((int32_t*)row[i]);
|
||||
} else if ((8 == i) && (fields[i].type == TSDB_DATA_TYPE_BINARY)) {
|
||||
memcpy(g_stConfInfo.stThreads[numOfThread].autoOffsetRest, row[i], lengths[i]);
|
||||
}
|
||||
}
|
||||
numOfThread++;
|
||||
|
|
|
|||
Loading…
Reference in New Issue