From 4cc3b019b362c42c168eb3b72815ce031b9d4382 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Fri, 22 Apr 2022 17:06:08 +0800 Subject: [PATCH] [test: add test cases for tmq] --- tests/script/tsim/tmq/basic1.sim | 72 +++-- tests/script/tsim/tmq/basic2.sim | 219 ++++++++++++++ tests/script/tsim/tmq/basic3.sim | 278 ++++++++++++++++++ tests/script/tsim/tmq/basic4.sim | 216 ++++++++++++++ .../script/tsim/tmq/prepareBasicEnv-1vgrp.sim | 88 ++++++ .../script/tsim/tmq/prepareBasicEnv-4vgrp.sim | 88 ++++++ tests/script/tsim/tmq/prepareBasicEnv.sim | 2 +- tests/test/c/tmqSim.c | 13 +- 8 files changed, 947 insertions(+), 29 deletions(-) create mode 100644 tests/script/tsim/tmq/basic2.sim create mode 100644 tests/script/tsim/tmq/basic3.sim create mode 100644 tests/script/tsim/tmq/basic4.sim create mode 100644 tests/script/tsim/tmq/prepareBasicEnv-1vgrp.sim create mode 100644 tests/script/tsim/tmq/prepareBasicEnv-4vgrp.sim diff --git a/tests/script/tsim/tmq/basic1.sim b/tests/script/tsim/tmq/basic1.sim index df6a553d1a..f4e8f08c37 100644 --- a/tests/script/tsim/tmq/basic1.sim +++ b/tests/script/tsim/tmq/basic1.sim @@ -1,15 +1,16 @@ #### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 -# vgroups=1, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# vgroups=1, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# vgroups=4, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# vgroups=4, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +#basic1.sim: vgroups=1, one topic for one consumer, firstly insert data, then start consume. Include six topics +#basic2.sim: vgroups=1, multi topics for one consumer, firstly insert data, then start consume. Include six topics +#basic3.sim: vgroups=4, one topic for one consumer, firstly insert data, then start consume. Include six topics +#basic4.sim: vgroups=4, multi topics for one consumer, firstly insert data, then start consume. Include six topics + # notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN # The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; # # notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). # -run tsim/tmq/prepareBasicEnv.sim +run tsim/tmq/prepareBasicEnv-1vgrp.sim #---- global parameters start ----# $dbName = db @@ -20,11 +21,11 @@ $ntbPrefix = ntb $stbNum = 1 $ctbNum = 10 $ntbNum = 10 -$rowsPerCtb = 100 +$rowsPerCtb = 10 $tstart = 1640966400000 # 2022-01-01 00:00:00.000 #---- global parameters end ----# -$pullDelay = 3 +$pullDelay = 5 $ifcheckdata = 1 $showMsg = 1 $showRow = 0 @@ -55,9 +56,33 @@ sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 $keyList = ' . group.id:cgrp1 $keyList = $keyList . ' +$cdb_index = 0 +#=============================== start consume =============================# + print ================ test consume from stb $loop_cnt = 0 loop_consume_diff_topic_from_stb: + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdb_index = $cdb_index + 1 +$cdbName = cdb . $cdb_index +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + if $loop_cnt == 0 then print == scenario 1: topic_stb_column $topicList = ' . topic_stb_column @@ -76,13 +101,12 @@ endi $consumerId = 0 $totalMsgOfStb = $ctbNum * $rowsPerCtb -#$expectmsgcnt = $totalMsgOfStb + 1 -$expectmsgcnt = 110 +$expectmsgcnt = $totalMsgOfStb sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) print == start consumer to pull msgs from stb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start print == check consume result wait_consumer_end_from_stb: @@ -106,11 +130,16 @@ $loop_cnt = $loop_cnt + 1 goto loop_consume_diff_topic_from_stb loop_consume_diff_topic_from_stb_end: +print ================ test consume from ctb +$loop_cnt = 0 +loop_consume_diff_topic_from_ctb: + ####################################################################################### # clear consume info and consume result #run tsim/tmq/clearConsume.sim # because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdbName = cdb1 +$cdb_index = $cdb_index + 1 +$cdbName = cdb . $cdb_index sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName @@ -125,10 +154,6 @@ if $rows != 2 then endi ####################################################################################### - -print ================ test consume from ctb -$loop_cnt = 0 -loop_consume_diff_topic_from_ctb: if $loop_cnt == 0 then print == scenario 1: topic_ctb_column $topicList = ' . topic_ctb_column @@ -147,7 +172,7 @@ endi $consumerId = 0 $totalMsgOfCtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfCtb + 1 +$expectmsgcnt = $totalMsgOfCtb sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) print == start consumer to pull msgs from stb @@ -176,11 +201,16 @@ $loop_cnt = $loop_cnt + 1 goto loop_consume_diff_topic_from_ctb loop_consume_diff_topic_from_ctb_end: +print ================ test consume from ntb +$loop_cnt = 0 +loop_consume_diff_topic_from_ntb: + ####################################################################################### # clear consume info and consume result #run tsim/tmq/clearConsume.sim # because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdbName = cdb2 +$cdb_index = $cdb_index + 1 +$cdbName = cdb . $cdb_index sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName @@ -195,10 +225,6 @@ if $rows != 2 then endi ####################################################################################### - -print ================ test consume from ntb -$loop_cnt = 0 -loop_consume_diff_topic_from_ntb: if $loop_cnt == 0 then print == scenario 1: topic_ntb_column $topicList = ' . topic_ntb_column @@ -217,7 +243,7 @@ endi $consumerId = 0 $totalMsgOfNtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfNtb + 1 +$expectmsgcnt = $totalMsgOfNtb sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) print == start consumer to pull msgs from stb diff --git a/tests/script/tsim/tmq/basic2.sim b/tests/script/tsim/tmq/basic2.sim new file mode 100644 index 0000000000..b3e9a67c62 --- /dev/null +++ b/tests/script/tsim/tmq/basic2.sim @@ -0,0 +1,219 @@ +#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 +#basic1.sim: vgroups=1, one topic for one consumer, firstly insert data, then start consume. Include six topics +#basic2.sim: vgroups=1, multi topics for one consumer, firstly insert data, then start consume. Include six topics +#basic3.sim: vgroups=4, one topic for one consumer, firstly insert data, then start consume. Include six topics +#basic4.sim: vgroups=4, multi topics for one consumer, firstly insert data, then start consume. Include six topics + +# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN +# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; +# +# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). +# + +run tsim/tmq/prepareBasicEnv-1vgrp.sim + +#---- global parameters start ----# +$dbName = db +$vgroups = 1 +$stbPrefix = stb +$ctbPrefix = ctb +$ntbPrefix = ntb +$stbNum = 1 +$ctbNum = 10 +$ntbNum = 10 +$rowsPerCtb = 10 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +#---- global parameters end ----# + +$pullDelay = 5 +$ifcheckdata = 1 +$showMsg = 1 +$showRow = 0 + +sql connect +sql use $dbName + +print == create topics from super table +sql create topic topic_stb_column as select ts, c3 from stb +sql create topic topic_stb_all as select ts, c1, c2, c3 from stb +sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb + +print == create topics from child table +sql create topic topic_ctb_column as select ts, c3 from ctb0 +sql create topic topic_ctb_all as select * from ctb0 +sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 + +print == create topics from normal table +sql create topic topic_ntb_column as select ts, c3 from ntb0 +sql create topic topic_ntb_all as select * from ntb0 +sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 + +#sql show topics +#if $rows != 9 then +# return -1 +#endi + +$keyList = ' . group.id:cgrp1 +$keyList = $keyList . ' + +$topicNum = 3 + +#=============================== start consume =============================# + + +print ================ test consume from stb +print == multi toipcs: topic_stb_column + topic_stb_all + topic_stb_function +$topicList = ' . topic_stb_column +$topicList = $topicList . , +$topicList = $topicList . topic_stb_all +$topicList = $topicList . , +$topicList = $topicList . topic_stb_function +$topicList = $topicList . ' + +$consumerId = 0 +$totalMsgOfStb = $ctbNum * $rowsPerCtb +$totalMsgOfStb = $totalMsgOfStb * $topicNum +$expectmsgcnt = $totalMsgOfStb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) + +print == start consumer to pull msgs from stb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start + +print == check consume result +wait_consumer_end_from_stb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_stb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $expectmsgcnt then + return -1 +endi +if $data[0][3] != $expectmsgcnt then + return -1 +endi + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdbName = cdb1 +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + + +print ================ test consume from ctb +print == multi toipcs: topic_ctb_column + topic_ctb_all + topic_ctb_function +$topicList = ' . topic_ctb_column +$topicList = $topicList . , +$topicList = $topicList . topic_ctb_all +$topicList = $topicList . , +$topicList = $topicList . topic_ctb_function +$topicList = $topicList . ' + +$consumerId = 0 +$totalMsgOfCtb = $rowsPerCtb * $topicNum +$expectmsgcnt = $totalMsgOfCtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) + +print == start consumer to pull msgs from stb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result +wait_consumer_end_from_ctb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_ctb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $totalMsgOfCtb then + return -1 +endi +if $data[0][3] != $totalMsgOfCtb then + return -1 +endi + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdbName = cdb2 +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + + +print ================ test consume from ntb +print == multi toipcs: topic_ntb_column + topic_ntb_all + topic_ntb_function +$topicList = ' . topic_ntb_column +$topicList = $topicList . , +$topicList = $topicList . topic_ntb_all +$topicList = $topicList . , +$topicList = $topicList . topic_ntb_function +$topicList = $topicList . ' + +$consumerId = 0 +$totalMsgOfNtb = $rowsPerCtb * $topicNum +$expectmsgcnt = $totalMsgOfNtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) + +print == start consumer to pull msgs from stb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result from ntb +wait_consumer_end_from_ntb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_ntb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $totalMsgOfNtb then + return -1 +endi +if $data[0][3] != $totalMsgOfNtb then + return -1 +endi + +#------ not need stop consumer, because it exit after pull msg overthan expect msg +#system tsim/tmq/consume.sh -s stop -x SIGINT + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/basic3.sim b/tests/script/tsim/tmq/basic3.sim new file mode 100644 index 0000000000..7a0ee6f917 --- /dev/null +++ b/tests/script/tsim/tmq/basic3.sim @@ -0,0 +1,278 @@ +#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 +#basic1.sim: vgroups=1, one topic for one consumer, firstly insert data, then start consume. Include six topics +#basic2.sim: vgroups=1, multi topics for one consumer, firstly insert data, then start consume. Include six topics +#basic3.sim: vgroups=4, one topic for one consumer, firstly insert data, then start consume. Include six topics +#basic4.sim: vgroups=4, multi topics for one consumer, firstly insert data, then start consume. Include six topics + +# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN +# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; +# +# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). +# + +run tsim/tmq/prepareBasicEnv-4vgrp.sim + +#---- global parameters start ----# +$dbName = db +$vgroups = 4 +$stbPrefix = stb +$ctbPrefix = ctb +$ntbPrefix = ntb +$stbNum = 1 +$ctbNum = 10 +$ntbNum = 10 +$rowsPerCtb = 10 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +#---- global parameters end ----# + +$pullDelay = 5 +$ifcheckdata = 1 +$showMsg = 1 +$showRow = 0 + +sql connect +sql use $dbName + +print == create topics from super table +sql create topic topic_stb_column as select ts, c3 from stb +sql create topic topic_stb_all as select ts, c1, c2, c3 from stb +sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb + +print == create topics from child table +sql create topic topic_ctb_column as select ts, c3 from ctb0 +sql create topic topic_ctb_all as select * from ctb0 +sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 + +print == create topics from normal table +sql create topic topic_ntb_column as select ts, c3 from ntb0 +sql create topic topic_ntb_all as select * from ntb0 +sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 + +#sql show topics +#if $rows != 9 then +# return -1 +#endi + +$keyList = ' . group.id:cgrp1 +$keyList = $keyList . ' + +$cdb_index = 0 +#=============================== start consume =============================# + +print ================ test consume from stb +$loop_cnt = 0 +loop_consume_diff_topic_from_stb: + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdb_index = $cdb_index + 1 +$cdbName = cdb . $cdb_index +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + +if $loop_cnt == 0 then + print == scenario 1: topic_stb_column + $topicList = ' . topic_stb_column + $topicList = $topicList . ' +elif $loop_cnt == 1 then + print == scenario 2: topic_stb_all + $topicList = ' . topic_stb_all + $topicList = $topicList . ' +elif $loop_cnt == 2 then + print == scenario 3: topic_stb_function + $topicList = ' . topic_stb_function + $topicList = $topicList . ' +else + goto loop_consume_diff_topic_from_stb_end +endi + +$consumerId = 0 +$totalMsgOfStb = $ctbNum * $rowsPerCtb +$expectmsgcnt = $totalMsgOfStb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) + +print == start consumer to pull msgs from stb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result +wait_consumer_end_from_stb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_stb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $expectmsgcnt then + return -1 +endi +if $data[0][3] != $expectmsgcnt then + return -1 +endi +$loop_cnt = $loop_cnt + 1 +goto loop_consume_diff_topic_from_stb +loop_consume_diff_topic_from_stb_end: + +print ================ test consume from ctb +$loop_cnt = 0 +loop_consume_diff_topic_from_ctb: + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdb_index = $cdb_index + 1 +$cdbName = cdb . $cdb_index +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + +if $loop_cnt == 0 then + print == scenario 1: topic_ctb_column + $topicList = ' . topic_ctb_column + $topicList = $topicList . ' +elif $loop_cnt == 1 then + print == scenario 2: topic_ctb_all + $topicList = ' . topic_ctb_all + $topicList = $topicList . ' +elif $loop_cnt == 2 then + print == scenario 3: topic_ctb_function + $topicList = ' . topic_ctb_function + $topicList = $topicList . ' +else + goto loop_consume_diff_topic_from_ctb_end +endi + +$consumerId = 0 +$totalMsgOfCtb = $rowsPerCtb +$expectmsgcnt = $totalMsgOfCtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) + +print == start consumer to pull msgs from stb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result +wait_consumer_end_from_ctb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_ctb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $totalMsgOfCtb then + return -1 +endi +if $data[0][3] != $totalMsgOfCtb then + return -1 +endi +$loop_cnt = $loop_cnt + 1 +goto loop_consume_diff_topic_from_ctb +loop_consume_diff_topic_from_ctb_end: + +print ================ test consume from ntb +$loop_cnt = 0 +loop_consume_diff_topic_from_ntb: + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdb_index = $cdb_index + 1 +$cdbName = cdb . $cdb_index +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + +if $loop_cnt == 0 then + print == scenario 1: topic_ntb_column + $topicList = ' . topic_ntb_column + $topicList = $topicList . ' +elif $loop_cnt == 1 then + print == scenario 2: topic_ntb_all + $topicList = ' . topic_ntb_all + $topicList = $topicList . ' +elif $loop_cnt == 2 then + print == scenario 3: topic_ntb_function + $topicList = ' . topic_ntb_function + $topicList = $topicList . ' +else + goto loop_consume_diff_topic_from_ntb_end +endi + +$consumerId = 0 +$totalMsgOfNtb = $rowsPerCtb +$expectmsgcnt = $totalMsgOfNtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) + +print == start consumer to pull msgs from stb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result from ntb +wait_consumer_end_from_ntb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_ntb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $totalMsgOfNtb then + return -1 +endi +if $data[0][3] != $totalMsgOfNtb then + return -1 +endi +$loop_cnt = $loop_cnt + 1 +goto loop_consume_diff_topic_from_ntb +loop_consume_diff_topic_from_ntb_end: + +#------ not need stop consumer, because it exit after pull msg overthan expect msg +#system tsim/tmq/consume.sh -s stop -x SIGINT + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/basic4.sim b/tests/script/tsim/tmq/basic4.sim new file mode 100644 index 0000000000..1af02751b1 --- /dev/null +++ b/tests/script/tsim/tmq/basic4.sim @@ -0,0 +1,216 @@ +#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 +#basic1.sim: vgroups=1, one topic for one consumer, firstly insert data, then start consume. Include six topics +#basic2.sim: vgroups=1, multi topics for one consumer, firstly insert data, then start consume. Include six topics +#basic3.sim: vgroups=4, one topic for one consumer, firstly insert data, then start consume. Include six topics +#basic4.sim: vgroups=4, multi topics for one consumer, firstly insert data, then start consume. Include six topics + +# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN +# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; +# +# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). +# + +run tsim/tmq/prepareBasicEnv-4vgrp.sim + +#---- global parameters start ----# +$dbName = db +$vgroups = 4 +$stbPrefix = stb +$ctbPrefix = ctb +$ntbPrefix = ntb +$stbNum = 1 +$ctbNum = 10 +$ntbNum = 10 +$rowsPerCtb = 10 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +#---- global parameters end ----# + +$pullDelay = 5 +$ifcheckdata = 1 +$showMsg = 1 +$showRow = 0 + +sql connect +sql use $dbName + +print == create topics from super table +sql create topic topic_stb_column as select ts, c3 from stb +sql create topic topic_stb_all as select ts, c1, c2, c3 from stb +sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb + +print == create topics from child table +sql create topic topic_ctb_column as select ts, c3 from ctb0 +sql create topic topic_ctb_all as select * from ctb0 +sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 + +print == create topics from normal table +sql create topic topic_ntb_column as select ts, c3 from ntb0 +sql create topic topic_ntb_all as select * from ntb0 +sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 + +#sql show topics +#if $rows != 9 then +# return -1 +#endi + +$keyList = ' . group.id:cgrp1 +$keyList = $keyList . ' + +$topicNum = 3 + +print ================ test consume from stb +print == multi toipcs: topic_stb_column + topic_stb_all + topic_stb_function +$topicList = ' . topic_stb_column +$topicList = $topicList . , +$topicList = $topicList . topic_stb_all +$topicList = $topicList . , +$topicList = $topicList . topic_stb_function +$topicList = $topicList . ' + +$consumerId = 0 +$totalMsgOfStb = $ctbNum * $rowsPerCtb +$totalMsgOfStb = $totalMsgOfStb * $topicNum +$expectmsgcnt = $totalMsgOfStb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) + +print == start consumer to pull msgs from stb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start + +print == check consume result +wait_consumer_end_from_stb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_stb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $expectmsgcnt then + return -1 +endi +if $data[0][3] != $expectmsgcnt then + return -1 +endi + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdbName = cdb1 +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + + +print ================ test consume from ctb +print == multi toipcs: topic_ctb_column + topic_ctb_all + topic_ctb_function +$topicList = ' . topic_ctb_column +$topicList = $topicList . , +$topicList = $topicList . topic_ctb_all +$topicList = $topicList . , +$topicList = $topicList . topic_ctb_function +$topicList = $topicList . ' + +$consumerId = 0 +$totalMsgOfCtb = $rowsPerCtb * $topicNum +$expectmsgcnt = $totalMsgOfCtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) + +print == start consumer to pull msgs from stb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result +wait_consumer_end_from_ctb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_ctb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $totalMsgOfCtb then + return -1 +endi +if $data[0][3] != $totalMsgOfCtb then + return -1 +endi + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdbName = cdb2 +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + + +print ================ test consume from ntb +print == multi toipcs: topic_ntb_column + topic_ntb_all + topic_ntb_function +$topicList = ' . topic_ntb_column +$topicList = $topicList . , +$topicList = $topicList . topic_ntb_all +$topicList = $topicList . , +$topicList = $topicList . topic_ntb_function +$topicList = $topicList . ' + +$consumerId = 0 +$totalMsgOfNtb = $rowsPerCtb * $topicNum +$expectmsgcnt = $totalMsgOfNtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) + +print == start consumer to pull msgs from stb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result from ntb +wait_consumer_end_from_ntb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_ntb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $totalMsgOfNtb then + return -1 +endi +if $data[0][3] != $totalMsgOfNtb then + return -1 +endi + +#------ not need stop consumer, because it exit after pull msg overthan expect msg +#system tsim/tmq/consume.sh -s stop -x SIGINT + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/prepareBasicEnv-1vgrp.sim b/tests/script/tsim/tmq/prepareBasicEnv-1vgrp.sim new file mode 100644 index 0000000000..db56bcf743 --- /dev/null +++ b/tests/script/tsim/tmq/prepareBasicEnv-1vgrp.sim @@ -0,0 +1,88 @@ +# stop all dnodes before start this case +system sh/stop_dnodes.sh + +# deploy dnode 1 +system sh/deploy.sh -n dnode1 -i 1 + +# add some config items for this case +#system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 + +# start dnode 1 +system sh/exec.sh -n dnode1 -s start + +sql connect + +#---- global parameters start ----# +$dbName = db +$vgroups = 1 +$stbPrefix = stb +$ctbPrefix = ctb +$ntbPrefix = ntb +$stbNum = 1 +$ctbNum = 10 +$ntbNum = 10 +$rowsPerCtb = 10 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +#---- global parameters end ----# + +print == create database $dbName vgroups $vgroups +sql create database $dbName vgroups $vgroups + +#wait database ready +$loop_cnt = 0 +check_db_ready: +if $loop_cnt == 10 then + print ====> database not ready! + return -1 +endi +sql show databases +print ==> rows: $rows +print ==> $data(db)[0] $data(db)[1] $data(db)[2] $data(db)[3] $data(db)[4] $data(db)[5] $data(db)[6] $data(db)[7] $data(db)[8] $data(db)[9] $data(db)[10] $data(db)[11] $data(db)[12] +print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $data(db)[18] $data(db)[19] $data(db)[20] +if $data(db)[20] != nostrict then + sleep 100 + $loop_cnt = $loop_cnt + 1 + goto check_db_ready +endi + +sql use $dbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi + +print == create super table +sql create table $stbPrefix (ts timestamp, c1 int, c2 float, c3 binary(16)) tags (t1 int) +sql show stables +if $rows != 1 then + return -1 +endi + +print == create child table, normal table and insert data +$i = 0 +while $i < $ctbNum + $ctb = $ctbPrefix . $i + $ntb = $ntbPrefix . $i + sql create table $ctb using $stbPrefix tags( $i ) + sql create table $ntb (ts timestamp, c1 int, c2 float, c3 binary(16)) + + $x = 0 + while $x < $rowsPerCtb + $binary = ' . binary- + $binary = $binary . $i + $binary = $binary . ' + + sql insert into $ctb values ($tstart , $i , $x , $binary ) + sql insert into $ntb values ($tstart , $i , $x , $binary ) + $tstart = $tstart + 1 + $x = $x + 1 + endw + + $i = $i + 1 + $tstart = 1640966400000 +endw diff --git a/tests/script/tsim/tmq/prepareBasicEnv-4vgrp.sim b/tests/script/tsim/tmq/prepareBasicEnv-4vgrp.sim new file mode 100644 index 0000000000..d7fa58558f --- /dev/null +++ b/tests/script/tsim/tmq/prepareBasicEnv-4vgrp.sim @@ -0,0 +1,88 @@ +# stop all dnodes before start this case +system sh/stop_dnodes.sh + +# deploy dnode 1 +system sh/deploy.sh -n dnode1 -i 1 + +# add some config items for this case +#system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 + +# start dnode 1 +system sh/exec.sh -n dnode1 -s start + +sql connect + +#---- global parameters start ----# +$dbName = db +$vgroups = 4 +$stbPrefix = stb +$ctbPrefix = ctb +$ntbPrefix = ntb +$stbNum = 1 +$ctbNum = 10 +$ntbNum = 10 +$rowsPerCtb = 10 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +#---- global parameters end ----# + +print == create database $dbName vgroups $vgroups +sql create database $dbName vgroups $vgroups + +#wait database ready +$loop_cnt = 0 +check_db_ready: +if $loop_cnt == 10 then + print ====> database not ready! + return -1 +endi +sql show databases +print ==> rows: $rows +print ==> $data(db)[0] $data(db)[1] $data(db)[2] $data(db)[3] $data(db)[4] $data(db)[5] $data(db)[6] $data(db)[7] $data(db)[8] $data(db)[9] $data(db)[10] $data(db)[11] $data(db)[12] +print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $data(db)[18] $data(db)[19] $data(db)[20] +if $data(db)[20] != nostrict then + sleep 100 + $loop_cnt = $loop_cnt + 1 + goto check_db_ready +endi + +sql use $dbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi + +print == create super table +sql create table $stbPrefix (ts timestamp, c1 int, c2 float, c3 binary(16)) tags (t1 int) +sql show stables +if $rows != 1 then + return -1 +endi + +print == create child table, normal table and insert data +$i = 0 +while $i < $ctbNum + $ctb = $ctbPrefix . $i + $ntb = $ntbPrefix . $i + sql create table $ctb using $stbPrefix tags( $i ) + sql create table $ntb (ts timestamp, c1 int, c2 float, c3 binary(16)) + + $x = 0 + while $x < $rowsPerCtb + $binary = ' . binary- + $binary = $binary . $i + $binary = $binary . ' + + sql insert into $ctb values ($tstart , $i , $x , $binary ) + sql insert into $ntb values ($tstart , $i , $x , $binary ) + $tstart = $tstart + 1 + $x = $x + 1 + endw + + $i = $i + 1 + $tstart = 1640966400000 +endw diff --git a/tests/script/tsim/tmq/prepareBasicEnv.sim b/tests/script/tsim/tmq/prepareBasicEnv.sim index 066d7d4ab0..c4f706e262 100644 --- a/tests/script/tsim/tmq/prepareBasicEnv.sim +++ b/tests/script/tsim/tmq/prepareBasicEnv.sim @@ -14,7 +14,7 @@ sql connect #---- global parameters start ----# $dbName = db -$vgroups = 1 +$vgroups = 4 $stbPrefix = stb $ctbPrefix = ctb $ntbPrefix = ntb diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index f9ba0e418c..9f6c1a59eb 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -14,13 +14,14 @@ */ #include +#include #include #include #include #include #include #include -// #include +#include #include "taos.h" #include "taoserror.h" @@ -99,11 +100,11 @@ void initLogFile() { // FILE *fp = fopen(g_stConfInfo.resultFileName, "a"); char file[256]; sprintf(file, "%s/../log/tmqlog.txt", configDir); - TdFilePtr pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND | TD_FILE_STREAM); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_TEXT | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM); if (NULL == pFile) { fprintf(stderr, "Failed to open %s for save result\n", "./tmqlog.txt"); - exit(-1); - } + exit -1; + }; g_fp = pFile; } @@ -119,6 +120,7 @@ void saveConfigToLogFile() { taosFprintfFile(g_fp, "# showMsgFlag: %d\n", g_stConfInfo.showMsgFlag); taosFprintfFile(g_fp, "# showRowFlag: %d\n", g_stConfInfo.showRowFlag); taosFprintfFile(g_fp, "# consumeDelay: %d\n", g_stConfInfo.consumeDelay); + taosFprintfFile(g_fp, "# numOfThread: %d\n", g_stConfInfo.numOfThread); for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { taosFprintfFile(g_fp, "# consumer %d info:\n", g_stConfInfo.stThreads[i].consumerId); @@ -483,6 +485,7 @@ int main(int32_t argc, char* argv[]) { taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); // pthread_create one thread to consume + taosFprintfFile(g_fp, "==== create %d consume thread ====\n", g_stConfInfo.numOfThread); for (int32_t i = 0; i < g_stConfInfo.numOfThread; ++i) { taosThreadCreate(&(g_stConfInfo.stThreads[i].thread), &thattr, consumeThreadFunc, (void *)(&(g_stConfInfo.stThreads[i]))); } @@ -493,7 +496,7 @@ int main(int32_t argc, char* argv[]) { //printf("consumer: %d, cosumer1: %d\n", totalMsgs, pInfo->consumeMsgCnt); - taosFprintfFile(g_fp, "\n"); + taosFprintfFile(g_fp, "==== close tmqlog ====\n"); taosCloseFile(&g_fp); return 0;