xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' of github.com:taosdata/TDengine into feature/udf

This commit is contained in:
slzhou 2022-05-16 19:03:53 +08:00
parent c245309e6e 3a1c12f8fd
commit 7bb38bed2f
16 changed files with 2731 additions and 2621 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
source/libs/executor/inc/executorimpl.h
View File

@ -658,6 +658,7 @@ void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWin
void cleanupAggSup(SAggSupporter* pAggSup); void cleanupAggSup(SAggSupporter* pAggSup);
void destroyBasicOperatorInfo(void* param, int32_t numOfOutput); void destroyBasicOperatorInfo(void* param, int32_t numOfOutput);
void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle); void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle);
void setTbNameColData(void* pMeta, const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId);
SInterval extractIntervalInfo(const STableScanPhysiNode* pTableScanNode); SInterval extractIntervalInfo(const STableScanPhysiNode* pTableScanNode);
SColumn extractColumnFromColumnNode(SColumnNode* pColNode); SColumn extractColumnFromColumnNode(SColumnNode* pColNode);

11
source/libs/executor/src/executorimpl.c
View File

@ -2055,6 +2055,11 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowR
SColumnInfoData* pDst = taosArrayGet(px->pDataBlock, i); SColumnInfoData* pDst = taosArrayGet(px->pDataBlock, i);
SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, i);
// it is a reserved column for scalar function, and no data in this column yet.
if (pSrc->pData == NULL) {
continue;
}
int32_t numOfRows = 0; int32_t numOfRows = 0;
for (int32_t j = 0; j < totalRows; ++j) { for (int32_t j = 0; j < totalRows; ++j) {
if (rowRes[j] == 0) { if (rowRes[j] == 0) {
@ -4372,9 +4377,9 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* p
goto _error; goto _error;
} }
pInfo->limit = *pLimit; pInfo->limit = *pLimit;
pInfo->slimit = *pSlimit; pInfo->slimit = *pSlimit;
pInfo->curOffset = pLimit->offset; pInfo->curOffset = pLimit->offset;
pInfo->curSOffset = pSlimit->offset; pInfo->curSOffset = pSlimit->offset;
pInfo->binfo.pRes = pResBlock; pInfo->binfo.pRes = pResBlock;

32
source/libs/executor/src/scanoperator.c
View File

@ -291,20 +291,7 @@ void addTagPseudoColumnData(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock)
// this is to handle the tbname // this is to handle the tbname
if (fmIsScanPseudoColumnFunc(functionId)) { if (fmIsScanPseudoColumnFunc(functionId)) {
struct SScalarFuncExecFuncs fpSet = {0}; setTbNameColData(pTableScanInfo->readHandle.meta, pBlock, pColInfoData, functionId);
fmGetScalarFuncExecFuncs(functionId, &fpSet);
SColumnInfoData infoData = {0};
infoData.info.type = TSDB_DATA_TYPE_BIGINT;
infoData.info.bytes = sizeof(uint64_t);
colInfoDataEnsureCapacity(&infoData, 0, 1);
colDataAppendInt64(&infoData, 0, &pBlock->info.uid);
SScalarParam srcParam = {
.numOfRows = pBlock->info.rows, .param = pTableScanInfo->readHandle.meta, .columnData = &infoData};
SScalarParam param = {.columnData = pColInfoData};
fpSet.process(&srcParam, 1, &param);
} else { // these are tags } else { // these are tags
const char* p = metaGetTableTagVal(&mr.me, pExpr->base.pParam[0].pCol->colId); const char* p = metaGetTableTagVal(&mr.me, pExpr->base.pParam[0].pCol->colId);
for (int32_t i = 0; i < pBlock->info.rows; ++i) { for (int32_t i = 0; i < pBlock->info.rows; ++i) {
@ -316,6 +303,23 @@ void addTagPseudoColumnData(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock)
metaReaderClear(&mr); metaReaderClear(&mr);
} }
void setTbNameColData(void* pMeta, const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId) {
struct SScalarFuncExecFuncs fpSet = {0};
fmGetScalarFuncExecFuncs(functionId, &fpSet);
SColumnInfoData infoData = {0};
infoData.info.type = TSDB_DATA_TYPE_BIGINT;
infoData.info.bytes = sizeof(uint64_t);
colInfoDataEnsureCapacity(&infoData, 0, 1);
colDataAppendInt64(&infoData, 0, (int64_t*) &pBlock->info.uid);
SScalarParam srcParam = {
.numOfRows = pBlock->info.rows, .param = pMeta, .columnData = &infoData};
SScalarParam param = {.columnData = pColInfoData};
fpSet.process(&srcParam, 1, &param);
}
static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) {
STableScanInfo* pTableScanInfo = pOperator->info; STableScanInfo* pTableScanInfo = pOperator->info;
SSDataBlock* pBlock = pTableScanInfo->pResBlock; SSDataBlock* pBlock = pTableScanInfo->pResBlock;

566
tests/script/tsim/tmq/basic1.sim
View File

@ -1,278 +1,288 @@
#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 #### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406
#basic1.sim: vgroups=1, one topic for one consumer, firstly insert data, then start consume. Include six topics #basic1.sim: vgroups=1, one topic for one consumer, firstly insert data, then start consume. Include six topics
#basic2.sim: vgroups=1, multi topics for one consumer, firstly insert data, then start consume. Include six topics #basic2.sim: vgroups=1, multi topics for one consumer, firstly insert data, then start consume. Include six topics
#basic3.sim: vgroups=4, one topic for one consumer, firstly insert data, then start consume. Include six topics #basic3.sim: vgroups=4, one topic for one consumer, firstly insert data, then start consume. Include six topics
#basic4.sim: vgroups=4, multi topics for one consumer, firstly insert data, then start consume. Include six topics #basic4.sim: vgroups=4, multi topics for one consumer, firstly insert data, then start consume. Include six topics
# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN # notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN
# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; # The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5;
# #
# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). # notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval).
# #
run tsim/tmq/prepareBasicEnv-1vgrp.sim run tsim/tmq/prepareBasicEnv-1vgrp.sim
#---- global parameters start ----# #---- global parameters start ----#
$dbName = db $dbName = db
$vgroups = 1 $vgroups = 1
$stbPrefix = stb $stbPrefix = stb
$ctbPrefix = ctb $ctbPrefix = ctb
$ntbPrefix = ntb $ntbPrefix = ntb
$stbNum = 1 $stbNum = 1
$ctbNum = 10 $ctbNum = 10
$ntbNum = 10 $ntbNum = 10
$rowsPerCtb = 10 $rowsPerCtb = 10
$tstart = 1640966400000 # 2022-01-01 00:00:00.000 $tstart = 1640966400000 # 2022-01-01 00:00:00.000
#---- global parameters end ----# #---- global parameters end ----#
$pullDelay = 3 $pullDelay = 3
$ifcheckdata = 1 $ifcheckdata = 1
$showMsg = 1 $ifmanualcommit = 1
$showRow = 0 $showMsg = 1
$showRow = 0
sql connect
sql use $dbName sql connect
sql use $dbName
print == create topics from super table
sql create topic topic_stb_column as select ts, c3 from stb print == create topics from super table
sql create topic topic_stb_all as select ts, c1, c2, c3 from stb sql create topic topic_stb_column as select ts, c3 from stb
sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb sql create topic topic_stb_all as select ts, c1, c2, c3 from stb
sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb
print == create topics from child table
sql create topic topic_ctb_column as select ts, c3 from ctb0 print == create topics from child table
sql create topic topic_ctb_all as select * from ctb0 sql create topic topic_ctb_column as select ts, c3 from ctb0
sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 sql create topic topic_ctb_all as select * from ctb0
sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0
print == create topics from normal table
sql create topic topic_ntb_column as select ts, c3 from ntb0 print == create topics from normal table
sql create topic topic_ntb_all as select * from ntb0 sql create topic topic_ntb_column as select ts, c3 from ntb0
sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 sql create topic topic_ntb_all as select * from ntb0
sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0
#sql show topics
#if $rows != 9 then #sql show topics
# return -1 #if $rows != 9 then
#endi # return -1
#endi
$keyList = ' . group.id:cgrp1
$keyList = $keyList . ' #'group.id:cgrp1,enable.auto.commit:false,auto.commit.interval.ms:6000,auto.offset.reset:earliest'
$keyList = ' . group.id:cgrp1
$cdb_index = 0 $keyList = $keyList . ,
#=============================== start consume =============================# $keyList = $keyList . enable.auto.commit:false
#$keyList = $keyList . ,
print ================ test consume from stb #$keyList = $keyList . auto.commit.interval.ms:6000
$loop_cnt = 0 #$keyList = $keyList . ,
loop_consume_diff_topic_from_stb: #$keyList = $keyList . auto.offset.reset:earliest
$keyList = $keyList . '
####################################################################################### print ========== key list: $keyList
# clear consume info and consume result
#run tsim/tmq/clearConsume.sim
# because drop table function no stable, so by create new db for consume info and result. Modify it later $cdb_index = 0
$cdb_index = $cdb_index + 1 #=============================== start consume =============================#
$cdbName = cdb . $cdb_index
sql create database $cdbName vgroups 1 print ================ test consume from stb
sleep 500 $loop_cnt = 0
sql use $cdbName loop_consume_diff_topic_from_stb:
print == create consume info table and consume result table #######################################################################################
sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) # clear consume info and consume result
sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) #run tsim/tmq/clearConsume.sim
# because drop table function no stable, so by create new db for consume info and result. Modify it later
sql show tables $cdb_index = $cdb_index + 1
if $rows != 2 then $cdbName = cdb . $cdb_index
return -1 sql create database $cdbName vgroups 1
endi sleep 500
####################################################################################### sql use $cdbName
if $loop_cnt == 0 then print == create consume info table and consume result table
print == scenario 1: topic_stb_column sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
$topicList = ' . topic_stb_column sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
$topicList = $topicList . '
elif $loop_cnt == 1 then sql show tables
print == scenario 2: topic_stb_all if $rows != 2 then
$topicList = ' . topic_stb_all return -1
$topicList = $topicList . ' endi
elif $loop_cnt == 2 then #######################################################################################
print == scenario 3: topic_stb_function
$topicList = ' . topic_stb_function if $loop_cnt == 0 then
$topicList = $topicList . ' print == scenario 1: topic_stb_column
else $topicList = ' . topic_stb_column
goto loop_consume_diff_topic_from_stb_end $topicList = $topicList . '
endi elif $loop_cnt == 1 then
print == scenario 2: topic_stb_all
$consumerId = 0 $topicList = ' . topic_stb_all
$totalMsgOfStb = $ctbNum * $rowsPerCtb $topicList = $topicList . '
$expectmsgcnt = $totalMsgOfStb elif $loop_cnt == 2 then
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) print == scenario 3: topic_stb_function
$topicList = ' . topic_stb_function
print == start consumer to pull msgs from stb $topicList = $topicList . '
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start else
system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start goto loop_consume_diff_topic_from_stb_end
endi
print == check consume result
wait_consumer_end_from_stb: $consumerId = 0
sql select * from consumeresult $totalMsgOfStb = $ctbNum * $rowsPerCtb
print ==> rows: $rows $expectmsgcnt = $totalMsgOfStb
print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
if $rows != 1 then
sleep 1000 print == start consumer to pull msgs from stb
goto wait_consumer_end_from_stb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
endi system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
if $data[0][1] != $consumerId then
return -1 print == check consume result
endi wait_consumer_end_from_stb:
if $data[0][2] != $expectmsgcnt then sql select * from consumeresult
return -1 print ==> rows: $rows
endi print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
if $data[0][3] != $expectmsgcnt then if $rows != 1 then
return -1 sleep 1000
endi goto wait_consumer_end_from_stb
$loop_cnt = $loop_cnt + 1 endi
goto loop_consume_diff_topic_from_stb if $data[0][1] != $consumerId then
loop_consume_diff_topic_from_stb_end: return -1
endi
print ================ test consume from ctb if $data[0][2] != $expectmsgcnt then
$loop_cnt = 0 return -1
loop_consume_diff_topic_from_ctb: endi
if $data[0][3] != $expectmsgcnt then
####################################################################################### return -1
# clear consume info and consume result endi
#run tsim/tmq/clearConsume.sim $loop_cnt = $loop_cnt + 1
# because drop table function no stable, so by create new db for consume info and result. Modify it later goto loop_consume_diff_topic_from_stb
$cdb_index = $cdb_index + 1 loop_consume_diff_topic_from_stb_end:
$cdbName = cdb . $cdb_index
sql create database $cdbName vgroups 1 print ================ test consume from ctb
sleep 500 $loop_cnt = 0
sql use $cdbName loop_consume_diff_topic_from_ctb:
print == create consume info table and consume result table #######################################################################################
sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) # clear consume info and consume result
sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) #run tsim/tmq/clearConsume.sim
# because drop table function no stable, so by create new db for consume info and result. Modify it later
sql show tables $cdb_index = $cdb_index + 1
if $rows != 2 then $cdbName = cdb . $cdb_index
return -1 sql create database $cdbName vgroups 1
endi sleep 500
####################################################################################### sql use $cdbName
if $loop_cnt == 0 then print == create consume info table and consume result table
print == scenario 1: topic_ctb_column sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
$topicList = ' . topic_ctb_column sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
$topicList = $topicList . '
elif $loop_cnt == 1 then sql show tables
print == scenario 2: topic_ctb_all if $rows != 2 then
$topicList = ' . topic_ctb_all return -1
$topicList = $topicList . ' endi
elif $loop_cnt == 2 then #######################################################################################
print == scenario 3: topic_ctb_function
$topicList = ' . topic_ctb_function if $loop_cnt == 0 then
$topicList = $topicList . ' print == scenario 1: topic_ctb_column
else $topicList = ' . topic_ctb_column
goto loop_consume_diff_topic_from_ctb_end $topicList = $topicList . '
endi elif $loop_cnt == 1 then
print == scenario 2: topic_ctb_all
$consumerId = 0 $topicList = ' . topic_ctb_all
$totalMsgOfCtb = $rowsPerCtb $topicList = $topicList . '
$expectmsgcnt = $totalMsgOfCtb elif $loop_cnt == 2 then
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) print == scenario 3: topic_ctb_function
$topicList = ' . topic_ctb_function
print == start consumer to pull msgs from ctb $topicList = $topicList . '
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start else
system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start goto loop_consume_diff_topic_from_ctb_end
endi
print == check consume result
wait_consumer_end_from_ctb: $consumerId = 0
sql select * from consumeresult $totalMsgOfCtb = $rowsPerCtb
print ==> rows: $rows $expectmsgcnt = $totalMsgOfCtb
print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
if $rows != 1 then
sleep 1000 print == start consumer to pull msgs from ctb
goto wait_consumer_end_from_ctb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
endi system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
if $data[0][1] != $consumerId then
return -1 print == check consume result
endi wait_consumer_end_from_ctb:
if $data[0][2] != $totalMsgOfCtb then sql select * from consumeresult
return -1 print ==> rows: $rows
endi print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
if $data[0][3] != $totalMsgOfCtb then if $rows != 1 then
return -1 sleep 1000
endi goto wait_consumer_end_from_ctb
$loop_cnt = $loop_cnt + 1 endi
goto loop_consume_diff_topic_from_ctb if $data[0][1] != $consumerId then
loop_consume_diff_topic_from_ctb_end: return -1
endi
print ================ test consume from ntb if $data[0][2] != $totalMsgOfCtb then
$loop_cnt = 0 return -1
loop_consume_diff_topic_from_ntb: endi
if $data[0][3] != $totalMsgOfCtb then
####################################################################################### return -1
# clear consume info and consume result endi
#run tsim/tmq/clearConsume.sim $loop_cnt = $loop_cnt + 1
# because drop table function no stable, so by create new db for consume info and result. Modify it later goto loop_consume_diff_topic_from_ctb
$cdb_index = $cdb_index + 1 loop_consume_diff_topic_from_ctb_end:
$cdbName = cdb . $cdb_index
sql create database $cdbName vgroups 1 print ================ test consume from ntb
sleep 500 $loop_cnt = 0
sql use $cdbName loop_consume_diff_topic_from_ntb:
print == create consume info table and consume result table #######################################################################################
sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) # clear consume info and consume result
sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) #run tsim/tmq/clearConsume.sim
# because drop table function no stable, so by create new db for consume info and result. Modify it later
sql show tables $cdb_index = $cdb_index + 1
if $rows != 2 then $cdbName = cdb . $cdb_index
return -1 sql create database $cdbName vgroups 1
endi sleep 500
####################################################################################### sql use $cdbName
if $loop_cnt == 0 then print == create consume info table and consume result table
print == scenario 1: topic_ntb_column sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
$topicList = ' . topic_ntb_column sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
$topicList = $topicList . '
elif $loop_cnt == 1 then sql show tables
print == scenario 2: topic_ntb_all if $rows != 2 then
$topicList = ' . topic_ntb_all return -1
$topicList = $topicList . ' endi
elif $loop_cnt == 2 then #######################################################################################
print == scenario 3: topic_ntb_function
$topicList = ' . topic_ntb_function if $loop_cnt == 0 then
$topicList = $topicList . ' print == scenario 1: topic_ntb_column
else $topicList = ' . topic_ntb_column
goto loop_consume_diff_topic_from_ntb_end $topicList = $topicList . '
endi elif $loop_cnt == 1 then
print == scenario 2: topic_ntb_all
$consumerId = 0 $topicList = ' . topic_ntb_all
$totalMsgOfNtb = $rowsPerCtb $topicList = $topicList . '
$expectmsgcnt = $totalMsgOfNtb elif $loop_cnt == 2 then
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) print == scenario 3: topic_ntb_function
$topicList = ' . topic_ntb_function
print == start consumer to pull msgs from ntb $topicList = $topicList . '
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start else
system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start goto loop_consume_diff_topic_from_ntb_end
endi
print == check consume result from ntb
wait_consumer_end_from_ntb: $consumerId = 0
sql select * from consumeresult $totalMsgOfNtb = $rowsPerCtb
print ==> rows: $rows $expectmsgcnt = $totalMsgOfNtb
print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
if $rows != 1 then
sleep 1000 print == start consumer to pull msgs from ntb
goto wait_consumer_end_from_ntb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
endi system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
if $data[0][1] != $consumerId then
return -1 print == check consume result from ntb
endi wait_consumer_end_from_ntb:
if $data[0][2] != $totalMsgOfNtb then sql select * from consumeresult
return -1 print ==> rows: $rows
endi print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
if $data[0][3] != $totalMsgOfNtb then if $rows != 1 then
return -1 sleep 1000
endi goto wait_consumer_end_from_ntb
$loop_cnt = $loop_cnt + 1 endi
goto loop_consume_diff_topic_from_ntb if $data[0][1] != $consumerId then
loop_consume_diff_topic_from_ntb_end: return -1
endi
#------ not need stop consumer, because it exit after pull msg overthan expect msg if $data[0][2] != $totalMsgOfNtb then
#system tsim/tmq/consume.sh -s stop -x SIGINT return -1
endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT if $data[0][3] != $totalMsgOfNtb then
return -1
endi
$loop_cnt = $loop_cnt + 1
goto loop_consume_diff_topic_from_ntb
loop_consume_diff_topic_from_ntb_end:
#------ not need stop consumer, because it exit after pull msg overthan expect msg
#system tsim/tmq/consume.sh -s stop -x SIGINT
system sh/exec.sh -n dnode1 -s stop -x SIGINT

754
tests/script/tsim/tmq/basic1Of2Cons.sim
View File

@ -1,372 +1,382 @@
#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 #### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406
#basic1Of2Cons.sim: vgroups=1, one topic for 2 consumers, firstly insert data, then start consume. Include six topics #basic1Of2Cons.sim: vgroups=1, one topic for 2 consumers, firstly insert data, then start consume. Include six topics
#basic2Of2Cons.sim: vgroups=1, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics #basic2Of2Cons.sim: vgroups=1, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics
#basic3Of2Cons.sim: vgroups=4, one topic for 2 consumers, firstly insert data, then start consume. Include six topics #basic3Of2Cons.sim: vgroups=4, one topic for 2 consumers, firstly insert data, then start consume. Include six topics
#basic4Of2Cons.sim: vgroups=4, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics #basic4Of2Cons.sim: vgroups=4, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics
# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN # notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN
# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; # The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5;
# #
# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). # notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval).
# #
run tsim/tmq/prepareBasicEnv-1vgrp.sim run tsim/tmq/prepareBasicEnv-1vgrp.sim
#---- global parameters start ----# #---- global parameters start ----#
$dbName = db $dbName = db
$vgroups = 1 $vgroups = 1
$stbPrefix = stb $stbPrefix = stb
$ctbPrefix = ctb $ctbPrefix = ctb
$ntbPrefix = ntb $ntbPrefix = ntb
$stbNum = 1 $stbNum = 1
$ctbNum = 10 $ctbNum = 10
$ntbNum = 10 $ntbNum = 10
$rowsPerCtb = 10 $rowsPerCtb = 10
$tstart = 1640966400000 # 2022-01-01 00:00:00.000 $tstart = 1640966400000 # 2022-01-01 00:00:00.000
#---- global parameters end ----# #---- global parameters end ----#
$pullDelay = 5 $pullDelay = 5
$ifcheckdata = 1 $ifcheckdata = 1
$showMsg = 1 $ifmanualcommit = 1
$showRow = 0 $showMsg = 1
$showRow = 0
sql connect
sql use $dbName sql connect
sql use $dbName
print == create topics from super table
sql create topic topic_stb_column as select ts, c3 from stb print == create topics from super table
sql create topic topic_stb_all as select ts, c1, c2, c3 from stb sql create topic topic_stb_column as select ts, c3 from stb
sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb sql create topic topic_stb_all as select ts, c1, c2, c3 from stb
sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb
print == create topics from child table
sql create topic topic_ctb_column as select ts, c3 from ctb0 print == create topics from child table
sql create topic topic_ctb_all as select * from ctb0 sql create topic topic_ctb_column as select ts, c3 from ctb0
sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 sql create topic topic_ctb_all as select * from ctb0
sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0
print == create topics from normal table
sql create topic topic_ntb_column as select ts, c3 from ntb0 print == create topics from normal table
sql create topic topic_ntb_all as select * from ntb0 sql create topic topic_ntb_column as select ts, c3 from ntb0
sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 sql create topic topic_ntb_all as select * from ntb0
sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0
#sql show topics
#if $rows != 9 then #sql show topics
# return -1 #if $rows != 9 then
#endi # return -1
#endi
$keyList = ' . group.id:cgrp1
$keyList = $keyList . ' #'group.id:cgrp1,enable.auto.commit:false,auto.commit.interval.ms:6000,auto.offset.reset:earliest'
$keyList = ' . group.id:cgrp1
$cdb_index = 0 $keyList = $keyList . ,
#=============================== start consume =============================# $keyList = $keyList . enable.auto.commit:false
#$keyList = $keyList . ,
print ================ test consume from stb #$keyList = $keyList . auto.commit.interval.ms:6000
$loop_cnt = 0 #$keyList = $keyList . ,
loop_consume_diff_topic_from_stb: #$keyList = $keyList . auto.offset.reset:earliest
$keyList = $keyList . '
####################################################################################### print ========== key list: $keyList
# clear consume info and consume result
#run tsim/tmq/clearConsume.sim $cdb_index = 0
# because drop table function no stable, so by create new db for consume info and result. Modify it later
$cdb_index = $cdb_index + 1 #=============================== start consume =============================#
$cdbName = cdb . $cdb_index
sql create database $cdbName vgroups 1 print ================ test consume from stb
sleep 500 $loop_cnt = 0
sql use $cdbName loop_consume_diff_topic_from_stb:
print == create consume info table and consume result table for stb #######################################################################################
sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) # clear consume info and consume result
sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) #run tsim/tmq/clearConsume.sim
# because drop table function no stable, so by create new db for consume info and result. Modify it later
sql show tables $cdb_index = $cdb_index + 1
if $rows != 2 then $cdbName = cdb . $cdb_index
return -1 sql create database $cdbName vgroups 1
endi sleep 500
####################################################################################### sql use $cdbName
if $loop_cnt == 0 then print == create consume info table and consume result table for stb
print == scenario 1: topic_stb_column sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
$topicList = ' . topic_stb_column sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
$topicList = $topicList . '
elif $loop_cnt == 1 then sql show tables
print == scenario 2: topic_stb_all if $rows != 2 then
$topicList = ' . topic_stb_all return -1
$topicList = $topicList . ' endi
elif $loop_cnt == 2 then #######################################################################################
print == scenario 3: topic_stb_function
$topicList = ' . topic_stb_function if $loop_cnt == 0 then
$topicList = $topicList . ' print == scenario 1: topic_stb_column
else $topicList = ' . topic_stb_column
goto loop_consume_diff_topic_from_stb_end $topicList = $topicList . '
endi elif $loop_cnt == 1 then
print == scenario 2: topic_stb_all
$consumerId = 0 $topicList = ' . topic_stb_all
$totalMsgOfStb = $ctbNum * $rowsPerCtb $topicList = $topicList . '
$expectmsgcnt = $totalMsgOfStb elif $loop_cnt == 2 then
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) print == scenario 3: topic_stb_function
$topicList = ' . topic_stb_function
$consumerId = 1 $topicList = $topicList . '
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) else
goto loop_consume_diff_topic_from_stb_end
print == start consumer to pull msgs from stb endi
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start $consumerId = 0
$totalMsgOfStb = $ctbNum * $rowsPerCtb
print == check consume result $expectmsgcnt = $totalMsgOfStb
wait_consumer_end_from_stb: sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
sql select * from consumeresult
print ==> rows: $rows $consumerId = 1
print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
if $rows != 2 then print == start consumer to pull msgs from stb
sleep 1000 print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
goto wait_consumer_end_from_stb system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
endi
if $data[0][1] == 0 then print == check consume result
if $data[1][1] != 1 then wait_consumer_end_from_stb:
return -1 sql select * from consumeresult
endi print ==> rows: $rows
endi print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
if $data[0][1] == 1 then print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
if $data[1][1] != 0 then if $rows != 2 then
return -1 sleep 1000
endi goto wait_consumer_end_from_stb
endi endi
if $data[0][1] == 0 then
# either $data[0][2] == $totalMsgOfStb and $data[1][2] == 0 if $data[1][1] != 1 then
# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfStb return -1
if $data[0][2] == $totalMsgOfStb then endi
if $data[1][2] == 0 then endi
goto check_ok_0 if $data[0][1] == 1 then
endi if $data[1][1] != 0 then
elif $data[0][2] == 0 then return -1
if $data[1][2] == $totalMsgOfStb then endi
goto check_ok_0 endi
endi
endi # either $data[0][2] == $totalMsgOfStb and $data[1][2] == 0
return -1 # or $data[0][2] == 0 and $data[1][2] == $totalMsgOfStb
check_ok_0: if $data[0][2] == $totalMsgOfStb then
if $data[1][2] == 0 then
if $data[0][3] == $totalMsgOfStb then goto check_ok_0
if $data[1][3] == 0 then endi
goto check_ok_1 elif $data[0][2] == 0 then
endi if $data[1][2] == $totalMsgOfStb then
elif $data[0][3] == 0 then goto check_ok_0
if $data[1][3] == $totalMsgOfStb then endi
goto check_ok_1 endi
endi return -1
endi check_ok_0:
return -1
check_ok_1: if $data[0][3] == $totalMsgOfStb then
if $data[1][3] == 0 then
$loop_cnt = $loop_cnt + 1 goto check_ok_1
goto loop_consume_diff_topic_from_stb endi
loop_consume_diff_topic_from_stb_end: elif $data[0][3] == 0 then
if $data[1][3] == $totalMsgOfStb then
print ================ test consume from ctb goto check_ok_1
$loop_cnt = 0 endi
loop_consume_diff_topic_from_ctb: endi
return -1
####################################################################################### check_ok_1:
# clear consume info and consume result
#run tsim/tmq/clearConsume.sim $loop_cnt = $loop_cnt + 1
# because drop table function no stable, so by create new db for consume info and result. Modify it later goto loop_consume_diff_topic_from_stb
$cdb_index = $cdb_index + 1 loop_consume_diff_topic_from_stb_end:
$cdbName = cdb . $cdb_index
sql create database $cdbName vgroups 1 print ================ test consume from ctb
sleep 500 $loop_cnt = 0
sql use $cdbName loop_consume_diff_topic_from_ctb:
print == create consume info table and consume result table for ctb #######################################################################################
sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) # clear consume info and consume result
sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) #run tsim/tmq/clearConsume.sim
# because drop table function no stable, so by create new db for consume info and result. Modify it later
sql show tables $cdb_index = $cdb_index + 1
if $rows != 2 then $cdbName = cdb . $cdb_index
return -1 sql create database $cdbName vgroups 1
endi sleep 500
####################################################################################### sql use $cdbName
if $loop_cnt == 0 then print == create consume info table and consume result table for ctb
print == scenario 1: topic_ctb_column sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
$topicList = ' . topic_ctb_column sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
$topicList = $topicList . '
elif $loop_cnt == 1 then sql show tables
print == scenario 2: topic_ctb_all if $rows != 2 then
$topicList = ' . topic_ctb_all return -1
$topicList = $topicList . ' endi
elif $loop_cnt == 2 then #######################################################################################
print == scenario 3: topic_ctb_function
$topicList = ' . topic_ctb_function if $loop_cnt == 0 then
$topicList = $topicList . ' print == scenario 1: topic_ctb_column
else $topicList = ' . topic_ctb_column
goto loop_consume_diff_topic_from_ctb_end $topicList = $topicList . '
endi elif $loop_cnt == 1 then
print == scenario 2: topic_ctb_all
$consumerId = 0 $topicList = ' . topic_ctb_all
$totalMsgOfCtb = $rowsPerCtb $topicList = $topicList . '
$expectmsgcnt = $totalMsgOfCtb elif $loop_cnt == 2 then
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) print == scenario 3: topic_ctb_function
$consumerId = 1 $topicList = ' . topic_ctb_function
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) $topicList = $topicList . '
else
print == start consumer to pull msgs from ctb goto loop_consume_diff_topic_from_ctb_end
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start endi
system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
$consumerId = 0
print == check consume result $totalMsgOfCtb = $rowsPerCtb
wait_consumer_end_from_ctb: $expectmsgcnt = $totalMsgOfCtb
sql select * from consumeresult sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
print ==> rows: $rows $consumerId = 1
print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
if $rows != 2 then print == start consumer to pull msgs from ctb
sleep 1000 print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
goto wait_consumer_end_from_ctb system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
endi
if $data[0][1] == 0 then print == check consume result
if $data[1][1] != 1 then wait_consumer_end_from_ctb:
return -1 sql select * from consumeresult
endi print ==> rows: $rows
endi print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
if $data[0][1] == 1 then print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
if $data[1][1] != 0 then if $rows != 2 then
return -1 sleep 1000
endi goto wait_consumer_end_from_ctb
endi endi
if $data[0][1] == 0 then
# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0 if $data[1][1] != 1 then
# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb return -1
if $data[0][2] == $totalMsgOfCtb then endi
if $data[1][2] == 0 then endi
goto check_ok_2 if $data[0][1] == 1 then
endi if $data[1][1] != 0 then
elif $data[0][2] == 0 then return -1
if $data[1][2] == $totalMsgOfCtb then endi
goto check_ok_2 endi
endi
endi # either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0
return -1 # or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb
check_ok_2: if $data[0][2] == $totalMsgOfCtb then
if $data[1][2] == 0 then
if $data[0][3] == $totalMsgOfCtb then goto check_ok_2
if $data[1][3] == 0 then endi
goto check_ok_3 elif $data[0][2] == 0 then
endi if $data[1][2] == $totalMsgOfCtb then
elif $data[0][3] == 0 then goto check_ok_2
if $data[1][3] == $totalMsgOfCtb then endi
goto check_ok_3 endi
endi return -1
endi check_ok_2:
return -1
check_ok_3: if $data[0][3] == $totalMsgOfCtb then
if $data[1][3] == 0 then
$loop_cnt = $loop_cnt + 1 goto check_ok_3
goto loop_consume_diff_topic_from_ctb endi
loop_consume_diff_topic_from_ctb_end: elif $data[0][3] == 0 then
if $data[1][3] == $totalMsgOfCtb then
print ================ test consume from ntb goto check_ok_3
$loop_cnt = 0 endi
loop_consume_diff_topic_from_ntb: endi
return -1
####################################################################################### check_ok_3:
# clear consume info and consume result
#run tsim/tmq/clearConsume.sim $loop_cnt = $loop_cnt + 1
# because drop table function no stable, so by create new db for consume info and result. Modify it later goto loop_consume_diff_topic_from_ctb
$cdb_index = $cdb_index + 1 loop_consume_diff_topic_from_ctb_end:
$cdbName = cdb . $cdb_index
sql create database $cdbName vgroups 1 print ================ test consume from ntb
sleep 500 $loop_cnt = 0
sql use $cdbName loop_consume_diff_topic_from_ntb:
print == create consume info table and consume result table for ntb #######################################################################################
sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) # clear consume info and consume result
sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) #run tsim/tmq/clearConsume.sim
# because drop table function no stable, so by create new db for consume info and result. Modify it later
sql show tables $cdb_index = $cdb_index + 1
if $rows != 2 then $cdbName = cdb . $cdb_index
return -1 sql create database $cdbName vgroups 1
endi sleep 500
####################################################################################### sql use $cdbName
if $loop_cnt == 0 then print == create consume info table and consume result table for ntb
print == scenario 1: topic_ntb_column sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
$topicList = ' . topic_ntb_column sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
$topicList = $topicList . '
elif $loop_cnt == 1 then sql show tables
print == scenario 2: topic_ntb_all if $rows != 2 then
$topicList = ' . topic_ntb_all return -1
$topicList = $topicList . ' endi
elif $loop_cnt == 2 then #######################################################################################
print == scenario 3: topic_ntb_function
$topicList = ' . topic_ntb_function if $loop_cnt == 0 then
$topicList = $topicList . ' print == scenario 1: topic_ntb_column
else $topicList = ' . topic_ntb_column
goto loop_consume_diff_topic_from_ntb_end $topicList = $topicList . '
endi elif $loop_cnt == 1 then
print == scenario 2: topic_ntb_all
$consumerId = 0 $topicList = ' . topic_ntb_all
$totalMsgOfNtb = $rowsPerCtb $topicList = $topicList . '
$expectmsgcnt = $totalMsgOfNtb elif $loop_cnt == 2 then
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) print == scenario 3: topic_ntb_function
$consumerId = 1 $topicList = ' . topic_ntb_function
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) $topicList = $topicList . '
else
print == start consumer to pull msgs from ntb goto loop_consume_diff_topic_from_ntb_end
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start endi
system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
$consumerId = 0
print == check consume result from ntb $totalMsgOfNtb = $rowsPerCtb
wait_consumer_end_from_ntb: $expectmsgcnt = $totalMsgOfNtb
sql select * from consumeresult sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
print ==> rows: $rows $consumerId = 1
print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
if $rows != 2 then print == start consumer to pull msgs from ntb
sleep 1000 print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
goto wait_consumer_end_from_ntb system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
endi
if $data[0][1] == 0 then print == check consume result from ntb
if $data[1][1] != 1 then wait_consumer_end_from_ntb:
return -1 sql select * from consumeresult
endi print ==> rows: $rows
endi print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
if $data[1][1] == 0 then print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
if $data[0][1] != 1 then if $rows != 2 then
return -1 sleep 1000
endi goto wait_consumer_end_from_ntb
endi endi
if $data[0][1] == 0 then
# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0 if $data[1][1] != 1 then
# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb return -1
if $data[0][2] == $totalMsgOfNtb then endi
if $data[1][2] == 0 then endi
goto check_ok_4 if $data[1][1] == 0 then
endi if $data[0][1] != 1 then
elif $data[0][2] == 0 then return -1
if $data[1][2] == $totalMsgOfNtb then endi
goto check_ok_4 endi
endi
endi # either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0
return -1 # or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb
check_ok_4: if $data[0][2] == $totalMsgOfNtb then
if $data[1][2] == 0 then
if $data[0][3] == $totalMsgOfNtb then goto check_ok_4
if $data[1][3] == 0 then endi
goto check_ok_5 elif $data[0][2] == 0 then
endi if $data[1][2] == $totalMsgOfNtb then
elif $data[0][3] == 0 then goto check_ok_4
if $data[1][3] == $totalMsgOfNtb then endi
goto check_ok_5 endi
endi return -1
endi check_ok_4:
return -1
check_ok_5: if $data[0][3] == $totalMsgOfNtb then
if $data[1][3] == 0 then
$loop_cnt = $loop_cnt + 1 goto check_ok_5
goto loop_consume_diff_topic_from_ntb endi
loop_consume_diff_topic_from_ntb_end: elif $data[0][3] == 0 then
if $data[1][3] == $totalMsgOfNtb then
#------ not need stop consumer, because it exit after pull msg overthan expect msg goto check_ok_5
#system tsim/tmq/consume.sh -s stop -x SIGINT endi
endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT return -1
check_ok_5:
$loop_cnt = $loop_cnt + 1
goto loop_consume_diff_topic_from_ntb
loop_consume_diff_topic_from_ntb_end:
#------ not need stop consumer, because it exit after pull msg overthan expect msg
#system tsim/tmq/consume.sh -s stop -x SIGINT
system sh/exec.sh -n dnode1 -s stop -x SIGINT

448
tests/script/tsim/tmq/basic2.sim
View File

@ -1,219 +1,229 @@
#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 #### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406
#basic1.sim: vgroups=1, one topic for one consumer, firstly insert data, then start consume. Include six topics #basic1.sim: vgroups=1, one topic for one consumer, firstly insert data, then start consume. Include six topics
#basic2.sim: vgroups=1, multi topics for one consumer, firstly insert data, then start consume. Include six topics #basic2.sim: vgroups=1, multi topics for one consumer, firstly insert data, then start consume. Include six topics
#basic3.sim: vgroups=4, one topic for one consumer, firstly insert data, then start consume. Include six topics #basic3.sim: vgroups=4, one topic for one consumer, firstly insert data, then start consume. Include six topics
#basic4.sim: vgroups=4, multi topics for one consumer, firstly insert data, then start consume. Include six topics #basic4.sim: vgroups=4, multi topics for one consumer, firstly insert data, then start consume. Include six topics
# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN # notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN
# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; # The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5;
# #
# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). # notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval).
# #
run tsim/tmq/prepareBasicEnv-1vgrp.sim run tsim/tmq/prepareBasicEnv-1vgrp.sim
#---- global parameters start ----# #---- global parameters start ----#
$dbName = db $dbName = db
$vgroups = 1 $vgroups = 1
$stbPrefix = stb $stbPrefix = stb
$ctbPrefix = ctb $ctbPrefix = ctb
$ntbPrefix = ntb $ntbPrefix = ntb
$stbNum = 1 $stbNum = 1
$ctbNum = 10 $ctbNum = 10
$ntbNum = 10 $ntbNum = 10
$rowsPerCtb = 10 $rowsPerCtb = 10
$tstart = 1640966400000 # 2022-01-01 00:00:00.000 $tstart = 1640966400000 # 2022-01-01 00:00:00.000
#---- global parameters end ----# #---- global parameters end ----#
$pullDelay = 3 $pullDelay = 3
$ifcheckdata = 1 $ifcheckdata = 1
$showMsg = 1 $ifmanualcommit = 1
$showRow = 0 $showMsg = 1
$showRow = 0
sql connect
sql use $dbName sql connect
sql use $dbName
print == create topics from super table
sql create topic topic_stb_column as select ts, c3 from stb print == create topics from super table
sql create topic topic_stb_all as select ts, c1, c2, c3 from stb sql create topic topic_stb_column as select ts, c3 from stb
sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb sql create topic topic_stb_all as select ts, c1, c2, c3 from stb
sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb
print == create topics from child table
sql create topic topic_ctb_column as select ts, c3 from ctb0 print == create topics from child table
sql create topic topic_ctb_all as select * from ctb0 sql create topic topic_ctb_column as select ts, c3 from ctb0
sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 sql create topic topic_ctb_all as select * from ctb0
sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0
print == create topics from normal table
sql create topic topic_ntb_column as select ts, c3 from ntb0 print == create topics from normal table
sql create topic topic_ntb_all as select * from ntb0 sql create topic topic_ntb_column as select ts, c3 from ntb0
sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 sql create topic topic_ntb_all as select * from ntb0
sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0
#sql show topics
#if $rows != 9 then #sql show topics
# return -1 #if $rows != 9 then
#endi # return -1
#endi
$keyList = ' . group.id:cgrp1
$keyList = $keyList . ' #'group.id:cgrp1,enable.auto.commit:false,auto.commit.interval.ms:6000,auto.offset.reset:earliest'
$keyList = ' . group.id:cgrp1
$topicNum = 3 $keyList = $keyList . ,
$keyList = $keyList . enable.auto.commit:false
#=============================== start consume =============================# #$keyList = $keyList . ,
#$keyList = $keyList . auto.commit.interval.ms:6000
#$keyList = $keyList . ,
print ================ test consume from stb #$keyList = $keyList . auto.offset.reset:earliest
print == multi toipcs: topic_stb_column + topic_stb_all + topic_stb_function $keyList = $keyList . '
$topicList = ' . topic_stb_column print ========== key list: $keyList
$topicList = $topicList . ,
$topicList = $topicList . topic_stb_all
$topicList = $topicList . , $topicNum = 3
$topicList = $topicList . topic_stb_function
$topicList = $topicList . ' #=============================== start consume =============================#
$consumerId = 0
$totalMsgOfStb = $ctbNum * $rowsPerCtb print ================ test consume from stb
$totalMsgOfStb = $totalMsgOfStb * $topicNum print == multi toipcs: topic_stb_column + topic_stb_all + topic_stb_function
$expectmsgcnt = $totalMsgOfStb $topicList = ' . topic_stb_column
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) $topicList = $topicList . ,
$topicList = $topicList . topic_stb_all
print == start consumer to pull msgs from stb $topicList = $topicList . ,
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start $topicList = $topicList . topic_stb_function
system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start $topicList = $topicList . '
print == check consume result $consumerId = 0
wait_consumer_end_from_stb: $totalMsgOfStb = $ctbNum * $rowsPerCtb
sql select * from consumeresult $totalMsgOfStb = $totalMsgOfStb * $topicNum
print ==> rows: $rows $expectmsgcnt = $totalMsgOfStb
print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
if $rows != 1 then
sleep 1000 print == start consumer to pull msgs from stb
goto wait_consumer_end_from_stb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start
endi system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start
if $data[0][1] != $consumerId then
return -1 print == check consume result
endi wait_consumer_end_from_stb:
if $data[0][2] != $expectmsgcnt then sql select * from consumeresult
return -1 print ==> rows: $rows
endi print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
if $data[0][3] != $expectmsgcnt then if $rows != 1 then
return -1 sleep 1000
endi goto wait_consumer_end_from_stb
endi
####################################################################################### if $data[0][1] != $consumerId then
# clear consume info and consume result return -1
#run tsim/tmq/clearConsume.sim endi
# because drop table function no stable, so by create new db for consume info and result. Modify it later if $data[0][2] != $expectmsgcnt then
$cdbName = cdb1 return -1
sql create database $cdbName vgroups 1 endi
sleep 500 if $data[0][3] != $expectmsgcnt then
sql use $cdbName return -1
endi
print == create consume info table and consume result table
sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) #######################################################################################
sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) # clear consume info and consume result
#run tsim/tmq/clearConsume.sim
sql show tables # because drop table function no stable, so by create new db for consume info and result. Modify it later
if $rows != 2 then $cdbName = cdb1
return -1 sql create database $cdbName vgroups 1
endi sleep 500
####################################################################################### sql use $cdbName
print == create consume info table and consume result table
print ================ test consume from ctb sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
print == multi toipcs: topic_ctb_column + topic_ctb_all + topic_ctb_function sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
$topicList = ' . topic_ctb_column
$topicList = $topicList . , sql show tables
$topicList = $topicList . topic_ctb_all if $rows != 2 then
$topicList = $topicList . , return -1
$topicList = $topicList . topic_ctb_function endi
$topicList = $topicList . ' #######################################################################################
$consumerId = 0
$totalMsgOfCtb = $rowsPerCtb * $topicNum print ================ test consume from ctb
$expectmsgcnt = $totalMsgOfCtb print == multi toipcs: topic_ctb_column + topic_ctb_all + topic_ctb_function
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) $topicList = ' . topic_ctb_column
$topicList = $topicList . ,
print == start consumer to pull msgs from ctb $topicList = $topicList . topic_ctb_all
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start $topicList = $topicList . ,
system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start $topicList = $topicList . topic_ctb_function
$topicList = $topicList . '
print == check consume result
wait_consumer_end_from_ctb: $consumerId = 0
sql select * from consumeresult $totalMsgOfCtb = $rowsPerCtb * $topicNum
print ==> rows: $rows $expectmsgcnt = $totalMsgOfCtb
print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
if $rows != 1 then
sleep 1000 print == start consumer to pull msgs from ctb
goto wait_consumer_end_from_ctb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
endi system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
if $data[0][1] != $consumerId then
return -1 print == check consume result
endi wait_consumer_end_from_ctb:
if $data[0][2] != $totalMsgOfCtb then sql select * from consumeresult
return -1 print ==> rows: $rows
endi print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
if $data[0][3] != $totalMsgOfCtb then if $rows != 1 then
return -1 sleep 1000
endi goto wait_consumer_end_from_ctb
endi
####################################################################################### if $data[0][1] != $consumerId then
# clear consume info and consume result return -1
#run tsim/tmq/clearConsume.sim endi
# because drop table function no stable, so by create new db for consume info and result. Modify it later if $data[0][2] != $totalMsgOfCtb then
$cdbName = cdb2 return -1
sql create database $cdbName vgroups 1 endi
sleep 500 if $data[0][3] != $totalMsgOfCtb then
sql use $cdbName return -1
endi
print == create consume info table and consume result table
sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) #######################################################################################
sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) # clear consume info and consume result
#run tsim/tmq/clearConsume.sim
sql show tables # because drop table function no stable, so by create new db for consume info and result. Modify it later
if $rows != 2 then $cdbName = cdb2
return -1 sql create database $cdbName vgroups 1
endi sleep 500
####################################################################################### sql use $cdbName
print == create consume info table and consume result table
print ================ test consume from ntb sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
print == multi toipcs: topic_ntb_column + topic_ntb_all + topic_ntb_function sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
$topicList = ' . topic_ntb_column
$topicList = $topicList . , sql show tables
$topicList = $topicList . topic_ntb_all if $rows != 2 then
$topicList = $topicList . , return -1
$topicList = $topicList . topic_ntb_function endi
$topicList = $topicList . ' #######################################################################################
$consumerId = 0
$totalMsgOfNtb = $rowsPerCtb * $topicNum print ================ test consume from ntb
$expectmsgcnt = $totalMsgOfNtb print == multi toipcs: topic_ntb_column + topic_ntb_all + topic_ntb_function
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) $topicList = ' . topic_ntb_column
$topicList = $topicList . ,
print == start consumer to pull msgs from ntb $topicList = $topicList . topic_ntb_all
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start $topicList = $topicList . ,
system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start $topicList = $topicList . topic_ntb_function
$topicList = $topicList . '
print == check consume result from ntb
wait_consumer_end_from_ntb: $consumerId = 0
sql select * from consumeresult $totalMsgOfNtb = $rowsPerCtb * $topicNum
print ==> rows: $rows $expectmsgcnt = $totalMsgOfNtb
print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
if $rows != 1 then
sleep 1000 print == start consumer to pull msgs from ntb
goto wait_consumer_end_from_ntb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
endi system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
if $data[0][1] != $consumerId then
return -1 print == check consume result from ntb
endi wait_consumer_end_from_ntb:
if $data[0][2] != $totalMsgOfNtb then sql select * from consumeresult
return -1 print ==> rows: $rows
endi print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
if $data[0][3] != $totalMsgOfNtb then if $rows != 1 then
return -1 sleep 1000
endi goto wait_consumer_end_from_ntb
endi
#------ not need stop consumer, because it exit after pull msg overthan expect msg if $data[0][1] != $consumerId then
#system tsim/tmq/consume.sh -s stop -x SIGINT return -1
endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT if $data[0][2] != $totalMsgOfNtb then
return -1
endi
if $data[0][3] != $totalMsgOfNtb then
return -1
endi
#------ not need stop consumer, because it exit after pull msg overthan expect msg
#system tsim/tmq/consume.sh -s stop -x SIGINT
system sh/exec.sh -n dnode1 -s stop -x SIGINT

627
tests/script/tsim/tmq/basic2Of2Cons.sim
View File

@ -1,309 +1,318 @@
#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 #### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406
#basic1Of2Cons.sim: vgroups=1, one topic for 2 consumers, firstly insert data, then start consume. Include six topics #basic1Of2Cons.sim: vgroups=1, one topic for 2 consumers, firstly insert data, then start consume. Include six topics
#basic2Of2Cons.sim: vgroups=1, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics #basic2Of2Cons.sim: vgroups=1, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics
#basic3Of2Cons.sim: vgroups=4, one topic for 2 consumers, firstly insert data, then start consume. Include six topics #basic3Of2Cons.sim: vgroups=4, one topic for 2 consumers, firstly insert data, then start consume. Include six topics
#basic4Of2Cons.sim: vgroups=4, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics #basic4Of2Cons.sim: vgroups=4, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics
# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN # notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN
# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; # The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5;
# #
# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). # notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval).
# #
run tsim/tmq/prepareBasicEnv-1vgrp.sim run tsim/tmq/prepareBasicEnv-1vgrp.sim
#---- global parameters start ----# #---- global parameters start ----#
$dbName = db $dbName = db
$vgroups = 1 $vgroups = 1
$stbPrefix = stb $stbPrefix = stb
$ctbPrefix = ctb $ctbPrefix = ctb
$ntbPrefix = ntb $ntbPrefix = ntb
$stbNum = 1 $stbNum = 1
$ctbNum = 10 $ctbNum = 10
$ntbNum = 10 $ntbNum = 10
$rowsPerCtb = 10 $rowsPerCtb = 10
$tstart = 1640966400000 # 2022-01-01 00:00:00.000 $tstart = 1640966400000 # 2022-01-01 00:00:00.000
#---- global parameters end ----# #---- global parameters end ----#
$pullDelay = 5 $pullDelay = 5
$ifcheckdata = 1 $ifcheckdata = 1
$showMsg = 1 $ifmanualcommit = 1
$showRow = 0 $showMsg = 1
$showRow = 0
sql connect
sql use $dbName sql connect
sql use $dbName
print == create topics from super table
sql create topic topic_stb_column as select ts, c3 from stb print == create topics from super table
sql create topic topic_stb_all as select ts, c1, c2, c3 from stb sql create topic topic_stb_column as select ts, c3 from stb
sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb sql create topic topic_stb_all as select ts, c1, c2, c3 from stb
sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb
print == create topics from child table
sql create topic topic_ctb_column as select ts, c3 from ctb0 print == create topics from child table
sql create topic topic_ctb_all as select * from ctb0 sql create topic topic_ctb_column as select ts, c3 from ctb0
sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 sql create topic topic_ctb_all as select * from ctb0
sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0
print == create topics from normal table
sql create topic topic_ntb_column as select ts, c3 from ntb0 print == create topics from normal table
sql create topic topic_ntb_all as select * from ntb0 sql create topic topic_ntb_column as select ts, c3 from ntb0
sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 sql create topic topic_ntb_all as select * from ntb0
sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0
#sql show topics
#if $rows != 9 then #sql show topics
# return -1 #if $rows != 9 then
#endi # return -1
#endi
$keyList = ' . group.id:cgrp1
$keyList = $keyList . ' #'group.id:cgrp1,enable.auto.commit:false,auto.commit.interval.ms:6000,auto.offset.reset:earliest'
$keyList = ' . group.id:cgrp1
$topicNum = 3 $keyList = $keyList . ,
$keyList = $keyList . enable.auto.commit:false
#=============================== start consume =============================# #$keyList = $keyList . ,
#$keyList = $keyList . auto.commit.interval.ms:6000
#$keyList = $keyList . ,
print ================ test consume from stb #$keyList = $keyList . auto.offset.reset:earliest
print == multi toipcs: topic_stb_column + topic_stb_all + topic_stb_function $keyList = $keyList . '
$topicList = ' . topic_stb_column print ========== key list: $keyList
$topicList = $topicList . ,
$topicList = $topicList . topic_stb_all $topicNum = 3
$topicList = $topicList . ,
$topicList = $topicList . topic_stb_function #=============================== start consume =============================#
$topicList = $topicList . '
$consumerId = 0 print ================ test consume from stb
$totalMsgOfStb = $ctbNum * $rowsPerCtb print == multi toipcs: topic_stb_column + topic_stb_all + topic_stb_function
$totalMsgOfStb = $totalMsgOfStb * $topicNum $topicList = ' . topic_stb_column
$expectmsgcnt = $totalMsgOfStb $topicList = $topicList . ,
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) $topicList = $topicList . topic_stb_all
$consumerId = 1 $topicList = $topicList . ,
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) $topicList = $topicList . topic_stb_function
$topicList = $topicList . '
print == start consumer to pull msgs from stb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start $consumerId = 0
system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start $totalMsgOfStb = $ctbNum * $rowsPerCtb
$totalMsgOfStb = $totalMsgOfStb * $topicNum
print == check consume result $expectmsgcnt = $totalMsgOfStb
wait_consumer_end_from_stb: sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
sql select * from consumeresult $consumerId = 1
print ==> rows: $rows sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] print == start consumer to pull msgs from stb
if $rows != 2 then print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start
sleep 1000 system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start
goto wait_consumer_end_from_stb
endi print == check consume result
if $data[0][1] == 0 then wait_consumer_end_from_stb:
if $data[1][1] != 1 then sql select * from consumeresult
return -1 print ==> rows: $rows
endi print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
endi print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
if $data[0][1] == 1 then if $rows != 2 then
if $data[1][1] != 0 then sleep 1000
return -1 goto wait_consumer_end_from_stb
endi endi
endi if $data[0][1] == 0 then
if $data[1][1] != 1 then
# either $data[0][2] == $totalMsgOfStb and $data[1][2] == 0 return -1
# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfStb endi
if $data[0][2] == $totalMsgOfStb then endi
if $data[1][2] == 0 then if $data[0][1] == 1 then
goto check_ok_0 if $data[1][1] != 0 then
endi return -1
elif $data[0][2] == 0 then endi
if $data[1][2] == $totalMsgOfStb then endi
goto check_ok_0
endi # either $data[0][2] == $totalMsgOfStb and $data[1][2] == 0
endi # or $data[0][2] == 0 and $data[1][2] == $totalMsgOfStb
return -1 if $data[0][2] == $totalMsgOfStb then
check_ok_0: if $data[1][2] == 0 then
goto check_ok_0
if $data[0][3] == $totalMsgOfStb then endi
if $data[1][3] == 0 then elif $data[0][2] == 0 then
goto check_ok_1 if $data[1][2] == $totalMsgOfStb then
endi goto check_ok_0
elif $data[0][3] == 0 then endi
if $data[1][3] == $totalMsgOfStb then endi
goto check_ok_1 return -1
endi check_ok_0:
endi
return -1 if $data[0][3] == $totalMsgOfStb then
check_ok_1: if $data[1][3] == 0 then
goto check_ok_1
####################################################################################### endi
# clear consume info and consume result elif $data[0][3] == 0 then
#run tsim/tmq/clearConsume.sim if $data[1][3] == $totalMsgOfStb then
# because drop table function no stable, so by create new db for consume info and result. Modify it later goto check_ok_1
$cdbName = cdb1 endi
sql create database $cdbName vgroups 1 endi
sleep 500 return -1
sql use $cdbName check_ok_1:
print == create consume info table and consume result table for ctb #######################################################################################
sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) # clear consume info and consume result
sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) #run tsim/tmq/clearConsume.sim
# because drop table function no stable, so by create new db for consume info and result. Modify it later
sql show tables $cdbName = cdb1
if $rows != 2 then sql create database $cdbName vgroups 1
return -1 sleep 500
endi sql use $cdbName
#######################################################################################
print == create consume info table and consume result table for ctb
sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
print ================ test consume from ctb sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
print == multi toipcs: topic_ctb_column + topic_ctb_all + topic_ctb_function
$topicList = ' . topic_ctb_column sql show tables
$topicList = $topicList . , if $rows != 2 then
$topicList = $topicList . topic_ctb_all return -1
$topicList = $topicList . , endi
$topicList = $topicList . topic_ctb_function #######################################################################################
$topicList = $topicList . '
$consumerId = 0 print ================ test consume from ctb
$totalMsgOfCtb = $rowsPerCtb * $topicNum print == multi toipcs: topic_ctb_column + topic_ctb_all + topic_ctb_function
$expectmsgcnt = $totalMsgOfCtb $topicList = ' . topic_ctb_column
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) $topicList = $topicList . ,
$consumerId = 1 $topicList = $topicList . topic_ctb_all
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) $topicList = $topicList . ,
$topicList = $topicList . topic_ctb_function
print == start consumer to pull msgs from ctb $topicList = $topicList . '
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start $consumerId = 0
$totalMsgOfCtb = $rowsPerCtb * $topicNum
print == check consume result $expectmsgcnt = $totalMsgOfCtb
wait_consumer_end_from_ctb: sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
sql select * from consumeresult $consumerId = 1
print ==> rows: $rows sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] print == start consumer to pull msgs from ctb
if $rows != 2 then print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
sleep 1000 system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
goto wait_consumer_end_from_ctb
endi print == check consume result
if $data[0][1] == 0 then wait_consumer_end_from_ctb:
if $data[1][1] != 1 then sql select * from consumeresult
return -1 print ==> rows: $rows
endi print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
endi print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
if $data[0][1] == 1 then if $rows != 2 then
if $data[1][1] != 0 then sleep 1000
return -1 goto wait_consumer_end_from_ctb
endi endi
endi if $data[0][1] == 0 then
if $data[1][1] != 1 then
# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0 return -1
# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb endi
if $data[0][2] == $totalMsgOfCtb then endi
if $data[1][2] == 0 then if $data[0][1] == 1 then
goto check_ok_2 if $data[1][1] != 0 then
endi return -1
elif $data[0][2] == 0 then endi
if $data[1][2] == $totalMsgOfCtb then endi
goto check_ok_2
endi # either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0
endi # or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb
return -1 if $data[0][2] == $totalMsgOfCtb then
check_ok_2: if $data[1][2] == 0 then
goto check_ok_2
if $data[0][3] == $totalMsgOfCtb then endi
if $data[1][3] == 0 then elif $data[0][2] == 0 then
goto check_ok_3 if $data[1][2] == $totalMsgOfCtb then
endi goto check_ok_2
elif $data[0][3] == 0 then endi
if $data[1][3] == $totalMsgOfCtb then endi
goto check_ok_3 return -1
endi check_ok_2:
endi
return -1 if $data[0][3] == $totalMsgOfCtb then
check_ok_3: if $data[1][3] == 0 then
goto check_ok_3
####################################################################################### endi
# clear consume info and consume result elif $data[0][3] == 0 then
#run tsim/tmq/clearConsume.sim if $data[1][3] == $totalMsgOfCtb then
# because drop table function no stable, so by create new db for consume info and result. Modify it later goto check_ok_3
$cdbName = cdb2 endi
sql create database $cdbName vgroups 1 endi
sleep 500 return -1
sql use $cdbName check_ok_3:
print == create consume info table and consume result table for ntb #######################################################################################
sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) # clear consume info and consume result
sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) #run tsim/tmq/clearConsume.sim
# because drop table function no stable, so by create new db for consume info and result. Modify it later
sql show tables $cdbName = cdb2
if $rows != 2 then sql create database $cdbName vgroups 1
return -1 sleep 500
endi sql use $cdbName
#######################################################################################
print == create consume info table and consume result table for ntb
sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
print ================ test consume from ntb sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
print == multi toipcs: topic_ntb_column + topic_ntb_all + topic_ntb_function
$topicList = ' . topic_ntb_column sql show tables
$topicList = $topicList . , if $rows != 2 then
$topicList = $topicList . topic_ntb_all return -1
$topicList = $topicList . , endi
$topicList = $topicList . topic_ntb_function #######################################################################################
$topicList = $topicList . '
$consumerId = 0 print ================ test consume from ntb
$totalMsgOfNtb = $rowsPerCtb * $topicNum print == multi toipcs: topic_ntb_column + topic_ntb_all + topic_ntb_function
$expectmsgcnt = $totalMsgOfNtb $topicList = ' . topic_ntb_column
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) $topicList = $topicList . ,
$consumerId = 1 $topicList = $topicList . topic_ntb_all
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) $topicList = $topicList . ,
$topicList = $topicList . topic_ntb_function
print == start consumer to pull msgs from ntb $topicList = $topicList . '
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start $consumerId = 0
$totalMsgOfNtb = $rowsPerCtb * $topicNum
print == check consume result from ntb $expectmsgcnt = $totalMsgOfNtb
wait_consumer_end_from_ntb: sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
sql select * from consumeresult $consumerId = 1
print ==> rows: $rows sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] print == start consumer to pull msgs from ntb
if $rows != 2 then print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
sleep 1000 system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
goto wait_consumer_end_from_ntb
endi print == check consume result from ntb
if $data[0][1] == 0 then wait_consumer_end_from_ntb:
if $data[1][1] != 1 then sql select * from consumeresult
return -1 print ==> rows: $rows
endi print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
endi print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
if $data[1][1] == 0 then if $rows != 2 then
if $data[0][1] != 1 then sleep 1000
return -1 goto wait_consumer_end_from_ntb
endi endi
endi if $data[0][1] == 0 then
if $data[1][1] != 1 then
# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0 return -1
# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb endi
if $data[0][2] == $totalMsgOfNtb then endi
if $data[1][2] == 0 then if $data[1][1] == 0 then
goto check_ok_4 if $data[0][1] != 1 then
endi return -1
elif $data[0][2] == 0 then endi
if $data[1][2] == $totalMsgOfNtb then endi
goto check_ok_4
endi # either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0
endi # or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb
return -1 if $data[0][2] == $totalMsgOfNtb then
check_ok_4: if $data[1][2] == 0 then
goto check_ok_4
if $data[0][3] == $totalMsgOfNtb then endi
if $data[1][3] == 0 then elif $data[0][2] == 0 then
goto check_ok_5 if $data[1][2] == $totalMsgOfNtb then
endi goto check_ok_4
elif $data[0][3] == 0 then endi
if $data[1][3] == $totalMsgOfNtb then endi
goto check_ok_5 return -1
endi check_ok_4:
endi
return -1 if $data[0][3] == $totalMsgOfNtb then
check_ok_5: if $data[1][3] == 0 then
goto check_ok_5
#------ not need stop consumer, because it exit after pull msg overthan expect msg endi
#system tsim/tmq/consume.sh -s stop -x SIGINT elif $data[0][3] == 0 then
if $data[1][3] == $totalMsgOfNtb then
system sh/exec.sh -n dnode1 -s stop -x SIGINT goto check_ok_5
endi
endi
return -1
check_ok_5:
#------ not need stop consumer, because it exit after pull msg overthan expect msg
#system tsim/tmq/consume.sh -s stop -x SIGINT
system sh/exec.sh -n dnode1 -s stop -x SIGINT

25
tests/script/tsim/tmq/basic2Of2ConsOverlap.sim
View File

@ -27,6 +27,7 @@ $tstart = 1640966400000 # 2022-01-01 00:00:00.000
$pullDelay = 5 $pullDelay = 5
$ifcheckdata = 1 $ifcheckdata = 1
$ifmanualcommit = 1
$showMsg = 1 $showMsg = 1
$showRow = 0 $showRow = 0
@ -53,8 +54,16 @@ sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0
# return -1 # return -1
#endi #endi
#'group.id:cgrp1,enable.auto.commit:false,auto.commit.interval.ms:6000,auto.offset.reset:earliest'
$keyList = ' . group.id:cgrp1 $keyList = ' . group.id:cgrp1
$keyList = $keyList . ,
$keyList = $keyList . enable.auto.commit:false
#$keyList = $keyList . ,
#$keyList = $keyList . auto.commit.interval.ms:6000
#$keyList = $keyList . ,
#$keyList = $keyList . auto.offset.reset:earliest
$keyList = $keyList . ' $keyList = $keyList . '
print ========== key list: $keyList
$topicNum = 2 $topicNum = 2
@ -72,7 +81,7 @@ $consumerId = 0
$totalMsgOfOneTopic = $ctbNum * $rowsPerCtb $totalMsgOfOneTopic = $ctbNum * $rowsPerCtb
$totalMsgOfStb = $totalMsgOfOneTopic * $topicNum $totalMsgOfStb = $totalMsgOfOneTopic * $topicNum
$expectmsgcnt = $totalMsgOfStb $expectmsgcnt = $totalMsgOfStb
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
$topicList = ' . topic_stb_all $topicList = ' . topic_stb_all
@ -80,7 +89,7 @@ $topicList = $topicList . ,
$topicList = $topicList . topic_stb_function $topicList = $topicList . topic_stb_function
$topicList = $topicList . ' $topicList = $topicList . '
$consumerId = 1 $consumerId = 1
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
print == start consumer to pull msgs from stb print == start consumer to pull msgs from stb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start
@ -158,7 +167,7 @@ sleep 500
sql use $cdbName sql use $cdbName
print == create consume info table and consume result table for ctb print == create consume info table and consume result table for ctb
sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
sql show tables sql show tables
@ -179,14 +188,14 @@ $consumerId = 0
$totalMsgOfOneTopic = $rowsPerCtb $totalMsgOfOneTopic = $rowsPerCtb
$totalMsgOfCtb = $totalMsgOfOneTopic * $topicNum $totalMsgOfCtb = $totalMsgOfOneTopic * $topicNum
$expectmsgcnt = $totalMsgOfCtb $expectmsgcnt = $totalMsgOfCtb
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
$topicList = ' . topic_ctb_function $topicList = ' . topic_ctb_function
$topicList = $topicList . , $topicList = $topicList . ,
$topicList = $topicList . topic_ctb_all $topicList = $topicList . topic_ctb_all
$topicList = $topicList . ' $topicList = $topicList . '
$consumerId = 1 $consumerId = 1
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
print == start consumer to pull msgs from ctb print == start consumer to pull msgs from ctb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
@ -249,7 +258,7 @@ sleep 500
sql use $cdbName sql use $cdbName
print == create consume info table and consume result table for ntb print == create consume info table and consume result table for ntb
sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
sql show tables sql show tables
@ -270,7 +279,7 @@ $consumerId = 0
$totalMsgOfOneTopic = $rowsPerCtb $totalMsgOfOneTopic = $rowsPerCtb
$totalMsgOfNtb = $totalMsgOfOneTopic * $topicNum $totalMsgOfNtb = $totalMsgOfOneTopic * $topicNum
$expectmsgcnt = $totalMsgOfNtb $expectmsgcnt = $totalMsgOfNtb
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
$topicList = ' . topic_ntb_function $topicList = ' . topic_ntb_function
@ -278,7 +287,7 @@ $topicList = $topicList . ,
$topicList = $topicList . topic_ntb_all $topicList = $topicList . topic_ntb_all
$topicList = $topicList . ' $topicList = $topicList . '
$consumerId = 1 $consumerId = 1
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
print == start consumer to pull msgs from ntb print == start consumer to pull msgs from ntb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start

566
tests/script/tsim/tmq/basic3.sim
View File

@ -1,278 +1,288 @@
#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 #### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406
#basic1.sim: vgroups=1, one topic for one consumer, firstly insert data, then start consume. Include six topics #basic1.sim: vgroups=1, one topic for one consumer, firstly insert data, then start consume. Include six topics
#basic2.sim: vgroups=1, multi topics for one consumer, firstly insert data, then start consume. Include six topics #basic2.sim: vgroups=1, multi topics for one consumer, firstly insert data, then start consume. Include six topics
#basic3.sim: vgroups=4, one topic for one consumer, firstly insert data, then start consume. Include six topics #basic3.sim: vgroups=4, one topic for one consumer, firstly insert data, then start consume. Include six topics
#basic4.sim: vgroups=4, multi topics for one consumer, firstly insert data, then start consume. Include six topics #basic4.sim: vgroups=4, multi topics for one consumer, firstly insert data, then start consume. Include six topics
# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN # notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN
# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; # The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5;
# #
# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). # notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval).
# #
run tsim/tmq/prepareBasicEnv-4vgrp.sim run tsim/tmq/prepareBasicEnv-4vgrp.sim
#---- global parameters start ----# #---- global parameters start ----#
$dbName = db $dbName = db
$vgroups = 4 $vgroups = 4
$stbPrefix = stb $stbPrefix = stb
$ctbPrefix = ctb $ctbPrefix = ctb
$ntbPrefix = ntb $ntbPrefix = ntb
$stbNum = 1 $stbNum = 1
$ctbNum = 10 $ctbNum = 10
$ntbNum = 10 $ntbNum = 10
$rowsPerCtb = 10 $rowsPerCtb = 10
$tstart = 1640966400000 # 2022-01-01 00:00:00.000 $tstart = 1640966400000 # 2022-01-01 00:00:00.000
#---- global parameters end ----# #---- global parameters end ----#
$pullDelay = 3 $pullDelay = 3
$ifcheckdata = 1 $ifcheckdata = 1
$showMsg = 1 $ifmanualcommit = 1
$showRow = 0 $showMsg = 1
$showRow = 0
sql connect
sql use $dbName sql connect
sql use $dbName
print == create topics from super table
sql create topic topic_stb_column as select ts, c3 from stb print == create topics from super table
sql create topic topic_stb_all as select ts, c1, c2, c3 from stb sql create topic topic_stb_column as select ts, c3 from stb
sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb sql create topic topic_stb_all as select ts, c1, c2, c3 from stb
sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb
print == create topics from child table
sql create topic topic_ctb_column as select ts, c3 from ctb0 print == create topics from child table
sql create topic topic_ctb_all as select * from ctb0 sql create topic topic_ctb_column as select ts, c3 from ctb0
sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 sql create topic topic_ctb_all as select * from ctb0
sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0
print == create topics from normal table
sql create topic topic_ntb_column as select ts, c3 from ntb0 print == create topics from normal table
sql create topic topic_ntb_all as select * from ntb0 sql create topic topic_ntb_column as select ts, c3 from ntb0
sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 sql create topic topic_ntb_all as select * from ntb0
sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0
#sql show topics
#if $rows != 9 then #sql show topics
# return -1 #if $rows != 9 then
#endi # return -1
#endi
$keyList = ' . group.id:cgrp1
$keyList = $keyList . ' #'group.id:cgrp1,enable.auto.commit:false,auto.commit.interval.ms:6000,auto.offset.reset:earliest'
$keyList = ' . group.id:cgrp1
$cdb_index = 0 $keyList = $keyList . ,
#=============================== start consume =============================# $keyList = $keyList . enable.auto.commit:false
#$keyList = $keyList . ,
print ================ test consume from stb #$keyList = $keyList . auto.commit.interval.ms:6000
$loop_cnt = 0 #$keyList = $keyList . ,
loop_consume_diff_topic_from_stb: #$keyList = $keyList . auto.offset.reset:earliest
$keyList = $keyList . '
####################################################################################### print ========== key list: $keyList
# clear consume info and consume result
#run tsim/tmq/clearConsume.sim
# because drop table function no stable, so by create new db for consume info and result. Modify it later $cdb_index = 0
$cdb_index = $cdb_index + 1 #=============================== start consume =============================#
$cdbName = cdb . $cdb_index
sql create database $cdbName vgroups 1 print ================ test consume from stb
sleep 500 $loop_cnt = 0
sql use $cdbName loop_consume_diff_topic_from_stb:
print == create consume info table and consume result table #######################################################################################
sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) # clear consume info and consume result
sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) #run tsim/tmq/clearConsume.sim
# because drop table function no stable, so by create new db for consume info and result. Modify it later
sql show tables $cdb_index = $cdb_index + 1
if $rows != 2 then $cdbName = cdb . $cdb_index
return -1 sql create database $cdbName vgroups 1
endi sleep 500
####################################################################################### sql use $cdbName
if $loop_cnt == 0 then print == create consume info table and consume result table
print == scenario 1: topic_stb_column sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
$topicList = ' . topic_stb_column sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
$topicList = $topicList . '
elif $loop_cnt == 1 then sql show tables
print == scenario 2: topic_stb_all if $rows != 2 then
$topicList = ' . topic_stb_all return -1
$topicList = $topicList . ' endi
elif $loop_cnt == 2 then #######################################################################################
print == scenario 3: topic_stb_function
$topicList = ' . topic_stb_function if $loop_cnt == 0 then
$topicList = $topicList . ' print == scenario 1: topic_stb_column
else $topicList = ' . topic_stb_column
goto loop_consume_diff_topic_from_stb_end $topicList = $topicList . '
endi elif $loop_cnt == 1 then
print == scenario 2: topic_stb_all
$consumerId = 0 $topicList = ' . topic_stb_all
$totalMsgOfStb = $ctbNum * $rowsPerCtb $topicList = $topicList . '
$expectmsgcnt = $totalMsgOfStb elif $loop_cnt == 2 then
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) print == scenario 3: topic_stb_function
$topicList = ' . topic_stb_function
print == start consumer to pull msgs from stb $topicList = $topicList . '
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start else
system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start goto loop_consume_diff_topic_from_stb_end
endi
print == check consume result
wait_consumer_end_from_stb: $consumerId = 0
sql select * from consumeresult $totalMsgOfStb = $ctbNum * $rowsPerCtb
print ==> rows: $rows $expectmsgcnt = $totalMsgOfStb
print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
if $rows != 1 then
sleep 1000 print == start consumer to pull msgs from stb
goto wait_consumer_end_from_stb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
endi system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
if $data[0][1] != $consumerId then
return -1 print == check consume result
endi wait_consumer_end_from_stb:
if $data[0][2] != $expectmsgcnt then sql select * from consumeresult
return -1 print ==> rows: $rows
endi print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
if $data[0][3] != $expectmsgcnt then if $rows != 1 then
return -1 sleep 1000
endi goto wait_consumer_end_from_stb
$loop_cnt = $loop_cnt + 1 endi
goto loop_consume_diff_topic_from_stb if $data[0][1] != $consumerId then
loop_consume_diff_topic_from_stb_end: return -1
endi
print ================ test consume from ctb if $data[0][2] != $expectmsgcnt then
$loop_cnt = 0 return -1
loop_consume_diff_topic_from_ctb: endi
if $data[0][3] != $expectmsgcnt then
####################################################################################### return -1
# clear consume info and consume result endi
#run tsim/tmq/clearConsume.sim $loop_cnt = $loop_cnt + 1
# because drop table function no stable, so by create new db for consume info and result. Modify it later goto loop_consume_diff_topic_from_stb
$cdb_index = $cdb_index + 1 loop_consume_diff_topic_from_stb_end:
$cdbName = cdb . $cdb_index
sql create database $cdbName vgroups 1 print ================ test consume from ctb
sleep 500 $loop_cnt = 0
sql use $cdbName loop_consume_diff_topic_from_ctb:
print == create consume info table and consume result table #######################################################################################
sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) # clear consume info and consume result
sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) #run tsim/tmq/clearConsume.sim
# because drop table function no stable, so by create new db for consume info and result. Modify it later
sql show tables $cdb_index = $cdb_index + 1
if $rows != 2 then $cdbName = cdb . $cdb_index
return -1 sql create database $cdbName vgroups 1
endi sleep 500
####################################################################################### sql use $cdbName
if $loop_cnt == 0 then print == create consume info table and consume result table
print == scenario 1: topic_ctb_column sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
$topicList = ' . topic_ctb_column sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
$topicList = $topicList . '
elif $loop_cnt == 1 then sql show tables
print == scenario 2: topic_ctb_all if $rows != 2 then
$topicList = ' . topic_ctb_all return -1
$topicList = $topicList . ' endi
elif $loop_cnt == 2 then #######################################################################################
print == scenario 3: topic_ctb_function
$topicList = ' . topic_ctb_function if $loop_cnt == 0 then
$topicList = $topicList . ' print == scenario 1: topic_ctb_column
else $topicList = ' . topic_ctb_column
goto loop_consume_diff_topic_from_ctb_end $topicList = $topicList . '
endi elif $loop_cnt == 1 then
print == scenario 2: topic_ctb_all
$consumerId = 0 $topicList = ' . topic_ctb_all
$totalMsgOfCtb = $rowsPerCtb $topicList = $topicList . '
$expectmsgcnt = $totalMsgOfCtb elif $loop_cnt == 2 then
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) print == scenario 3: topic_ctb_function
$topicList = ' . topic_ctb_function
print == start consumer to pull msgs from ctb $topicList = $topicList . '
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start else
system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start goto loop_consume_diff_topic_from_ctb_end
endi
print == check consume result
wait_consumer_end_from_ctb: $consumerId = 0
sql select * from consumeresult $totalMsgOfCtb = $rowsPerCtb
print ==> rows: $rows $expectmsgcnt = $totalMsgOfCtb
print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
if $rows != 1 then
sleep 1000 print == start consumer to pull msgs from ctb
goto wait_consumer_end_from_ctb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
endi system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
if $data[0][1] != $consumerId then
return -1 print == check consume result
endi wait_consumer_end_from_ctb:
if $data[0][2] != $totalMsgOfCtb then sql select * from consumeresult
return -1 print ==> rows: $rows
endi print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
if $data[0][3] != $totalMsgOfCtb then if $rows != 1 then
return -1 sleep 1000
endi goto wait_consumer_end_from_ctb
$loop_cnt = $loop_cnt + 1 endi
goto loop_consume_diff_topic_from_ctb if $data[0][1] != $consumerId then
loop_consume_diff_topic_from_ctb_end: return -1
endi
print ================ test consume from ntb if $data[0][2] != $totalMsgOfCtb then
$loop_cnt = 0 return -1
loop_consume_diff_topic_from_ntb: endi
if $data[0][3] != $totalMsgOfCtb then
####################################################################################### return -1
# clear consume info and consume result endi
#run tsim/tmq/clearConsume.sim $loop_cnt = $loop_cnt + 1
# because drop table function no stable, so by create new db for consume info and result. Modify it later goto loop_consume_diff_topic_from_ctb
$cdb_index = $cdb_index + 1 loop_consume_diff_topic_from_ctb_end:
$cdbName = cdb . $cdb_index
sql create database $cdbName vgroups 1 print ================ test consume from ntb
sleep 500 $loop_cnt = 0
sql use $cdbName loop_consume_diff_topic_from_ntb:
print == create consume info table and consume result table #######################################################################################
sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) # clear consume info and consume result
sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) #run tsim/tmq/clearConsume.sim
# because drop table function no stable, so by create new db for consume info and result. Modify it later
sql show tables $cdb_index = $cdb_index + 1
if $rows != 2 then $cdbName = cdb . $cdb_index
return -1 sql create database $cdbName vgroups 1
endi sleep 500
####################################################################################### sql use $cdbName
if $loop_cnt == 0 then print == create consume info table and consume result table
print == scenario 1: topic_ntb_column sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
$topicList = ' . topic_ntb_column sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
$topicList = $topicList . '
elif $loop_cnt == 1 then sql show tables
print == scenario 2: topic_ntb_all if $rows != 2 then
$topicList = ' . topic_ntb_all return -1
$topicList = $topicList . ' endi
elif $loop_cnt == 2 then #######################################################################################
print == scenario 3: topic_ntb_function
$topicList = ' . topic_ntb_function if $loop_cnt == 0 then
$topicList = $topicList . ' print == scenario 1: topic_ntb_column
else $topicList = ' . topic_ntb_column
goto loop_consume_diff_topic_from_ntb_end $topicList = $topicList . '
endi elif $loop_cnt == 1 then
print == scenario 2: topic_ntb_all
$consumerId = 0 $topicList = ' . topic_ntb_all
$totalMsgOfNtb = $rowsPerCtb $topicList = $topicList . '
$expectmsgcnt = $totalMsgOfNtb elif $loop_cnt == 2 then
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) print == scenario 3: topic_ntb_function
$topicList = ' . topic_ntb_function
print == start consumer to pull msgs from ntb $topicList = $topicList . '
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start else
system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start goto loop_consume_diff_topic_from_ntb_end
endi
print == check consume result from ntb
wait_consumer_end_from_ntb: $consumerId = 0
sql select * from consumeresult $totalMsgOfNtb = $rowsPerCtb
print ==> rows: $rows $expectmsgcnt = $totalMsgOfNtb
print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
if $rows != 1 then
sleep 1000 print == start consumer to pull msgs from ntb
goto wait_consumer_end_from_ntb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
endi system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
if $data[0][1] != $consumerId then
return -1 print == check consume result from ntb
endi wait_consumer_end_from_ntb:
if $data[0][2] != $totalMsgOfNtb then sql select * from consumeresult
return -1 print ==> rows: $rows
endi print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
if $data[0][3] != $totalMsgOfNtb then if $rows != 1 then
return -1 sleep 1000
endi goto wait_consumer_end_from_ntb
$loop_cnt = $loop_cnt + 1 endi
goto loop_consume_diff_topic_from_ntb if $data[0][1] != $consumerId then
loop_consume_diff_topic_from_ntb_end: return -1
endi
#------ not need stop consumer, because it exit after pull msg overthan expect msg if $data[0][2] != $totalMsgOfNtb then
#system tsim/tmq/consume.sh -s stop -x SIGINT return -1
endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT if $data[0][3] != $totalMsgOfNtb then
return -1
endi
$loop_cnt = $loop_cnt + 1
goto loop_consume_diff_topic_from_ntb
loop_consume_diff_topic_from_ntb_end:
#------ not need stop consumer, because it exit after pull msg overthan expect msg
#system tsim/tmq/consume.sh -s stop -x SIGINT
system sh/exec.sh -n dnode1 -s stop -x SIGINT

777
tests/script/tsim/tmq/basic3Of2Cons.sim
View File

@ -1,384 +1,393 @@
#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 #### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406
#basic1Of2Cons.sim: vgroups=1, one topic for 2 consumers, firstly insert data, then start consume. Include six topics #basic1Of2Cons.sim: vgroups=1, one topic for 2 consumers, firstly insert data, then start consume. Include six topics
#basic2Of2Cons.sim: vgroups=1, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics #basic2Of2Cons.sim: vgroups=1, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics
#basic3Of2Cons.sim: vgroups=4, one topic for 2 consumers, firstly insert data, then start consume. Include six topics #basic3Of2Cons.sim: vgroups=4, one topic for 2 consumers, firstly insert data, then start consume. Include six topics
#basic4Of2Cons.sim: vgroups=4, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics #basic4Of2Cons.sim: vgroups=4, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics
# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN # notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN
# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; # The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5;
# #
# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). # notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval).
# #
run tsim/tmq/prepareBasicEnv-4vgrp.sim run tsim/tmq/prepareBasicEnv-4vgrp.sim
#---- global parameters start ----# #---- global parameters start ----#
$dbName = db $dbName = db
$vgroups = 4 $vgroups = 4
$stbPrefix = stb $stbPrefix = stb
$ctbPrefix = ctb $ctbPrefix = ctb
$ntbPrefix = ntb $ntbPrefix = ntb
$stbNum = 1 $stbNum = 1
$ctbNum = 10 $ctbNum = 10
$ntbNum = 10 $ntbNum = 10
$rowsPerCtb = 10 $rowsPerCtb = 10
$tstart = 1640966400000 # 2022-01-01 00:00:00.000 $tstart = 1640966400000 # 2022-01-01 00:00:00.000
#---- global parameters end ----# #---- global parameters end ----#
$pullDelay = 5 $pullDelay = 5
$ifcheckdata = 1 $ifcheckdata = 1
$showMsg = 1 $ifmanualcommit = 1
$showRow = 0 $showMsg = 1
$showRow = 0
sql connect
sql use $dbName sql connect
sql use $dbName
print == create topics from super table
sql create topic topic_stb_column as select ts, c3 from stb print == create topics from super table
sql create topic topic_stb_all as select ts, c1, c2, c3 from stb sql create topic topic_stb_column as select ts, c3 from stb
sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb sql create topic topic_stb_all as select ts, c1, c2, c3 from stb
sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb
print == create topics from child table
sql create topic topic_ctb_column as select ts, c3 from ctb0 print == create topics from child table
sql create topic topic_ctb_all as select * from ctb0 sql create topic topic_ctb_column as select ts, c3 from ctb0
sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 sql create topic topic_ctb_all as select * from ctb0
sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0
print == create topics from normal table
sql create topic topic_ntb_column as select ts, c3 from ntb0 print == create topics from normal table
sql create topic topic_ntb_all as select * from ntb0 sql create topic topic_ntb_column as select ts, c3 from ntb0
sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 sql create topic topic_ntb_all as select * from ntb0
sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0
#sql show topics
#if $rows != 9 then #sql show topics
# return -1 #if $rows != 9 then
#endi # return -1
#endi
$keyList = ' . group.id:cgrp1
$keyList = $keyList . ' #'group.id:cgrp1,enable.auto.commit:false,auto.commit.interval.ms:6000,auto.offset.reset:earliest'
$keyList = ' . group.id:cgrp1
$cdb_index = 0 $keyList = $keyList . ,
#=============================== start consume =============================# $keyList = $keyList . enable.auto.commit:false
#$keyList = $keyList . ,
print ================ test consume from stb #$keyList = $keyList . auto.commit.interval.ms:6000
$loop_cnt = 0 #$keyList = $keyList . ,
loop_consume_diff_topic_from_stb: #$keyList = $keyList . auto.offset.reset:earliest
$keyList = $keyList . '
####################################################################################### print ========== key list: $keyList
# clear consume info and consume result
#run tsim/tmq/clearConsume.sim $cdb_index = 0
# because drop table function no stable, so by create new db for consume info and result. Modify it later #=============================== start consume =============================#
$cdb_index = $cdb_index + 1
$cdbName = cdb . $cdb_index print ================ test consume from stb
sql create database $cdbName vgroups 1 $loop_cnt = 0
sleep 500 loop_consume_diff_topic_from_stb:
sql use $cdbName
#######################################################################################
print == create consume info table and consume result table # clear consume info and consume result
sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) #run tsim/tmq/clearConsume.sim
sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) # because drop table function no stable, so by create new db for consume info and result. Modify it later
$cdb_index = $cdb_index + 1
sql show tables $cdbName = cdb . $cdb_index
if $rows != 2 then sql create database $cdbName vgroups 1
return -1 sleep 500
endi sql use $cdbName
#######################################################################################
print == create consume info table and consume result table
if $loop_cnt == 0 then sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
print == scenario 1: topic_stb_column sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
$topicList = ' . topic_stb_column
$topicList = $topicList . ' sql show tables
elif $loop_cnt == 1 then if $rows != 2 then
print == scenario 2: topic_stb_all return -1
$topicList = ' . topic_stb_all endi
$topicList = $topicList . ' #######################################################################################
elif $loop_cnt == 2 then
print == scenario 3: topic_stb_function if $loop_cnt == 0 then
$topicList = ' . topic_stb_function print == scenario 1: topic_stb_column
$topicList = $topicList . ' $topicList = ' . topic_stb_column
else $topicList = $topicList . '
goto loop_consume_diff_topic_from_stb_end elif $loop_cnt == 1 then
endi print == scenario 2: topic_stb_all
$topicList = ' . topic_stb_all
$consumerId = 0 $topicList = $topicList . '
$totalMsgOfStb = $ctbNum * $rowsPerCtb elif $loop_cnt == 2 then
$expectmsgcnt = $totalMsgOfStb print == scenario 3: topic_stb_function
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) $topicList = ' . topic_stb_function
$consumerId = 1 $topicList = $topicList . '
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) else
goto loop_consume_diff_topic_from_stb_end
print == start consumer to pull msgs from stb endi
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start $consumerId = 0
$totalMsgOfStb = $ctbNum * $rowsPerCtb
print == check consume result $expectmsgcnt = $totalMsgOfStb
wait_consumer_end_from_stb: sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
sql select * from consumeresult $consumerId = 1
print ==> rows: $rows sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] print == start consumer to pull msgs from stb
if $rows != 2 then print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
sleep 1000 system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
goto wait_consumer_end_from_stb
endi print == check consume result
if $data[0][1] == 0 then wait_consumer_end_from_stb:
if $data[1][1] != 1 then sql select * from consumeresult
return -1 print ==> rows: $rows
endi print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
endi print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
if $data[0][1] == 1 then if $rows != 2 then
if $data[1][1] != 0 then sleep 1000
return -1 goto wait_consumer_end_from_stb
endi endi
endi if $data[0][1] == 0 then
if $data[1][1] != 1 then
if $data[0][2] <= 0 then return -1
return -1 endi
endi endi
if $data[0][2] >= $expectmsgcnt then if $data[0][1] == 1 then
return -1 if $data[1][1] != 0 then
endi return -1
endi
if $data[1][2] <= 0 then endi
return -1
endi if $data[0][2] <= 0 then
if $data[1][2] >= $expectmsgcnt then return -1
return -1 endi
endi if $data[0][2] >= $expectmsgcnt then
return -1
$sumOfMsgCnt = $data[0][2] + $data[1][2] endi
if $sumOfMsgCnt != $expectmsgcnt then
return -1 if $data[1][2] <= 0 then
endi return -1
endi
if $data[1][2] >= $expectmsgcnt then
if $data[0][3] <= 0 then return -1
return -1 endi
endi
if $data[0][3] >= $expectmsgcnt then $sumOfMsgCnt = $data[0][2] + $data[1][2]
return -1 if $sumOfMsgCnt != $expectmsgcnt then
endi return -1
endi
if $data[1][3] <= 0 then
return -1
endi if $data[0][3] <= 0 then
if $data[1][3] >= $expectmsgcnt then return -1
return -1 endi
endi if $data[0][3] >= $expectmsgcnt then
return -1
$sumOfMsgRows = $data[0][3] + $data[1][3] endi
if $sumOfMsgRows != $expectmsgcnt then
return -1 if $data[1][3] <= 0 then
endi return -1
endi
$loop_cnt = $loop_cnt + 1 if $data[1][3] >= $expectmsgcnt then
goto loop_consume_diff_topic_from_stb return -1
loop_consume_diff_topic_from_stb_end: endi
print ================ test consume from ctb $sumOfMsgRows = $data[0][3] + $data[1][3]
$loop_cnt = 0 if $sumOfMsgRows != $expectmsgcnt then
loop_consume_diff_topic_from_ctb: return -1
endi
#######################################################################################
# clear consume info and consume result $loop_cnt = $loop_cnt + 1
#run tsim/tmq/clearConsume.sim goto loop_consume_diff_topic_from_stb
# because drop table function no stable, so by create new db for consume info and result. Modify it later loop_consume_diff_topic_from_stb_end:
$cdb_index = $cdb_index + 1
$cdbName = cdb . $cdb_index print ================ test consume from ctb
sql create database $cdbName vgroups 1 $loop_cnt = 0
sleep 500 loop_consume_diff_topic_from_ctb:
sql use $cdbName
#######################################################################################
print == create consume info table and consume result table # clear consume info and consume result
sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) #run tsim/tmq/clearConsume.sim
sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) # because drop table function no stable, so by create new db for consume info and result. Modify it later
$cdb_index = $cdb_index + 1
sql show tables $cdbName = cdb . $cdb_index
if $rows != 2 then sql create database $cdbName vgroups 1
return -1 sleep 500
endi sql use $cdbName
#######################################################################################
print == create consume info table and consume result table
if $loop_cnt == 0 then sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
print == scenario 1: topic_ctb_column sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
$topicList = ' . topic_ctb_column
$topicList = $topicList . ' sql show tables
elif $loop_cnt == 1 then if $rows != 2 then
print == scenario 2: topic_ctb_all return -1
$topicList = ' . topic_ctb_all endi
$topicList = $topicList . ' #######################################################################################
elif $loop_cnt == 2 then
print == scenario 3: topic_ctb_function if $loop_cnt == 0 then
$topicList = ' . topic_ctb_function print == scenario 1: topic_ctb_column
$topicList = $topicList . ' $topicList = ' . topic_ctb_column
else $topicList = $topicList . '
goto loop_consume_diff_topic_from_ctb_end elif $loop_cnt == 1 then
endi print == scenario 2: topic_ctb_all
$topicList = ' . topic_ctb_all
$consumerId = 0 $topicList = $topicList . '
$totalMsgOfCtb = $rowsPerCtb elif $loop_cnt == 2 then
$expectmsgcnt = $totalMsgOfCtb print == scenario 3: topic_ctb_function
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) $topicList = ' . topic_ctb_function
$consumerId = 1 $topicList = $topicList . '
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) else
goto loop_consume_diff_topic_from_ctb_end
print == start consumer to pull msgs from ctb endi
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start $consumerId = 0
$totalMsgOfCtb = $rowsPerCtb
print == check consume result $expectmsgcnt = $totalMsgOfCtb
wait_consumer_end_from_ctb: sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
sql select * from consumeresult $consumerId = 1
print ==> rows: $rows sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] print == start consumer to pull msgs from ctb
if $rows != 2 then print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
sleep 1000 system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
goto wait_consumer_end_from_ctb
endi print == check consume result
if $data[0][1] == 0 then wait_consumer_end_from_ctb:
if $data[1][1] != 1 then sql select * from consumeresult
return -1 print ==> rows: $rows
endi print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
endi print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
if $data[0][1] == 1 then if $rows != 2 then
if $data[1][1] != 0 then sleep 1000
return -1 goto wait_consumer_end_from_ctb
endi endi
endi if $data[0][1] == 0 then
if $data[1][1] != 1 then
# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0 return -1
# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb endi
if $data[0][2] == $totalMsgOfCtb then endi
if $data[1][2] == 0 then if $data[0][1] == 1 then
goto check_ok_0 if $data[1][1] != 0 then
endi return -1
elif $data[1][2] == $totalMsgOfCtb then endi
if $data[0][2] == 0 then endi
goto check_ok_0
endi # either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0
endi # or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb
return -1 if $data[0][2] == $totalMsgOfCtb then
check_ok_0: if $data[1][2] == 0 then
goto check_ok_0
if $data[0][3] == $totalMsgOfCtb then endi
if $data[1][3] == 0 then elif $data[1][2] == $totalMsgOfCtb then
goto check_ok_1 if $data[0][2] == 0 then
endi goto check_ok_0
elif $data[1][3] == $totalMsgOfCtb then endi
if $data[0][3] == 0 then endi
goto check_ok_1 return -1
endi check_ok_0:
endi
return -1 if $data[0][3] == $totalMsgOfCtb then
check_ok_1: if $data[1][3] == 0 then
goto check_ok_1
$loop_cnt = $loop_cnt + 1 endi
goto loop_consume_diff_topic_from_ctb elif $data[1][3] == $totalMsgOfCtb then
loop_consume_diff_topic_from_ctb_end: if $data[0][3] == 0 then
goto check_ok_1
print ================ test consume from ntb endi
$loop_cnt = 0 endi
loop_consume_diff_topic_from_ntb: return -1
check_ok_1:
#######################################################################################
# clear consume info and consume result $loop_cnt = $loop_cnt + 1
#run tsim/tmq/clearConsume.sim goto loop_consume_diff_topic_from_ctb
# because drop table function no stable, so by create new db for consume info and result. Modify it later loop_consume_diff_topic_from_ctb_end:
$cdb_index = $cdb_index + 1
$cdbName = cdb . $cdb_index print ================ test consume from ntb
sql create database $cdbName vgroups 1 $loop_cnt = 0
sleep 500 loop_consume_diff_topic_from_ntb:
sql use $cdbName
#######################################################################################
print == create consume info table and consume result table # clear consume info and consume result
sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) #run tsim/tmq/clearConsume.sim
sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) # because drop table function no stable, so by create new db for consume info and result. Modify it later
$cdb_index = $cdb_index + 1
sql show tables $cdbName = cdb . $cdb_index
if $rows != 2 then sql create database $cdbName vgroups 1
return -1 sleep 500
endi sql use $cdbName
#######################################################################################
print == create consume info table and consume result table
if $loop_cnt == 0 then sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
print == scenario 1: topic_ntb_column sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
$topicList = ' . topic_ntb_column
$topicList = $topicList . ' sql show tables
elif $loop_cnt == 1 then if $rows != 2 then
print == scenario 2: topic_ntb_all return -1
$topicList = ' . topic_ntb_all endi
$topicList = $topicList . ' #######################################################################################
elif $loop_cnt == 2 then
print == scenario 3: topic_ntb_function if $loop_cnt == 0 then
$topicList = ' . topic_ntb_function print == scenario 1: topic_ntb_column
$topicList = $topicList . ' $topicList = ' . topic_ntb_column
else $topicList = $topicList . '
goto loop_consume_diff_topic_from_ntb_end elif $loop_cnt == 1 then
endi print == scenario 2: topic_ntb_all
$topicList = ' . topic_ntb_all
$consumerId = 0 $topicList = $topicList . '
$totalMsgOfNtb = $rowsPerCtb elif $loop_cnt == 2 then
$expectmsgcnt = $totalMsgOfNtb print == scenario 3: topic_ntb_function
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) $topicList = ' . topic_ntb_function
$consumerId = 1 $topicList = $topicList . '
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) else
goto loop_consume_diff_topic_from_ntb_end
print == start consumer to pull msgs from ntb endi
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start $consumerId = 0
$totalMsgOfNtb = $rowsPerCtb
print == check consume result from ntb $expectmsgcnt = $totalMsgOfNtb
wait_consumer_end_from_ntb: sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
sql select * from consumeresult $consumerId = 1
print ==> rows: $rows sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] print == start consumer to pull msgs from ntb
if $rows != 2 then print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
sleep 1000 system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
goto wait_consumer_end_from_ntb
endi print == check consume result from ntb
if $data[0][1] == 0 then wait_consumer_end_from_ntb:
if $data[1][1] != 1 then sql select * from consumeresult
return -1 print ==> rows: $rows
endi print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
endi print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
if $data[1][1] == 0 then if $rows != 2 then
if $data[0][1] != 1 then sleep 1000
return -1 goto wait_consumer_end_from_ntb
endi endi
endi if $data[0][1] == 0 then
if $data[1][1] != 1 then
# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0 return -1
# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb endi
if $data[0][2] == $totalMsgOfNtb then endi
if $data[1][2] == 0 then if $data[1][1] == 0 then
goto check_ok_2 if $data[0][1] != 1 then
endi return -1
elif $data[1][2] == $totalMsgOfNtb then endi
if $data[0][2] == 0 then endi
goto check_ok_2
endi # either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0
endi # or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb
return -1 if $data[0][2] == $totalMsgOfNtb then
check_ok_2: if $data[1][2] == 0 then
goto check_ok_2
if $data[0][3] == $totalMsgOfNtb then endi
if $data[1][3] == 0 then elif $data[1][2] == $totalMsgOfNtb then
goto check_ok_3 if $data[0][2] == 0 then
endi goto check_ok_2
elif $data[1][3] == $totalMsgOfNtb then endi
if $data[0][3] == 0 then endi
goto check_ok_3 return -1
endi check_ok_2:
endi
return -1 if $data[0][3] == $totalMsgOfNtb then
check_ok_3: if $data[1][3] == 0 then
goto check_ok_3
$loop_cnt = $loop_cnt + 1 endi
goto loop_consume_diff_topic_from_ntb elif $data[1][3] == $totalMsgOfNtb then
loop_consume_diff_topic_from_ntb_end: if $data[0][3] == 0 then
goto check_ok_3
#------ not need stop consumer, because it exit after pull msg overthan expect msg endi
#system tsim/tmq/consume.sh -s stop -x SIGINT endi
return -1
system sh/exec.sh -n dnode1 -s stop -x SIGINT check_ok_3:
$loop_cnt = $loop_cnt + 1
goto loop_consume_diff_topic_from_ntb
loop_consume_diff_topic_from_ntb_end:
#------ not need stop consumer, because it exit after pull msg overthan expect msg
#system tsim/tmq/consume.sh -s stop -x SIGINT
system sh/exec.sh -n dnode1 -s stop -x SIGINT

442
tests/script/tsim/tmq/basic4.sim
View File

@ -1,216 +1,226 @@
#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 #### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406
#basic1.sim: vgroups=1, one topic for one consumer, firstly insert data, then start consume. Include six topics #basic1.sim: vgroups=1, one topic for one consumer, firstly insert data, then start consume. Include six topics
#basic2.sim: vgroups=1, multi topics for one consumer, firstly insert data, then start consume. Include six topics #basic2.sim: vgroups=1, multi topics for one consumer, firstly insert data, then start consume. Include six topics
#basic3.sim: vgroups=4, one topic for one consumer, firstly insert data, then start consume. Include six topics #basic3.sim: vgroups=4, one topic for one consumer, firstly insert data, then start consume. Include six topics
#basic4.sim: vgroups=4, multi topics for one consumer, firstly insert data, then start consume. Include six topics #basic4.sim: vgroups=4, multi topics for one consumer, firstly insert data, then start consume. Include six topics
# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN # notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN
# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; # The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5;
# #
# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). # notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval).
# #
run tsim/tmq/prepareBasicEnv-4vgrp.sim run tsim/tmq/prepareBasicEnv-4vgrp.sim
#---- global parameters start ----# #---- global parameters start ----#
$dbName = db $dbName = db
$vgroups = 4 $vgroups = 4
$stbPrefix = stb $stbPrefix = stb
$ctbPrefix = ctb $ctbPrefix = ctb
$ntbPrefix = ntb $ntbPrefix = ntb
$stbNum = 1 $stbNum = 1
$ctbNum = 10 $ctbNum = 10
$ntbNum = 10 $ntbNum = 10
$rowsPerCtb = 10 $rowsPerCtb = 10
$tstart = 1640966400000 # 2022-01-01 00:00:00.000 $tstart = 1640966400000 # 2022-01-01 00:00:00.000
#---- global parameters end ----# #---- global parameters end ----#
$pullDelay = 3 $pullDelay = 3
$ifcheckdata = 1 $ifcheckdata = 1
$showMsg = 1 $ifmanualcommit = 1
$showRow = 0 $showMsg = 1
$showRow = 0
sql connect
sql use $dbName sql connect
sql use $dbName
print == create topics from super table
sql create topic topic_stb_column as select ts, c3 from stb print == create topics from super table
sql create topic topic_stb_all as select ts, c1, c2, c3 from stb sql create topic topic_stb_column as select ts, c3 from stb
sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb sql create topic topic_stb_all as select ts, c1, c2, c3 from stb
sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb
print == create topics from child table
sql create topic topic_ctb_column as select ts, c3 from ctb0 print == create topics from child table
sql create topic topic_ctb_all as select * from ctb0 sql create topic topic_ctb_column as select ts, c3 from ctb0
sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 sql create topic topic_ctb_all as select * from ctb0
sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0
print == create topics from normal table
sql create topic topic_ntb_column as select ts, c3 from ntb0 print == create topics from normal table
sql create topic topic_ntb_all as select * from ntb0 sql create topic topic_ntb_column as select ts, c3 from ntb0
sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 sql create topic topic_ntb_all as select * from ntb0
sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0
#sql show topics
#if $rows != 9 then #sql show topics
# return -1 #if $rows != 9 then
#endi # return -1
#endi
$keyList = ' . group.id:cgrp1
$keyList = $keyList . ' #'group.id:cgrp1,enable.auto.commit:false,auto.commit.interval.ms:6000,auto.offset.reset:earliest'
$keyList = ' . group.id:cgrp1
$topicNum = 3 $keyList = $keyList . ,
$keyList = $keyList . enable.auto.commit:false
print ================ test consume from stb #$keyList = $keyList . ,
print == multi toipcs: topic_stb_column + topic_stb_all + topic_stb_function #$keyList = $keyList . auto.commit.interval.ms:6000
$topicList = ' . topic_stb_column #$keyList = $keyList . ,
$topicList = $topicList . , #$keyList = $keyList . auto.offset.reset:earliest
$topicList = $topicList . topic_stb_all $keyList = $keyList . '
$topicList = $topicList . , print ========== key list: $keyList
$topicList = $topicList . topic_stb_function
$topicList = $topicList . '
$topicNum = 3
$consumerId = 0
$totalMsgOfStb = $ctbNum * $rowsPerCtb print ================ test consume from stb
$totalMsgOfStb = $totalMsgOfStb * $topicNum print == multi toipcs: topic_stb_column + topic_stb_all + topic_stb_function
$expectmsgcnt = $totalMsgOfStb $topicList = ' . topic_stb_column
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) $topicList = $topicList . ,
$topicList = $topicList . topic_stb_all
print == start consumer to pull msgs from stb $topicList = $topicList . ,
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start $topicList = $topicList . topic_stb_function
system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start $topicList = $topicList . '
print == check consume result $consumerId = 0
wait_consumer_end_from_stb: $totalMsgOfStb = $ctbNum * $rowsPerCtb
sql select * from consumeresult $totalMsgOfStb = $totalMsgOfStb * $topicNum
print ==> rows: $rows $expectmsgcnt = $totalMsgOfStb
print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
if $rows != 1 then
sleep 1000 print == start consumer to pull msgs from stb
goto wait_consumer_end_from_stb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start
endi system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start
if $data[0][1] != $consumerId then
return -1 print == check consume result
endi wait_consumer_end_from_stb:
if $data[0][2] != $expectmsgcnt then sql select * from consumeresult
return -1 print ==> rows: $rows
endi print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
if $data[0][3] != $expectmsgcnt then if $rows != 1 then
return -1 sleep 1000
endi goto wait_consumer_end_from_stb
endi
####################################################################################### if $data[0][1] != $consumerId then
# clear consume info and consume result return -1
#run tsim/tmq/clearConsume.sim endi
# because drop table function no stable, so by create new db for consume info and result. Modify it later if $data[0][2] != $expectmsgcnt then
$cdbName = cdb1 return -1
sql create database $cdbName vgroups 1 endi
sleep 500 if $data[0][3] != $expectmsgcnt then
sql use $cdbName return -1
endi
print == create consume info table and consume result table
sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) #######################################################################################
sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) # clear consume info and consume result
#run tsim/tmq/clearConsume.sim
sql show tables # because drop table function no stable, so by create new db for consume info and result. Modify it later
if $rows != 2 then $cdbName = cdb1
return -1 sql create database $cdbName vgroups 1
endi sleep 500
####################################################################################### sql use $cdbName
print == create consume info table and consume result table
print ================ test consume from ctb sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
print == multi toipcs: topic_ctb_column + topic_ctb_all + topic_ctb_function sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
$topicList = ' . topic_ctb_column
$topicList = $topicList . , sql show tables
$topicList = $topicList . topic_ctb_all if $rows != 2 then
$topicList = $topicList . , return -1
$topicList = $topicList . topic_ctb_function endi
$topicList = $topicList . ' #######################################################################################
$consumerId = 0
$totalMsgOfCtb = $rowsPerCtb * $topicNum print ================ test consume from ctb
$expectmsgcnt = $totalMsgOfCtb print == multi toipcs: topic_ctb_column + topic_ctb_all + topic_ctb_function
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) $topicList = ' . topic_ctb_column
$topicList = $topicList . ,
print == start consumer to pull msgs from ctb $topicList = $topicList . topic_ctb_all
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start $topicList = $topicList . ,
system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start $topicList = $topicList . topic_ctb_function
$topicList = $topicList . '
print == check consume result
wait_consumer_end_from_ctb: $consumerId = 0
sql select * from consumeresult $totalMsgOfCtb = $rowsPerCtb * $topicNum
print ==> rows: $rows $expectmsgcnt = $totalMsgOfCtb
print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
if $rows != 1 then
sleep 1000 print == start consumer to pull msgs from ctb
goto wait_consumer_end_from_ctb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
endi system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
if $data[0][1] != $consumerId then
return -1 print == check consume result
endi wait_consumer_end_from_ctb:
if $data[0][2] != $totalMsgOfCtb then sql select * from consumeresult
return -1 print ==> rows: $rows
endi print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
if $data[0][3] != $totalMsgOfCtb then if $rows != 1 then
return -1 sleep 1000
endi goto wait_consumer_end_from_ctb
endi
####################################################################################### if $data[0][1] != $consumerId then
# clear consume info and consume result return -1
#run tsim/tmq/clearConsume.sim endi
# because drop table function no stable, so by create new db for consume info and result. Modify it later if $data[0][2] != $totalMsgOfCtb then
$cdbName = cdb2 return -1
sql create database $cdbName vgroups 1 endi
sleep 500 if $data[0][3] != $totalMsgOfCtb then
sql use $cdbName return -1
endi
print == create consume info table and consume result table
sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) #######################################################################################
sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) # clear consume info and consume result
#run tsim/tmq/clearConsume.sim
sql show tables # because drop table function no stable, so by create new db for consume info and result. Modify it later
if $rows != 2 then $cdbName = cdb2
return -1 sql create database $cdbName vgroups 1
endi sleep 500
####################################################################################### sql use $cdbName
print == create consume info table and consume result table
print ================ test consume from ntb sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
print == multi toipcs: topic_ntb_column + topic_ntb_all + topic_ntb_function sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
$topicList = ' . topic_ntb_column
$topicList = $topicList . , sql show tables
$topicList = $topicList . topic_ntb_all if $rows != 2 then
$topicList = $topicList . , return -1
$topicList = $topicList . topic_ntb_function endi
$topicList = $topicList . ' #######################################################################################
$consumerId = 0
$totalMsgOfNtb = $rowsPerCtb * $topicNum print ================ test consume from ntb
$expectmsgcnt = $totalMsgOfNtb print == multi toipcs: topic_ntb_column + topic_ntb_all + topic_ntb_function
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) $topicList = ' . topic_ntb_column
$topicList = $topicList . ,
print == start consumer to pull msgs from ntb $topicList = $topicList . topic_ntb_all
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start $topicList = $topicList . ,
system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start $topicList = $topicList . topic_ntb_function
$topicList = $topicList . '
print == check consume result from ntb
wait_consumer_end_from_ntb: $consumerId = 0
sql select * from consumeresult $totalMsgOfNtb = $rowsPerCtb * $topicNum
print ==> rows: $rows $expectmsgcnt = $totalMsgOfNtb
print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
if $rows != 1 then
sleep 1000 print == start consumer to pull msgs from ntb
goto wait_consumer_end_from_ntb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
endi system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
if $data[0][1] != $consumerId then
return -1 print == check consume result from ntb
endi wait_consumer_end_from_ntb:
if $data[0][2] != $totalMsgOfNtb then sql select * from consumeresult
return -1 print ==> rows: $rows
endi print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
if $data[0][3] != $totalMsgOfNtb then if $rows != 1 then
return -1 sleep 1000
endi goto wait_consumer_end_from_ntb
endi
#------ not need stop consumer, because it exit after pull msg overthan expect msg if $data[0][1] != $consumerId then
#system tsim/tmq/consume.sh -s stop -x SIGINT return -1
endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT if $data[0][2] != $totalMsgOfNtb then
return -1
endi
if $data[0][3] != $totalMsgOfNtb then
return -1
endi
#------ not need stop consumer, because it exit after pull msg overthan expect msg
#system tsim/tmq/consume.sh -s stop -x SIGINT
system sh/exec.sh -n dnode1 -s stop -x SIGINT

647
tests/script/tsim/tmq/basic4Of2Cons.sim
View File

@ -1,319 +1,328 @@
#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 #### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406
#basic1Of2Cons.sim: vgroups=1, one topic for 2 consumers, firstly insert data, then start consume. Include six topics #basic1Of2Cons.sim: vgroups=1, one topic for 2 consumers, firstly insert data, then start consume. Include six topics
#basic2Of2Cons.sim: vgroups=1, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics #basic2Of2Cons.sim: vgroups=1, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics
#basic3Of2Cons.sim: vgroups=4, one topic for 2 consumers, firstly insert data, then start consume. Include six topics #basic3Of2Cons.sim: vgroups=4, one topic for 2 consumers, firstly insert data, then start consume. Include six topics
#basic4Of2Cons.sim: vgroups=4, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics #basic4Of2Cons.sim: vgroups=4, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics
# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN # notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN
# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; # The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5;
# #
# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). # notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval).
# #
run tsim/tmq/prepareBasicEnv-4vgrp.sim run tsim/tmq/prepareBasicEnv-4vgrp.sim
#---- global parameters start ----# #---- global parameters start ----#
$dbName = db $dbName = db
$vgroups = 4 $vgroups = 4
$stbPrefix = stb $stbPrefix = stb
$ctbPrefix = ctb $ctbPrefix = ctb
$ntbPrefix = ntb $ntbPrefix = ntb
$stbNum = 1 $stbNum = 1
$ctbNum = 10 $ctbNum = 10
$ntbNum = 10 $ntbNum = 10
$rowsPerCtb = 10 $rowsPerCtb = 10
$tstart = 1640966400000 # 2022-01-01 00:00:00.000 $tstart = 1640966400000 # 2022-01-01 00:00:00.000
#---- global parameters end ----# #---- global parameters end ----#
$pullDelay = 5 $pullDelay = 5
$ifcheckdata = 1 $ifcheckdata = 1
$showMsg = 1 $ifmanualcommit = 1
$showRow = 0 $showMsg = 1
$showRow = 0
sql connect
sql use $dbName sql connect
sql use $dbName
print == create topics from super table
sql create topic topic_stb_column as select ts, c3 from stb print == create topics from super table
sql create topic topic_stb_all as select ts, c1, c2, c3 from stb sql create topic topic_stb_column as select ts, c3 from stb
sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb sql create topic topic_stb_all as select ts, c1, c2, c3 from stb
sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb
print == create topics from child table
sql create topic topic_ctb_column as select ts, c3 from ctb0 print == create topics from child table
sql create topic topic_ctb_all as select * from ctb0 sql create topic topic_ctb_column as select ts, c3 from ctb0
sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 sql create topic topic_ctb_all as select * from ctb0
sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0
print == create topics from normal table
sql create topic topic_ntb_column as select ts, c3 from ntb0 print == create topics from normal table
sql create topic topic_ntb_all as select * from ntb0 sql create topic topic_ntb_column as select ts, c3 from ntb0
sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 sql create topic topic_ntb_all as select * from ntb0
sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0
#sql show topics
#if $rows != 9 then #sql show topics
# return -1 #if $rows != 9 then
#endi # return -1
#endi
$keyList = ' . group.id:cgrp1
$keyList = $keyList . ' #'group.id:cgrp1,enable.auto.commit:false,auto.commit.interval.ms:6000,auto.offset.reset:earliest'
$keyList = ' . group.id:cgrp1
$topicNum = 3 $keyList = $keyList . ,
$keyList = $keyList . enable.auto.commit:false
print ================ test consume from stb #$keyList = $keyList . ,
print == multi toipcs: topic_stb_column + topic_stb_all + topic_stb_function #$keyList = $keyList . auto.commit.interval.ms:6000
$topicList = ' . topic_stb_column #$keyList = $keyList . ,
$topicList = $topicList . , #$keyList = $keyList . auto.offset.reset:earliest
$topicList = $topicList . topic_stb_all $keyList = $keyList . '
$topicList = $topicList . , print ========== key list: $keyList
$topicList = $topicList . topic_stb_function
$topicList = $topicList . ' $topicNum = 3
$consumerId = 0 print ================ test consume from stb
$totalMsgOfStb = $ctbNum * $rowsPerCtb print == multi toipcs: topic_stb_column + topic_stb_all + topic_stb_function
$totalMsgOfStb = $totalMsgOfStb * $topicNum $topicList = ' . topic_stb_column
$expectmsgcnt = $totalMsgOfStb $topicList = $topicList . ,
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) $topicList = $topicList . topic_stb_all
$consumerId = 1 $topicList = $topicList . ,
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) $topicList = $topicList . topic_stb_function
$topicList = $topicList . '
print == start consumer to pull msgs from stb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start $consumerId = 0
system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start $totalMsgOfStb = $ctbNum * $rowsPerCtb
$totalMsgOfStb = $totalMsgOfStb * $topicNum
print == check consume result $expectmsgcnt = $totalMsgOfStb
wait_consumer_end_from_stb: sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
sql select * from consumeresult $consumerId = 1
print ==> rows: $rows sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] print == start consumer to pull msgs from stb
if $rows != 2 then print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start
sleep 1000 system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start
goto wait_consumer_end_from_stb
endi print == check consume result
if $data[0][1] == 0 then wait_consumer_end_from_stb:
if $data[1][1] != 1 then sql select * from consumeresult
return -1 print ==> rows: $rows
endi print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
endi print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
if $data[0][1] == 1 then if $rows != 2 then
if $data[1][1] != 0 then sleep 1000
return -1 goto wait_consumer_end_from_stb
endi endi
endi if $data[0][1] == 0 then
if $data[1][1] != 1 then
if $data[0][2] <= 0 then return -1
return -1 endi
endi endi
if $data[0][2] >= $expectmsgcnt then if $data[0][1] == 1 then
return -1 if $data[1][1] != 0 then
endi return -1
endi
if $data[1][2] <= 0 then endi
return -1
endi if $data[0][2] <= 0 then
if $data[1][2] >= $expectmsgcnt then return -1
return -1 endi
endi if $data[0][2] >= $expectmsgcnt then
return -1
$sumOfConsMsg = $data[0][2] + $data[1][2] endi
if $sumOfConsMsg != $expectmsgcnt then
return -1 if $data[1][2] <= 0 then
endi return -1
endi
if $data[0][3] <= 0 then if $data[1][2] >= $expectmsgcnt then
return -1 return -1
endi endi
if $data[0][3] >= $expectmsgcnt then
return -1 $sumOfConsMsg = $data[0][2] + $data[1][2]
endi if $sumOfConsMsg != $expectmsgcnt then
return -1
if $data[1][3] <= 0 then endi
return -1
endi if $data[0][3] <= 0 then
if $data[1][3] >= $expectmsgcnt then return -1
return -1 endi
endi if $data[0][3] >= $expectmsgcnt then
return -1
$sumOfConsRow = $data[0][3] + $data[1][3] endi
if $sumOfConsRow != $expectmsgcnt then
return -1 if $data[1][3] <= 0 then
endi return -1
endi
####################################################################################### if $data[1][3] >= $expectmsgcnt then
# clear consume info and consume result return -1
#run tsim/tmq/clearConsume.sim endi
# because drop table function no stable, so by create new db for consume info and result. Modify it later
$cdbName = cdb1 $sumOfConsRow = $data[0][3] + $data[1][3]
sql create database $cdbName vgroups 1 if $sumOfConsRow != $expectmsgcnt then
sleep 500 return -1
sql use $cdbName endi
print == create consume info table and consume result table #######################################################################################
sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) # clear consume info and consume result
sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) #run tsim/tmq/clearConsume.sim
# because drop table function no stable, so by create new db for consume info and result. Modify it later
sql show tables $cdbName = cdb1
if $rows != 2 then sql create database $cdbName vgroups 1
return -1 sleep 500
endi sql use $cdbName
#######################################################################################
print == create consume info table and consume result table
sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
print ================ test consume from ctb sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
print == multi toipcs: topic_ctb_column + topic_ctb_all + topic_ctb_function
$topicList = ' . topic_ctb_column sql show tables
$topicList = $topicList . , if $rows != 2 then
$topicList = $topicList . topic_ctb_all return -1
$topicList = $topicList . , endi
$topicList = $topicList . topic_ctb_function #######################################################################################
$topicList = $topicList . '
$consumerId = 0 print ================ test consume from ctb
$totalMsgOfCtb = $rowsPerCtb * $topicNum print == multi toipcs: topic_ctb_column + topic_ctb_all + topic_ctb_function
$expectmsgcnt = $totalMsgOfCtb $topicList = ' . topic_ctb_column
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) $topicList = $topicList . ,
$consumerId = 1 $topicList = $topicList . topic_ctb_all
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) $topicList = $topicList . ,
$topicList = $topicList . topic_ctb_function
print == start consumer to pull msgs from ctb $topicList = $topicList . '
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start $consumerId = 0
$totalMsgOfCtb = $rowsPerCtb * $topicNum
print == check consume result $expectmsgcnt = $totalMsgOfCtb
wait_consumer_end_from_ctb: sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
sql select * from consumeresult $consumerId = 1
print ==> rows: $rows sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] print == start consumer to pull msgs from ctb
if $rows != 2 then print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
sleep 1000 system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
goto wait_consumer_end_from_ctb
endi print == check consume result
if $data[0][1] == 0 then wait_consumer_end_from_ctb:
if $data[1][1] != 1 then sql select * from consumeresult
return -1 print ==> rows: $rows
endi print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
endi print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
if $data[0][1] == 1 then if $rows != 2 then
if $data[1][1] != 0 then sleep 1000
return -1 goto wait_consumer_end_from_ctb
endi endi
endi if $data[0][1] == 0 then
if $data[1][1] != 1 then
# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0 return -1
# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb endi
if $data[0][2] == $totalMsgOfCtb then endi
if $data[1][2] == 0 then if $data[0][1] == 1 then
goto check_ok_0 if $data[1][1] != 0 then
endi return -1
elif $data[0][2] == 0 then endi
if $data[1][2] == $totalMsgOfCtb then endi
goto check_ok_0
endi # either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0
endi # or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb
return -1 if $data[0][2] == $totalMsgOfCtb then
check_ok_0: if $data[1][2] == 0 then
goto check_ok_0
if $data[0][3] == $totalMsgOfCtb then endi
if $data[1][3] == 0 then elif $data[0][2] == 0 then
goto check_ok_1 if $data[1][2] == $totalMsgOfCtb then
endi goto check_ok_0
elif $data[0][3] == 0 then endi
if $data[1][3] == $totalMsgOfCtb then endi
goto check_ok_1 return -1
endi check_ok_0:
endi
return -1 if $data[0][3] == $totalMsgOfCtb then
check_ok_1: if $data[1][3] == 0 then
goto check_ok_1
endi
####################################################################################### elif $data[0][3] == 0 then
# clear consume info and consume result if $data[1][3] == $totalMsgOfCtb then
#run tsim/tmq/clearConsume.sim goto check_ok_1
# because drop table function no stable, so by create new db for consume info and result. Modify it later endi
$cdbName = cdb2 endi
sql create database $cdbName vgroups 1 return -1
sleep 500 check_ok_1:
sql use $cdbName
print == create consume info table and consume result table #######################################################################################
sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) # clear consume info and consume result
sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) #run tsim/tmq/clearConsume.sim
# because drop table function no stable, so by create new db for consume info and result. Modify it later
sql show tables $cdbName = cdb2
if $rows != 2 then sql create database $cdbName vgroups 1
return -1 sleep 500
endi sql use $cdbName
#######################################################################################
print == create consume info table and consume result table
sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
print ================ test consume from ntb sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
print == multi toipcs: topic_ntb_column + topic_ntb_all + topic_ntb_function
$topicList = ' . topic_ntb_column sql show tables
$topicList = $topicList . , if $rows != 2 then
$topicList = $topicList . topic_ntb_all return -1
$topicList = $topicList . , endi
$topicList = $topicList . topic_ntb_function #######################################################################################
$topicList = $topicList . '
$consumerId = 0 print ================ test consume from ntb
$totalMsgOfNtb = $rowsPerCtb * $topicNum print == multi toipcs: topic_ntb_column + topic_ntb_all + topic_ntb_function
$expectmsgcnt = $totalMsgOfNtb $topicList = ' . topic_ntb_column
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) $topicList = $topicList . ,
$consumerId = 1 $topicList = $topicList . topic_ntb_all
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) $topicList = $topicList . ,
$topicList = $topicList . topic_ntb_function
print == start consumer to pull msgs from ntb $topicList = $topicList . '
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start $consumerId = 0
$totalMsgOfNtb = $rowsPerCtb * $topicNum
print == check consume result from ntb $expectmsgcnt = $totalMsgOfNtb
wait_consumer_end_from_ntb: sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
sql select * from consumeresult $consumerId = 1
print ==> rows: $rows sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] print == start consumer to pull msgs from ntb
if $rows != 2 then print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
sleep 1000 system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
goto wait_consumer_end_from_ntb
endi print == check consume result from ntb
if $data[0][1] == 0 then wait_consumer_end_from_ntb:
if $data[1][1] != 1 then sql select * from consumeresult
return -1 print ==> rows: $rows
endi print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
endi print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
if $data[1][1] == 0 then if $rows != 2 then
if $data[0][1] != 1 then sleep 1000
return -1 goto wait_consumer_end_from_ntb
endi endi
endi if $data[0][1] == 0 then
if $data[1][1] != 1 then
# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0 return -1
# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb endi
if $data[0][2] == $totalMsgOfNtb then endi
if $data[1][2] == 0 then if $data[1][1] == 0 then
goto check_ok_2 if $data[0][1] != 1 then
endi return -1
elif $data[0][2] == 0 then endi
if $data[1][2] == $totalMsgOfNtb then endi
goto check_ok_2
endi # either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0
endi # or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb
return -1 if $data[0][2] == $totalMsgOfNtb then
check_ok_2: if $data[1][2] == 0 then
goto check_ok_2
if $data[0][3] == $totalMsgOfNtb then endi
if $data[1][3] == 0 then elif $data[0][2] == 0 then
goto check_ok_3 if $data[1][2] == $totalMsgOfNtb then
endi goto check_ok_2
elif $data[0][3] == 0 then endi
if $data[1][3] == $totalMsgOfNtb then endi
goto check_ok_3 return -1
endi check_ok_2:
endi
return -1 if $data[0][3] == $totalMsgOfNtb then
check_ok_3: if $data[1][3] == 0 then
goto check_ok_3
#------ not need stop consumer, because it exit after pull msg overthan expect msg endi
#system tsim/tmq/consume.sh -s stop -x SIGINT elif $data[0][3] == 0 then
if $data[1][3] == $totalMsgOfNtb then
system sh/exec.sh -n dnode1 -s stop -x SIGINT goto check_ok_3
endi
endi
return -1
check_ok_3:
#------ not need stop consumer, because it exit after pull msg overthan expect msg
#system tsim/tmq/consume.sh -s stop -x SIGINT
system sh/exec.sh -n dnode1 -s stop -x SIGINT

176
tests/script/tsim/tmq/prepareBasicEnv-1vgrp.sim
View File

@ -1,88 +1,88 @@
# stop all dnodes before start this case # stop all dnodes before start this case
system sh/stop_dnodes.sh system sh/stop_dnodes.sh
# deploy dnode 1 # deploy dnode 1
system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode1 -i 1
# add some config items for this case # add some config items for this case
#system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 #system sh/cfg.sh -n dnode1 -c supportVnodes -v 0
# start dnode 1 # start dnode 1
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
sql connect sql connect
#---- global parameters start ----# #---- global parameters start ----#
$dbName = db $dbName = db
$vgroups = 1 $vgroups = 1
$stbPrefix = stb $stbPrefix = stb
$ctbPrefix = ctb $ctbPrefix = ctb
$ntbPrefix = ntb $ntbPrefix = ntb
$stbNum = 1 $stbNum = 1
$ctbNum = 10 $ctbNum = 10
$ntbNum = 10 $ntbNum = 10
$rowsPerCtb = 10 $rowsPerCtb = 10
$tstart = 1640966400000 # 2022-01-01 00:00:00.000 $tstart = 1640966400000 # 2022-01-01 00:00:00.000
#---- global parameters end ----# #---- global parameters end ----#
print == create database $dbName vgroups $vgroups print == create database $dbName vgroups $vgroups
sql create database $dbName vgroups $vgroups sql create database $dbName vgroups $vgroups
#wait database ready #wait database ready
$loop_cnt = 0 $loop_cnt = 0
check_db_ready: check_db_ready:
if $loop_cnt == 10 then if $loop_cnt == 10 then
print ====> database not ready! print ====> database not ready!
return -1 return -1
endi endi
sql show databases sql show databases
print ==> rows: $rows print ==> rows: $rows
print ==> $data(db)[0] $data(db)[1] $data(db)[2] $data(db)[3] $data(db)[4] $data(db)[5] $data(db)[6] $data(db)[7] $data(db)[8] $data(db)[9] $data(db)[10] $data(db)[11] $data(db)[12] print ==> $data(db)[0] $data(db)[1] $data(db)[2] $data(db)[3] $data(db)[4] $data(db)[5] $data(db)[6] $data(db)[7] $data(db)[8] $data(db)[9] $data(db)[10] $data(db)[11] $data(db)[12]
print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $data(db)[18] $data(db)[19] $data(db)[20] print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $data(db)[18] $data(db)[19] $data(db)[20]
if $data(db)[19] != nostrict then if $data(db)[19] != nostrict then
sleep 100 sleep 100
$loop_cnt = $loop_cnt + 1 $loop_cnt = $loop_cnt + 1
goto check_db_ready goto check_db_ready
endi endi
sql use $dbName sql use $dbName
print == create consume info table and consume result table print == create consume info table and consume result table
sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
sql show tables sql show tables
if $rows != 2 then if $rows != 2 then
return -1 return -1
endi endi
print == create super table print == create super table
sql create table $stbPrefix (ts timestamp, c1 int, c2 float, c3 binary(16)) tags (t1 int) sql create table $stbPrefix (ts timestamp, c1 int, c2 float, c3 binary(16)) tags (t1 int)
sql show stables sql show stables
if $rows != 1 then if $rows != 1 then
return -1 return -1
endi endi
print == create child table, normal table and insert data print == create child table, normal table and insert data
$i = 0 $i = 0
while $i < $ctbNum while $i < $ctbNum
$ctb = $ctbPrefix . $i $ctb = $ctbPrefix . $i
$ntb = $ntbPrefix . $i $ntb = $ntbPrefix . $i
sql create table $ctb using $stbPrefix tags( $i ) sql create table $ctb using $stbPrefix tags( $i )
sql create table $ntb (ts timestamp, c1 int, c2 float, c3 binary(16)) sql create table $ntb (ts timestamp, c1 int, c2 float, c3 binary(16))
$x = 0 $x = 0
while $x < $rowsPerCtb while $x < $rowsPerCtb
$binary = ' . binary- $binary = ' . binary-
$binary = $binary . $i $binary = $binary . $i
$binary = $binary . ' $binary = $binary . '
sql insert into $ctb values ($tstart , $i , $x , $binary ) sql insert into $ctb values ($tstart , $i , $x , $binary )
sql insert into $ntb values ($tstart , $i , $x , $binary ) sql insert into $ntb values ($tstart , $i , $x , $binary )
$tstart = $tstart + 1 $tstart = $tstart + 1
$x = $x + 1 $x = $x + 1
endw endw
$i = $i + 1 $i = $i + 1
$tstart = 1640966400000 $tstart = 1640966400000
endw endw

176
tests/script/tsim/tmq/prepareBasicEnv-4vgrp.sim
View File

@ -1,88 +1,88 @@
# stop all dnodes before start this case # stop all dnodes before start this case
system sh/stop_dnodes.sh system sh/stop_dnodes.sh
# deploy dnode 1 # deploy dnode 1
system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode1 -i 1
# add some config items for this case # add some config items for this case
#system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 #system sh/cfg.sh -n dnode1 -c supportVnodes -v 0
# start dnode 1 # start dnode 1
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
sql connect sql connect
#---- global parameters start ----# #---- global parameters start ----#
$dbName = db $dbName = db
$vgroups = 4 $vgroups = 4
$stbPrefix = stb $stbPrefix = stb
$ctbPrefix = ctb $ctbPrefix = ctb
$ntbPrefix = ntb $ntbPrefix = ntb
$stbNum = 1 $stbNum = 1
$ctbNum = 10 $ctbNum = 10
$ntbNum = 10 $ntbNum = 10
$rowsPerCtb = 10 $rowsPerCtb = 10
$tstart = 1640966400000 # 2022-01-01 00:00:00.000 $tstart = 1640966400000 # 2022-01-01 00:00:00.000
#---- global parameters end ----# #---- global parameters end ----#
print == create database $dbName vgroups $vgroups print == create database $dbName vgroups $vgroups
sql create database $dbName vgroups $vgroups sql create database $dbName vgroups $vgroups
#wait database ready #wait database ready
$loop_cnt = 0 $loop_cnt = 0
check_db_ready: check_db_ready:
if $loop_cnt == 10 then if $loop_cnt == 10 then
print ====> database not ready! print ====> database not ready!
return -1 return -1
endi endi
sql show databases sql show databases
print ==> rows: $rows print ==> rows: $rows
print ==> $data(db)[0] $data(db)[1] $data(db)[2] $data(db)[3] $data(db)[4] $data(db)[5] $data(db)[6] $data(db)[7] $data(db)[8] $data(db)[9] $data(db)[10] $data(db)[11] $data(db)[12] print ==> $data(db)[0] $data(db)[1] $data(db)[2] $data(db)[3] $data(db)[4] $data(db)[5] $data(db)[6] $data(db)[7] $data(db)[8] $data(db)[9] $data(db)[10] $data(db)[11] $data(db)[12]
print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $data(db)[18] $data(db)[19] $data(db)[20] print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $data(db)[18] $data(db)[19] $data(db)[20]
if $data(db)[19] != nostrict then if $data(db)[19] != nostrict then
sleep 100 sleep 100
$loop_cnt = $loop_cnt + 1 $loop_cnt = $loop_cnt + 1
goto check_db_ready goto check_db_ready
endi endi
sql use $dbName sql use $dbName
print == create consume info table and consume result table print == create consume info table and consume result table
sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
sql show tables sql show tables
if $rows != 2 then if $rows != 2 then
return -1 return -1
endi endi
print == create super table print == create super table
sql create table $stbPrefix (ts timestamp, c1 int, c2 float, c3 binary(16)) tags (t1 int) sql create table $stbPrefix (ts timestamp, c1 int, c2 float, c3 binary(16)) tags (t1 int)
sql show stables sql show stables
if $rows != 1 then if $rows != 1 then
return -1 return -1
endi endi
print == create child table, normal table and insert data print == create child table, normal table and insert data
$i = 0 $i = 0
while $i < $ctbNum while $i < $ctbNum
$ctb = $ctbPrefix . $i $ctb = $ctbPrefix . $i
$ntb = $ntbPrefix . $i $ntb = $ntbPrefix . $i
sql create table $ctb using $stbPrefix tags( $i ) sql create table $ctb using $stbPrefix tags( $i )
sql create table $ntb (ts timestamp, c1 int, c2 float, c3 binary(16)) sql create table $ntb (ts timestamp, c1 int, c2 float, c3 binary(16))
$x = 0 $x = 0
while $x < $rowsPerCtb while $x < $rowsPerCtb
$binary = ' . binary- $binary = ' . binary-
$binary = $binary . $i $binary = $binary . $i
$binary = $binary . ' $binary = $binary . '
sql insert into $ctb values ($tstart , $i , $x , $binary ) sql insert into $ctb values ($tstart , $i , $x , $binary )
sql insert into $ntb values ($tstart , $i , $x , $binary ) sql insert into $ntb values ($tstart , $i , $x , $binary )
$tstart = $tstart + 1 $tstart = $tstart + 1
$x = $x + 1 $x = $x + 1
endw endw
$i = $i + 1 $i = $i + 1
$tstart = 1640966400000 $tstart = 1640966400000
endw endw

52
tests/system-test/7-tmq/basic5.py
View File

@ -52,7 +52,7 @@ class TDTestCase:
def create_tables(self,tsql, dbName,vgroups,stbName,ctbNum,rowsPerTbl): def create_tables(self,tsql, dbName,vgroups,stbName,ctbNum,rowsPerTbl):
tsql.execute("create database if not exists %s vgroups %d"%(dbName, vgroups)) tsql.execute("create database if not exists %s vgroups %d"%(dbName, vgroups))
tsql.execute("use %s" %dbName) tsql.execute("use %s" %dbName)
tsql.execute("create table %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName) tsql.execute("create table if not exists %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName)
pre_create = "create table" pre_create = "create table"
sql = pre_create sql = pre_create
#tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname)) #tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname))
@ -345,11 +345,11 @@ class TDTestCase:
after starting consumer, create ctables ") after starting consumer, create ctables ")
# create and start thread # create and start thread
parameterDict = {'cfg': '', \ parameterDict = {'cfg': '', \
'dbName': 'db2', \ 'dbName': 'db3', \
'vgroups': 1, \ 'vgroups': 1, \
'stbName': 'stb', \ 'stbName': 'stb', \
'ctbNum': 10, \ 'ctbNum': 10, \
'rowsPerTbl': 10000, \ 'rowsPerTbl': 30000, \
'batchNum': 100, \ 'batchNum': 100, \
'startTs': 1640966400000} # 2022-01-01 00:00:00.000 'startTs': 1640966400000} # 2022-01-01 00:00:00.000
parameterDict['cfg'] = cfgPath parameterDict['cfg'] = cfgPath
@ -374,22 +374,33 @@ class TDTestCase:
break break
else: else:
time.sleep(1) time.sleep(1)
tdLog.info("create stable2 for the seconde topic")
parameterDict2 = {'cfg': '', \
'dbName': 'db3', \
'vgroups': 1, \
'stbName': 'stb2', \
'ctbNum': 10, \
'rowsPerTbl': 30000, \
'batchNum': 100, \
'startTs': 1640966400000} # 2022-01-01 00:00:00.000
parameterDict2['cfg'] = cfgPath
tdSql.execute("create stable if not exists %s.%s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%(parameterDict2['dbName'], parameterDict2['stbName']))
tdLog.info("create topics from super table") tdLog.info("create topics from super table")
topicFromStb = 'topic_stb_column2' topicFromStb = 'topic_stb_column3'
topicFromCtb = 'topic_ctb_column2' topicFromStb2 = 'topic_stb_column32'
tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb, parameterDict['dbName'], parameterDict['stbName'])) tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb, parameterDict['dbName'], parameterDict['stbName']))
tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s_0" %(topicFromCtb, parameterDict['dbName'], parameterDict['stbName'])) tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb2, parameterDict2['dbName'], parameterDict2['stbName']))
time.sleep(1)
tdSql.query("show topics") tdSql.query("show topics")
topic1 = tdSql.getData(0 , 0) topic1 = tdSql.getData(0 , 0)
topic2 = tdSql.getData(1 , 0) topic2 = tdSql.getData(1 , 0)
tdLog.info("show topics: %s, %s"%(topic1, topic2)) tdLog.info("show topics: %s, %s"%(topic1, topic2))
if topic1 != topicFromStb and topic1 != topicFromCtb: if topic1 != topicFromStb and topic1 != topicFromStb2:
tdLog.exit("topic error1") tdLog.exit("topic error1")
if topic2 != topicFromStb and topic2 != topicFromCtb: if topic2 != topicFromStb and topic2 != topicFromStb2:
tdLog.exit("topic error2") tdLog.exit("topic error2")
tdLog.info("create consume info table and consume result table") tdLog.info("create consume info table and consume result table")
@ -397,10 +408,9 @@ class TDTestCase:
tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)"%cdbName) tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)"%cdbName)
tdSql.query("create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"%cdbName) tdSql.query("create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"%cdbName)
rowsOfNewCtb = 1000
consumerId = 0 consumerId = 0
expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + rowsOfNewCtb expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + parameterDict2["rowsPerTbl"] * parameterDict2["ctbNum"]
topicList = topicFromStb topicList = topicFromStb + ',' + topicFromStb2
ifcheckdata = 0 ifcheckdata = 0
keyList = 'group.id:cgrp1,\ keyList = 'group.id:cgrp1,\
enable.auto.commit:false,\ enable.auto.commit:false,\
@ -432,17 +442,13 @@ class TDTestCase:
tdLog.info(shellCmd) tdLog.info(shellCmd)
os.system(shellCmd) os.system(shellCmd)
# create new child table and insert data # start the second thread to create new child table and insert data
newCtbName = 'newctb' prepareEnvThread2 = threading.Thread(target=self.prepareEnv, kwargs=parameterDict2)
tdSql.query("create table %s.%s using %s.%s tags(9999)"%(parameterDict["dbName"], newCtbName, parameterDict["dbName"], parameterDict["stbName"])) prepareEnvThread2.start()
startTs = parameterDict["startTs"]
for j in range(rowsOfNewCtb):
sql = "insert into %s.%s values (%d, %d, 'tmqrow_%d') "%(parameterDict["dbName"], newCtbName, startTs + j, j, j)
tdSql.execute(sql)
tdLog.debug("insert data into new child table ............ [OK]")
# wait for data ready # wait for data ready
prepareEnvThread.join() prepareEnvThread.join()
prepareEnvThread2.join()
tdLog.info("insert process end, and start to check consume result") tdLog.info("insert process end, and start to check consume result")
while 1: while 1:
@ -457,7 +463,7 @@ class TDTestCase:
tdSql.checkData(0 , 3, expectrowcnt) tdSql.checkData(0 , 3, expectrowcnt)
tdSql.query("drop topic %s"%topicFromStb) tdSql.query("drop topic %s"%topicFromStb)
tdSql.query("drop topic %s"%topicFromCtb) tdSql.query("drop topic %s"%topicFromStb2)
tdLog.printNoPrefix("======== test case 3 end ...... ") tdLog.printNoPrefix("======== test case 3 end ...... ")
@ -474,7 +480,7 @@ class TDTestCase:
self.tmqCase1(cfgPath, buildPath) self.tmqCase1(cfgPath, buildPath)
self.tmqCase2(cfgPath, buildPath) self.tmqCase2(cfgPath, buildPath)
#self.tmqCase3(cfgPath, buildPath) self.tmqCase3(cfgPath, buildPath)
def stop(self): def stop(self):
tdSql.close() tdSql.close()

52
tests/test/c/tmqSim.c
View File

@ -37,9 +37,10 @@ typedef struct {
TdThread thread; TdThread thread;
int32_t consumerId; int32_t consumerId;
int32_t autoCommitIntervalMs; // 1000 ms int32_t ifManualCommit;
char autoCommit[8]; // true, false //int32_t autoCommitIntervalMs; // 1000 ms
char autoOffsetRest[16]; // none, earliest, latest //char autoCommit[8]; // true, false
//char autoOffsetRest[16]; // none, earliest, latest
int32_t ifCheckData; int32_t ifCheckData;
int64_t expectMsgCnt; int64_t expectMsgCnt;
@ -136,9 +137,9 @@ void saveConfigToLogFile() {
for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) {
taosFprintfFile(g_fp, "# consumer %d info:\n", g_stConfInfo.stThreads[i].consumerId); taosFprintfFile(g_fp, "# consumer %d info:\n", g_stConfInfo.stThreads[i].consumerId);
taosFprintfFile(g_fp, " auto commit: %s\n", g_stConfInfo.stThreads[i].autoCommit); //taosFprintfFile(g_fp, " auto commit: %s\n", g_stConfInfo.stThreads[i].autoCommit);
taosFprintfFile(g_fp, " auto commit interval ms: %d\n", g_stConfInfo.stThreads[i].autoCommitIntervalMs); //taosFprintfFile(g_fp, " auto commit interval ms: %d\n", g_stConfInfo.stThreads[i].autoCommitIntervalMs);
taosFprintfFile(g_fp, " auto offset rest: %s\n", g_stConfInfo.stThreads[i].autoOffsetRest); //taosFprintfFile(g_fp, " auto offset rest: %s\n", g_stConfInfo.stThreads[i].autoOffsetRest);
taosFprintfFile(g_fp, " Topics: "); taosFprintfFile(g_fp, " Topics: ");
for (int j = 0; j < g_stConfInfo.stThreads[i].numOfTopic; j++) { for (int j = 0; j < g_stConfInfo.stThreads[i].numOfTopic; j++) {
taosFprintfFile(g_fp, "%s, ", g_stConfInfo.stThreads[i].topics[j]); taosFprintfFile(g_fp, "%s, ", g_stConfInfo.stThreads[i].topics[j]);
@ -232,13 +233,18 @@ static int32_t msg_process(TAOS_RES* msg, int64_t msgIndex, int32_t threadLable)
while (1) { while (1) {
TAOS_ROW row = taos_fetch_row(msg); TAOS_ROW row = taos_fetch_row(msg);
if (row == NULL) break;
if (0 != g_stConfInfo.showRowFlag) { if (row == NULL) break;
TAOS_FIELD* fields = taos_fetch_fields(msg);
int32_t numOfFields = taos_field_count(msg); TAOS_FIELD* fields = taos_fetch_fields(msg);
taos_print_row(buf, row, fields, numOfFields); int32_t numOfFields = taos_field_count(msg);
taos_print_row(buf, row, fields, numOfFields);
if (0 != g_stConfInfo.showRowFlag) {
taosFprintfFile(g_fp, "rows[%d]: %s\n", totalRows, buf); taosFprintfFile(g_fp, "rows[%d]: %s\n", totalRows, buf);
} }
totalRows++; totalRows++;
} }
@ -316,6 +322,8 @@ int32_t saveConsumeResult(SThreadInfo* pInfo) {
sprintf(sqlStr, "insert into %s.consumeresult values (now, %d, %" PRId64 ", %" PRId64 ", %d)", g_stConfInfo.cdbName, sprintf(sqlStr, "insert into %s.consumeresult values (now, %d, %" PRId64 ", %" PRId64 ", %d)", g_stConfInfo.cdbName,
pInfo->consumerId, pInfo->consumeMsgCnt, pInfo->consumeRowCnt, pInfo->checkresult); pInfo->consumerId, pInfo->consumeMsgCnt, pInfo->consumeRowCnt, pInfo->checkresult);
taosFprintfFile(g_fp, "== save result sql: %s \n", sqlStr);
TAOS_RES* pRes = taos_query(pConn, sqlStr); TAOS_RES* pRes = taos_query(pConn, sqlStr);
if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
pError("error in save consumeinfo, reason:%s\n", taos_errstr(pRes)); pError("error in save consumeinfo, reason:%s\n", taos_errstr(pRes));
@ -384,8 +392,12 @@ void* consumeThreadFunc(void* param) {
loop_consume(pInfo); loop_consume(pInfo);
tmq_commit(pInfo->tmq, NULL, 0); if (pInfo->ifManualCommit) {
taosFprintfFile(g_fp, "tmq_commit() manual commit when consume end.\n");
pPrint("tmq_commit() manual commit when consume end.\n");
tmq_commit(pInfo->tmq, NULL, 0);
}
err = tmq_unsubscribe(pInfo->tmq); err = tmq_unsubscribe(pInfo->tmq);
if (err) { if (err) {
pError("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err)); pError("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err));
@ -470,9 +482,9 @@ int32_t getConsumeInfo() {
int32_t* lengths = taos_fetch_lengths(pRes); int32_t* lengths = taos_fetch_lengths(pRes);
// set default value // set default value
g_stConfInfo.stThreads[numOfThread].autoCommitIntervalMs = 5000; //g_stConfInfo.stThreads[numOfThread].autoCommitIntervalMs = 5000;
memcpy(g_stConfInfo.stThreads[numOfThread].autoCommit, "true", strlen("true")); //memcpy(g_stConfInfo.stThreads[numOfThread].autoCommit, "true", strlen("true"));
memcpy(g_stConfInfo.stThreads[numOfThread].autoOffsetRest, "earlieast", strlen("earlieast")); //memcpy(g_stConfInfo.stThreads[numOfThread].autoOffsetRest, "earlieast", strlen("earlieast"));
for (int i = 0; i < num_fields; ++i) { for (int i = 0; i < num_fields; ++i) {
if (row[i] == NULL || 0 == i) { if (row[i] == NULL || 0 == i) {
@ -489,12 +501,8 @@ int32_t getConsumeInfo() {
g_stConfInfo.stThreads[numOfThread].expectMsgCnt = *((int64_t*)row[i]); g_stConfInfo.stThreads[numOfThread].expectMsgCnt = *((int64_t*)row[i]);
} else if ((5 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) { } else if ((5 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) {
g_stConfInfo.stThreads[numOfThread].ifCheckData = *((int32_t*)row[i]); g_stConfInfo.stThreads[numOfThread].ifCheckData = *((int32_t*)row[i]);
} else if ((6 == i) && (fields[i].type == TSDB_DATA_TYPE_BINARY)) { } else if ((6 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) {
memcpy(g_stConfInfo.stThreads[numOfThread].autoCommit, row[i], lengths[i]); g_stConfInfo.stThreads[numOfThread].ifManualCommit = *((int32_t*)row[i]);
} else if ((7 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) {
g_stConfInfo.stThreads[numOfThread].autoCommitIntervalMs = *((int32_t*)row[i]);
} else if ((8 == i) && (fields[i].type == TSDB_DATA_TYPE_BINARY)) {
memcpy(g_stConfInfo.stThreads[numOfThread].autoOffsetRest, row[i], lengths[i]);
} }
} }
numOfThread++; numOfThread++;