From b01865f6c45bdb0e8fa80408ae59fbe9f2da6f96 Mon Sep 17 00:00:00 2001 From: bryanchang0603 Date: Fri, 11 Jun 2021 22:17:38 +0800 Subject: [PATCH 1/9] [TD-4366] adding more ntp. try to make sure ntp is turned on --- tests/pytest/client/change_time_1_1.py | 4 +++- tests/pytest/client/change_time_1_2.py | 4 ++-- tests/pytest/fulltest.sh | 4 ++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/pytest/client/change_time_1_1.py b/tests/pytest/client/change_time_1_1.py index 24e0195a9c..abae49a41f 100644 --- a/tests/pytest/client/change_time_1_1.py +++ b/tests/pytest/client/change_time_1_1.py @@ -51,7 +51,8 @@ class TDTestCase: #move 5 days ahead to 2020/10/25. 4 oldest files should be removed during the new write #leaving 7 data files. os.system ('timedatectl set-time 2020-10-25') - os.system(f"{binPath}taosdemo -f tools/taosdemoAllTest/manual_change_time_1_1_B.json") + os.system(f"{binPath}taosdemo -f tools/taosdemoAllTest/manual_change_time_1_1_B.json") + os.system('sudo timedatectl set-ntp on') commandArray = ['ls', '-l', f'{TDenginePath}/sim/dnode1/data/vnode/vnode2/tsdb/data'] result = subprocess.run(commandArray, stdout=subprocess.PIPE).stdout.decode('utf-8') print(result.count('data')) @@ -61,6 +62,7 @@ class TDTestCase: tdLog.debug("data file number correct") tdSql.query('select first(ts) from stb_0') tdSql.checkData(0,0,datetime(2020,10,14,8,0,0,0)) #check the last data in the database + os.system('sudo timedatectl set-ntp on') def stop(self): os.system('sudo timedatectl set-ntp on') diff --git a/tests/pytest/client/change_time_1_2.py b/tests/pytest/client/change_time_1_2.py index cd1a17926d..855b04873c 100644 --- a/tests/pytest/client/change_time_1_2.py +++ b/tests/pytest/client/change_time_1_2.py @@ -62,6 +62,7 @@ class TDTestCase: tdDnodes.start(1) tdSql.query('select first(ts) from stb_0') tdSql.checkData(0,0,datetime(2020,10,14,8,0,0,0)) #check the last data in the database + os.system('sudo timedatectl set-ntp on') commandArray = ['ls', '-l', f'{TDenginePath}/sim/dnode1/data/vnode/vnode2/tsdb/data'] result = subprocess.run(commandArray, stdout=subprocess.PIPE).stdout.decode('utf-8') print(result.count('data')) @@ -69,8 +70,7 @@ class TDTestCase: tdLog.exit('wrong number of files') else: tdLog.debug("data file number correct") - - + os.system('sudo timedatectl set-ntp on') def stop(self): os.system('sudo timedatectl set-ntp on') diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index fe023acc4b..bab56fa657 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -256,8 +256,6 @@ python3 ./test.py -f client/client.py python3 ./test.py -f client/version.py python3 ./test.py -f client/alterDatabase.py python3 ./test.py -f client/noConnectionErrorTest.py -python3 test.py -f client/change_time_1_1.py -python3 test.py -f client/change_time_1_2.py # Misc python3 testCompress.py @@ -344,4 +342,6 @@ python3 test.py -f alter/alter_keep.py python3 test.py -f alter/alter_cacheLastRow.py python3 ./test.py -f query/querySession.py python3 test.py -f alter/alter_create_exception.py +python3 test.py -f client/change_time_1_1.py +python3 test.py -f client/change_time_1_2.py #======================p4-end=============== From 881a5506231700db5b9aeb1928dfd7a159ef905d Mon Sep 17 00:00:00 2001 From: bryanchang0603 Date: Tue, 15 Jun 2021 09:24:15 +0800 Subject: [PATCH 2/9] [TD-4366 adding sleep to prevent time problem] --- tests/pytest/alter/alter_keep.py | 20 ++++++++++---------- tests/pytest/client/change_time_1_1.py | 2 ++ tests/pytest/client/change_time_1_2.py | 1 + 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/tests/pytest/alter/alter_keep.py b/tests/pytest/alter/alter_keep.py index 9f66b08823..72ca635ac3 100644 --- a/tests/pytest/alter/alter_keep.py +++ b/tests/pytest/alter/alter_keep.py @@ -15,6 +15,7 @@ import sys from util.log import * from util.cases import * from util.sql import * +import time class TDTestCase: @@ -129,9 +130,6 @@ class TDTestCase: tdSql.prepare() - ##TODO: need to wait for TD-4445 to implement the following - ## tests - ## preset the keep tdSql.prepare() @@ -176,20 +174,22 @@ class TDTestCase: tdSql.error('insert into tb values (now-15d, 10)') tdSql.query('select * from tb') tdSql.checkRows(rowNum) - + tdLog.notice('testing keep will be altered if sudden change from small to big') - tdSql.execute('alter database db keep 14,14,14') - tdSql.execute('alter database db keep 15,15,15') - tdSql.execute('insert into tb values (now-15d, 10)') - tdSql.query('select * from tb') - tdSql.checkRows(rowNum + 1) + for i in range(30): + tdSql.execute('alter database db keep 14,14,14') + tdSql.execute('alter database db keep 16,16,16') + tdSql.execute('insert into tb values (now-15d, 10)') + tdSql.query('select * from tb') + rowNum += 1 + tdSql.checkRows(rowNum ) tdLog.notice('testing keep will be altered if sudden change from big to small') tdSql.execute('alter database db keep 16,16,16') tdSql.execute('alter database db keep 14,14,14') tdSql.error('insert into tb values (now-15d, 10)') tdSql.query('select * from tb') - tdSql.checkRows(rowNum + 1) + tdSql.checkRows(rowNum) diff --git a/tests/pytest/client/change_time_1_1.py b/tests/pytest/client/change_time_1_1.py index abae49a41f..54984747b5 100644 --- a/tests/pytest/client/change_time_1_1.py +++ b/tests/pytest/client/change_time_1_1.py @@ -19,6 +19,7 @@ from util.pathFinding import * from util.dnodes import tdDnodes from datetime import datetime import subprocess +import time ##TODO: this is now automatic, but not sure if this will run through jenkins class TDTestCase: @@ -63,6 +64,7 @@ class TDTestCase: tdSql.query('select first(ts) from stb_0') tdSql.checkData(0,0,datetime(2020,10,14,8,0,0,0)) #check the last data in the database os.system('sudo timedatectl set-ntp on') + time.sleep(5) def stop(self): os.system('sudo timedatectl set-ntp on') diff --git a/tests/pytest/client/change_time_1_2.py b/tests/pytest/client/change_time_1_2.py index 855b04873c..0937080b5c 100644 --- a/tests/pytest/client/change_time_1_2.py +++ b/tests/pytest/client/change_time_1_2.py @@ -71,6 +71,7 @@ class TDTestCase: else: tdLog.debug("data file number correct") os.system('sudo timedatectl set-ntp on') + time.sleep(5) def stop(self): os.system('sudo timedatectl set-ntp on') From 1a5aac6eeb76378868fe64dd5fd9999f16e8c69e Mon Sep 17 00:00:00 2001 From: bryanchang0603 Date: Tue, 15 Jun 2021 10:04:44 +0800 Subject: [PATCH 3/9] [TD-4366 adding sleep to prevent time problem] --- tests/pytest/client/change_time_1_1.py | 22 ++++++++++++++++------ tests/pytest/client/change_time_1_2.py | 17 ++++++++++++----- tests/pytest/fulltest.sh | 4 ++-- 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/tests/pytest/client/change_time_1_1.py b/tests/pytest/client/change_time_1_1.py index 54984747b5..32917ab2e4 100644 --- a/tests/pytest/client/change_time_1_1.py +++ b/tests/pytest/client/change_time_1_1.py @@ -40,11 +40,16 @@ class TDTestCase: #run taosdemo to insert data. one row per second from 2020/10/11 to 2020/10/20 #11 data files should be generated #vnode at TDinternal/community/sim/dnode1/data/vnode - os.system(f"{binPath}taosdemo -f tools/taosdemoAllTest/manual_change_time_1_1_A.json") - commandArray = ['ls', '-l', f'{TDenginePath}/sim/dnode1/data/vnode/vnode2/tsdb/data'] - result = subprocess.run(commandArray, stdout=subprocess.PIPE).stdout.decode('utf-8') - print(result.count('data')) + try: + os.system(f"{binPath}taosdemo -f tools/taosdemoAllTest/manual_change_time_1_1_A.json") + commandArray = ['ls', '-l', f'{TDenginePath}/sim/dnode1/data/vnode/vnode2/tsdb/data'] + result = subprocess.run(commandArray, stdout=subprocess.PIPE).stdout.decode('utf-8') + except BaseException: + os.system('sudo timedatectl set-ntp on') + tdLog.sleep(10) if result.count('data') != 11: + os.system('sudo timedatectl set-ntp on') + tdLog.sleep(10) tdLog.exit('wrong number of files') else: tdLog.debug("data file number correct") @@ -52,8 +57,13 @@ class TDTestCase: #move 5 days ahead to 2020/10/25. 4 oldest files should be removed during the new write #leaving 7 data files. os.system ('timedatectl set-time 2020-10-25') - os.system(f"{binPath}taosdemo -f tools/taosdemoAllTest/manual_change_time_1_1_B.json") + try: + os.system(f"{binPath}taosdemo -f tools/taosdemoAllTest/manual_change_time_1_1_B.json") + except BaseException: + os.system('sudo timedatectl set-ntp on') + tdLog.sleep(10) os.system('sudo timedatectl set-ntp on') + tdLog.sleep(10) commandArray = ['ls', '-l', f'{TDenginePath}/sim/dnode1/data/vnode/vnode2/tsdb/data'] result = subprocess.run(commandArray, stdout=subprocess.PIPE).stdout.decode('utf-8') print(result.count('data')) @@ -64,7 +74,7 @@ class TDTestCase: tdSql.query('select first(ts) from stb_0') tdSql.checkData(0,0,datetime(2020,10,14,8,0,0,0)) #check the last data in the database os.system('sudo timedatectl set-ntp on') - time.sleep(5) + tdLog.sleep(10) def stop(self): os.system('sudo timedatectl set-ntp on') diff --git a/tests/pytest/client/change_time_1_2.py b/tests/pytest/client/change_time_1_2.py index 0937080b5c..d693010be4 100644 --- a/tests/pytest/client/change_time_1_2.py +++ b/tests/pytest/client/change_time_1_2.py @@ -39,11 +39,17 @@ class TDTestCase: #run taosdemo to insert data. one row per second from 2020/10/11 to 2020/10/20 #11 data files should be generated #vnode at TDinternal/community/sim/dnode1/data/vnode - os.system(f"{binPath}taosdemo -f tools/taosdemoAllTest/manual_change_time_1_1_A.json") - commandArray = ['ls', '-l', f'{TDenginePath}/sim/dnode1/data/vnode/vnode2/tsdb/data'] - result = subprocess.run(commandArray, stdout=subprocess.PIPE).stdout.decode('utf-8') - print(result.count('data')) + try: + os.system(f"{binPath}taosdemo -f tools/taosdemoAllTest/manual_change_time_1_1_A.json") + commandArray = ['ls', '-l', f'{TDenginePath}/sim/dnode1/data/vnode/vnode2/tsdb/data'] + result = subprocess.run(commandArray, stdout=subprocess.PIPE).stdout.decode('utf-8') + except BaseException: + os.system('sudo timedatectl set-ntp on') + tdLog.sleep(10) + if result.count('data') != 11: + os.system('sudo timedatectl set-ntp on') + tdLog.sleep(10) tdLog.exit('wrong number of files') else: tdLog.debug("data file number correct") @@ -63,6 +69,7 @@ class TDTestCase: tdSql.query('select first(ts) from stb_0') tdSql.checkData(0,0,datetime(2020,10,14,8,0,0,0)) #check the last data in the database os.system('sudo timedatectl set-ntp on') + tdLog.sleep(10) commandArray = ['ls', '-l', f'{TDenginePath}/sim/dnode1/data/vnode/vnode2/tsdb/data'] result = subprocess.run(commandArray, stdout=subprocess.PIPE).stdout.decode('utf-8') print(result.count('data')) @@ -71,7 +78,7 @@ class TDTestCase: else: tdLog.debug("data file number correct") os.system('sudo timedatectl set-ntp on') - time.sleep(5) + tdLog.sleep(10) def stop(self): os.system('sudo timedatectl set-ntp on') diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index bab56fa657..6a8497d241 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -342,6 +342,6 @@ python3 test.py -f alter/alter_keep.py python3 test.py -f alter/alter_cacheLastRow.py python3 ./test.py -f query/querySession.py python3 test.py -f alter/alter_create_exception.py -python3 test.py -f client/change_time_1_1.py -python3 test.py -f client/change_time_1_2.py +# python3 test.py -f client/change_time_1_1.py +# python3 test.py -f client/change_time_1_2.py #======================p4-end=============== From 67d6339f1a4c2072a34d2d6afdc3b654feb774ac Mon Sep 17 00:00:00 2001 From: bryanchang0603 Date: Tue, 15 Jun 2021 12:48:02 +0800 Subject: [PATCH 4/9] [4711] uncomment the test case in fulltest.sh --- tests/pytest/fulltest.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 9ab13914de..9e31e930ad 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -346,6 +346,6 @@ python3 test.py -f alter/alter_keep.py python3 test.py -f alter/alter_cacheLastRow.py python3 ./test.py -f query/querySession.py python3 test.py -f alter/alter_create_exception.py -# python3 test.py -f client/change_time_1_1.py -# python3 test.py -f client/change_time_1_2.py +python3 test.py -f client/change_time_1_1.py +python3 test.py -f client/change_time_1_2.py #======================p4-end=============== From 08a9a1f765043c0c04a4e5baef57ed5e1e0b5239 Mon Sep 17 00:00:00 2001 From: bryanchang0603 Date: Tue, 15 Jun 2021 13:05:30 +0800 Subject: [PATCH 5/9] [TD-4711] trying to solve the problem of ntp not swtich on --- tests/pytest/client/change_time_1_1.py | 3 ++- tests/pytest/client/change_time_1_2.py | 26 +++++++++++++++++++------- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/tests/pytest/client/change_time_1_1.py b/tests/pytest/client/change_time_1_1.py index 32917ab2e4..1744c09132 100644 --- a/tests/pytest/client/change_time_1_1.py +++ b/tests/pytest/client/change_time_1_1.py @@ -56,8 +56,8 @@ class TDTestCase: #move 5 days ahead to 2020/10/25. 4 oldest files should be removed during the new write #leaving 7 data files. - os.system ('timedatectl set-time 2020-10-25') try: + os.system ('timedatectl set-time 2020-10-25') os.system(f"{binPath}taosdemo -f tools/taosdemoAllTest/manual_change_time_1_1_B.json") except BaseException: os.system('sudo timedatectl set-ntp on') @@ -78,6 +78,7 @@ class TDTestCase: def stop(self): os.system('sudo timedatectl set-ntp on') + tdLog.sleep(10) tdSql.close() tdLog.success("alter block manual check finish") diff --git a/tests/pytest/client/change_time_1_2.py b/tests/pytest/client/change_time_1_2.py index d693010be4..d8180136e6 100644 --- a/tests/pytest/client/change_time_1_2.py +++ b/tests/pytest/client/change_time_1_2.py @@ -54,8 +54,14 @@ class TDTestCase: else: tdLog.debug("data file number correct") - tdSql.query('select first(ts) from stb_0') #check the last data in the database - tdSql.checkData(0,0,datetime(2020,10,11,0,0,0,0)) + + try: + tdSql.query('select first(ts) from stb_0') #check the last data in the database + tdSql.checkData(0,0,datetime(2020,10,11,0,0,0,0)) + except BaseException: + os.system('sudo timedatectl set-ntp on') + tdLog.sleep(10) + os.system ('timedatectl set-time 2020-10-25') @@ -63,11 +69,17 @@ class TDTestCase: #4 oldest data file should be removed from tsdb/data #7 data file should be found #vnode at TDinternal/community/sim/dnode1/data/vnode - os.system ('timedatectl set-time 2020-10-25') - tdDnodes.stop(1) - tdDnodes.start(1) - tdSql.query('select first(ts) from stb_0') - tdSql.checkData(0,0,datetime(2020,10,14,8,0,0,0)) #check the last data in the database + + try: + os.system ('timedatectl set-time 2020-10-25') + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query('select first(ts) from stb_0') + tdSql.checkData(0,0,datetime(2020,10,14,8,0,0,0)) #check the last data in the database + except BaseException: + os.system('sudo timedatectl set-ntp on') + tdLog.sleep(10) + os.system('sudo timedatectl set-ntp on') tdLog.sleep(10) commandArray = ['ls', '-l', f'{TDenginePath}/sim/dnode1/data/vnode/vnode2/tsdb/data'] From 7247919b765009af0b77d08f0110438ce03d1ea7 Mon Sep 17 00:00:00 2001 From: bryanchang0603 Date: Tue, 15 Jun 2021 13:08:31 +0800 Subject: [PATCH 6/9] [TD-4711] trying to solve the problem of ntp not swtich on --- tests/pytest/client/change_time_1_2.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/pytest/client/change_time_1_2.py b/tests/pytest/client/change_time_1_2.py index d8180136e6..76400885f8 100644 --- a/tests/pytest/client/change_time_1_2.py +++ b/tests/pytest/client/change_time_1_2.py @@ -62,9 +62,6 @@ class TDTestCase: os.system('sudo timedatectl set-ntp on') tdLog.sleep(10) - - os.system ('timedatectl set-time 2020-10-25') - #moves 5 days ahead to 2020/10/25 and restart taosd #4 oldest data file should be removed from tsdb/data #7 data file should be found From 90cc682dd3167ba99722ddeec186dd8ae12b4ef9 Mon Sep 17 00:00:00 2001 From: bryanchang0603 Date: Tue, 15 Jun 2021 16:27:20 +0800 Subject: [PATCH 7/9] removing from the auto test --- tests/pytest/fulltest.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 9e31e930ad..9ab13914de 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -346,6 +346,6 @@ python3 test.py -f alter/alter_keep.py python3 test.py -f alter/alter_cacheLastRow.py python3 ./test.py -f query/querySession.py python3 test.py -f alter/alter_create_exception.py -python3 test.py -f client/change_time_1_1.py -python3 test.py -f client/change_time_1_2.py +# python3 test.py -f client/change_time_1_1.py +# python3 test.py -f client/change_time_1_2.py #======================p4-end=============== From 545f2127e11edf3ace3d79558127cf2f607565d9 Mon Sep 17 00:00:00 2001 From: bryanchang0603 Date: Tue, 15 Jun 2021 18:09:47 +0800 Subject: [PATCH 8/9] [TD-4366] retry to commit --- tests/pytest/client/change_time_1_1.py | 1 + tests/pytest/client/change_time_1_2.py | 1 + tests/pytest/fulltest.sh | 8 +++----- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/pytest/client/change_time_1_1.py b/tests/pytest/client/change_time_1_1.py index 1744c09132..acdea17fbf 100644 --- a/tests/pytest/client/change_time_1_1.py +++ b/tests/pytest/client/change_time_1_1.py @@ -35,6 +35,7 @@ class TDTestCase: ## change system time to 2020/10/20 os.system('sudo timedatectl set-ntp off') + tdLog.sleep(10) os.system('sudo timedatectl set-time 2020-10-20') #run taosdemo to insert data. one row per second from 2020/10/11 to 2020/10/20 diff --git a/tests/pytest/client/change_time_1_2.py b/tests/pytest/client/change_time_1_2.py index 76400885f8..ec483b00be 100644 --- a/tests/pytest/client/change_time_1_2.py +++ b/tests/pytest/client/change_time_1_2.py @@ -34,6 +34,7 @@ class TDTestCase: ## change system time to 2020/10/20 os.system ('timedatectl set-ntp off') + tdLog.sleep(10) os.system ('timedatectl set-time 2020-10-20') #run taosdemo to insert data. one row per second from 2020/10/11 to 2020/10/20 diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index e8b369e0cd..1a7a0b64ee 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -257,8 +257,8 @@ python3 ./test.py -f client/client.py python3 ./test.py -f client/version.py python3 ./test.py -f client/alterDatabase.py python3 ./test.py -f client/noConnectionErrorTest.py -#python3 test.py -f client/change_time_1_1.py -#python3 test.py -f client/change_time_1_2.py +python3 test.py -f client/change_time_1_1.py +python3 test.py -f client/change_time_1_2.py # Misc python3 testCompress.py @@ -342,10 +342,8 @@ python3 test.py -f tools/taosdemoAllTest/taosdemoTestInsertWithJson.py python3 test.py -f tools/taosdemoAllTest/taosdemoTestQueryWithJson.py python3 ./test.py -f tag_lite/drop_auto_create.py python3 test.py -f insert/insert_before_use_db.py -#python3 test.py -f alter/alter_keep.py +python3 test.py -f alter/alter_keep.py python3 test.py -f alter/alter_cacheLastRow.py python3 ./test.py -f query/querySession.py python3 test.py -f alter/alter_create_exception.py -# python3 test.py -f client/change_time_1_1.py -# python3 test.py -f client/change_time_1_2.py #======================p4-end=============== From 86414f03efaf9f20a58fee3df6cfd6fe3fcb31a3 Mon Sep 17 00:00:00 2001 From: bryanchang0603 Date: Wed, 16 Jun 2021 09:05:46 +0800 Subject: [PATCH 9/9] [TD-4366] removing time alter test case from auto test --- tests/pytest/fulltest.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 1a7a0b64ee..146c47c51d 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -257,8 +257,8 @@ python3 ./test.py -f client/client.py python3 ./test.py -f client/version.py python3 ./test.py -f client/alterDatabase.py python3 ./test.py -f client/noConnectionErrorTest.py -python3 test.py -f client/change_time_1_1.py -python3 test.py -f client/change_time_1_2.py +# python3 test.py -f client/change_time_1_1.py +# python3 test.py -f client/change_time_1_2.py # Misc python3 testCompress.py