From 1bc3c4a44914861cf01c74020735f3a56da6846d Mon Sep 17 00:00:00 2001 From: kailixu Date: Sat, 28 Dec 2024 19:28:07 +0800 Subject: [PATCH] test: add test case for auto compact --- source/libs/parser/inc/sql.y | 21 ++-- source/libs/parser/src/parTranslater.c | 4 + tests/parallel_test/cases.task | 1 + tests/system-test/0-others/compact_auto.py | 118 +++++++++++++++++++ tests/system-test/1-insert/alter_database.py | 25 +--- 5 files changed, 136 insertions(+), 33 deletions(-) create mode 100644 tests/system-test/0-others/compact_auto.py diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 38e93a7ce3..63a6b97bbb 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -290,8 +290,7 @@ db_options(A) ::= db_options(B) ENCRYPT_ALGORITHM NK_STRING(C). db_options(A) ::= db_options(B) DNODES NK_STRING(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_DNODES, &C); } db_options(A) ::= db_options(B) COMPACT_INTERVAL NK_INTEGER (C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_INTERVAL, &C); } db_options(A) ::= db_options(B) COMPACT_INTERVAL NK_VARIABLE(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_INTERVAL, &C); } -db_options(A) ::= db_options(B) COMPACT_TIME_RANGE signed_integer_list(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_TIME_RANGE, C); } -db_options(A) ::= db_options(B) COMPACT_TIME_RANGE signed_variable_list(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_TIME_RANGE, C); } +db_options(A) ::= db_options(B) COMPACT_TIME_RANGE signed_duration_list(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_TIME_RANGE, C); } db_options(A) ::= db_options(B) COMPACT_TIME_OFFSET NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_TIME_OFFSET, &C); } db_options(A) ::= db_options(B) COMPACT_TIME_OFFSET NK_VARIABLE(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_TIME_OFFSET, &C); } @@ -331,8 +330,7 @@ alter_db_option(A) ::= KEEP_TIME_OFFSET NK_INTEGER(B). alter_db_option(A) ::= ENCRYPT_ALGORITHM NK_STRING(B). { A.type = DB_OPTION_ENCRYPT_ALGORITHM; A.val = B; } alter_db_option(A) ::= COMPACT_INTERVAL NK_INTEGER(B). { A.type = DB_OPTION_COMPACT_INTERVAL; A.val = B; } alter_db_option(A) ::= COMPACT_INTERVAL NK_VARIABLE(B). { A.type = DB_OPTION_COMPACT_INTERVAL; A.val = B; } -alter_db_option(A) ::= COMPACT_TIME_RANGE signed_integer_list(B). { A.type = DB_OPTION_COMPACT_TIME_RANGE; A.pList = B; } -alter_db_option(A) ::= COMPACT_TIME_RANGE signed_variable_list(B). { A.type = DB_OPTION_COMPACT_TIME_RANGE; A.pList = B; } +alter_db_option(A) ::= COMPACT_TIME_RANGE signed_duration_list(B). { A.type = DB_OPTION_COMPACT_TIME_RANGE; A.pList = B; } alter_db_option(A) ::= COMPACT_TIME_OFFSET NK_INTEGER(B). { A.type = DB_OPTION_COMPACT_TIME_OFFSET; A.val = B; } alter_db_option(A) ::= COMPACT_TIME_OFFSET NK_VARIABLE(B). { A.type = DB_OPTION_COMPACT_TIME_OFFSET; A.val = B; } @@ -341,20 +339,17 @@ alter_db_option(A) ::= COMPACT_TIME_OFFSET NK_VARIABLE(B). integer_list(A) ::= NK_INTEGER(B). { A = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B)); } integer_list(A) ::= integer_list(B) NK_COMMA NK_INTEGER(C). { A = addNodeToList(pCxt, B, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C)); } -%type signed_integer_list { SNodeList* } -%destructor signed_integer_list { nodesDestroyList($$); } -signed_integer_list(A) ::= signed_integer(B). { A = createNodeList(pCxt, B); } -signed_integer_list(A) ::= signed_integer_list(B) NK_COMMA signed_integer(C). { A = addNodeToList(pCxt, B, C); } - %type variable_list { SNodeList* } %destructor variable_list { nodesDestroyList($$); } variable_list(A) ::= NK_VARIABLE(B). { A = createNodeList(pCxt, createDurationValueNode(pCxt, &B)); } variable_list(A) ::= variable_list(B) NK_COMMA NK_VARIABLE(C). { A = addNodeToList(pCxt, B, createDurationValueNode(pCxt, &C)); } -%type signed_variable_list { SNodeList* } -%destructor signed_variable_list { nodesDestroyList($$); } -signed_variable_list(A) ::= signed_variable(B). { A = createNodeList(pCxt, releaseRawExprNode(pCxt, B)); } -signed_variable_list(A) ::= signed_variable_list(B) NK_COMMA signed_variable(C). { A = addNodeToList(pCxt, B, releaseRawExprNode(pCxt, C)); } +%type signed_duration_list { SNodeList* } +%destructor signed_duration_list { nodesDestroyList($$); } +signed_duration_list(A) ::= signed_variable(B). { A = createNodeList(pCxt, releaseRawExprNode(pCxt, B)); } +signed_duration_list(A) ::= signed_integer(B). { A = createNodeList(pCxt, B); } +signed_duration_list(A) ::= signed_duration_list(B) NK_COMMA signed_integer(C). { A = addNodeToList(pCxt, B, C); } +signed_duration_list(A) ::= signed_duration_list(B) NK_COMMA signed_variable(C). { A = addNodeToList(pCxt, B, releaseRawExprNode(pCxt, C)); } %type retention_list { SNodeList* } %destructor retention_list { nodesDestroyList($$); } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 2e711efa89..abd0efa2d8 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -8252,6 +8252,10 @@ static int32_t checkDbCompactTimeRangeOption(STranslateContext* pCxt, const char pOptions->compactStartTime = getBigintFromValueNode(pStart); pOptions->compactEndTime = getBigintFromValueNode(pEnd); + if (pOptions->compactStartTime == 0 && pOptions->compactEndTime == 0) { + return TSDB_CODE_SUCCESS; + } + if (pOptions->compactStartTime >= pOptions->compactEndTime) { return generateSyntaxErrMsgExt( &pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 38dd080ef5..786f1f6dde 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -414,6 +414,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/persisit_config.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/qmemCtrl.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/compact_vgroups.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/compact_auto.py ,,n,system-test,python3 ./test.py -f 0-others/dumpsdb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/compact.py -N 3 diff --git a/tests/system-test/0-others/compact_auto.py b/tests/system-test/0-others/compact_auto.py new file mode 100644 index 0000000000..615c2bba5b --- /dev/null +++ b/tests/system-test/0-others/compact_auto.py @@ -0,0 +1,118 @@ +import taos +import sys +import time +import socket +import os +import threading +import psutil +import platform +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + self.default_compact_options = [ "0d", "0d,0d", "0h"] + self.compact_options = [["db00", "0m", "-0d,0", "0", "0d", "0d,0d", "0h"], + ["db01", "2880m", "-61d,-60", "0", "2d", "-61d,-60d", "0h"], + ["db02", "48h", "-87840m,-60", "1h", "2d", "-61d,-60d", "1h"], + ["db03", "2d", "-87840m,-1440h", "12", "2d", "-61d,-60d", "12h"], + ["db04", "2", "-61,-1440h", "23h", "2d", "-61d,-60d", "23h"], + ] + + def create_db_compact(self): + for item in self.compact_options: + tdSql.execute(f'create database {item[0]} compact_interval {item[1]} compact_time_range {item[2]} compact_time_offset {item[3]}') + tdSql.query(f'select * from information_schema.ins_databases where name = "{item[0]}"') + tdSql.checkEqual(tdSql.queryResult[0][34], item[4]) + tdSql.checkEqual(tdSql.queryResult[0][35], item[5]) + tdSql.checkEqual(tdSql.queryResult[0][36], item[6]) + tdSql.query(f'show create database {item[0]}') + tdSql.checkEqual(tdSql.queryResult[0][0], item[0]) + tdSql.checkEqual(True, f'COMPACT_INTERVAL {item[4]} COMPACT_TIME_RANGE {item[5]} COMPACT_TIME_OFFSET {item[6]}' in tdSql.queryResult[0][1]) + tdSql.execute(f'drop database {item[0]}') + + def checkShowCreateWithTimeout(self, db, expectResult, timeout=30): + result = False + for i in range(timeout): + tdSql.query(f'show create database `%s`' %(db)) + tdSql.checkEqual(tdSql.queryResult[0][0], db) + if expectResult in tdSql.queryResult[0][1]: + result = True + break + time.sleep(1) + if result == False: + raise Exception(f"Unexpected result of 'show create database `{db}`':{tdSql.queryResult[0][1]}") + + def alter_db_compact(self): + for item in self.compact_options: + tdSql.execute(f'create database {item[0]}') + tdSql.query(f'select * from information_schema.ins_databases where name = "{item[0]}"') + tdSql.checkEqual(tdSql.queryResult[0][34], self.default_compact_options[0]) + tdSql.checkEqual(tdSql.queryResult[0][35], self.default_compact_options[1]) + tdSql.checkEqual(tdSql.queryResult[0][36], self.default_compact_options[2]) + tdSql.query(f'show create database {item[0]}') + tdSql.checkEqual(tdSql.queryResult[0][0], item[0]) + tdSql.checkEqual(True, f'COMPACT_INTERVAL {self.default_compact_options[0]} COMPACT_TIME_RANGE {self.default_compact_options[1]} COMPACT_TIME_OFFSET {self.default_compact_options[2]}' in tdSql.queryResult[0][1]) + tdSql.execute(f'alter database {item[0]} compact_interval {item[1]} compact_time_range {item[2]} compact_time_offset {item[3]}') + tdSql.query(f'select * from information_schema.ins_databases where name = "{item[0]}"') + tdSql.checkEqual(tdSql.queryResult[0][34], item[4]) + tdSql.checkEqual(tdSql.queryResult[0][35], item[5]) + tdSql.checkEqual(tdSql.queryResult[0][36], item[6]) + for item in self.compact_options: + self.checkShowCreateWithTimeout(item[0], f'COMPACT_INTERVAL {item[4]} COMPACT_TIME_RANGE {item[5]} COMPACT_TIME_OFFSET {item[6]}') + tdSql.execute(f'drop database {item[0]}') + + def compact_error(self): + compact_err_list = [["compact_time_range 86400m,61d", "Invalid option compact_time_range: 86400m, start time should be in range: [-5256000m, -14400m]"], + ["compact_time_range 60,61", "Invalid option compact_time_range: 86400m, start time should be in range: [-5256000m, -14400m]"], + ["compact_time_range 60d,61d", "Invalid option compact_time_range: 86400m, start time should be in range: [-5256000m, -14400m]"], + ["compact_time_range -60,-60", "Invalid option compact_time_range: -86400m,-86400m, start time should be less than end time"], + ["compact_time_range -60,-1440h", "Invalid option compact_time_range: -86400m,-86400m, start time should be less than end time"], + ["compact_time_range -60d,-61d", "Invalid option compact_time_range: -86400m,-87840m, start time should be less than end time"], + ["compact_time_range -5256001m,-1", "Invalid option compact_time_range: -5256001m, start time should be in range: [-5256000m, -14400m]"], + ["compact_time_range -60d,-1", "Invalid option compact_time_range: -1440m, end time should be in range: [-5256000m, -14400m]"], + ["compact_interval 24h compact_time_range -60,61", "Invalid option compact_time_range: 87840m, end time should be in range: [-5256000m, -14400m]"], + ["compact_interval 100 compact_time_range -60d,61d", "Invalid option compact_time_range: 87840m, end time should be in range: [-5256000m, -14400m]"], + ["compact_time_range -60d,87840m", "Invalid option compact_time_range: 87840m, end time should be in range: [-5256000m, -14400m]"], + ["compact_interval 10m compact_time_range -120d,-14400m compact_time_offset -1", "syntax error near"], + ["compact_time_range -100,-99d compact_interval -1", "syntax error near"], + ["compact_time_range 0", "Invalid option compact_time_range, should have 2 value"], + ["compact_time_range -100", "Invalid option compact_time_range, should have 2 value"], + ["compact_time_range -100,-90,-80", "Invalid option compact_time_range, should have 2 value"], + ["compact_time_range -100;-90", "Invalid option compact_time_range, should have 2 value"], + ["compact_time_range -100:-90", "syntax error near"], + ["compact_time_range -100 -90", "syntax error near"], + ["compact_interval 1m", "Invalid option compact_interval: 1m, valid range: [10m, 5256000m]"], + ["compact_interval 5256001m", "Invalid option compact_interval: 5256001m, valid range: [10m, 5256000m]"], + ["compact_interval 3651", "Invalid option compact_interval: 5257440m, valid range: [10m, 5256000m]"], + ["compact_interval -1", "syntax error near"], + ["compact_time_offset -1", "syntax error near"], + ["compact_time_offset 1d", "Invalid option compact_time_offset unit: d, only h allowed"], + ["compact_time_offset 24", "Invalid option compact_time_offset: 24h, valid range: [0h, 23h]"], + ["compact_time_offset 24h", "Invalid option compact_time_offset: 24h, valid range: [0h, 23h]"], + ["compact_time_offset 1d", "Invalid option compact_time_offset unit: d, only h allowed"], + ["compact_interval 10m compact_time_range -120d,-60 compact_time_offset 1d", "Invalid option compact_time_offset unit: d, only h allowed"], + ] + tdSql.execute('create database if not exists db') + for item in compact_err_list: + tdSql.error(f"create database db {item[0]}", expectErrInfo=item[1], fullMatched=False) + tdSql.error(f"alter database db {item[0]}", expectErrInfo=item[1], fullMatched=False) + + def run(self): + self.create_db_compact() + self.alter_db_compact() + self.compact_error() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/1-insert/alter_database.py b/tests/system-test/1-insert/alter_database.py index c20ace6ff4..55b636f5ea 100644 --- a/tests/system-test/1-insert/alter_database.py +++ b/tests/system-test/1-insert/alter_database.py @@ -75,28 +75,14 @@ class TDTestCase: tdSql.checkEqual("Invalid option encrypt_algorithm: none ", tdSql.error('alter database db encrypt_algorithm \'none \'')) tdSql.execute('drop database db') - def alter_compact(self): - tdSql.execute('create database db') - tdSql.execute('alter database db compact_time_offset 2') - tdSql.execute('alter database db compact_time_offset 3h') - tdSql.error('create database db1 compact_time_range 60,61', expectErrInfo="Invalid option compact_time_range: 86400m, start_time should be in range: [-5256000m, -14400m]", fullMatched=False) - tdSql.error('create database db1 compact_time_offset -1', expectErrInfo="syntax error near", fullMatched=False) - tdSql.error('create database d3 compact_interval 1m; ', expectErrInfo="Invalid option compact_interval: 1m, valid range: [10m, 5256000m]", fullMatched=False) - tdSql.error('alter database db compact_time_offset -1', expectErrInfo="syntax error near", fullMatched=False) - tdSql.error('alter database db compact_time_offset 24', expectErrInfo="Invalid option compact_time_offset: 24h, valid range: [0h, 23h]", fullMatched=False) - tdSql.error('alter database db compact_time_offset 24h', expectErrInfo="Invalid option compact_time_offset: 24h, valid range: [0h, 23h]", fullMatched=False) - tdSql.error('alter database db compact_time_offset 1d', expectErrInfo="Invalid option compact_time_offset unit: d, only h allowed", fullMatched=False) - - - def alter_same_options(self): tdSql.execute('drop database if exists db') tdSql.execute('create database db') tdSql.query('select * from information_schema.ins_databases where name = "db"') db_options_items = ["replica","keep","buffer","pages","minrows","cachemodel","cachesize","wal_level","wal_fsync_period", - "wal_retention_period","wal_retention_size","stt_trigger"] - db_options_result_idx = [4,7,8,10,11,18,19,20,21,22,23,24] + "wal_retention_period","wal_retention_size","stt_trigger", "compact_interval", "compact_time_range", "compact_time_offset"] + db_options_result_idx = [4,7,8,10,11,18,19,20,21,22,23,24,34,35,36] self.option_result = [] for idx in db_options_result_idx: @@ -115,11 +101,10 @@ class TDTestCase: def run(self): - self.alter_buffer() - self.alter_pages() - self.alter_encrypt_alrogithm() + # self.alter_buffer() + # self.alter_pages() + # self.alter_encrypt_alrogithm() self.alter_same_options() - self.alter_compact() def stop(self): tdSql.close()